From f04433e9ad5b6521ddd4c150c5dc6d70d2b2a590 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 27 Aug 2026 15:24:04 -0700 Subject: [PATCH] eliminate snapshot detail query overhead --- archivebox/core/models.py | 27 ++++++++++++++------ archivebox/core/settings.py | 2 +- archivebox/core/views.py | 28 +++++---------------- archivebox/misc/db.py | 19 +++++++++++--- archivebox/templates/core/snapshot.html | 8 +++--- archivebox/tests/test_ui_public_snapshot.py | 13 ++++++++-- 6 files changed, 56 insertions(+), 41 deletions(-) diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 24322e28..8c3120c2 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -2417,15 +2417,26 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW if stored_title: return stored_title - title_results = ( - self.archiveresult_set.filter( - plugin="title", - status=ArchiveResult.StatusChoices.SUCCEEDED, + loaded_results = self.__dict__.get("_admin_archiveresults") + if loaded_results is None: + title_results = ( + self.archiveresult_set.filter( + plugin="title", + status=ArchiveResult.StatusChoices.SUCCEEDED, + ) + .exclude(output_str="") + .order_by("-start_ts", "-end_ts", "-created_at") + .only("output_str") ) - .exclude(output_str="") - .order_by("-start_ts", "-end_ts", "-created_at") - ) - for title_result in title_results.only("output_str"): + else: + title_results = reversed( + [ + result + for result in loaded_results + if result.plugin == "title" and result.status == ArchiveResult.StatusChoices.SUCCEEDED and result.output_str + ], + ) + for title_result in title_results: result_title = self._normalize_title_candidate(title_result.output_str, snapshot_url=self.url) if result_title: return result_title diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index d54be8a5..e09d2cb7 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -250,7 +250,7 @@ SQLITE_CONNECTION_OPTIONS = { }, } -if is_postgres(): +if is_postgres(CONFIG): DATABASES = { "default": { "ENGINE": "django.db.backends.postgresql", diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 1c181fe4..6d90bb11 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -273,7 +273,7 @@ class SnapshotView(View): # render static html index from filesystem archive//index.html @staticmethod - def find_snapshots_for_url(path: str): + def find_snapshots_for_url(path: str, *, allow_fallback: bool = True): """Return a queryset of snapshots matching a URL-ish path. URL only — never tries ID matching. Use ``find_snapshots_for_id`` separately if you also want to match by snapshot UUID. @@ -299,7 +299,7 @@ class SnapshotView(View): if path.startswith(("http://", "https://")): # exact url match (indexed) — fastest path qs = Snapshot.objects.filter(_fragmentless_url_query(path)) - if qs.exists(): + if not allow_fallback or qs.exists(): return qs normalized = normalized.split("://", 1)[1] @@ -359,18 +359,6 @@ class SnapshotView(View): if current is None or (output.get("size") or 0) > (current.get("size") or 0): archiveresults[output["name"]] = output hash_index = snapshot.hashes_index - accounted_entries: set[str] = set() - for output in outputs: - output_name = output.get("name") or "" - if output_name: - accounted_entries.add(output_name) - output_path = output.get("path") or "" - if not output_path: - continue - parts = Path(output_path).parts - if parts: - accounted_entries.add(parts[0]) - loose_items, failed_items = snapshot.get_detail_page_auxiliary_items( outputs, hidden_card_plugins=hidden_card_plugins, @@ -393,14 +381,10 @@ class SnapshotView(View): best_result = archiveresults[result_type] break - related_snapshots_qs = ( - SnapshotView.find_snapshots_for_url(snapshot.url) - .select_related("crawl", "crawl__created_by") - .annotate( - num_outputs_cached=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED), - num_failures_cached=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED), - ) - ) + related_snapshots_qs = SnapshotView.find_snapshots_for_url( + snapshot.url, + allow_fallback=False, + ).only("id", "url", "bookmarked_at", "created_at", "downloaded_at", "output_size") related_snapshots = list( related_snapshots_qs.exclude(id=snapshot.id).order_by("-bookmarked_at", "-created_at", "-timestamp")[:25], ) diff --git a/archivebox/misc/db.py b/archivebox/misc/db.py index 3b8f4e1e..1c07755a 100644 --- a/archivebox/misc/db.py +++ b/archivebox/misc/db.py @@ -31,11 +31,22 @@ from archivebox.misc.util import enforce_types # ``CONSTANTS.DATABASE_FILE`` directly. -def is_postgres() -> bool: - """True if DATABASE_ENGINE selects postgres (sqlite is the default).""" - from archivebox.config.common import get_config +_IS_POSTGRES: bool | None = None - return (get_config().DATABASE_ENGINE or "sqlite").strip().lower().startswith("postgres") + +def is_postgres(config: Any | None = None) -> bool: + """True if this process started with the PostgreSQL database backend.""" + global _IS_POSTGRES + + if _IS_POSTGRES is None: + if config is None: + from archivebox.config.common import get_config + + config = get_config() + + _IS_POSTGRES = (config.DATABASE_ENGINE or "sqlite").strip().lower().startswith("postgres") + + return _IS_POSTGRES def postgres_db_params() -> dict[str, str]: diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html index d41b165c..cbf6449e 100644 --- a/archivebox/templates/core/snapshot.html +++ b/archivebox/templates/core/snapshot.html @@ -1630,15 +1630,15 @@ {{ entry.year }}
{% for snap in entry.snapshots %} - - {{ snap.bookmarked_at|default:snap.created_at|default:snap.downloaded_at|date:"Y-m-d H:i:s" }}   📁 {{ snap.num_outputs }} + + {{ snap.bookmarked_at|default:snap.created_at|default:snap.downloaded_at|date:"Y-m-d H:i:s" }}   💾 {{ snap.output_size|filesizeformat }} {% endfor %}
{% else %}
- + {{ entry.year }}
@@ -1659,7 +1659,7 @@
{% for snap in related_snapshots %} - + {{ snap.bookmarked_at|default:snap.created_at|default:snap.downloaded_at|date:"Y-m-d H:i:s" }}   💾 {{ snap.output_size|filesizeformat }} {% endfor %} diff --git a/archivebox/tests/test_ui_public_snapshot.py b/archivebox/tests/test_ui_public_snapshot.py index 5c6438d5..6df5a159 100644 --- a/archivebox/tests/test_ui_public_snapshot.py +++ b/archivebox/tests/test_ui_public_snapshot.py @@ -219,7 +219,7 @@ def test_archive_url_with_multiple_snapshots_redirects_to_latest_snapshot(client first_crawl = Crawl.objects.create(urls=url, created_by=admin_user, config={"PERMISSIONS": "public"}) second_crawl = Crawl.objects.create(urls=url, created_by=admin_user, config={"PERMISSIONS": "public"}) first = Snapshot.objects.create(url=url, title="First copy", crawl=first_crawl, status=Snapshot.StatusChoices.SEALED) - second = Snapshot.objects.create(url=url, title="Second copy", crawl=second_crawl, status=Snapshot.StatusChoices.SEALED) + second = Snapshot.objects.create(url=url, title="", crawl=second_crawl, status=Snapshot.StatusChoices.SEALED) for plugin, output_size in (("screenshot", 1536), ("singlefile", 2560)): ArchiveResult.objects.create( snapshot=first, @@ -229,16 +229,24 @@ def test_archive_url_with_multiple_snapshots_redirects_to_latest_snapshot(client output_size=output_size, ) ArchiveResult.refresh_snapshot_output_sizes({first.id}) + ArchiveResult.objects.create( + snapshot=second, + plugin="title", + hook_name="on_Snapshot__10_title.py", + status=ArchiveResult.StatusChoices.SUCCEEDED, + output_str="Resolved second copy", + ) with CaptureQueriesContext(connection) as captured_queries: response = client.get(f"/archive/{url}", HTTP_HOST=WEB_TEST_HOST, follow=True) - assert len(captured_queries) <= 8 + assert len(captured_queries) <= 7 assert ( f"/snapshot/{second.id.hex}/index.html" in response.redirect_chain[0][0] or f"snap-{second.id.hex[-12:]}" in response.redirect_chain[0][0] ) assert response.status_code == 200 + assert b"Resolved second copy" in response.content assert b"Click to see other snapshots for this URL" in response.content assert re.search(rb'snapshot-count-badge">\s*1\s*', response.content) chooser = re.search( @@ -249,6 +257,7 @@ def test_archive_url_with_multiple_snapshots_redirects_to_latest_snapshot(client assert chooser assert b"4.0\xc2\xa0KB" in chooser.group() assert b"\xf0\x9f\x93\x81 2" not in chooser.group() + assert b"\xf0\x9f\x93\x81 2" not in response.content def _login_admin_session_over_http(port: int, host: str) -> requests.Session: