eliminate snapshot detail query overhead

archivebox/core/models.py

@@ -2417,15 +2417,26 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         if stored_title:
             return stored_title
 
-        title_results = (
-            self.archiveresult_set.filter(
-                plugin="title",
-                status=ArchiveResult.StatusChoices.SUCCEEDED,
+        loaded_results = self.__dict__.get("_admin_archiveresults")
+        if loaded_results is None:
+            title_results = (
+                self.archiveresult_set.filter(
+                    plugin="title",
+                    status=ArchiveResult.StatusChoices.SUCCEEDED,
+                )
+                .exclude(output_str="")
+                .order_by("-start_ts", "-end_ts", "-created_at")
+                .only("output_str")
             )
-            .exclude(output_str="")
-            .order_by("-start_ts", "-end_ts", "-created_at")
-        )
-        for title_result in title_results.only("output_str"):
+        else:
+            title_results = reversed(
+                [
+                    result
+                    for result in loaded_results
+                    if result.plugin == "title" and result.status == ArchiveResult.StatusChoices.SUCCEEDED and result.output_str
+                ],
+            )
+        for title_result in title_results:
             result_title = self._normalize_title_candidate(title_result.output_str, snapshot_url=self.url)
             if result_title:
                 return result_title

archivebox/core/settings.py

@@ -250,7 +250,7 @@ SQLITE_CONNECTION_OPTIONS = {
     },
 }
 
-if is_postgres():
+if is_postgres(CONFIG):
     DATABASES = {
         "default": {
             "ENGINE": "django.db.backends.postgresql",

archivebox/core/views.py

@@ -273,7 +273,7 @@ class SnapshotView(View):
     # render static html index from filesystem archive/<timestamp>/index.html
 
     @staticmethod
-    def find_snapshots_for_url(path: str):
+    def find_snapshots_for_url(path: str, *, allow_fallback: bool = True):
         """Return a queryset of snapshots matching a URL-ish path. URL only — never tries ID matching.
 
         Use ``find_snapshots_for_id`` separately if you also want to match by snapshot UUID.
@@ -299,7 +299,7 @@ class SnapshotView(View):
         if path.startswith(("http://", "https://")):
             # exact url match (indexed) — fastest path
             qs = Snapshot.objects.filter(_fragmentless_url_query(path))
-            if qs.exists():
+            if not allow_fallback or qs.exists():
                 return qs
             normalized = normalized.split("://", 1)[1]
 
@@ -359,18 +359,6 @@ class SnapshotView(View):
             if current is None or (output.get("size") or 0) > (current.get("size") or 0):
                 archiveresults[output["name"]] = output
         hash_index = snapshot.hashes_index
-        accounted_entries: set[str] = set()
-        for output in outputs:
-            output_name = output.get("name") or ""
-            if output_name:
-                accounted_entries.add(output_name)
-            output_path = output.get("path") or ""
-            if not output_path:
-                continue
-            parts = Path(output_path).parts
-            if parts:
-                accounted_entries.add(parts[0])
-
         loose_items, failed_items = snapshot.get_detail_page_auxiliary_items(
             outputs,
             hidden_card_plugins=hidden_card_plugins,
@@ -393,14 +381,10 @@ class SnapshotView(View):
                 best_result = archiveresults[result_type]
                 break
 
-        related_snapshots_qs = (
-            SnapshotView.find_snapshots_for_url(snapshot.url)
-            .select_related("crawl", "crawl__created_by")
-            .annotate(
-                num_outputs_cached=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
-                num_failures_cached=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
-            )
-        )
+        related_snapshots_qs = SnapshotView.find_snapshots_for_url(
+            snapshot.url,
+            allow_fallback=False,
+        ).only("id", "url", "bookmarked_at", "created_at", "downloaded_at", "output_size")
         related_snapshots = list(
             related_snapshots_qs.exclude(id=snapshot.id).order_by("-bookmarked_at", "-created_at", "-timestamp")[:25],
         )

archivebox/misc/db.py

@@ -31,11 +31,22 @@ from archivebox.misc.util import enforce_types
 # ``CONSTANTS.DATABASE_FILE`` directly.
 
 
-def is_postgres() -> bool:
-    """True if DATABASE_ENGINE selects postgres (sqlite is the default)."""
-    from archivebox.config.common import get_config
+_IS_POSTGRES: bool | None = None
 
-    return (get_config().DATABASE_ENGINE or "sqlite").strip().lower().startswith("postgres")
+
+def is_postgres(config: Any | None = None) -> bool:
+    """True if this process started with the PostgreSQL database backend."""
+    global _IS_POSTGRES
+
+    if _IS_POSTGRES is None:
+        if config is None:
+            from archivebox.config.common import get_config
+
+            config = get_config()
+
+        _IS_POSTGRES = (config.DATABASE_ENGINE or "sqlite").strip().lower().startswith("postgres")
+
+    return _IS_POSTGRES
 
 
 def postgres_db_params() -> dict[str, str]:

archivebox/templates/core/snapshot.html

@@ -1630,15 +1630,15 @@
                                         <summary class="badge badge-default">{{ entry.year }}</summary>
                                         <div class="snapshot-variants-list">
                                             {% for snap in entry.snapshots %}
-                                                <a href="{% web_base_url %}/{{ snap.archive_path }}/index.html" title="{{ snap.url }}">
-                                                    {{ snap.bookmarked_at|default:snap.created_at|default:snap.downloaded_at|date:"Y-m-d H:i:s" }} &nbsp; 📁 {{ snap.num_outputs }}
+                                                <a href="{% snapshot_base_url snap %}/index.html" title="{{ snap.url }}">
+                                                    {{ snap.bookmarked_at|default:snap.created_at|default:snap.downloaded_at|date:"Y-m-d H:i:s" }} &nbsp; 💾 {{ snap.output_size|filesizeformat }}
                                                 </a>
                                             {% endfor %}
                                         </div>
                                     </details>
                                 {% else %}
                                     <div class="badge badge-default">
-                                        <a href="{% web_base_url %}/{{ entry.latest.archive_path }}/index.html" title="{{ entry.latest.url }}">
+                                        <a href="{% snapshot_base_url entry.latest %}/index.html" title="{{ entry.latest.url }}">
                                             {{ entry.year }}
                                         </a>
                                     </div>
@@ -1659,7 +1659,7 @@
                                 </summary>
                                 <div class="snapshot-variants-list">
                                     {% for snap in related_snapshots %}
-                                        <a href="{% web_base_url %}/{{ snap.archive_path }}/index.html" title="{{ snap.url }}">
+                                        <a href="{% snapshot_base_url snap %}/index.html" title="{{ snap.url }}">
                                             {{ snap.bookmarked_at|default:snap.created_at|default:snap.downloaded_at|date:"Y-m-d H:i:s" }} &nbsp; 💾 {{ snap.output_size|filesizeformat }}
                                         </a>
                                     {% endfor %}

archivebox/tests/test_ui_public_snapshot.py

@@ -219,7 +219,7 @@ def test_archive_url_with_multiple_snapshots_redirects_to_latest_snapshot(client
     first_crawl = Crawl.objects.create(urls=url, created_by=admin_user, config={"PERMISSIONS": "public"})
     second_crawl = Crawl.objects.create(urls=url, created_by=admin_user, config={"PERMISSIONS": "public"})
     first = Snapshot.objects.create(url=url, title="First copy", crawl=first_crawl, status=Snapshot.StatusChoices.SEALED)
-    second = Snapshot.objects.create(url=url, title="Second copy", crawl=second_crawl, status=Snapshot.StatusChoices.SEALED)
+    second = Snapshot.objects.create(url=url, title="", crawl=second_crawl, status=Snapshot.StatusChoices.SEALED)
     for plugin, output_size in (("screenshot", 1536), ("singlefile", 2560)):
         ArchiveResult.objects.create(
             snapshot=first,
@@ -229,16 +229,24 @@ def test_archive_url_with_multiple_snapshots_redirects_to_latest_snapshot(client
             output_size=output_size,
         )
     ArchiveResult.refresh_snapshot_output_sizes({first.id})
+    ArchiveResult.objects.create(
+        snapshot=second,
+        plugin="title",
+        hook_name="on_Snapshot__10_title.py",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_str="Resolved second copy",
+    )
 
     with CaptureQueriesContext(connection) as captured_queries:
         response = client.get(f"/archive/{url}", HTTP_HOST=WEB_TEST_HOST, follow=True)
-    assert len(captured_queries) <= 8
+    assert len(captured_queries) <= 7
 
     assert (
         f"/snapshot/{second.id.hex}/index.html" in response.redirect_chain[0][0]
         or f"snap-{second.id.hex[-12:]}" in response.redirect_chain[0][0]
     )
     assert response.status_code == 200
+    assert b"Resolved second copy" in response.content
     assert b"Click to see other snapshots for this URL" in response.content
     assert re.search(rb'snapshot-count-badge">\s*1\s*</span>', response.content)
     chooser = re.search(
@@ -249,6 +257,7 @@ def test_archive_url_with_multiple_snapshots_redirects_to_latest_snapshot(client
     assert chooser
     assert b"4.0\xc2\xa0KB" in chooser.group()
     assert b"\xf0\x9f\x93\x81 2" not in chooser.group()
+    assert b"\xf0\x9f\x93\x81 2" not in response.content
 
 
 def _login_admin_session_over_http(port: int, host: str) -> requests.Session: