diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 414ce637..24322e28 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -3215,7 +3215,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW latest[plugin] = result.embed_path() if result else None return latest - def discover_outputs(self, include_filesystem_fallback: bool = True) -> list[dict]: + def discover_outputs( + self, + include_filesystem_fallback: bool = True, + archive_results: list["ArchiveResult"] | None = None, + ) -> list[dict]: """Discover output files from ArchiveResults and filesystem.""" from archivebox.misc.util import ts_to_date_str @@ -3235,7 +3239,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW return lower.endswith(text_exts) hashes_index = self.hashes_index if include_filesystem_fallback else {} - for result in self.archiveresult_set.all().order_by("start_ts"): + results = archive_results if archive_results is not None else self.archiveresult_set.all().order_by("start_ts") + for result in results: output_file_map = result.output_file_map() embed_path = result.embed_path_db(output_file_map=output_file_map) if not embed_path and include_filesystem_fallback: @@ -3528,6 +3533,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW self, outputs: list[dict] | None = None, hidden_card_plugins: set[str] | None = None, + archive_results: list["ArchiveResult"] | None = None, ) -> tuple[list[dict[str, object]], list[dict[str, object]]]: if outputs is None: outputs = self.discover_outputs(include_filesystem_fallback=True) @@ -3571,7 +3577,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW ArchiveResult = self.archiveresult_set.model failed_items: list[dict[str, object]] = [] seen_failed: set[str] = set() - for result in self.archiveresult_set.all().order_by("start_ts"): + results = archive_results if archive_results is not None else self.archiveresult_set.all().order_by("start_ts") + for result in results: if result.status != ArchiveResult.StatusChoices.FAILED: continue root = str(result.plugin or "").strip() diff --git a/archivebox/core/views.py b/archivebox/core/views.py index b99d5e8e..1c181fe4 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -113,18 +113,20 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None: if not lookup: return None + snapshots = Snapshot.objects.select_related("crawl", "crawl__created_by") + if len(lookup) == 12 and "-" not in lookup: - return Snapshot.objects.filter(id__endswith=lookup).order_by("-created_at", "-downloaded_at").first() + return snapshots.filter(id__endswith=lookup).order_by("-created_at", "-downloaded_at").first() try: - return Snapshot.objects.get(pk=lookup) + return snapshots.get(pk=lookup) except Snapshot.DoesNotExist: try: - return Snapshot.objects.get(id__startswith=lookup) + return snapshots.get(id__startswith=lookup) except Snapshot.DoesNotExist: return None except Snapshot.MultipleObjectsReturned: - return Snapshot.objects.filter(id__startswith=lookup).first() + return snapshots.filter(id__startswith=lookup).first() def _admin_login_redirect_or_forbidden(request: HttpRequest): @@ -339,10 +341,16 @@ class SnapshotView(View): runtime_config = get_request_config(request) snapshot._runtime_config = runtime_config snapshot_permissions = get_snapshot_permissions(snapshot) + archive_results = list(snapshot.archiveresult_set.all().order_by("start_ts")) + tags = list(snapshot.tags.all()) + snapshot.__dict__["_admin_archiveresults"] = archive_results + snapshot.__dict__["_tags_str_cached"] = ",".join(sorted(tag.name for tag in tags)) + snapshot.__dict__["num_outputs_cached"] = sum(result.status == ArchiveResult.StatusChoices.SUCCEEDED for result in archive_results) + snapshot.__dict__["num_failures_cached"] = sum(result.status == ArchiveResult.StatusChoices.FAILED for result in archive_results) hidden_card_plugins = {"archivedotorg", "favicon", "title"} outputs = [ out - for out in snapshot.discover_outputs(include_filesystem_fallback=True) + for out in snapshot.discover_outputs(include_filesystem_fallback=True, archive_results=archive_results) if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins ] archiveresults = {} @@ -363,7 +371,11 @@ class SnapshotView(View): if parts: accounted_entries.add(parts[0]) - loose_items, failed_items = snapshot.get_detail_page_auxiliary_items(outputs, hidden_card_plugins=hidden_card_plugins) + loose_items, failed_items = snapshot.get_detail_page_auxiliary_items( + outputs, + hidden_card_plugins=hidden_card_plugins, + archive_results=archive_results, + ) preview_priority = [ "singlefile", "screenshot", @@ -432,6 +444,7 @@ class SnapshotView(View): compact_outputs = [out for out in ordered_outputs if out.get("is_compact") or out.get("is_metadata")] tag_widget = TagEditorWidget() output_size = sum(int(out.get("size") or 0) for out in ordered_outputs) + archive_dates = [result.start_ts for result in archive_results if result.start_ts] has_outputs = bool(ordered_outputs) is_archived = has_outputs or snapshot.status == Snapshot.StatusChoices.SEALED snapshot_status = str(snapshot.status or "").lower() @@ -473,7 +486,7 @@ class SnapshotView(View): "downloaded_datestr": snapshot.downloaded_datestr, "num_outputs": snapshot.num_outputs, "num_failures": snapshot.num_failures, - "oldest_archive_date": ts_to_date_str(snapshot.oldest_archive_date), + "oldest_archive_date": ts_to_date_str(min(archive_dates) if archive_dates else None), "warc_path": warc_path, "archiveresults": [*non_compact_outputs, *compact_outputs], "best_result": best_result, @@ -483,7 +496,7 @@ class SnapshotView(View): "related_years": related_years, "loose_items": loose_items, "failed_items": failed_items, - "title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in snapshot.tags.all().order_by("name")], + "title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in sorted(tags, key=lambda tag: tag.name)], } return render(template_name="core/snapshot.html", request=request, context=context) diff --git a/archivebox/tests/test_ui_public_snapshot.py b/archivebox/tests/test_ui_public_snapshot.py index c2bb7873..5c6438d5 100644 --- a/archivebox/tests/test_ui_public_snapshot.py +++ b/archivebox/tests/test_ui_public_snapshot.py @@ -210,6 +210,8 @@ def _create_public_snapshot_with_cli(data_dir, url: str) -> str: @override_settings(PUBLIC_INDEX=True) def test_archive_url_with_multiple_snapshots_redirects_to_latest_snapshot(client, admin_user): + from django.db import connection + from django.test.utils import CaptureQueriesContext from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl @@ -228,7 +230,9 @@ def test_archive_url_with_multiple_snapshots_redirects_to_latest_snapshot(client ) ArchiveResult.refresh_snapshot_output_sizes({first.id}) - response = client.get(f"/archive/{url}", HTTP_HOST=WEB_TEST_HOST, follow=True) + with CaptureQueriesContext(connection) as captured_queries: + response = client.get(f"/archive/{url}", HTTP_HOST=WEB_TEST_HOST, follow=True) + assert len(captured_queries) <= 8 assert ( f"/snapshot/{second.id.hex}/index.html" in response.redirect_chain[0][0]