From 06659868afcd10dafbe3ffadeea36e15407849fb Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 27 Aug 2026 15:34:48 -0700 Subject: [PATCH] separate snapshot progress from output previews --- archivebox/core/admin_snapshots.py | 42 ++++++++++++- archivebox/tests/test_ui_admin_snapshot.py | 73 ++++++++++++++++++++++ 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index 8e2c7217..08c0c777 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -247,7 +247,21 @@ class SnapshotChangeList(SearchResultsChangeList): if not snapshot_ids: return - results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids} + status_counts_by_snapshot = { + row["snapshot_id"]: row + for row in ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids) + .values("snapshot_id") + .annotate( + total=Count("pk"), + succeeded=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.SUCCEEDED)), + failed=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.FAILED)), + running=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.STARTED)), + skipped=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.SKIPPED)), + noresults=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.NORESULTS)), + ) + } + + output_results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids} seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids} rows = ( ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0) @@ -258,12 +272,32 @@ class SnapshotChangeList(SearchResultsChangeList): if plugin in seen_plugins[snapshot_id]: continue seen_plugins[snapshot_id].add(plugin) - results_by_snapshot[snapshot_id].append( + output_results_by_snapshot[snapshot_id].append( SimpleNamespace(plugin=plugin, status=status, output_size=output_size, output_files=output_files), ) for obj in self.result_list: - obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk] + counts = status_counts_by_snapshot.get(obj.pk, {}) + total = int(counts.get("total") or 0) + succeeded = int(counts.get("succeeded") or 0) + failed = int(counts.get("failed") or 0) + running = int(counts.get("running") or 0) + skipped = int(counts.get("skipped") or 0) + noresults = int(counts.get("noresults") or 0) + completed = succeeded + failed + skipped + noresults + obj.__dict__["_admin_progress_stats"] = { + "total": total, + "succeeded": succeeded, + "failed": failed, + "running": running, + "pending": max(total - completed - running, 0), + "skipped": skipped, + "noresults": noresults, + "percent": int((completed / total * 100) if total else 0), + "output_size": obj.output_size or 0, + "is_sealed": obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED), + } + obj.__dict__["_admin_output_results"] = output_results_by_snapshot[obj.pk] def get_results(self, request): super().get_results(request) @@ -1277,6 +1311,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): return stats def _get_prefetched_results(self, obj): + if "_admin_output_results" in obj.__dict__: + return obj.__dict__["_admin_output_results"] if "_admin_archiveresults" in obj.__dict__: return obj.__dict__["_admin_archiveresults"] if "archiveresult_set" in obj.__dict__.get("_prefetched_objects_cache", {}): diff --git a/archivebox/tests/test_ui_admin_snapshot.py b/archivebox/tests/test_ui_admin_snapshot.py index a01ceaf4..187573de 100644 --- a/archivebox/tests/test_ui_admin_snapshot.py +++ b/archivebox/tests/test_ui_admin_snapshot.py @@ -755,6 +755,79 @@ class TestAdminSnapshotListView: assert response.status_code == 200 assert result.plugin.encode() in response.content + def test_list_view_uses_complete_bulk_progress_stats_without_per_snapshot_queries(self, client, admin_user, snapshot, crawl): + from django.db import connection + from django.test.utils import CaptureQueriesContext + + from archivebox.core.models import ArchiveResult, Snapshot + + snapshot.status = Snapshot.StatusChoices.STARTED + snapshot.save(update_fields=["status", "modified_at"]) + for plugin, status, output_size in ( + ("screenshot", ArchiveResult.StatusChoices.SUCCEEDED, 1024), + ("title", ArchiveResult.StatusChoices.FAILED, 0), + ("wget", ArchiveResult.StatusChoices.STARTED, 0), + ("hashes", ArchiveResult.StatusChoices.SKIPPED, 0), + ): + ArchiveResult.objects.create( + snapshot=snapshot, + plugin=plugin, + hook_name=f"on_Snapshot__50_{plugin}.py", + status=status, + output_size=output_size, + output_files={"screenshot.png": {"size": output_size}} if output_size else {}, + ) + + client.force_login(admin_user) + url = reverse("admin:core_snapshot_changelist") + with CaptureQueriesContext(connection) as single_snapshot_queries: + response = client.get(url, HTTP_HOST=ADMIN_TEST_HOST) + + assert response.status_code == 200 + rendered_snapshot = next(obj for obj in response.context["cl"].result_list if obj.pk == snapshot.pk) + assert rendered_snapshot.__dict__["_admin_progress_stats"] == { + "total": 4, + "succeeded": 1, + "failed": 1, + "running": 1, + "pending": 0, + "skipped": 1, + "noresults": 0, + "percent": 75, + "output_size": rendered_snapshot.output_size, + "is_sealed": False, + } + assert "_admin_archiveresults" not in rendered_snapshot.__dict__ + assert [result.plugin for result in rendered_snapshot.__dict__["_admin_output_results"]] == ["screenshot"] + + additional_snapshots = [ + Snapshot.objects.create( + url=f"https://progress-{index}.example.com", + crawl=crawl, + status=Snapshot.StatusChoices.STARTED, + title=f"Progress {index}", + ) + for index in range(5) + ] + ArchiveResult.objects.bulk_create( + [ + ArchiveResult( + snapshot=additional_snapshot, + plugin="wget", + hook_name="on_Snapshot__50_wget.py", + status=ArchiveResult.StatusChoices.STARTED, + ) + for additional_snapshot in additional_snapshots + ], + ) + with CaptureQueriesContext(connection) as many_snapshot_queries: + many_response = client.get(url, HTTP_HOST=ADMIN_TEST_HOST) + + assert many_response.status_code == 200 + single_result_queries = [query for query in single_snapshot_queries.captured_queries if 'FROM "core_archiveresult"' in query["sql"]] + many_result_queries = [query for query in many_snapshot_queries.captured_queries if 'FROM "core_archiveresult"' in query["sql"]] + assert len(single_result_queries) == len(many_result_queries) == 2 + def test_list_view_uses_prefetched_tags_without_row_queries(self, client, admin_user, crawl, db): """Changelist tag rendering should reuse the prefetched tag cache.""" from django.db import connection