The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

separate snapshot progress from output previews

Browse Source
This commit is contained in:
Nick Sweeting 2026-08-27 15:34:48 -07:00
parent 5273f99b9e
commit 06659868af
No known key found for this signature in database
2 changed files with 112 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
archivebox/core/admin_snapshots.py
View File

@ -247,7 +247,21 @@ class SnapshotChangeList(SearchResultsChangeList):
if not snapshot_ids:
return
results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
status_counts_by_snapshot = {
row["snapshot_id"]: row
for row in ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids)
.values("snapshot_id")
.annotate(
total=Count("pk"),
succeeded=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.SUCCEEDED)),
failed=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.FAILED)),
running=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.STARTED)),
skipped=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.SKIPPED)),
noresults=Count("pk", filter=Q(status=ArchiveResult.StatusChoices.NORESULTS)),
)
}
output_results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
rows = (
ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
@ -258,12 +272,32 @@ class SnapshotChangeList(SearchResultsChangeList):
if plugin in seen_plugins[snapshot_id]:
continue
seen_plugins[snapshot_id].add(plugin)
results_by_snapshot[snapshot_id].append(
output_results_by_snapshot[snapshot_id].append(
SimpleNamespace(plugin=plugin, status=status, output_size=output_size, output_files=output_files),
)
for obj in self.result_list:
obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
counts = status_counts_by_snapshot.get(obj.pk, {})
total = int(counts.get("total") or 0)
succeeded = int(counts.get("succeeded") or 0)
failed = int(counts.get("failed") or 0)
running = int(counts.get("running") or 0)
skipped = int(counts.get("skipped") or 0)
noresults = int(counts.get("noresults") or 0)
completed = succeeded + failed + skipped + noresults
obj.__dict__["_admin_progress_stats"] = {
"total": total,
"succeeded": succeeded,
"failed": failed,
"running": running,
"pending": max(total - completed - running, 0),
"skipped": skipped,
"noresults": noresults,
"percent": int((completed / total * 100) if total else 0),
"output_size": obj.output_size or 0,
"is_sealed": obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED),
}
obj.__dict__["_admin_output_results"] = output_results_by_snapshot[obj.pk]
def get_results(self, request):
super().get_results(request)
@ -1277,6 +1311,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
return stats
def _get_prefetched_results(self, obj):
if "_admin_output_results" in obj.__dict__:
return obj.__dict__["_admin_output_results"]
if "_admin_archiveresults" in obj.__dict__:
return obj.__dict__["_admin_archiveresults"]
if "archiveresult_set" in obj.__dict__.get("_prefetched_objects_cache", {}):

73
archivebox/tests/test_ui_admin_snapshot.py
View File

@ -755,6 +755,79 @@ class TestAdminSnapshotListView:
assert response.status_code == 200
assert result.plugin.encode() in response.content
def test_list_view_uses_complete_bulk_progress_stats_without_per_snapshot_queries(self, client, admin_user, snapshot, crawl):
from django.db import connection
from django.test.utils import CaptureQueriesContext
from archivebox.core.models import ArchiveResult, Snapshot
snapshot.status = Snapshot.StatusChoices.STARTED
snapshot.save(update_fields=["status", "modified_at"])
for plugin, status, output_size in (
("screenshot", ArchiveResult.StatusChoices.SUCCEEDED, 1024),
("title", ArchiveResult.StatusChoices.FAILED, 0),
("wget", ArchiveResult.StatusChoices.STARTED, 0),
("hashes", ArchiveResult.StatusChoices.SKIPPED, 0),
):
ArchiveResult.objects.create(
snapshot=snapshot,
plugin=plugin,
hook_name=f"on_Snapshot__50_{plugin}.py",
status=status,
output_size=output_size,
output_files={"screenshot.png": {"size": output_size}} if output_size else {},
)
client.force_login(admin_user)
url = reverse("admin:core_snapshot_changelist")
with CaptureQueriesContext(connection) as single_snapshot_queries:
response = client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200
rendered_snapshot = next(obj for obj in response.context["cl"].result_list if obj.pk == snapshot.pk)
assert rendered_snapshot.__dict__["_admin_progress_stats"] == {
"total": 4,
"succeeded": 1,
"failed": 1,
"running": 1,
"pending": 0,
"skipped": 1,
"noresults": 0,
"percent": 75,
"output_size": rendered_snapshot.output_size,
"is_sealed": False,
}
assert "_admin_archiveresults" not in rendered_snapshot.__dict__
assert [result.plugin for result in rendered_snapshot.__dict__["_admin_output_results"]] == ["screenshot"]
additional_snapshots = [
Snapshot.objects.create(
url=f"https://progress-{index}.example.com",
crawl=crawl,
status=Snapshot.StatusChoices.STARTED,
title=f"Progress {index}",
)
for index in range(5)
]
ArchiveResult.objects.bulk_create(
[
ArchiveResult(
snapshot=additional_snapshot,
plugin="wget",
hook_name="on_Snapshot__50_wget.py",
status=ArchiveResult.StatusChoices.STARTED,
)
for additional_snapshot in additional_snapshots
],
)
with CaptureQueriesContext(connection) as many_snapshot_queries:
many_response = client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
assert many_response.status_code == 200
single_result_queries = [query for query in single_snapshot_queries.captured_queries if 'FROM "core_archiveresult"' in query["sql"]]
many_result_queries = [query for query in many_snapshot_queries.captured_queries if 'FROM "core_archiveresult"' in query["sql"]]
assert len(single_result_queries) == len(many_result_queries) == 2
def test_list_view_uses_prefetched_tags_without_row_queries(self, client, admin_user, crawl, db):
"""Changelist tag rendering should reuse the prefetched tag cache."""
from django.db import connection