diff --git a/.github/scripts/clone_abx_repo.sh b/.github/scripts/clone_abx_repo.sh index 1ce85490..832b33d4 100755 --- a/.github/scripts/clone_abx_repo.sh +++ b/.github/scripts/clone_abx_repo.sh @@ -9,7 +9,7 @@ case "$repo_name" in version="1.11.230" ;; abx-plugins) - version="1.11.241" + version="1.11.243" ;; *) version="$( diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index 5384f1ce..693b8dcf 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -260,13 +260,15 @@ class SnapshotChangeList(SearchResultsChangeList): rows = ( ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0) .order_by("snapshot_id", "plugin") - .values_list("snapshot_id", "plugin", "status", "output_size") + .values_list("snapshot_id", "plugin", "status", "output_size", "output_files") ) - for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000): + for snapshot_id, plugin, status, output_size, output_files in rows.iterator(chunk_size=1000): if plugin in seen_plugins[snapshot_id]: continue seen_plugins[snapshot_id].add(plugin) - results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size)) + results_by_snapshot[snapshot_id].append( + SimpleNamespace(plugin=plugin, status=status, output_size=output_size, output_files=output_files), + ) for obj in self.result_list: obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk] diff --git a/archivebox/tests/test_recursive_crawl.py b/archivebox/tests/test_recursive_crawl.py index c1a9e339..ab8157ab 100644 --- a/archivebox/tests/test_recursive_crawl.py +++ b/archivebox/tests/test_recursive_crawl.py @@ -516,28 +516,12 @@ def test_add_archivewebpage_installs_required_chrome_dependency(initialized_arch @pytest.mark.timeout(1200) def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initialized_archive, free_tcp_port_factory): - """Run a bounded real depth=2 crawl with representative plugins and verify parallel snapshot execution.""" + """Run a bounded real depth=2 crawl with all plugins enabled and verify parallel snapshot execution.""" from abx_dl.models import discover_plugins root_url = "https://example.com/" - crawl_plugins = { - "archivedotorg", - "dom", - "favicon", - "headers", - "htmltotext", - "mercury", - "parse_html_urls", - "pdf", - "readability", - "screenshot", - "title", - "wget", - } - plugin_selection = ",".join( - sorted(plugin for plugin in discover_plugins().keys() if plugin in crawl_plugins), - ) + plugin_selection = ",".join(sorted(plugin for plugin in discover_plugins().keys() if not plugin.startswith("claude"))) env = os.environ.copy() env.pop("CHROME_BINARY", None) env.update( @@ -657,6 +641,7 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia "pdf", "screenshot", "dom", + "singlefile", "readability", "mercury", "htmltotext", @@ -672,6 +657,7 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia assert list(snapshot_root.rglob("pdf/**/*.pdf")) assert list(snapshot_root.rglob("screenshot/**/*.png")) assert list(snapshot_root.rglob("dom/**/*.html")) + assert list(snapshot_root.rglob("singlefile/**/*.html")) assert list(snapshot_root.rglob("readability/**/*.html")) assert list(snapshot_root.rglob("mercury/**/*.html")) assert list(snapshot_root.rglob("htmltotext/**/*.txt")) diff --git a/archivebox/tests/test_ui_admin_snapshot.py b/archivebox/tests/test_ui_admin_snapshot.py index d3d88c31..908b1df9 100644 --- a/archivebox/tests/test_ui_admin_snapshot.py +++ b/archivebox/tests/test_ui_admin_snapshot.py @@ -32,6 +32,25 @@ def test_snapshot_changelist_uses_stable_ordering_without_unordered_paginator_wa assert b"Searching matching snapshots..." in response.content +def test_snapshot_changelist_preview_uses_prefetched_output_files(admin_client, snapshot): + from archivebox.core.models import ArchiveResult + + ArchiveResult.objects.create( + snapshot=snapshot, + plugin="screenshot", + hook_name="on_Snapshot__40_screenshot.js", + status=ArchiveResult.StatusChoices.SUCCEEDED, + output_size=128, + output_files={"screenshot.png": {"size": 128, "root_relative": True}}, + ) + + response = admin_client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_TEST_HOST) + + assert response.status_code == 200 + assert b'class="snapshot-preview' in response.content + assert b"screenshot.png" in response.content + + def test_snapshot_admin_tag_editor_escapes_tag_json_script_breakout(admin_client, snapshot): from archivebox.core.models import Tag diff --git a/bin/deploy_dev_demo.sh b/bin/deploy_dev_demo.sh index bf9ed6cb..dc571504 100755 --- a/bin/deploy_dev_demo.sh +++ b/bin/deploy_dev_demo.sh @@ -60,6 +60,9 @@ cat > .archivebox-deploy.override.yml < "$OVERRIDE_FILE" <