release: archivebox 0.9.33rc39

archivebox/cli/archivebox_archiveresult.py

@@ -152,7 +152,7 @@ def create_archiveresults(
             config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
             hooks = discover_hooks("Snapshot", config=config)
             for hook_path in hooks:
-                hook_name = hook_path.name
+                hook_name = hook_path.stem
                 plugin_name = hook_path.parent.name
                 if not is_tty:
                     write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))

archivebox/core/forms.py

@@ -685,7 +685,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         ),
     )
     persona = forms.ModelChoiceField(
-        label="Persona (authentication profile)",
+        label="Persona (configuration profile)",
         required=False,
         queryset=Persona.objects.none(),
         empty_label=None,

archivebox/core/models.py

@@ -2565,7 +2565,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         archiveresults = []
 
         for hook_path in hooks:
-            hook_name = hook_path.name  # e.g., 'on_Snapshot__50_wget.py'
+            hook_name = hook_path.stem  # e.g., 'on_Snapshot__50_wget'
             plugin = hook_path.parent.name  # e.g., 'wget'
 
             # ArchiveResult output is one filesystem directory per plugin hook, so

archivebox/core/views.py

@@ -1132,6 +1132,7 @@ class AddView(UserPassesTestMixin, FormView):
                 "effective_config": effective_config_json,
                 "binary_urls": binary_urls,
             }
+        recent_personas = list(persona_queryset.order_by("-created_at", "name")[:5])
         plugin_dependency_map = {}
         if can_override_crawl_config:
             plugin_dependency_map = {
@@ -1153,6 +1154,7 @@ class AddView(UserPassesTestMixin, FormView):
             "required_search_plugin": required_search_plugin,
             "plugin_dependency_map_json": json.dumps(plugin_dependency_map, sort_keys=True),
             "persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str),
+            "recent_personas": recent_personas,
             "can_override_crawl_config": can_override_crawl_config,
             "stdout": "",
         }
@@ -1621,22 +1623,20 @@ def live_progress_view(request):
         snapshots_paused = snapshot_status_counts.get(Snapshot.StatusChoices.PAUSED, 0)
 
         download_plugin_names, indexing_plugin_names = _live_progress_plugin_names()
-        archiveresult_status_counts: dict[str, int] = {}
-        download_status_counts: dict[str, int] = {}
-        indexing_status_counts: dict[str, int] = {}
-        for status in (
+        result_statuses = (
             ArchiveResult.StatusChoices.QUEUED,
             ArchiveResult.StatusChoices.STARTED,
             ArchiveResult.StatusChoices.PAUSED,
-        ):
-            for row in archiveresult_scope.filter(status=status).values("plugin").annotate(count=Count("id")).order_by():
-                plugin = row["plugin"]
-                count = row["count"]
-                archiveresult_status_counts[status] = archiveresult_status_counts.get(status, 0) + count
-                if plugin in indexing_plugin_names:
-                    indexing_status_counts[status] = indexing_status_counts.get(status, 0) + count
-                elif plugin in download_plugin_names:
-                    download_status_counts[status] = download_status_counts.get(status, 0) + count
+        )
+        archiveresult_status_counts = count_statuses(archiveresult_scope, result_statuses)
+        download_scope = archiveresult_scope.filter(
+            plugin__in=download_plugin_names,
+            snapshot__status__in=(Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED),
+            snapshot__crawl__status__in=(Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED),
+        )
+        indexing_scope = archiveresult_scope.filter(plugin__in=indexing_plugin_names)
+        download_status_counts = count_statuses(download_scope, result_statuses)
+        indexing_status_counts = count_statuses(indexing_scope, result_statuses)
         archiveresults_pending = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
         archiveresults_started = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
         archiveresults_paused = archiveresult_status_counts.get(ArchiveResult.StatusChoices.PAUSED, 0)

archivebox/services/runner.py

@@ -13,6 +13,7 @@ import time
 from collections.abc import Mapping
 from contextlib import nullcontext
 from datetime import timedelta
+from functools import lru_cache
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any
@@ -1185,19 +1186,43 @@ def run_binary(binary_id: str) -> None:
     asyncio.run(_run_binary(binary_id))
 
 
+@lru_cache(maxsize=1)
+def _snapshot_hook_names_by_plugin() -> dict[str, frozenset[str]]:
+    return {plugin.name: frozenset(hook.name for hook in plugin.filter_hooks("Snapshot")) for plugin in discover_plugins().values()}
+
+
 def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
     from archivebox.core.models import ArchiveResult
 
-    queued_plugins = sorted(
-        set(
-            ArchiveResult.objects.filter(
-                snapshot_id=snapshot_id,
-                status=ArchiveResult.StatusChoices.QUEUED,
-            )
-            .exclude(plugin="")
-            .values_list("plugin", flat=True),
-        ),
+    queued_results = list(
+        ArchiveResult.objects.filter(
+            snapshot_id=snapshot_id,
+            status=ArchiveResult.StatusChoices.QUEUED,
+        )
+        .exclude(plugin="")
+        .only("id", "plugin", "hook_name"),
     )
+    hooks_by_plugin = _snapshot_hook_names_by_plugin()
+    obsolete_result_ids = [
+        result.id
+        for result in queued_results
+        if result.hook_name and result.hook_name not in hooks_by_plugin.get(result.plugin, frozenset())
+    ]
+    if obsolete_result_ids:
+        # Hook names are the scheduler identity for ArchiveResults. If an old
+        # queued row names a hook that the current plugin model cannot run, mark
+        # only that row final so the Snapshot scheduler can drain normally instead
+        # of re-running the whole plugin forever.
+        ArchiveResult.objects.filter(
+            id__in=obsolete_result_ids,
+            status=ArchiveResult.StatusChoices.QUEUED,
+        ).update(
+            status=ArchiveResult.StatusChoices.SKIPPED,
+            output_str="Hook no longer exists in the current plugin set.",
+            modified_at=timezone.now(),
+        )
+
+    queued_plugins = sorted({result.plugin for result in queued_results if result.id not in obsolete_result_ids})
     if queued_plugins:
         return queued_plugins
     return None

archivebox/templates/core/add.html

@@ -100,9 +100,9 @@
                                 <div class="error">{{ form.persona.errors }}</div>
                             {% endif %}
                             <div class="help-text">
-                                Authentication profile (Chrome profile, cookies, etc.) to use when accessing URLs.
+                                Authentication + configuration settings to use when saving URLs (cookies, user agent, resolution, timeouts, etc.)
                                 {% if can_override_crawl_config %}
-                                <a href="/admin/personas/persona/add/" target="_blank">Create new persona / import from Chrome →</a>
+                                <a href="/admin/personas/persona/add/" target="_blank">Create new profile / import from Chrome -&gt;</a>
                                 {% endif %}
                             </div>
                         </div>
@@ -247,6 +247,9 @@
 
                     <div class="plugin-presets">
                         <span class="preset-label">Quick Select:</span>
+                        {% for persona in recent_personas %}
+                        <button type="button" class="preset-btn persona-preset-btn" data-persona="{{ persona.name }}" title="Use {{ persona.name }} persona">👤 {{ persona.name }}</button>
+                        {% endfor %}
                         <button type="button" class="preset-btn" data-preset="text-only">📄 Text Only</button>
                         <button type="button" class="preset-btn" data-preset="select-all">✓ Select All</button>
                         <button type="button" class="preset-btn" data-preset="clear-all">✗ Clear All</button>
@@ -1151,7 +1154,16 @@
                     'text-only': ['wget', 'readability', 'mercury', 'htmltotext', 'title', 'favicon']
                 };
 
-                document.querySelectorAll('.preset-btn').forEach(btn => {
+                document.querySelectorAll('.persona-preset-btn').forEach(btn => {
+                    btn.addEventListener('click', function() {
+                        if (!personaSelect) return;
+                        personaSelect.value = this.dataset.persona || '';
+                        dispatchChange(personaSelect);
+                        saveFormState();
+                    });
+                });
+
+                document.querySelectorAll('.preset-btn[data-preset]').forEach(btn => {
                     btn.addEventListener('click', function() {
                         const preset = this.dataset.preset;
                         const allCheckboxes = document.querySelectorAll('.plugin-section-toggle');

archivebox/templates/static/add.css

@@ -766,6 +766,12 @@ select {
   box-shadow: none;
 }
 
+.persona-preset-btn {
+  color: #004882;
+  border-color: #b7d3ea;
+  background-color: #f5fbff;
+}
+
 /* Advanced section (collapsible) */
 .advanced-section {
   background-color: white;

archivebox/tests/test_cli_run.py

@@ -797,6 +797,32 @@ class TestRecoverOrchestratorState:
         assert snapshot.retry_at is None
         assert snapshot.downloaded_at is None
 
+    def test_create_pending_archiveresults_uses_canonical_hook_names(self):
+        from django.utils import timezone
+
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import ArchiveResult, Snapshot
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            status=Crawl.StatusChoices.STARTED,
+            retry_at=timezone.now(),
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+        )
+
+        snapshot.create_pending_archiveresults()
+
+        hook_names = list(ArchiveResult.objects.filter(snapshot=snapshot).values_list("hook_name", flat=True))
+        assert hook_names
+        assert all(not hook_name.endswith((".py", ".js", ".sh")) for hook_name in hook_names)
+
     def test_run_due_snapshot_pauses_child_when_parent_is_paused(self):
         from django.utils import timezone
 
@@ -1194,6 +1220,40 @@ class TestRecoverOrchestratorState:
         assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
         assert snapshot.retry_at is None
 
+    def test_run_due_snapshot_skips_obsolete_queued_hook_name(self):
+        from django.utils import timezone
+
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import ArchiveResult, Snapshot
+        from archivebox.services.runner import run_due_snapshot
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=timezone.now(),
+        )
+        result = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="singlefile",
+            hook_name="on_Snapshot__50_singlefile.py",
+            status=ArchiveResult.StatusChoices.QUEUED,
+        )
+
+        assert run_due_snapshot(snapshot, lock_seconds=60) is True
+
+        result.refresh_from_db()
+        snapshot.refresh_from_db()
+        assert result.status == ArchiveResult.StatusChoices.SKIPPED
+        assert snapshot.retry_at is None
+
     def test_recover_orchestrator_state_ignores_sealed_downloaded_snapshot_without_results(self):
         from django.utils import timezone
 

archivebox/tests/test_ui_admin_views.py

@@ -1259,7 +1259,7 @@ class TestArchiveResultAdminListView:
 
 class TestLiveProgressView:
     def test_live_progress_hides_finished_cancelled_crawl(self, client, admin_user, crawl, snapshot):
-        from archivebox.core.models import Snapshot
+        from archivebox.core.models import ArchiveResult, Snapshot
         from archivebox.crawls.models import Crawl
 
         now = timezone.now()
@@ -1274,12 +1274,20 @@ class TestLiveProgressView:
             downloaded_at=None,
             modified_at=now,
         )
+        ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="singlefile",
+            hook_name="on_Snapshot__50_singlefile",
+            status=ArchiveResult.StatusChoices.QUEUED,
+        )
 
         client.login(username="testadmin", password="testpassword")
         response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST)
 
         assert response.status_code == 200
-        assert response.json()["active_crawls"] == []
+        payload = response.json()
+        assert payload["active_crawls"] == []
+        assert payload["downloads_queued"] == 0
 
     def test_live_progress_reports_real_orchestrator_process_running(self, client, admin_user, db):
         import archivebox.machine.models as machine_models

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.33rc38",
+  "version": "0.9.33rc39",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.33rc38"
+version = "0.9.33rc39"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,9 +79,9 @@ dependencies = [
     ### Extractor dependencies (optional binaries detected at runtime via shutil.which)
     ### Binary/Package Management
     "abxbus==2.5.8",          # EventBus API
-    "abxpkg>=1.11.64",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins>=1.11.70",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl>=1.11.70",       # shared ArchiveBox downloader package with blocking install preflight
+    "abxpkg>=1.11.65",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
+    "abx-plugins>=1.11.71",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl>=1.11.71",       # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]