release: archivebox 0.9.34rc39

Dockerfile

@@ -313,7 +313,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
     && export PERSONAS_DIR="$LIB_DIR/personas" \
     && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
-    && export ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \
+    && export ABX_RUNTIME=archivebox ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \
     && mkdir -p "$LIB_DIR" \
     && apt-get update -qq \
     && apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \
@@ -327,7 +327,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
         accessibility archivedotorg archivewebpage base chrome chrome_mhtml chrome_screencast \
         claudechrome claudecode claudecodecleanup claudecodeextract consolelog defuddle dns dom \
         favicon forumdl gallerydl git hashes headers htmltotext infiniscroll \
-        istilldontcareaboutcookies liteparse media mercury modalcloser opendataloader papersdl \
+        istilldontcareaboutcookies liteparse media mercury modalcloser opencode opendataloader papersdl \
         parse_dom_outlinks parse_html_urls parse_jsonl_urls parse_netscape_urls parse_rss_urls \
         parse_txt_urls pdf readability redirects responses screenshot search_backend_ripgrep \
         search_backend_sonic search_backend_sqlite seo singlefile ssl sslcerts staticfile title \

archivebox/cli/archivebox_run.py

@@ -320,7 +320,12 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
                 interactive_interrupts=interactive_interrupts,
             )
         return 0
-    except (KeyboardInterrupt, asyncio.CancelledError):
+    except KeyboardInterrupt:
+        return 0
+    except asyncio.CancelledError as e:
+        if daemon:
+            rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+            return 1
         return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)

archivebox/core/admin_archiveresults.py

@@ -75,6 +75,12 @@ def get_plugin_admin_url(plugin_name: str) -> str:
     return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/"
 
 
+def get_process_link_label(process) -> str:
+    if process.pid:
+        return str(process.pid)
+    return str(process.id)[-8:]
+
+
 def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
     """Render a nice inline list view of archive results with status, plugin, output, and actions."""
 
@@ -139,7 +145,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
             process_display = f'''
                 <a href="{reverse("admin:machine_process_change", args=[process.id])}"
                    style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{process.pid or "-"}</a>
+                   title="View process">{get_process_link_label(process)}</a>
             '''
 
         machine_display = "-"
@@ -659,7 +665,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
         process = result.process_record
         if not process:
             return "-"
-        process_label = process.pid or "-"
+        process_label = get_process_link_label(process)
         return format_html(
             '<a href="{}"><code>{}</code></a>',
             reverse("admin:machine_process_change", args=[process.id]),
@@ -673,9 +679,8 @@ class ArchiveResultAdmin(BaseModelAdmin):
             return "-"
         machine = process.machine
         return format_html(
-            '<a href="{}"><code>{}</code> {}</a>',
+            '<a href="{}">{}</a>',
             reverse("admin:machine_machine_change", args=[machine.id]),
-            str(machine.id)[:8],
             machine.hostname,
         )
 

archivebox/core/admin_tags.py

@@ -117,7 +117,7 @@ class TagAdmin(BaseModelAdmin):
         sort = normalize_tag_sort((request.GET.get("sort") or "created_desc").strip())
         created_by = normalize_created_by_filter((request.GET.get("created_by") or "").strip())
         year = normalize_created_year_filter((request.GET.get("year") or "").strip())
-        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "yes").strip())
+        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "all").strip())
         context = {
             **self.admin_site.each_context(request),
             **(extra_context or {}),

archivebox/core/settings.py

@@ -72,6 +72,7 @@ INSTALLED_APPS = [
     "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
     "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
     "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
+    "abx_plugins.plugins.opencode",
     # 3rd-party apps from PyPI that need to be loaded last
     "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
     "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)

archivebox/core/tag_util.py

@@ -114,9 +114,9 @@ def add_snapshot_counts(tags: list[Tag], snapshot_queryset: QuerySet[Snapshot] |
     queryset = SnapshotTag.objects.filter(tag_id__in=tag_ids)
     if snapshot_queryset is not None:
         queryset = queryset.filter(snapshot_id__in=snapshot_queryset.values("id"))
-    counts = {row["tag_id"]: row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
+    counts = {int(row["tag_id"]): row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
     for tag in tags:
-        tag.num_snapshots = counts.get(tag.pk, 0)
+        tag.num_snapshots = counts.get(int(tag.pk), 0)
 
 
 def get_tag_creator_choices() -> list[tuple[str, str]]:

archivebox/core/urls.py

@@ -41,6 +41,7 @@ urlpatterns = [
     path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
     path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
     path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
+    re_path(r"^admin/agent/?(?=$|opencode)", include("abx_plugins.plugins.opencode.urls")),
     path("public/", PublicIndexView.as_view(), name="public-index"),
     path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),
     path("archive/", RedirectView.as_view(url="/")),

archivebox/machine/admin.py

@@ -434,6 +434,9 @@ class ProcessAdmin(BaseModelAdmin):
         "crawl_link",
         "cmd_display",
         "env_display",
+        "stdout_display",
+        "stderr_display",
+        "archiveresult_output_display",
         "timeout",
         "pid",
         "exit_code",
@@ -475,7 +478,7 @@ class ProcessAdmin(BaseModelAdmin):
         (
             "Output",
             {
-                "fields": ("stdout", "stderr"),
+                "fields": ("stdout_display", "stderr_display", "archiveresult_output_display"),
                 "classes": ("card", "wide", "collapse"),
             },
         ),
@@ -740,6 +743,28 @@ class ProcessAdmin(BaseModelAdmin):
             return "-"
         return _render_copy_block(env_text, multiline=True)
 
+    @admin.display(description="Stdout")
+    def stdout_display(self, process):
+        if not process.stdout:
+            return "-"
+        return _render_copy_block(process.stdout, multiline=True)
+
+    @admin.display(description="Stderr")
+    def stderr_display(self, process):
+        if not process.stderr:
+            return "-"
+        return _render_copy_block(process.stderr, multiline=True)
+
+    @admin.display(description="ArchiveResult Output")
+    def archiveresult_output_display(self, process):
+        try:
+            output = process.archiveresult.output_str
+        except Process.archiveresult.RelatedObjectDoesNotExist:
+            return "-"
+        if not output:
+            return "-"
+        return _render_copy_block(output, multiline=True)
+
 
 def register_admin(admin_site):
     admin_site.register(Machine, MachineAdmin)

archivebox/machine/migrations/0020_repair_process_binary_iface_links.py

@@ -0,0 +1,130 @@
+import json
+from pathlib import Path
+
+from django.db import migrations
+from django.utils import timezone
+
+
+def _cmd_array(value):
+    if isinstance(value, list):
+        return value
+    if isinstance(value, str):
+        try:
+            parsed = json.loads(value)
+        except json.JSONDecodeError:
+            return [value] if value else []
+        return parsed if isinstance(parsed, list) else []
+    return []
+
+
+def _ensure_placeholder_iface(NetworkInterface, machine_id, hostname):
+    iface = NetworkInterface.objects.filter(machine_id=machine_id).order_by("-modified_at", "-created_at").first()
+    if iface is not None:
+        return iface
+
+    now = timezone.now()
+    return NetworkInterface.objects.create(
+        machine_id=machine_id,
+        created_at=now,
+        modified_at=now,
+        mac_address="00:00:00:00:00:00",
+        ip_public="0.0.0.0",
+        ip_local="0.0.0.0",
+        dns_server="0.0.0.0",
+        hostname=(hostname or "unknown")[:63],
+        iface="unknown",
+        isp="",
+        city="",
+        region="",
+        country="",
+    )
+
+
+def _get_or_create_binary(Binary, machine_id, reference):
+    reference = str(reference or "").strip()
+    if not reference:
+        return None
+
+    name = Path(reference).name or reference
+    qs = Binary.objects.filter(machine_id=machine_id)
+    binary = qs.filter(abspath=reference).order_by("-modified_at", "-created_at").first()
+    if binary is None:
+        binary = qs.filter(name=name).order_by("-modified_at", "-created_at").first()
+    if binary is not None:
+        return binary
+
+    now = timezone.now()
+    return Binary.objects.create(
+        machine_id=machine_id,
+        created_at=now,
+        modified_at=now,
+        name=name[:63],
+        binproviders="env",
+        overrides={},
+        binprovider="env",
+        abspath=reference[:255],
+        version="",
+        sha256="",
+        status="installed",
+        retry_at=None,
+    )
+
+
+def repair_process_binary_iface_links(apps, schema_editor):
+    Binary = apps.get_model("machine", "Binary")
+    Machine = apps.get_model("machine", "Machine")
+    NetworkInterface = apps.get_model("machine", "NetworkInterface")
+    Process = apps.get_model("machine", "Process")
+
+    machines = {machine.id: machine for machine in Machine.objects.only("id", "hostname").iterator(chunk_size=100)}
+    iface_by_machine = {}
+    binary_by_key = {}
+
+    qs = Process.objects.filter(machine_id__isnull=False).filter(binary_id__isnull=True) | Process.objects.filter(
+        machine_id__isnull=False,
+        iface_id__isnull=True,
+    )
+    for process in qs.distinct().only("id", "machine_id", "binary_id", "iface_id", "cmd").iterator(chunk_size=500):
+        update_fields = []
+        machine = machines.get(process.machine_id)
+
+        if process.iface_id is None:
+            iface = iface_by_machine.get(process.machine_id)
+            if iface is None:
+                iface = _ensure_placeholder_iface(
+                    NetworkInterface,
+                    process.machine_id,
+                    machine.hostname if machine is not None else "",
+                )
+                iface_by_machine[process.machine_id] = iface
+            process.iface_id = iface.id
+            update_fields.append("iface_id")
+
+        if process.binary_id is None:
+            cmd = _cmd_array(process.cmd)
+            reference = str(cmd[0]).strip() if cmd else ""
+            if reference:
+                key = (process.machine_id, reference)
+                binary = binary_by_key.get(key)
+                if binary is None:
+                    binary = _get_or_create_binary(Binary, process.machine_id, reference)
+                    binary_by_key[key] = binary
+                if binary is not None:
+                    process.binary_id = binary.id
+                    update_fields.append("binary_id")
+
+        if update_fields:
+            process.modified_at = timezone.now()
+            process.save(update_fields=[*update_fields, "modified_at"])
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("machine", "0019_single_active_runner_constraint"),
+    ]
+
+    operations = [
+        migrations.RunPython(repair_process_binary_iface_links, migrations.RunPython.noop),
+    ]

archivebox/machine/models.py

@@ -650,22 +650,6 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
         binary_overrides = record.get("overrides", {})
         normalized_overrides = binary_overrides if isinstance(binary_overrides, dict) else {}
 
-        # abx-plugins currently emits a GitHub install URL for readability-extractor,
-        # but the package is published on npm. Prefer the registry package to avoid
-        # long git-based installs in CI while still using canonical install_args.
-        if (
-            name == "readability-extractor"
-            and isinstance(normalized_overrides.get("npm"), dict)
-            and normalized_overrides["npm"].get("install_args") == ["https://github.com/ArchiveBox/readability-extractor"]
-        ):
-            normalized_overrides = {
-                **normalized_overrides,
-                "npm": {
-                    **normalized_overrides["npm"],
-                    "install_args": ["readability-extractor"],
-                },
-            }
-
         # Case 1: Already installed (from on_Crawl hooks) - has abspath AND binproviders
         # This happens when on_Crawl hooks detect already-installed binaries
         abspath = record.get("abspath")

archivebox/personas/importers.py

@@ -612,7 +612,7 @@ def export_browser_state(
     if not state_script.exists():
         return False, None, f"Browser state export script not found at {state_script}"
 
-    node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules"
+    node_modules_dir = get_config().LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules"
     chrome_plugin_dir = Path(get_plugins_dir()).resolve()
 
     env = os.environ.copy()

archivebox/plugins/forms.py

@@ -130,6 +130,7 @@ HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
     "env",
     "media",
     "npm",
+    "opencode",
     "pip",
     "puppeteer",
     "search_backend_ripgrep",

archivebox/plugins/hooks.py

@@ -285,7 +285,6 @@ def run_hook(
     """
     from archivebox.machine.models import Process, Machine, NetworkInterface
     from archivebox.config.common import get_config, normalize_runtime_config
-    import sys
 
     config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
     config_overrides = _config_to_overrides(config)
@@ -333,12 +332,15 @@ def run_hook(
         )
         return process
 
-    # Determine the interpreter based on file extension
+    # Python hooks carry their runtime contract in the shebang
+    # (usually `abxpkg run --script python3`), so execute them directly.
+    # For shell/JS hooks we still dispatch through the conventional
+    # interpreter because those hooks do not need per-script Python env setup.
     ext = script.suffix.lower()
     if ext == ".sh":
         cmd = ["bash", str(script)]
     elif ext == ".py":
-        cmd = [sys.executable, str(script)]
+        cmd = [str(script)]
     elif ext == ".js":
         cmd = ["node", str(script)]
     else:
@@ -388,7 +390,7 @@ def run_hook(
     # NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory.
     node_modules_dir = hook_config.get("NODE_MODULES_DIR")
     if not node_modules_dir and lib_dir:
-        node_modules_dir = Path(lib_dir) / "npm" / "node_modules"
+        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
 
     node_path_parts = [part for part in str(hook_config.get("NODE_PATH") or "").split(os.pathsep) if part]
     if node_modules_dir:

archivebox/progressmonitor/views.py

@@ -327,8 +327,20 @@ def live_progress_view(request):
             .order_by("-modified_at")[:max_active_crawls],
         )
         paused_crawls = list(
-            crawl_scope.filter(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+            crawl_scope.filter(
+                Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+                | Q(
+                    status=Crawl.StatusChoices.PAUSED,
+                    snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
+                    snapshot_set__retry_at__lte=now,
+                )
+                | Q(
+                    status=Crawl.StatusChoices.PAUSED,
+                    snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
+                ),
+            )
             .values(*active_crawl_fields)
+            .distinct()
             .order_by("-modified_at")[:max_active_crawls],
         )
         queued_crawls = list(

archivebox/services/archive_result_service.py

@@ -432,6 +432,9 @@ class ArchiveResultService(BaseService):
                     ).now()
             return
 
+        # TODO: consider moving this fallback derivation into abx-dl itself.
+        # First try both patterns: if the whole abx-dl process crashes, restarting
+        # the snapshot may be enough, but don't guess before validating it.
         process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED)
         with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"):
             await event.emit(
@@ -439,7 +442,7 @@ class ArchiveResultService(BaseService):
                     snapshot_id=snapshot_event.snapshot_id,
                     plugin=event.plugin_name,
                     hook_name=event.hook_name,
-                    status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "skipped"),
+                    status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "noresult"),
                     output_str=event.stderr if process_failed else "",
                     output_files=event.output_files,
                     start_ts=event.start_ts,

archivebox/services/binary_service.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import json
 from collections.abc import Mapping
 from pathlib import Path
@@ -186,38 +187,109 @@ class ArchiveBoxDBBinaryCacheBackend:
 class ArchiveBoxBinaryService(BaseService):
     """Preserve ArchiveBox's legacy Binary Process rows around abxpkg requests."""
 
-    LISTENS_TO = [BinaryRequestEvent]
+    LISTENS_TO = [BinaryRequestEvent, BinaryEvent]
     EMITS: list[type[BaseEvent]] = []
 
     def __init__(self, bus: EventBus):
         super().__init__(bus)
+        self.process_ids_by_request_id: dict[str, str] = {}
         self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__project_process)
+        self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__schedule_missing_finalize)
+        self.bus.on(BinaryEvent, self.on_BinaryEvent__finalize_process)
 
     async def on_BinaryRequestEvent__project_process(self, request: BinaryRequestEvent) -> None:
-        from archivebox.machine.models import Binary, Machine, Process, _canonical_binary_name
-        from archivebox.services.process_service import current_network_interface_with_machine
+        from archivebox.machine.models import Machine, Process, _canonical_binary_name
 
         machine = await sync_to_async(Machine.current, thread_sensitive=True)()
         binary_name = _canonical_binary_name(request.name)
         if not binary_name:
             return
-        binary = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
-        if binary is None:
-            return
+        binary = await self._get_or_create_binary(machine, binary_name, request)
+        started_at = timezone.now()
+        output_dir = self._process_output_dir(binary, request)
+        await sync_to_async(output_dir.mkdir, thread_sensitive=True)(parents=True, exist_ok=True)
+        process = await Process.objects.acreate(
+            machine=machine,
+            iface=None,
+            process_type=Process.TypeChoices.BINARY,
+            worker_type="",
+            pwd=str(output_dir),
+            cmd=self._process_cmd(request),
+            env={},
+            timeout=int(request.event_timeout or request.install_timeout or 600),
+            pid=None,
+            url=None,
+            started_at=started_at,
+            ended_at=None,
+            stdout="",
+            stderr="",
+            exit_code=None,
+            status=Process.StatusChoices.RUNNING,
+            retry_at=None,
+            binary=binary,
+        )
+        self.process_ids_by_request_id[request.event_id] = str(process.id)
 
-        binary_event = await self.bus.find(
-            BinaryEvent,
-            child_of=request,
+    async def on_BinaryEvent__finalize_process(self, event: BinaryEvent) -> None:
+        from archivebox.machine.models import Binary, Process, _canonical_binary_name
+
+        request = await self.bus.find(
+            BinaryRequestEvent,
             past=True,
             future=False,
-            name=request.name,
-            where=lambda candidate: bool(candidate.abspath),
+            where=lambda candidate: self.bus.event_is_child_of(event, candidate),
         )
-        iface = await sync_to_async(current_network_interface_with_machine, thread_sensitive=True)()
-        now = timezone.now()
-        success = isinstance(binary_event, BinaryEvent)
-        output_dir = self._process_output_dir(binary, request)
-        output_dir.mkdir(parents=True, exist_ok=True)
+        request = request if isinstance(request, BinaryRequestEvent) else None
+        process_id = self.process_ids_by_request_id.pop(request.event_id, "") if request is not None else ""
+        if not process_id:
+            return
+        process = await Process.objects.filter(id=process_id).select_related("binary").afirst()
+        if process is None:
+            return
+        binary_name = _canonical_binary_name(event.name)
+        binary = process.binary
+        if binary is not None and binary_name:
+            binary.abspath = event.abspath
+            if event.version:
+                binary.version = str(event.version)
+            if event.sha256:
+                binary.sha256 = str(event.sha256)
+            binary.binproviders = event.binproviders or binary.binproviders
+            binary.binprovider = event.binprovider or binary.binprovider
+            binary.status = Binary.StatusChoices.INSTALLED
+            binary.retry_at = None
+            await binary.asave(
+                update_fields=["abspath", "version", "sha256", "binproviders", "binprovider", "status", "retry_at", "modified_at"],
+            )
+        process.ended_at = timezone.now()
+        process.stdout = json.dumps(self._binary_event_json(event, binary)) + "\n"
+        process.stderr = ""
+        process.exit_code = 0
+        process.status = Process.StatusChoices.EXITED
+        await process.asave(update_fields=["ended_at", "stdout", "stderr", "exit_code", "status", "modified_at"])
+        if binary is not None:
+            await sync_to_async(self._write_binary_index, thread_sensitive=True)(binary, process, Path(process.pwd))
+
+    async def _get_or_create_binary(self, machine, binary_name: str, request: BinaryRequestEvent):
+        from archivebox.machine.models import Binary
+
+        binary_id = str(request.extra_context.get("binary_id") or "")
+        if binary_id:
+            binary = await Binary.objects.filter(id=binary_id).afirst()
+            if binary is not None:
+                return binary
+        binary = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
+        if binary is not None:
+            return binary
+        return await Binary.objects.acreate(
+            machine=machine,
+            name=binary_name,
+            binproviders=_binproviders_to_str(request.binproviders),
+            overrides=_persisted_overrides_for_request(request),
+            status=Binary.StatusChoices.QUEUED,
+        )
+
+    def _process_cmd(self, request: BinaryRequestEvent) -> list[str]:
         cmd = [
             "abxpkg",
             "install",
@@ -226,29 +298,65 @@ class ArchiveBoxBinaryService(BaseService):
         ]
         if request.overrides:
             cmd.append(f"--overrides={json.dumps(request.overrides, sort_keys=True)}")
-        stdout = json.dumps(binary.to_json()) + "\n" if success else ""
-        stderr = "" if success else f"Binary request did not resolve: {request.name}"
-        process = await Process.objects.acreate(
-            machine=iface.machine,
-            iface=iface,
-            process_type=Process.TypeChoices.BINARY,
-            worker_type="",
-            pwd=str(output_dir),
-            cmd=cmd,
-            env={},
-            timeout=int(request.event_timeout or request.install_timeout or 600),
-            pid=None,
-            url=None,
-            started_at=now,
-            ended_at=now,
-            stdout=stdout,
-            stderr=stderr,
-            exit_code=0 if success else 1,
-            status=Process.StatusChoices.EXITED,
-            retry_at=None,
-            binary=binary,
+        return cmd
+
+    def _binary_event_json(self, event: BinaryEvent, binary) -> dict[str, Any]:
+        if binary is not None:
+            data = binary.to_json()
+        else:
+            data = {"type": "Binary", "name": event.name}
+        data.update(
+            {
+                "type": "Binary",
+                "name": event.name,
+                "binproviders": event.binproviders,
+                "binprovider": event.binprovider,
+                "abspath": event.abspath,
+                "version": str(event.version or ""),
+                "sha256": event.sha256 or "",
+                "status": "installed",
+            },
         )
-        self._write_binary_index(binary, process, output_dir)
+        return data
+
+    async def _finalize_missing_process(self, request: BinaryRequestEvent) -> None:
+        from archivebox.machine.models import Process
+
+        process_id = self.process_ids_by_request_id.pop(request.event_id, "")
+        if not process_id:
+            return
+        process = await Process.objects.filter(id=process_id).afirst()
+        if process is None or process.status == Process.StatusChoices.EXITED:
+            return
+        process.ended_at = timezone.now()
+        process.stderr = f"Binary request did not resolve: {request.name}"
+        process.exit_code = 1
+        process.status = Process.StatusChoices.EXITED
+        await process.asave(update_fields=["ended_at", "stderr", "exit_code", "status", "modified_at"])
+
+    async def _finalize_request_when_done(self, request: BinaryRequestEvent) -> None:
+        try:
+            await request.wait(timeout=request.event_timeout)
+        except TimeoutError:
+            await self._finalize_missing_process(request)
+            return
+        binary_event = await self.bus.find(
+            BinaryEvent,
+            child_of=request,
+            past=True,
+            future=False,
+            name=request.name,
+            where=lambda candidate: bool(candidate.abspath),
+        )
+        if not isinstance(binary_event, BinaryEvent):
+            await self._finalize_missing_process(request)
+
+    def _schedule_missing_finalize(self, request: BinaryRequestEvent) -> None:
+        task = asyncio.create_task(self._finalize_request_when_done(request))
+        task.add_done_callback(lambda done: None if done.cancelled() else done.exception())
+
+    async def on_BinaryRequestEvent__schedule_missing_finalize(self, request: BinaryRequestEvent) -> None:
+        self._schedule_missing_finalize(request)
 
     def _process_output_dir(self, binary, request: BinaryRequestEvent) -> Path:
         raw_output_dir = str(request.extra_context.get("output_dir") or "").strip()

archivebox/services/machine_service.py

@@ -7,8 +7,6 @@ from asgiref.sync import sync_to_async
 from abx_dl.events import MachineEvent
 from abx_dl.services.base import BaseService
 
-_BINARY_EVENT_ALLOWED_KEYS = frozenset({"ABX_INSTALL_CACHE"})
-
 
 def _is_binary_event_key(key: str) -> bool:
     """``MachineEvent`` projector only ever writes binary-related state.
@@ -20,7 +18,7 @@ def _is_binary_event_key(key: str) -> bool:
     mirror is a security boundary), so the projector strips anything that
     isn't a binary path or the binary install cache.
     """
-    if key in _BINARY_EVENT_ALLOWED_KEYS:
+    if key.startswith("ABX_") and key.endswith("CACHE"):
         return True
     return key.endswith("_BINARY")
 

archivebox/services/runner.py

@@ -121,6 +121,10 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str
     return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
 
 
+def _discover_archivebox_plugins() -> dict[str, Plugin]:
+    return discover_plugins(runtime="archivebox")
+
+
 def _runner_task_context() -> contextvars.Context:
     context = contextvars.copy_context()
     context.run(EventBus.current_event_context.set, None)
@@ -213,13 +217,13 @@ class CrawlRunner:
     ):
         self.crawl = crawl
         self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0)
-        self.plugins = discover_plugins()
+        self.plugins = _discover_archivebox_plugins()
         HookProcessService(self.bus, emit_jsonl=False, interactive_tty=interactive_interrupts)
         register_sonic_daemon_event_handler(self.bus)
         PersistedProcessService(self.bus)
+        ArchiveBoxBinaryService(self.bus)
         BinaryCacheService(self.bus, backend=ArchiveBoxDBBinaryCacheBackend())
         BinaryService(self.bus)
-        ArchiveBoxBinaryService(self.bus)
         TagService(self.bus)
         CrawlService(self.bus, crawl_id=str(crawl.id))
         MachineService(self.bus)
@@ -839,7 +843,7 @@ class CrawlRunner:
             url=snapshot["url"],
             snapshot=abx_snapshot,
             output_dir=output_dir,
-            install_enabled=False,
+            install_enabled=True,
             crawl_setup_enabled=True,
             crawl_event_enabled=False,
             crawl_start_enabled=False,
@@ -1179,7 +1183,7 @@ async def _run_binary(binary_id: str) -> None:
     from archivebox.machine.models import Binary, Machine
 
     binary = await Binary.objects.aget(id=binary_id)
-    plugins = discover_plugins()
+    plugins = _discover_archivebox_plugins()
     config = get_config(include_machine=False)
     machine = await sync_to_async(Machine.current, thread_sensitive=True)()
     derived_config = normalize_runtime_config(machine.config)
@@ -1187,9 +1191,9 @@ async def _run_binary(binary_id: str) -> None:
     config = normalize_runtime_config(config)
     bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0)
     process_service = PersistedProcessService(bus)
+    ArchiveBoxBinaryService(bus)
     BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
     BinaryService(bus)
-    ArchiveBoxBinaryService(bus)
     TagService(bus)
     ArchiveResultService(bus)
     MachineService(bus)
@@ -1238,7 +1242,9 @@ def run_binary(binary_id: str) -> None:
 
 @lru_cache(maxsize=1)
 def _snapshot_hook_names_by_plugin() -> dict[str, frozenset[str]]:
-    return {plugin.name: frozenset(hook.name for hook in plugin.filter_hooks("Snapshot")) for plugin in discover_plugins().values()}
+    return {
+        plugin.name: frozenset(hook.name for hook in plugin.filter_hooks("Snapshot")) for plugin in _discover_archivebox_plugins().values()
+    }
 
 
 def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
@@ -1566,7 +1572,7 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
     from archivebox.config.common import get_config
     from archivebox.machine.models import Machine
 
-    plugins = discover_plugins()
+    plugins = _discover_archivebox_plugins()
     config = get_config(include_machine=False)
     machine = await sync_to_async(Machine.current, thread_sensitive=True)()
     derived_config = normalize_runtime_config(machine.config)
@@ -1574,9 +1580,9 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
     config = normalize_runtime_config(config)
     bus = create_bus(name="ArchiveBox_install", total_timeout=3600.0)
     PersistedProcessService(bus)
+    ArchiveBoxBinaryService(bus)
     BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
     BinaryService(bus)
-    ArchiveBoxBinaryService(bus)
     TagService(bus)
     ArchiveResultService(bus)
     MachineService(bus)

archivebox/templates/admin/core/tag/change_list.html

@@ -517,7 +517,7 @@ document.addEventListener('DOMContentLoaded', function () {
             sort: typeof next.sort === 'string' ? next.sort : (sortSelect?.value || 'created_desc'),
             created_by: typeof next.created_by === 'string' ? next.created_by : (createdBySelect?.value || ''),
             year: typeof next.year === 'string' ? next.year : (yearSelect?.value || ''),
-            has_snapshots: typeof next.has_snapshots === 'string' ? next.has_snapshots : (shell.dataset.initialHasSnapshots || 'yes'),
+            has_snapshots: typeof next.has_snapshots === 'string' ? next.has_snapshots : (shell.dataset.initialHasSnapshots || 'all'),
         };
     }
 
@@ -554,7 +554,7 @@ document.addEventListener('DOMContentLoaded', function () {
             url.searchParams.delete('year');
         }
 
-        if (state.has_snapshots && state.has_snapshots !== 'yes') {
+        if (state.has_snapshots && state.has_snapshots !== 'all') {
             url.searchParams.set('has_snapshots', state.has_snapshots);
         } else {
             url.searchParams.delete('has_snapshots');
@@ -570,9 +570,7 @@ document.addEventListener('DOMContentLoaded', function () {
     }
 
     function renderCards(nextCards, state) {
-        cards = Array.isArray(nextCards) ? nextCards.filter(function (card) {
-            return Number(card && card.num_snapshots || 0) > 0;
-        }) : [];
+        cards = Array.isArray(nextCards) ? nextCards : [];
         setMeta(state || getCurrentState(), cards.length);
 
         if (!cards.length) {

archivebox/templates/core/navigation.html

@@ -6,6 +6,9 @@
     <a href="{% url 'Home' %}">Snapshots</a> |
     <a href="/admin/core/archiveresult/?o=-1">Log</a> |
     <a href="/admin/core/tag/">Tags</a> &nbsp; &nbsp;
+    {% if user.is_authenticated and user.is_superuser %}
+        <a href="/admin/agent">💬 AI</a> |
+    {% endif %}
     <a href="{% url 'Docs' %}" target="_blank" rel="noopener noreferrer">Docs</a> |
     <a href="/api/v1/docs">API</a> |
     <a href="/admin/">Admin</a>

archivebox/tests/conftest.py

@@ -26,6 +26,7 @@ import requests
 from django.utils import timezone
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
+WORKSPACE_ROOT = REPO_ROOT.parent
 PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
 SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve()
 (SESSION_DATA_DIR / "tests").mkdir(parents=True, exist_ok=True)
@@ -55,6 +56,29 @@ def _assert_safe_runtime_paths(*, cwd: Path | None = None, env: dict[str, str] |
             _assert_not_repo_path(Path(value), label=key)
 
 
+def _test_source_pythonpath() -> str:
+    entries: list[str] = []
+    for repo_name in ("abxpkg", "abx-plugins", "abx-dl"):
+        repo_path = WORKSPACE_ROOT / repo_name
+        if repo_path.exists():
+            entries.append(str(repo_path.resolve(strict=False)))
+    return os.pathsep.join(entries)
+
+
+def _set_test_source_pythonpath(env: dict[str, str]) -> None:
+    source_pythonpath = _test_source_pythonpath()
+    existing_entries = [
+        str(Path(entry).expanduser().resolve(strict=False))
+        for entry in (env.get("PYTHONPATH") or "").split(os.pathsep)
+        if entry and Path(entry).expanduser().is_absolute()
+    ]
+    entries = [entry for entry in [*source_pythonpath.split(os.pathsep), *existing_entries] if entry]
+    if entries:
+        env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(entries))
+    else:
+        env.pop("PYTHONPATH", None)
+
+
 def _sync_archivebox_test_data_dir(data_dir: Path) -> None:
     from archivebox.config import constants as constants_mod
     from archivebox.config import paths as paths_mod
@@ -170,9 +194,8 @@ def run_archivebox_cmd(
 
     _assert_not_repo_path(cwd, label="cwd")
 
-    run_env: dict[str, str] | None = None
+    run_env = {} if replace_env else os.environ.copy()
     if default_cli_env or disable_extractors or env is not None:
-        run_env = {} if replace_env else os.environ.copy()
         if default_cli_env:
             run_env["USE_COLOR"] = "False"
             run_env["SHOW_PROGRESS"] = "False"
@@ -198,8 +221,9 @@ def run_archivebox_cmd(
             )
         if env:
             run_env.update(env)
+    _set_test_source_pythonpath(run_env)
 
-    _assert_safe_runtime_paths(cwd=cwd, env=run_env or os.environ)
+    _assert_safe_runtime_paths(cwd=cwd, env=run_env)
 
     if stdin is not None:
         assert input is None, "pass either input or stdin, not both"
@@ -562,6 +586,7 @@ def cli_env(
     **extra: str,
 ) -> dict[str, str]:
     env = {} if replace else os.environ.copy()
+    _set_test_source_pythonpath(env)
     env.update({"USE_COLOR": "False", "SHOW_PROGRESS": "False"})
 
     if disable_extractors or live or server:
@@ -1373,7 +1398,7 @@ def _find_cached_chrome(lib_dir: Path) -> Path | None:
         lib_dir / "puppeteer" / "chromium",
         lib_dir / "puppeteer",
         lib_dir / "ms-playwright",
-        lib_dir / "npm" / "node_modules" / "puppeteer" / ".local-chromium",
+        lib_dir / "pnpm" / "packages" / "chrome" / "node_modules" / "puppeteer" / ".local-chromium",
     ]
     for base in candidates:
         if not base.exists():
@@ -1401,17 +1426,17 @@ def _find_system_browser() -> Path | None:
 
 
 def _ensure_puppeteer(shared_lib: Path) -> None:
-    npm_prefix = shared_lib / "npm"
-    node_modules = npm_prefix / "node_modules"
+    pnpm_prefix = shared_lib / "pnpm" / "packages" / "chrome"
+    node_modules = pnpm_prefix / "node_modules"
     puppeteer_dir = node_modules / "puppeteer"
     if puppeteer_dir.exists():
         return
-    npm_prefix.mkdir(parents=True, exist_ok=True)
+    pnpm_prefix.mkdir(parents=True, exist_ok=True)
     env = os.environ.copy()
     env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
     subprocess.run(
-        ["npm", "install", "puppeteer"],
-        cwd=str(npm_prefix),
+        ["pnpm", "add", "--dir", str(pnpm_prefix), "puppeteer"],
+        cwd=str(pnpm_prefix),
         env=env,
         check=True,
         capture_output=True,

archivebox/tests/test_api_v1_core_tags_search.py

@@ -41,6 +41,26 @@ def test_tag_search_api_returns_card_payload(client, api_token, tagged_data):
     assert {snap.url for snap in snapshots} == {"https://example.com/one", "https://example.com/two"}
 
 
+def test_tag_search_api_default_includes_empty_tags_and_counts_linked_snapshots(client, api_token, tagged_data, api_admin_user):
+    linked_tag, _snapshots = tagged_data
+    empty_tag = Tag.objects.create(name="Empty Tag", created_by=api_admin_user)
+
+    response = client.get(
+        "/api/v1/core/tags/search/",
+        {"api_key": api_token.token},
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+
+    assert response.status_code == 200
+    payload = response.json()
+    cards_by_name = {tag["name"]: tag for tag in payload["tags"]}
+    assert payload["has_snapshots"] == "all"
+    assert cards_by_name["Alpha Research"]["id"] == linked_tag.id
+    assert cards_by_name["Alpha Research"]["num_snapshots"] == 2
+    assert cards_by_name["Empty Tag"]["id"] == empty_tag.id
+    assert cards_by_name["Empty Tag"]["num_snapshots"] == 0
+
+
 def test_tag_search_api_respects_sort_and_filters(client, api_token, admin_user, crawl, tagged_data):
     from datetime import datetime
 

archivebox/tests/test_binary_service.py

@@ -134,6 +134,8 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert binary_processes
     assert binary_processes[-1].status == Process.StatusChoices.EXITED
     assert binary_processes[-1].exit_code == 0
+    assert binary_processes[-1].ended_at is not None
+    assert binary_processes[-1].started_at < binary_processes[-1].ended_at
     assert any(f"--name={name}" in arg for arg in binary_processes[-1].cmd)
 
     _cmd_result = run_archivebox_cmd(

archivebox/tests/test_cli_server.py

@@ -139,6 +139,7 @@ def test_runner_worker_uses_current_interpreter():
     from archivebox.workers.supervisord_util import RUNNER_WORKER
 
     assert RUNNER_WORKER["command"] == f"{sys.executable} -m archivebox run --daemon"
+    assert RUNNER_WORKER["autorestart"] == "true"
     assert 'ARCHIVEBOX_RUNNER_DAEMON="1"' in RUNNER_WORKER["environment"]
 
 

archivebox/tests/test_crawl_runner.py

@@ -435,6 +435,8 @@ def test_machine_service_persists_only_derived_config_events(tmp_path, hermetic_
                     config={
                         "WGET_BINARY": str(wget_binary),
                         "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
+                        "ABX_UV_CACHE": "/tmp/uv-cache",
+                        "ABX_PNPM_CACHE": "/tmp/pnpm-cache",
                         "CHROME_USER_DATA_DIR": "/tmp/stale-derived-profile",
                     },
                     config_type="derived",
@@ -466,6 +468,8 @@ def test_machine_service_persists_only_derived_config_events(tmp_path, hermetic_
     # LIB_DIR) then the unset removed it; ABX_INSTALL_CACHE survives.
     assert machine.config == {
         "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
+        "ABX_PNPM_CACHE": "/tmp/pnpm-cache",
+        "ABX_UV_CACHE": "/tmp/uv-cache",
     }
 
 

archivebox/tests/test_hooks.py

@@ -18,11 +18,13 @@ import textwrap
 from pathlib import Path
 
 import pytest
+import rich_click as click
 
 # Set up Django before importing any Django-dependent modules
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.settings")
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
+WORKSPACE_ROOT = REPO_ROOT.parent
 RESULT_PREFIX = "__ARCHIVEBOX_TEST_RESULT__="
 
 
@@ -50,7 +52,12 @@ def run_plugin_discovery_subprocess(tmp_path: Path, plugins_dir: Path, script: s
     cwd_plugins_dir = data_dir / "custom_plugins"
     if plugins_dir != cwd_plugins_dir:
         shutil.copytree(plugins_dir, cwd_plugins_dir)
-    env["PYTHONPATH"] = str(REPO_ROOT) + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "")
+    existing_pythonpath = [
+        str(Path(entry).expanduser().resolve(strict=False))
+        for entry in env.get("PYTHONPATH", "").split(os.pathsep)
+        if entry and Path(entry).expanduser().is_absolute()
+    ]
+    env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys([str(REPO_ROOT), *existing_pythonpath]))
     subprocess_script = "\n".join(
         [
             "import json",
@@ -84,6 +91,28 @@ def run_plugin_discovery_subprocess(tmp_path: Path, plugins_dir: Path, script: s
     raise AssertionError(f"Subprocess did not emit a result line.\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}")
 
 
+def test_cli_env_does_not_emit_relative_pythonpath_entries():
+    from archivebox.tests.conftest import cli_env
+
+    old_pythonpath = os.environ.get("PYTHONPATH")
+    try:
+        os.environ["PYTHONPATH"] = os.pathsep.join(["../abxpkg", "../abx-plugins", "../abx-dl"])
+
+        env = cli_env()
+    finally:
+        if old_pythonpath is None:
+            os.environ.pop("PYTHONPATH", None)
+        else:
+            os.environ["PYTHONPATH"] = old_pythonpath
+
+    pythonpath_entries = env["PYTHONPATH"].split(os.pathsep)
+    assert str((WORKSPACE_ROOT / "abxpkg").resolve(strict=False)) in pythonpath_entries
+    assert str((WORKSPACE_ROOT / "abx-plugins").resolve(strict=False)) in pythonpath_entries
+    assert str((WORKSPACE_ROOT / "abx-dl").resolve(strict=False)) in pythonpath_entries
+    assert all(Path(entry).is_absolute() for entry in pythonpath_entries)
+    assert not any(entry.startswith("..") for entry in pythonpath_entries)
+
+
 class TestBackgroundHookDetection:
     """Test that background hooks are detected by .bg. suffix."""
 
@@ -704,13 +733,13 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat
     from archivebox.plugins.hooks import run_hook
 
     lib_dir = tmp_path / "lib"
-    node_modules_dir = lib_dir / "npm" / "node_modules"
+    node_modules_dir = lib_dir / "pnpm" / "packages" / "chrome" / "node_modules"
     configured_node_path = os.pathsep.join(
         [
-            "/home/archivebox/.npm/lib/node_modules",
+            "/home/archivebox/.pnpm/packages/chrome/node_modules",
             "/usr/lib/node_modules",
             str(node_modules_dir),
-            "/usr/share/archivebox/lib/npm/node_modules",
+            "/usr/share/archivebox/lib/pnpm/packages/chrome/node_modules",
         ],
     )
 
@@ -750,3 +779,47 @@ print(json.dumps({
     assert payload["NODE_MODULE_DIR"] == str(node_modules_dir)
     assert payload["NODE_PATH"].split(os.pathsep) == configured_node_path.split(os.pathsep)
     assert process.env["NODE_MODULES_DIR"] == str(node_modules_dir)
+
+
+@pytest.mark.django_db(transaction=True)
+def test_run_hook_executes_python_hooks_through_script_shebang(tmp_path):
+    """Python hooks must use their abxpkg script header instead of sys.executable."""
+    from archivebox.plugins.hooks import run_hook
+
+    plugin_dir = tmp_path / "plugins" / "shebangprobe"
+    plugin_dir.mkdir(parents=True)
+    hook_path = plugin_dir / "on_Snapshot__99_shebangprobe.py"
+    hook_path.write_text(
+        """#!/usr/bin/env -S abxpkg run --script python3
+# /// script
+# requires-python = ">=3.12"
+# ///
+import json
+import os
+import rich_click
+
+print(json.dumps({
+    "ABXPKG_FAST_SCRIPT": os.environ.get("ABXPKG_FAST_SCRIPT"),
+    "RICH_CLICK_FILE": rich_click.__file__,
+}))
+""",
+        encoding="utf-8",
+    )
+    hook_path.chmod(0o755)
+
+    output_dir = tmp_path / "archive" / "users" / "system" / "snapshots" / "20260603" / "example.com" / "test" / "shebangprobe"
+    process = run_hook(
+        hook_path,
+        output_dir,
+        config={
+            "LIB_DIR": str(tmp_path / "lib"),
+        },
+        timeout=10,
+    )
+    process.refresh_from_db()
+
+    assert process.cmd[0] == str(hook_path)
+    assert process.exit_code == 0, process.stderr
+    payload = json.loads(process.stdout.strip())
+    assert payload["ABXPKG_FAST_SCRIPT"] == "1"
+    assert Path(payload["RICH_CLICK_FILE"]).resolve() == Path(click.__file__).resolve()

archivebox/tests/test_machine_models.py

@@ -382,14 +382,14 @@ class TestBinaryModel:
         """Binary.from_json() should persist provider overrides unchanged."""
         overrides = {
             "apt": {"install_args": ["chromium"]},
-            "npm": {"install_args": "puppeteer"},
+            "pnpm": {"install_args": "puppeteer"},
             "custom": {"install": "bash -lc 'echo ok'"},
         }
 
         binary = Binary.from_json(
             {
                 "name": "chrome",
-                "binproviders": "apt,npm,custom",
+                "binproviders": "apt,pnpm,custom",
                 "overrides": overrides,
             },
         )
@@ -414,13 +414,13 @@ class TestBinaryModel:
         """Binary.from_json() should no longer translate legacy non-dict provider overrides."""
         overrides = {
             "apt": ["chromium"],
-            "npm": "puppeteer",
+            "pnpm": "puppeteer",
         }
 
         binary = Binary.from_json(
             {
                 "name": "chrome",
-                "binproviders": "apt,npm",
+                "binproviders": "apt,pnpm",
                 "overrides": overrides,
             },
         )
@@ -428,15 +428,15 @@ class TestBinaryModel:
         assert binary is not None
         assert binary.overrides == overrides
 
-    def test_binary_from_json_prefers_published_readability_package(self):
-        """Binary.from_json() should rewrite readability's npm git URL to the published package."""
+    def test_binary_from_json_preserves_readability_package_metadata(self):
+        """Binary.from_json() should preserve readability's pnpm package metadata."""
         binary = Binary.from_json(
             {
                 "name": "readability-extractor",
-                "binproviders": "env,npm",
+                "binproviders": "env,pnpm",
                 "overrides": {
-                    "npm": {
-                        "install_args": ["https://github.com/ArchiveBox/readability-extractor"],
+                    "pnpm": {
+                        "install_args": ["readability-extractor"],
                     },
                 },
             },
@@ -444,7 +444,7 @@ class TestBinaryModel:
 
         assert binary is not None
         assert binary.overrides == {
-            "npm": {
+            "pnpm": {
                 "install_args": ["readability-extractor"],
             },
         }

archivebox/tests/test_machine_service.py

@@ -105,6 +105,7 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
                     "LITEPARSE_BINARY": {str(installed_liteparse_path)!r},
                     "NODE_BINARY": {str(external_tool)!r},
                     "ABX_INSTALL_CACHE": {{"lit": "cached"}},
+                    "ABX_UV_CACHE": "/tmp/uv-cache",
                     "CHROME_USER_DATA_DIR": "/tmp/derived-profile",
                 }}, config_type="derived")).now()
                 await bus.emit(MachineEvent(method="unset", key="config/LITEPARSE_BINARY", config_type="derived")).now()
@@ -137,6 +138,8 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
 
     assert machine.config["LITEPARSE_BINARY"] == str(installed_liteparse_path)
     assert machine.config["LITEPARSE_BINARY"] != "/tmp/user-config-must-not-persist"
+    assert machine.config["ABX_INSTALL_CACHE"] == {"lit": "cached"}
+    assert machine.config["ABX_UV_CACHE"] == "/tmp/uv-cache"
 
     _cmd_result = run_archivebox_cmd(
         ["version"],

archivebox/tests/test_migrations_08_to_09.py

@@ -404,7 +404,7 @@ def test_migration_creates_process_records(migration_08_data):
 
 
 def test_migration_creates_binary_records(migration_08_data):
-    """Migration should create Binary records from cmd_version data."""
+    """Migration should create and link Binary/NetworkInterface records from migrated Process data."""
     work_dir, db_path, original_data = migration_08_data
     result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
     assert result.returncode == 0, f"Init failed: {result.stderr}"
@@ -420,6 +420,31 @@ def test_migration_creates_binary_records(migration_08_data):
     extractors = {ar["extractor"] for ar in original_data["archiveresults"]}
     assert binary_count >= len(extractors), f"Expected at least {len(extractors)} Binaries, got {binary_count}"
 
+    cursor.execute("""
+        SELECT COUNT(*)
+        FROM machine_process
+        WHERE cmd != '[]' AND binary_id IS NULL
+    """)
+    missing_binary_count = cursor.fetchone()[0]
+    assert missing_binary_count == 0
+
+    cursor.execute("""
+        SELECT p.cmd, b.name, b.abspath
+        FROM machine_process p
+        JOIN machine_binary b ON p.binary_id = b.id
+        WHERE p.cmd != '[]'
+    """)
+    rows = cursor.fetchall()
+    assert rows
+    for cmd_raw, binary_name, binary_abspath in rows:
+        cmd = json.loads(cmd_raw)
+        assert binary_name == cmd[0]
+        assert binary_abspath == cmd[0]
+
+    cursor.execute("SELECT COUNT(*) FROM machine_process WHERE iface_id IS NULL")
+    missing_iface_count = cursor.fetchone()[0]
+    assert missing_iface_count == 0
+
     conn.close()
 
 
@@ -746,7 +771,12 @@ def test_archiveresult_files_preserved_after_migration(tmp_path):
             files_before.extend([f for f in d.rglob("*") if f.is_file()])
     files_before_count = len(files_before)
     generated_metadata_names = {"index.html", "index.json", "index.jsonl"}
-    original_payloads = sorted(path.read_text() for path in files_before if path.name not in generated_metadata_names)
+    generated_search_backends = {"search_backend_sqlite", "search_backend_sonic"}
+
+    def is_generated_file(path) -> bool:
+        return path.name in generated_metadata_names or any(part in generated_search_backends for part in path.parts)
+
+    original_payloads = sorted(path.read_text() for path in files_before if not is_generated_file(path))
 
     # Sample some specific files to check they're preserved
     sample_paths_before = {}
@@ -881,9 +911,7 @@ def test_archiveresult_files_preserved_after_migration(tmp_path):
     # the hydrated DB row, so raw file counts are allowed to increase; compare
     # the legacy payload contents after excluding those generated metadata
     # files to keep the no-data-loss assertion strict.
-    migrated_payloads = sorted(
-        path.read_text() for path in [*files_new_structure, *old_files_remaining] if path.name not in generated_metadata_names
-    )
+    migrated_payloads = sorted(path.read_text() for path in [*files_new_structure, *old_files_remaining] if not is_generated_file(path))
     assert original_payloads == migrated_payloads, "Legacy payload files changed or were lost during reorganization"
     assert files_new_count >= files_before_count, "New 0.9 metadata should not replace legacy payload files"
 

archivebox/tests/test_opencode_agent.py

@@ -0,0 +1,344 @@
+import os
+import sqlite3
+
+import pytest
+import requests
+from asgiref.sync import async_to_sync
+
+from archivebox.tests.conftest import ADMIN_TEST_HOST
+
+
+pytestmark = pytest.mark.django_db
+
+
+def test_opencode_disabled_route_does_not_start_server(client, monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": False})
+    monkeypatch.setattr(views, "_ensure_opencode", lambda settings: pytest.fail("opencode should not start when disabled"))
+
+    response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+
+    assert response.status_code == 404
+
+
+def test_opencode_agent_requires_superuser(client, db, monkeypatch, django_user_model):
+    from abx_plugins.plugins.opencode import views
+
+    monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True})
+    monkeypatch.setattr(views, "_ensure_opencode", lambda settings: pytest.fail("opencode should not start before auth passes"))
+
+    response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 302
+    assert "/admin/login/" in response.headers["Location"]
+
+    user = django_user_model.objects.create_user(username="regular", password="testpassword")
+    client.force_login(user)
+    response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 403
+
+
+def test_opencode_agent_superuser_gets_wrapper(admin_client, db, monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True})
+    monkeypatch.setattr(views, "_ensure_opencode", lambda settings: (True, ""))
+
+    response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+
+    assert response.status_code == 200
+    assert b'<iframe src="/admin/agent/opencode/' in response.content
+    assert b'/session"' in response.content
+    assert b'id="header"' in response.content
+    assert b'id="progress-monitor"' in response.content
+
+
+def test_opencode_proxy_blocks_cross_origin_mutation(admin_client, db, monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True})
+    monkeypatch.setattr(views, "_ensure_opencode", lambda settings: pytest.fail("opencode should not start before origin check passes"))
+
+    response = admin_client.post(
+        "/admin/agent/opencode/session",
+        data=b"{}",
+        content_type="application/json",
+        HTTP_HOST=ADMIN_TEST_HOST,
+        HTTP_ORIGIN="https://evil.example",
+    )
+
+    assert response.status_code == 403
+
+
+def test_opencode_proxy_allows_same_origin_fetch_metadata(admin_client, db, monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True})
+    monkeypatch.setattr(views, "_ensure_opencode", lambda settings: (True, ""))
+
+    def fake_request(method, url, **kwargs):
+        upstream = requests.Response()
+        upstream.status_code = 200
+        upstream._content = b"{}"
+        upstream.headers["Content-Type"] = "application/json"
+        return upstream
+
+    monkeypatch.setattr(views.requests, "request", fake_request)
+
+    response = admin_client.post(
+        "/admin/agent/opencode/pty/test/connect-token",
+        data=b"{}",
+        content_type="application/json",
+        HTTP_HOST=ADMIN_TEST_HOST,
+        HTTP_SEC_FETCH_SITE="same-origin",
+    )
+
+    assert response.status_code == 200
+
+
+def test_opencode_proxy_allows_pty_connect_token_without_origin(admin_client, db, monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True})
+    monkeypatch.setattr(views, "_ensure_opencode", lambda settings: (True, ""))
+
+    def fake_request(method, url, **kwargs):
+        upstream = requests.Response()
+        upstream.status_code = 200
+        upstream._content = b"{}"
+        upstream.headers["Content-Type"] = "application/json"
+        return upstream
+
+    monkeypatch.setattr(views.requests, "request", fake_request)
+
+    response = admin_client.post(
+        "/admin/agent/opencode/pty/test/connect-token",
+        data=b"{}",
+        content_type="application/json",
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+
+    assert response.status_code == 200
+
+
+def test_opencode_proxy_blocks_cross_site_fetch_metadata(admin_client, db, monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True})
+    monkeypatch.setattr(
+        views,
+        "_ensure_opencode",
+        lambda settings: pytest.fail("opencode should not start before fetch metadata check passes"),
+    )
+
+    response = admin_client.post(
+        "/admin/agent/opencode/session",
+        data=b"{}",
+        content_type="application/json",
+        HTTP_HOST=ADMIN_TEST_HOST,
+        HTTP_SEC_FETCH_SITE="cross-site",
+    )
+
+    assert response.status_code == 403
+
+
+def test_opencode_project_current_is_seeded_data_project(admin_client, tmp_path, db, monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    workdir = tmp_path / "data"
+    monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True, "OPENCODE_WORKDIR": str(workdir)})
+    monkeypatch.setattr(views, "_ensure_opencode", lambda settings: (True, ""))
+
+    response = admin_client.get(
+        f"/admin/agent/opencode/project/current?directory={workdir}",
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+
+    assert response.status_code == 200
+    assert response.json()["id"] == "global"
+    assert response.json()["worktree"] == str(workdir.resolve())
+    assert response.json()["name"] == "data"
+
+
+def test_opencode_path_reports_data_as_worktree(admin_client, tmp_path, db, monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    workdir = tmp_path / "data"
+    monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True, "OPENCODE_WORKDIR": str(workdir)})
+    monkeypatch.setattr(views, "_ensure_opencode", lambda settings: (True, ""))
+
+    response = admin_client.get(
+        f"/admin/agent/opencode/path?directory={workdir}",
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+
+    assert response.status_code == 200
+    assert response.json()["directory"] == str(workdir.resolve())
+    assert response.json()["worktree"] == str(workdir.resolve())
+
+
+def test_opencode_proxy_does_not_use_basic_auth(admin_client, db, monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True})
+    monkeypatch.setattr(views, "_ensure_opencode", lambda settings: (True, ""))
+
+    def fake_request(method, url, **kwargs):
+        assert "Authorization" not in kwargs["headers"]
+        upstream = requests.Response()
+        upstream.status_code = 200
+        upstream._content = b"{}"
+        upstream.headers["Content-Type"] = "application/json"
+        return upstream
+
+    monkeypatch.setattr(views.requests, "request", fake_request)
+
+    response = admin_client.get("/admin/agent/opencode/global/config", HTTP_HOST=ADMIN_TEST_HOST)
+
+    assert response.status_code == 200
+
+
+def test_opencode_rewrites_vite_preload_assets():
+    from abx_plugins.plugins.opencode import views
+
+    body = b'const BL="modulepreload",UL=function(t){return"/"+t};const icon="/assets/sprite.svg#anthropic"'
+    rewritten = views._rewrite_text(body, {"origin": "http://127.0.0.1:4096"}).decode()
+
+    assert 'return"/"+t' not in rewritten
+    assert 'return"/admin/agent/opencode/"+t' in rewritten
+    assert '"/admin/agent/opencode/assets/sprite.svg#anthropic"' in rewritten
+
+
+def test_opencode_proxy_streams_sse_without_large_buffer(admin_client, db, monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    async def fake_event_chunks(request, settings, path):
+        assert path == "global/event"
+        yield b"data: {}\n\n"
+
+    async def collect(response):
+        return b"".join([chunk async for chunk in response.streaming_content])
+
+    monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True})
+    monkeypatch.setattr(views, "_ensure_opencode", lambda settings: (True, ""))
+    monkeypatch.setattr(views, "_event_chunks", fake_event_chunks)
+
+    response = admin_client.get("/admin/agent/opencode/global/event", HTTP_HOST=ADMIN_TEST_HOST)
+    assert async_to_sync(collect)(response) == b"data: {}\n\n"
+    assert response.headers["X-Accel-Buffering"] == "no"
+
+
+def test_opencode_sse_chunks_are_not_rewritten(monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    class FakeRequest:
+        method = "GET"
+        GET = {}
+        headers = {}
+
+    class FakeUpstream:
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *args):
+            return None
+
+        async def aiter_raw(self, chunk_size=None):
+            assert chunk_size == 512
+            yield b'event: message.part.updated\ndata: {"delta":"a\\nb"}\n\n'
+
+    class FakeClient:
+        def __init__(self, **kwargs):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *args):
+            return None
+
+        def stream(self, *args, **kwargs):
+            return FakeUpstream()
+
+    monkeypatch.setattr(views.httpx, "AsyncClient", FakeClient)
+
+    async def collect():
+        settings = {"timeout": 30, "origin": "http://127.0.0.1:4096"}
+        return b"".join([chunk async for chunk in views._event_chunks(FakeRequest(), settings, "global/event")])
+
+    assert async_to_sync(collect)() == b'event: message.part.updated\ndata: {"delta":"a\\nb"}\n\n'
+
+
+def test_opencode_starts_without_opening_browser(tmp_path, monkeypatch):
+    from abx_plugins.plugins.opencode import views
+
+    popen_kwargs = {}
+    health_checks = iter([False, False, True])
+
+    class FakeProcess:
+        def poll(self):
+            return None
+
+    def fake_popen(cmd, **kwargs):
+        popen_kwargs.update(kwargs)
+        return FakeProcess()
+
+    monkeypatch.setattr(views, "_health", lambda settings: next(health_checks))
+    monkeypatch.setattr(views.shutil, "which", lambda binary: "/usr/bin/false")
+    monkeypatch.setattr(views.subprocess, "Popen", fake_popen)
+
+    settings = views._settings({"OPENCODE_WORKDIR": str(tmp_path)})
+    ok, error = views._ensure_opencode(settings)
+
+    assert ok, error
+    assert popen_kwargs["cwd"] == tmp_path.resolve()
+    assert popen_kwargs["env"]["BROWSER"] == "false"
+    assert popen_kwargs["env"]["GIT_CEILING_DIRECTORIES"] == f"{tmp_path.resolve()}{os.pathsep}{tmp_path.parent.resolve()}"
+    assert popen_kwargs["env"]["OPENCODE_DISABLE_PROJECT_CONFIG"] == "true"
+    assert popen_kwargs["env"]["XDG_DATA_HOME"] == str(tmp_path / "opencode" / "data")
+
+
+def test_opencode_state_dir_is_separate_from_workdir(tmp_path):
+    from abx_plugins.plugins.opencode import views
+
+    workdir = tmp_path / "data"
+    settings = views._settings({"OPENCODE_WORKDIR": str(workdir)})
+    views._ensure_project_files(settings)
+
+    assert settings["workdir"] == workdir
+    assert settings["opencode_dir"] == workdir / "opencode"
+    assert settings["config_home"] == workdir / "opencode" / "config"
+    assert settings["data_home"] == workdir / "opencode" / "data"
+    assert settings["state_home"] == workdir / "opencode" / "state"
+    skill = workdir / "opencode" / "config" / "opencode" / "skills" / "archivebox" / "SKILL.md"
+    assert skill.exists()
+    assert f"ArchiveBox collection directory: {workdir.resolve()}" in skill.read_text()
+
+
+def test_opencode_seeds_global_project_under_embedded_data_dir(tmp_path):
+    from abx_plugins.plugins.opencode import views
+
+    workdir = tmp_path / "data"
+    settings = views._settings({"OPENCODE_WORKDIR": str(workdir)})
+    db_path = settings["data_home"] / "opencode" / "opencode.db"
+    db_path.parent.mkdir(parents=True)
+    with sqlite3.connect(db_path) as db:
+        db.execute(
+            """
+            CREATE TABLE project (
+                id text PRIMARY KEY,
+                worktree text NOT NULL,
+                name text,
+                time_created integer NOT NULL,
+                time_updated integer NOT NULL,
+                sandboxes text NOT NULL
+            )
+            """,
+        )
+
+    views._ensure_global_project(settings)
+
+    with sqlite3.connect(db_path) as db:
+        row = db.execute("SELECT id, worktree, name, sandboxes FROM project").fetchone()
+
+    assert row == ("global", str(workdir.resolve()), "data", "[]")

archivebox/tests/test_recursive_crawl.py

@@ -11,7 +11,7 @@ import pytest
 
 from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.crawls.models import Crawl
-from archivebox.machine.models import Process
+from archivebox.machine.models import Binary, Process
 from archivebox.tests.conftest import run_archivebox_cmd, cli_env
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
@@ -415,6 +415,295 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p
     assert any("wget" in (pwd or "") or "wget" in (cmd or "") for *_rest, pwd, cmd in process_rows)
 
 
+@pytest.mark.timeout(1200)
+def test_add_archivewebpage_installs_required_chrome_dependency(initialized_archive):
+    """archivebox add should install selected plugins' required_plugins and binaries before hooks run."""
+
+    env = os.environ.copy()
+    env.pop("CHROME_BINARY", None)
+    env.update(
+        {
+            "USE_COLOR": "false",
+            "SHOW_PROGRESS": "false",
+            "TIMEOUT": "120",
+            "ABXPKG_INSTALL_TIMEOUT": "900",
+            "LIB_DIR": str(initialized_archive / "lib"),
+            "ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
+            "CHROME_HEADLESS": "true",
+            "CHROME_SANDBOX": "false",
+            "CHROME_ISOLATION": "snapshot",
+            "CHROME_EXTENSIONS_DIR": str(initialized_archive / "lib/chromewebstore/extensions"),
+        },
+    )
+
+    result = run_archivebox_cmd(
+        [
+            "add",
+            "--depth=0",
+            "--max-urls=1",
+            "--tag=archivewebpage-required-plugin-preflight",
+            "--parser=url_list",
+            "--plugins=archivewebpage",
+            "https://example.com/",
+        ],
+        cwd=initialized_archive,
+        env=env,
+        timeout=1200,
+    )
+    stdout, stderr = result.stdout, result.stderr
+
+    if stderr:
+        print(f"\n=== STDERR ===\n{stderr}\n=== END STDERR ===\n")
+    if stdout:
+        print(f"\n=== STDOUT (last 4000 chars) ===\n{stdout[-4000:]}\n=== END STDOUT ===\n")
+    assert result.returncode == 0, stderr or stdout
+
+    with use_archivebox_db(initialized_archive):
+        binaries = {
+            row["name"]: row for row in Binary.objects.order_by("name").values("name", "status", "binprovider", "abspath", "version")
+        }
+        archive_results = list(
+            ArchiveResult.objects.order_by("plugin", "hook_name").values_list(
+                "plugin",
+                "hook_name",
+                "status",
+                "output_str",
+                "output_files",
+            ),
+        )
+        process_rows = list(
+            Process.objects.order_by("process_type", "created_at").values_list("process_type", "status", "exit_code", "cmd", "env"),
+        )
+        snapshot_output_dirs = [snapshot.output_dir for snapshot in Snapshot.objects.order_by("created_at")]
+
+    assert "chromium" in binaries
+    assert binaries["chromium"]["status"] == Binary.StatusChoices.INSTALLED
+    assert binaries["chromium"]["binprovider"] == "puppeteer"
+    assert Path(binaries["chromium"]["abspath"]).exists()
+    chromium_version_parts = [int(part) for part in binaries["chromium"]["version"].split(".")[:3]]
+    assert chromium_version_parts >= [149, 0, 0]
+
+    assert "archivewebpage" in binaries
+    assert binaries["archivewebpage"]["status"] == Binary.StatusChoices.INSTALLED
+    assert binaries["archivewebpage"]["binprovider"] == "chromewebstore"
+    archivewebpage_manifest = Path(binaries["archivewebpage"]["abspath"])
+    assert archivewebpage_manifest.exists()
+    assert archivewebpage_manifest.name == "manifest.json"
+
+    plugins_seen = {plugin for plugin, _hook_name, _status, _output_str, _output_files in archive_results}
+    assert {"chrome", "archivewebpage"}.issubset(plugins_seen)
+    assert all(
+        status == ArchiveResult.StatusChoices.SUCCEEDED
+        for plugin, _hook_name, status, _output_str, _output_files in archive_results
+        if plugin in {"chrome", "archivewebpage"}
+    ), archive_results
+    assert snapshot_output_dirs
+    archivewebpage_wacz = Path(snapshot_output_dirs[0]) / "archivewebpage" / "archivewebpage.wacz"
+    assert archivewebpage_wacz.exists()
+    assert archivewebpage_wacz.stat().st_size > 0
+    chrome_hook_envs = [
+        env
+        for process_type, _status, _exit_code, cmd, env in process_rows
+        if process_type == Process.TypeChoices.HOOK and "chrome_launch" in str(cmd)
+    ]
+    assert chrome_hook_envs
+    assert all("{LIB_DIR}" not in str(env) for env in chrome_hook_envs)
+    assert any(process_type == Process.TypeChoices.BINARY for process_type, _status, _exit_code, _cmd, _env in process_rows)
+    assert all(
+        status == Process.StatusChoices.EXITED and exit_code == 0
+        for process_type, status, exit_code, _cmd, _env in process_rows
+        if process_type == Process.TypeChoices.BINARY
+    )
+
+
+@pytest.mark.timeout(1200)
+def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initialized_archive, free_tcp_port_factory):
+    """Run a bounded real depth=2 crawl with all plugins enabled and verify parallel snapshot execution."""
+
+    from abx_dl.models import discover_plugins
+
+    root_url = "https://example.com/"
+    plugin_selection = ",".join(
+        sorted(plugin for plugin in discover_plugins().keys() if not plugin.startswith("claude")),
+    )
+    env = os.environ.copy()
+    env.pop("CHROME_BINARY", None)
+    env.update(
+        {
+            "USE_COLOR": "false",
+            "SHOW_PROGRESS": "false",
+            "LIB_DIR": str(initialized_archive / "lib"),
+            "ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
+            "TIMEOUT": "90",
+            "ABXPKG_INSTALL_TIMEOUT": "900",
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
+            "SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1",
+            "SEARCH_BACKEND_SONIC_PORT": str(free_tcp_port_factory()),
+            "CHROME_HEADLESS": "true",
+            "CHROME_SANDBOX": "false",
+            "CHROME_ISOLATION": "snapshot",
+        },
+    )
+
+    result = run_archivebox_cmd(
+        [
+            "add",
+            "--depth=2",
+            "--max-urls=8",
+            "--crawl-max-size=100mb",
+            "--tag=recursive-all-plugins",
+            "--parser=url_list",
+            f"--plugins={plugin_selection}",
+            root_url,
+        ],
+        cwd=initialized_archive,
+        env=env,
+        timeout=1200,
+    )
+    stdout, stderr = result.stdout, result.stderr
+
+    if stderr:
+        print(f"\n=== STDERR ===\n{stderr}\n=== END STDERR ===\n")
+    if stdout:
+        print(f"\n=== STDOUT (last 4000 chars) ===\n{stdout[-4000:]}\n=== END STDOUT ===\n")
+    assert result.returncode == 0, stderr or stdout
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.get(tags_str="recursive-all-plugins")
+        snapshots = list(
+            Snapshot.objects.filter(crawl=crawl)
+            .order_by("depth", "url")
+            .values_list("id", "url", "depth", "status", "parent_snapshot_id", "downloaded_at"),
+        )
+        snapshot_ids_by_output_dir = {
+            str(snapshot.output_dir): str(snapshot.id) for snapshot in Snapshot.objects.filter(crawl=crawl).order_by("depth", "url")
+        }
+        archive_results = list(
+            ArchiveResult.objects.filter(snapshot__crawl=crawl)
+            .select_related("snapshot")
+            .order_by("snapshot__depth", "snapshot__url", "plugin", "hook_name")
+            .values_list(
+                "snapshot_id",
+                "snapshot__url",
+                "snapshot__depth",
+                "plugin",
+                "hook_name",
+                "status",
+                "output_files",
+                "output_size",
+                "output_str",
+            ),
+        )
+        processes = list(
+            Process.objects.filter(process_type=Process.TypeChoices.HOOK, pwd__contains=str(crawl.output_dir))
+            .order_by("started_at")
+            .values_list("pwd", "cmd", "status", "exit_code", "started_at", "ended_at"),
+        )
+
+    assert crawl.max_depth == 2
+    assert crawl.config["CRAWL_MAX_URLS"] == 8
+    assert crawl.config["CRAWL_MAX_SIZE"] == 100 * 1024 * 1024
+    assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 3
+    assert crawl.status == Crawl.StatusChoices.SEALED
+    assert crawl.retry_at is None
+
+    assert len(snapshots) == 8
+    assert any(url == root_url and depth == 0 for _id, url, depth, _status, _parent, _downloaded_at in snapshots)
+    assert any("iana.org" in url and depth == 1 for _id, url, depth, _status, _parent, _downloaded_at in snapshots)
+    assert any(depth == 2 for _id, _url, depth, _status, _parent, _downloaded_at in snapshots)
+    assert all(status == Snapshot.StatusChoices.SEALED for _id, _url, _depth, status, _parent, _downloaded_at in snapshots)
+    assert all(downloaded_at is not None for _id, _url, _depth, _status, _parent, downloaded_at in snapshots)
+
+    assert archive_results
+    allowed_statuses = {
+        ArchiveResult.StatusChoices.SUCCEEDED,
+        ArchiveResult.StatusChoices.NORESULTS,
+        ArchiveResult.StatusChoices.SKIPPED,
+    }
+    unexpected_results = [
+        {
+            "url": url,
+            "depth": depth,
+            "plugin": plugin,
+            "hook_name": hook_name,
+            "status": status,
+            "output_str": output_str,
+        }
+        for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
+        if not (status in allowed_statuses or (plugin == "archivedotorg" and status == ArchiveResult.StatusChoices.FAILED))
+    ]
+    assert not unexpected_results
+
+    plugins_seen = {plugin for _snapshot_id, _url, _depth, plugin, _hook_name, _status, _files, _size, _output in archive_results}
+    assert {
+        "wget",
+        "headers",
+        "title",
+        "pdf",
+        "screenshot",
+        "dom",
+        "singlefile",
+        "readability",
+        "mercury",
+        "htmltotext",
+        "favicon",
+        "parse_html_urls",
+        "archivedotorg",
+    }.issubset(plugins_seen)
+
+    snapshot_root = initialized_archive / "archive/users/system/snapshots"
+    assert list(snapshot_root.rglob("wget/**/*.html"))
+    assert list(snapshot_root.rglob("headers/**/headers.json"))
+    assert list(snapshot_root.rglob("title/title.txt"))
+    assert list(snapshot_root.rglob("pdf/**/*.pdf"))
+    assert list(snapshot_root.rglob("screenshot/**/*.png"))
+    assert list(snapshot_root.rglob("dom/**/*.html"))
+    assert list(snapshot_root.rglob("singlefile/**/*.html"))
+    assert list(snapshot_root.rglob("readability/**/*.html"))
+    assert list(snapshot_root.rglob("mercury/**/*.html"))
+    assert list(snapshot_root.rglob("htmltotext/**/*.txt"))
+    assert list(snapshot_root.rglob("favicon/**/*"))
+    urls_jsonl_files = list(snapshot_root.rglob("parse_html_urls/urls.jsonl"))
+    assert urls_jsonl_files
+    assert any("iana.org" in path.read_text(errors="ignore") for path in urls_jsonl_files)
+
+    assert processes
+    failed_hook_results = [
+        {
+            "url": url,
+            "depth": depth,
+            "plugin": plugin,
+            "hook_name": hook_name,
+            "status": status,
+            "output_str": output_str,
+        }
+        for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
+        if status == ArchiveResult.StatusChoices.FAILED and plugin != "archivedotorg"
+    ]
+    assert not failed_hook_results
+    assert all(status == Process.StatusChoices.EXITED for _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
+
+    intervals = []
+    for pwd, cmd, _status, _exit_code, started_at, ended_at in processes:
+        if not started_at or not ended_at:
+            continue
+        process_snapshot_id = next(
+            (snapshot_id for output_dir, snapshot_id in snapshot_ids_by_output_dir.items() if str(pwd).startswith(output_dir)),
+            None,
+        )
+        if process_snapshot_id is None:
+            continue
+        intervals.append((process_snapshot_id, started_at, ended_at, pwd, cmd))
+
+    overlapping = [
+        (left, right)
+        for index, left in enumerate(intervals)
+        for right in intervals[index + 1 :]
+        if left[0] != right[0] and left[1] < right[2] and right[1] < left[2]
+    ]
+    assert overlapping, f"Expected hook processes from different snapshots to overlap, got intervals: {intervals}"
+
+
 def test_crawl_snapshot_has_parent_snapshot_field(tmp_path, initialized_archive):
     """Test that Snapshot model has parent_snapshot field."""
 

archivebox/tests/test_server_security_browser.py

@@ -292,7 +292,7 @@ def _resolve_browser(shared_lib: Path) -> Path | None:
 @pytest.fixture(scope="session")
 def browser_runtime(tmp_path_factory):
     assert shutil.which("node") is not None, "Node.js is required for browser security tests"
-    assert shutil.which("npm") is not None, "npm is required for browser security tests"
+    assert shutil.which("pnpm") is not None, "pnpm is required for browser security tests"
 
     shared_lib = tmp_path_factory.mktemp("archivebox_browser_lib")
     _ensure_puppeteer(shared_lib)
@@ -302,7 +302,7 @@ def browser_runtime(tmp_path_factory):
 
     return {
         "lib_dir": shared_lib,
-        "node_modules_dir": shared_lib / "npm" / "node_modules",
+        "node_modules_dir": shared_lib / "pnpm" / "packages" / "chrome" / "node_modules",
         "chrome_binary": browser,
     }
 

archivebox/tests/test_ui_admin_links.py

@@ -118,7 +118,7 @@ def running_process_record():
 
 
 def test_archiveresult_admin_links_plugin_and_process():
-    from archivebox.core.admin_archiveresults import ArchiveResultAdmin
+    from archivebox.core.admin_archiveresults import ArchiveResultAdmin, render_archiveresults_list
     from archivebox.core.models import ArchiveResult
     from archivebox.machine.models import Process
 
@@ -147,6 +147,17 @@ def test_archiveresult_admin_links_plugin_and_process():
 
     assert "/admin/environment/plugins/builtin.wget/" in plugin_html
     assert f"/admin/machine/process/{process.id}/change" in process_html
+    assert f"<code>{str(process.id)[-8:]}</code>" in process_html
+    assert "<code>-</code>" not in process_html
+
+    machine_html = str(admin.machine_link(result))
+    assert f"/admin/machine/machine/{iface.machine.id}/change" in machine_html
+    assert machine_html == f'<a href="/admin/machine/machine/{iface.machine.id}/change/">{iface.machine.hostname}</a>'
+
+    inline_html = str(render_archiveresults_list(ArchiveResult.objects.filter(id=result.id)))
+    assert f"/admin/machine/process/{process.id}/change" in inline_html
+    assert f">{str(process.id)[-8:]}</a>" in inline_html
+    assert ">-</a>" not in inline_html
 
 
 def test_deleting_binary_and_process_records_preserves_results():

archivebox/tests/test_ui_admin_machine.py

@@ -83,6 +83,8 @@ class TestMachineAdmin:
             timeout=90,
             pid=54321,
             exit_code=0,
+            stdout="job stdout\nline 2",
+            stderr="job stderr\nline 2",
             url="https://example.com/status",
             started_at=timezone.now() - timedelta(seconds=52),
             ended_at=timezone.now(),
@@ -96,6 +98,8 @@ class TestMachineAdmin:
         assert b"Kill" in response.content
         assert b"python /tmp/job.py --url=https://example.com" in response.content
         assert b"ENABLED=True" in response.content
+        assert b"job stdout" in response.content
+        assert b"job stderr" in response.content
         assert b"52s" in response.content
         assert b"API_KEY=" not in response.content
         assert b"ACCESS_TOKEN=" not in response.content
@@ -107,6 +111,8 @@ class TestMachineAdmin:
         assert b'name="timeout"' not in response.content
         assert b'name="pid"' not in response.content
         assert b'name="exit_code"' not in response.content
+        assert b'name="stdout"' not in response.content
+        assert b'name="stderr"' not in response.content
         assert b'name="url"' not in response.content
         assert b'name="started_at"' not in response.content
         assert b'name="ended_at"' not in response.content

archivebox/tests/test_ui_live_progress.py

@@ -53,7 +53,7 @@ class TestLiveProgressView:
 
         response = client.get("/progress.json", HTTP_HOST=ADMIN_TEST_HOST)
 
-        assert response.status_code == 200
+        assert response.status_code == 200, response.content
         payload = response.json()
         active_crawl = next(item for item in payload["active_crawls"] if item["id"] == str(crawl.pk))
         active_snapshot = next(item for item in active_crawl["active_snapshots"] if item["id"] == str(snapshot.pk))
@@ -92,7 +92,7 @@ class TestLiveProgressView:
 
         response = client.get("/progress.json", HTTP_HOST=ADMIN_TEST_HOST)
 
-        assert response.status_code == 200
+        assert response.status_code == 200, response.content
         payload = response.json()
         active_crawl = next(item for item in payload["active_crawls"] if item["id"] == str(crawl.pk))
         active_snapshot = next(item for item in active_crawl["active_snapshots"] if item["id"] == str(snapshot.pk))
@@ -163,6 +163,39 @@ class TestLiveProgressView:
             assert payload["scope"]["crawl_id"] == compact_id
             assert payload["active_crawls"]
 
+    def test_live_progress_shows_old_paused_crawl_with_due_snapshot_work(self, client, admin_user, crawl, snapshot):
+        from datetime import timedelta
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import Snapshot
+
+        old_timestamp = timezone.now() - timedelta(days=2)
+        Crawl.objects.filter(pk=crawl.pk).update(
+            status=Crawl.StatusChoices.PAUSED,
+            created_at=old_timestamp,
+            modified_at=old_timestamp,
+            retry_at=None,
+        )
+        Snapshot.objects.filter(pk=snapshot.pk).update(
+            status=Snapshot.StatusChoices.QUEUED,
+            retry_at=timezone.now(),
+            modified_at=timezone.now(),
+        )
+
+        client.force_login(admin_user)
+        response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
+
+        assert response.status_code == 200, response.content
+        payload = response.json()
+        active_crawl = next(item for item in payload["active_crawls"] if item["id"] == str(crawl.pk))
+        assert active_crawl["status"] == Crawl.StatusChoices.PAUSED
+        assert active_crawl["pending_snapshots"] == 1
+        assert active_crawl["active_snapshots"] == [
+            [
+                str(snapshot.pk),
+                "https://example.com",
+            ],
+        ]
+
     def test_live_progress_reports_real_orchestrator_process_running(self, client, admin_user, db):
         import archivebox.machine.models as machine_models
         from archivebox.machine.models import Machine, Process, psutil

archivebox/tests/test_urls.py

@@ -6,6 +6,8 @@ from pathlib import Path
 
 import pytest
 
+from archivebox.tests.conftest import run_archivebox_cmd
+
 
 REPO_ROOT = Path(__file__).resolve().parents[3]
 
@@ -253,26 +255,35 @@ class TestUrlRouting:
         )
 
     def test_api_archive_redirect_uses_public_web_base_url(self) -> None:
-        self._run(
-            """
-            client = Client()
-
-            resp = client.get(
-                "/api/archive/https://example.com/",
-                HTTP_HOST="api.archivebox.io",
-                secure=True,
+        try:
+            config_result = run_archivebox_cmd(
+                ["config", "--set", "BASE_URL=https://archivebox.io"],
+                cwd=self.data_dir,
             )
+            assert config_result.returncode == 0, config_result.stderr
+            self._run(
+                """
+                client = Client()
 
-            assert resp.status_code in (301, 302)
-            assert resp["Location"] == "https://web.archivebox.io/web/https://example.com/"
+                resp = client.get(
+                    "/api/archive/https://example.com/",
+                    HTTP_HOST="api.archivebox.io",
+                    secure=True,
+                )
 
-            print("OK")
-            """,
-            mode="safe-subdomains-fullreplay",
-            env_overrides={
-                "BASE_URL": "https://archivebox.io",
-            },
-        )
+                assert resp.status_code in (301, 302)
+                assert resp["Location"] == "https://web.archivebox.io/web/https://example.com/"
+
+                print("OK")
+                """,
+                mode="safe-subdomains-fullreplay",
+            )
+        finally:
+            reset_result = run_archivebox_cmd(
+                ["config", "--set", "BASE_URL=http://archivebox.localhost:8000"],
+                cwd=self.data_dir,
+            )
+            assert reset_result.returncode == 0, reset_result.stderr
 
     def test_web_admin_routing(self) -> None:
         self._run(

archivebox/workers/supervisord_util.py

@@ -158,7 +158,7 @@ RUNNER_WORKER = {
     "name": "worker_runner",
     "command": _shell_join([sys.executable, "-m", "archivebox", "run", "--daemon"]),
     "autostart": "false",
-    "autorestart": "unexpected",
+    "autorestart": "true",
     "environment": 'PYTHONUNBUFFERED="1",COLUMNS="200",ARCHIVEBOX_RUNNER_DAEMON="1"',
     "stopasgroup": "true",
     "killasgroup": "true",

bin/deploy_dev_demo.sh

@@ -69,8 +69,16 @@ echo "[+] Pulling $DEPLOY_IMAGE..."
 echo "[+] Restarting $DEPLOY_SERVICE..."
 "${COMPOSE[@]}" up -d "$DEPLOY_SERVICE"
 
+if "${COMPOSE[@]}" config --services | grep -qx argo; then
+    echo "[+] Ensuring argo tunnel is running..."
+    "${COMPOSE[@]}" up -d argo
+fi
+
 echo "[+] Container status:"
 "${COMPOSE[@]}" ps "$DEPLOY_SERVICE"
+if "${COMPOSE[@]}" config --services | grep -qx argo; then
+    "${COMPOSE[@]}" ps argo
+fi
 
 echo "[+] ArchiveBox version:"
 "${COMPOSE[@]}" exec -T "$DEPLOY_SERVICE" archivebox version | sed -n '1,40p'

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.34rc38",
+  "version": "0.9.34rc39",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.34rc38"
+version = "0.9.34rc39"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,9 +79,9 @@ dependencies = [
     ### Extractor dependencies (optional binaries detected at runtime via shutil.which)
     ### Binary/Package Management
     "abxbus==2.5.9",          # EventBus API
-    "abxpkg>=1.11.154",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins>=1.11.156",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl>=1.11.156",       # shared ArchiveBox downloader package with blocking install preflight
+    "abxpkg>=1.11.158",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
+    "abx-plugins>=1.11.158",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl>=1.11.158",       # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]