release: archivebox 0.9.34rc2

archivebox/services/binary_service.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 
@@ -42,6 +43,8 @@ class ArchiveBoxDBBinaryCacheBackend:
             return None
 
         existing = await Binary.objects.filter(machine=machine, name=binary_name).afirst()
+        persisted_overrides = _persisted_overrides_for_request(request)
+        native_overrides = request.overrides or {}
         cache_invalidated = False
         if existing and existing.status == Binary.StatusChoices.INSTALLED:
             changed = False
@@ -49,8 +52,8 @@ class ArchiveBoxDBBinaryCacheBackend:
             if requested_binproviders and existing.binproviders != requested_binproviders:
                 existing.binproviders = requested_binproviders
                 changed = True
-            if request.overrides and existing.overrides != request.overrides:
-                existing.overrides = request.overrides
+            if persisted_overrides and existing.overrides != persisted_overrides:
+                existing.overrides = persisted_overrides
                 changed = True
             if changed:
                 existing.status = Binary.StatusChoices.QUEUED
@@ -62,7 +65,7 @@ class ArchiveBoxDBBinaryCacheBackend:
                 machine=machine,
                 name=binary_name,
                 binproviders=_binproviders_to_str(request.binproviders),
-                overrides=request.overrides or {},
+                overrides=persisted_overrides,
                 status=Binary.StatusChoices.QUEUED,
             )
 
@@ -80,7 +83,7 @@ class ArchiveBoxDBBinaryCacheBackend:
                 installed.retry_at = None
                 await installed.asave(update_fields=["status", "retry_at", "modified_at"])
                 installed = None
-            if installed is not None and request.overrides and installed.overrides != request.overrides:
+            if installed is not None and persisted_overrides and installed.overrides != persisted_overrides:
                 installed.status = Binary.StatusChoices.QUEUED
                 installed.retry_at = None
                 await installed.asave(update_fields=["status", "retry_at", "modified_at"])
@@ -102,7 +105,7 @@ class ArchiveBoxDBBinaryCacheBackend:
                 await installed.asave(update_fields=["status", "retry_at", "modified_at"])
                 return None
 
-        provider = _provider_for_name(provider_name, installed.name, installed.overrides)
+        provider = _provider_for_name(provider_name, installed.name, native_overrides)
         binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {}
         provider_names = _provider_names(installed.binproviders or request.binproviders or "env")
         return AbxBinary.model_validate(
@@ -110,7 +113,7 @@ class ArchiveBoxDBBinaryCacheBackend:
                 "name": request.name,
                 "description": request.description,
                 "binproviders": _providers_for_names(provider_names),
-                "overrides": installed.overrides or request.overrides or {},
+                "overrides": native_overrides,
                 "loaded_binprovider": provider,
                 "loaded_abspath": installed.abspath,
                 "loaded_version": installed.version or None,
@@ -150,7 +153,7 @@ class ArchiveBoxDBBinaryCacheBackend:
         )
         if binary.loaded_binprovider is not None:
             existing.binprovider = binary.loaded_binprovider.name
-        existing.overrides = request.overrides if request is not None and request.overrides is not None else binary.overrides
+        existing.overrides = _persisted_overrides_for_request(request) if request is not None else binary.overrides
         existing.status = Binary.StatusChoices.INSTALLED
         existing.retry_at = None
         await existing.asave(
@@ -302,3 +305,12 @@ def _provider_for_name(provider_name: str, binary_name: str, overrides: dict[str
             overrides={binary_name: provider_overrides},
         )
     return provider
+
+
+def _persisted_overrides_for_request(request: BinaryRequestEvent | None) -> dict[str, Any]:
+    if request is None:
+        return {}
+    raw_overrides = request.extra_context.get("raw_overrides")
+    if isinstance(raw_overrides, Mapping):
+        return dict(raw_overrides)
+    return dict(request.overrides or {})

archivebox/services/runner.py

@@ -51,7 +51,7 @@ from abx_dl.orchestrator import (
     setup_services as setup_abx_services,
 )
 from abx_dl.services.process_service import ProcessService as HookProcessService
-from abx_dl.services.binary_service import PluginBinariesService
+from abx_dl.services.binary_service import PluginBinariesService, split_abxpkg_binary_request_overrides
 from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService
 from abx_dl.cli import LiveBusUI
 from abxbus import BaseEvent
@@ -708,7 +708,12 @@ class CrawlRunner:
         from archivebox.config.common import get_config
 
         snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-        config = get_config(crawl=snapshot.crawl, snapshot=snapshot, include_machine=False)
+        config = get_config(
+            crawl=snapshot.crawl,
+            snapshot=snapshot,
+            base_config=self.base_config or None,
+            include_machine=False,
+        )
         config["CRAWL_DIR"] = self.crawl_output_dir
         config["SNAP_DIR"] = str(snapshot.output_dir)
         extra_context: dict[str, Any] = {}
@@ -1187,18 +1192,21 @@ async def _run_binary(binary_id: str) -> None:
     )
     await _emit_machine_config(bus, config=config, derived_config=derived_config)
 
+    native_overrides, override_extra_context = split_abxpkg_binary_request_overrides(binary.overrides or None)
+
     try:
         await bus.emit(
             BinaryRequestEvent(
                 name=binary.name,
                 binproviders=binary.binproviders,
-                overrides=binary.overrides or None,
+                overrides=native_overrides or None,
                 extra_context={
                     "plugin_name": "archivebox",
                     "hook_name": "archivebox_binary_run",
                     "output_dir": str(binary.output_dir),
                     "binary_id": str(binary.id),
                     "machine_id": str(binary.machine_id),
+                    **override_extra_context,
                 },
             ),
         ).now(first_result=True)

archivebox/tests/test_binary_service.py

@@ -2,6 +2,7 @@ import json
 import os
 import shutil
 import uuid
+import asyncio
 from pathlib import Path
 
 import pytest
@@ -36,6 +37,62 @@ def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]:
     }
 
 
+def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event():
+    from abxpkg.binary_service import BinaryCacheService, BinaryEvent, BinaryRequestEvent, BinaryService
+    from abx_dl.orchestrator import create_bus
+    from archivebox.services.binary_service import ArchiveBoxDBBinaryCacheBackend
+
+    machine = Machine.current()
+    raw_overrides = {
+        "pip": {
+            "install_args": ["imagesize>=2.0.0"],
+            "module_name": "imagesize",
+        },
+    }
+    binary = Binary.objects.create(
+        machine=machine,
+        name="sh",
+        abspath="/bin/sh",
+        version="1.0.0",
+        binprovider="env",
+        binproviders="env,pip",
+        overrides=raw_overrides,
+        status=Binary.StatusChoices.INSTALLED,
+    )
+    bus = create_bus(name=f"test_binary_raw_overrides_{uuid.uuid4().hex[:8]}")
+    BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
+    BinaryService(bus)
+    binary_events: list[BinaryEvent] = []
+
+    async def on_BinaryEvent(event: BinaryEvent) -> None:
+        binary_events.append(event)
+
+    bus.on(BinaryEvent, on_BinaryEvent)
+
+    async def run_event() -> None:
+        await bus.emit(
+            BinaryRequestEvent(
+                name="sh",
+                binproviders="env,pip",
+                overrides={"pip": {"install_args": ["imagesize>=2.0.0"]}},
+                extra_context={
+                    "raw_overrides": raw_overrides,
+                    "provider_metadata": {"pip": {"module_name": "imagesize"}},
+                },
+            ),
+        ).now()
+        await bus.wait_until_idle()
+
+    asyncio.run(run_event())
+
+    binary.refresh_from_db()
+    assert binary.status == Binary.StatusChoices.INSTALLED
+    assert binary.overrides == raw_overrides
+    assert binary_events
+    assert binary_events[-1].overrides == {"pip": {"install_args": ["imagesize>=2.0.0"]}}
+    assert binary_events[-1].extra_context["raw_overrides"] == raw_overrides
+
+
 def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized_archive, tmp_path):
     name = f"abx-e2e-rg-{uuid.uuid4().hex[:8]}"
     bootstrap_bin_dir = tmp_path / "realbin"

archivebox/tests/test_crawl_runner.py

@@ -104,6 +104,35 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
     ]
 
 
+@pytest.mark.django_db(transaction=True)
+def test_snapshot_payload_uses_crawl_persona_runtime_dirs():
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
+    from archivebox.personas.models import Persona
+    from archivebox.services.runner import CrawlRunner
+
+    persona = Persona.objects.create(name="RuntimePersona")
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        persona_id=persona.id,
+        created_by_id=get_or_create_system_user_pk(),
+    )
+    snapshot = Snapshot.objects.create(url="https://example.com", crawl=crawl)
+
+    runner = CrawlRunner(crawl)
+    runner.load_run_state()
+    payload = runner.load_snapshot_payload(str(snapshot.id))
+    config = payload["config"]
+
+    assert Path(config["CHROME_USER_DATA_DIR"]).is_relative_to(crawl.output_dir)
+    assert Path(config["CHROME_DOWNLOADS_DIR"]).is_relative_to(crawl.output_dir)
+    assert Path(config["CHROME_USER_DATA_DIR"]).name == "chrome_profile"
+    assert Path(config["CHROME_DOWNLOADS_DIR"]).name == "chrome_downloads"
+    assert Path(config["CRAWL_DIR"]) == crawl.output_dir
+    assert Path(config["SNAP_DIR"]) == snapshot.output_dir
+
+
 def test_ensure_background_runner_skips_under_pytest_guard():
     from archivebox.services.runner import ensure_background_runner
 

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.34rc1",
+  "version": "0.9.34rc2",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.34rc1"
+version = "0.9.34rc2"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,9 +79,9 @@ dependencies = [
     ### Extractor dependencies (optional binaries detected at runtime via shutil.which)
     ### Binary/Package Management
     "abxbus==2.5.8",          # EventBus API
-    "abxpkg>=1.11.114",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins>=1.11.117",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl>=1.11.117",       # shared ArchiveBox downloader package with blocking install preflight
+    "abxpkg>=1.11.116",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
+    "abx-plugins>=1.11.119",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl>=1.11.119",       # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]