From 96ffc765c0441cb136d36f67c4caee310c4cf225 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 1 Jun 2026 03:36:54 -0700 Subject: [PATCH] release: archivebox 0.9.34rc2 --- archivebox/services/binary_service.py | 26 ++++++++--- archivebox/services/runner.py | 14 ++++-- archivebox/tests/test_binary_service.py | 57 +++++++++++++++++++++++++ archivebox/tests/test_crawl_runner.py | 29 +++++++++++++ etc/package.json | 2 +- pyproject.toml | 8 ++-- 6 files changed, 121 insertions(+), 15 deletions(-) diff --git a/archivebox/services/binary_service.py b/archivebox/services/binary_service.py index 8e62f4e8..c2feaf3a 100644 --- a/archivebox/services/binary_service.py +++ b/archivebox/services/binary_service.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -42,6 +43,8 @@ class ArchiveBoxDBBinaryCacheBackend: return None existing = await Binary.objects.filter(machine=machine, name=binary_name).afirst() + persisted_overrides = _persisted_overrides_for_request(request) + native_overrides = request.overrides or {} cache_invalidated = False if existing and existing.status == Binary.StatusChoices.INSTALLED: changed = False @@ -49,8 +52,8 @@ class ArchiveBoxDBBinaryCacheBackend: if requested_binproviders and existing.binproviders != requested_binproviders: existing.binproviders = requested_binproviders changed = True - if request.overrides and existing.overrides != request.overrides: - existing.overrides = request.overrides + if persisted_overrides and existing.overrides != persisted_overrides: + existing.overrides = persisted_overrides changed = True if changed: existing.status = Binary.StatusChoices.QUEUED @@ -62,7 +65,7 @@ class ArchiveBoxDBBinaryCacheBackend: machine=machine, name=binary_name, binproviders=_binproviders_to_str(request.binproviders), - overrides=request.overrides or {}, + overrides=persisted_overrides, status=Binary.StatusChoices.QUEUED, ) @@ -80,7 +83,7 @@ class ArchiveBoxDBBinaryCacheBackend: installed.retry_at = None await installed.asave(update_fields=["status", "retry_at", "modified_at"]) installed = None - if installed is not None and request.overrides and installed.overrides != request.overrides: + if installed is not None and persisted_overrides and installed.overrides != persisted_overrides: installed.status = Binary.StatusChoices.QUEUED installed.retry_at = None await installed.asave(update_fields=["status", "retry_at", "modified_at"]) @@ -102,7 +105,7 @@ class ArchiveBoxDBBinaryCacheBackend: await installed.asave(update_fields=["status", "retry_at", "modified_at"]) return None - provider = _provider_for_name(provider_name, installed.name, installed.overrides) + provider = _provider_for_name(provider_name, installed.name, native_overrides) binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {} provider_names = _provider_names(installed.binproviders or request.binproviders or "env") return AbxBinary.model_validate( @@ -110,7 +113,7 @@ class ArchiveBoxDBBinaryCacheBackend: "name": request.name, "description": request.description, "binproviders": _providers_for_names(provider_names), - "overrides": installed.overrides or request.overrides or {}, + "overrides": native_overrides, "loaded_binprovider": provider, "loaded_abspath": installed.abspath, "loaded_version": installed.version or None, @@ -150,7 +153,7 @@ class ArchiveBoxDBBinaryCacheBackend: ) if binary.loaded_binprovider is not None: existing.binprovider = binary.loaded_binprovider.name - existing.overrides = request.overrides if request is not None and request.overrides is not None else binary.overrides + existing.overrides = _persisted_overrides_for_request(request) if request is not None else binary.overrides existing.status = Binary.StatusChoices.INSTALLED existing.retry_at = None await existing.asave( @@ -302,3 +305,12 @@ def _provider_for_name(provider_name: str, binary_name: str, overrides: dict[str overrides={binary_name: provider_overrides}, ) return provider + + +def _persisted_overrides_for_request(request: BinaryRequestEvent | None) -> dict[str, Any]: + if request is None: + return {} + raw_overrides = request.extra_context.get("raw_overrides") + if isinstance(raw_overrides, Mapping): + return dict(raw_overrides) + return dict(request.overrides or {}) diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 3ce6bb7f..eed1461d 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -51,7 +51,7 @@ from abx_dl.orchestrator import ( setup_services as setup_abx_services, ) from abx_dl.services.process_service import ProcessService as HookProcessService -from abx_dl.services.binary_service import PluginBinariesService +from abx_dl.services.binary_service import PluginBinariesService, split_abxpkg_binary_request_overrides from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService from abx_dl.cli import LiveBusUI from abxbus import BaseEvent @@ -708,7 +708,12 @@ class CrawlRunner: from archivebox.config.common import get_config snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id) - config = get_config(crawl=snapshot.crawl, snapshot=snapshot, include_machine=False) + config = get_config( + crawl=snapshot.crawl, + snapshot=snapshot, + base_config=self.base_config or None, + include_machine=False, + ) config["CRAWL_DIR"] = self.crawl_output_dir config["SNAP_DIR"] = str(snapshot.output_dir) extra_context: dict[str, Any] = {} @@ -1187,18 +1192,21 @@ async def _run_binary(binary_id: str) -> None: ) await _emit_machine_config(bus, config=config, derived_config=derived_config) + native_overrides, override_extra_context = split_abxpkg_binary_request_overrides(binary.overrides or None) + try: await bus.emit( BinaryRequestEvent( name=binary.name, binproviders=binary.binproviders, - overrides=binary.overrides or None, + overrides=native_overrides or None, extra_context={ "plugin_name": "archivebox", "hook_name": "archivebox_binary_run", "output_dir": str(binary.output_dir), "binary_id": str(binary.id), "machine_id": str(binary.machine_id), + **override_extra_context, }, ), ).now(first_result=True) diff --git a/archivebox/tests/test_binary_service.py b/archivebox/tests/test_binary_service.py index 4745f992..9fe1e277 100644 --- a/archivebox/tests/test_binary_service.py +++ b/archivebox/tests/test_binary_service.py @@ -2,6 +2,7 @@ import json import os import shutil import uuid +import asyncio from pathlib import Path import pytest @@ -36,6 +37,62 @@ def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]: } +def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event(): + from abxpkg.binary_service import BinaryCacheService, BinaryEvent, BinaryRequestEvent, BinaryService + from abx_dl.orchestrator import create_bus + from archivebox.services.binary_service import ArchiveBoxDBBinaryCacheBackend + + machine = Machine.current() + raw_overrides = { + "pip": { + "install_args": ["imagesize>=2.0.0"], + "module_name": "imagesize", + }, + } + binary = Binary.objects.create( + machine=machine, + name="sh", + abspath="/bin/sh", + version="1.0.0", + binprovider="env", + binproviders="env,pip", + overrides=raw_overrides, + status=Binary.StatusChoices.INSTALLED, + ) + bus = create_bus(name=f"test_binary_raw_overrides_{uuid.uuid4().hex[:8]}") + BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend()) + BinaryService(bus) + binary_events: list[BinaryEvent] = [] + + async def on_BinaryEvent(event: BinaryEvent) -> None: + binary_events.append(event) + + bus.on(BinaryEvent, on_BinaryEvent) + + async def run_event() -> None: + await bus.emit( + BinaryRequestEvent( + name="sh", + binproviders="env,pip", + overrides={"pip": {"install_args": ["imagesize>=2.0.0"]}}, + extra_context={ + "raw_overrides": raw_overrides, + "provider_metadata": {"pip": {"module_name": "imagesize"}}, + }, + ), + ).now() + await bus.wait_until_idle() + + asyncio.run(run_event()) + + binary.refresh_from_db() + assert binary.status == Binary.StatusChoices.INSTALLED + assert binary.overrides == raw_overrides + assert binary_events + assert binary_events[-1].overrides == {"pip": {"install_args": ["imagesize>=2.0.0"]}} + assert binary_events[-1].extra_context["raw_overrides"] == raw_overrides + + def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized_archive, tmp_path): name = f"abx-e2e-rg-{uuid.uuid4().hex[:8]}" bootstrap_bin_dir = tmp_path / "realbin" diff --git a/archivebox/tests/test_crawl_runner.py b/archivebox/tests/test_crawl_runner.py index 1aad6567..cd28c32c 100644 --- a/archivebox/tests/test_crawl_runner.py +++ b/archivebox/tests/test_crawl_runner.py @@ -104,6 +104,35 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path): ] +@pytest.mark.django_db(transaction=True) +def test_snapshot_payload_uses_crawl_persona_runtime_dirs(): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.personas.models import Persona + from archivebox.services.runner import CrawlRunner + + persona = Persona.objects.create(name="RuntimePersona") + crawl = Crawl.objects.create( + urls="https://example.com", + persona_id=persona.id, + created_by_id=get_or_create_system_user_pk(), + ) + snapshot = Snapshot.objects.create(url="https://example.com", crawl=crawl) + + runner = CrawlRunner(crawl) + runner.load_run_state() + payload = runner.load_snapshot_payload(str(snapshot.id)) + config = payload["config"] + + assert Path(config["CHROME_USER_DATA_DIR"]).is_relative_to(crawl.output_dir) + assert Path(config["CHROME_DOWNLOADS_DIR"]).is_relative_to(crawl.output_dir) + assert Path(config["CHROME_USER_DATA_DIR"]).name == "chrome_profile" + assert Path(config["CHROME_DOWNLOADS_DIR"]).name == "chrome_downloads" + assert Path(config["CRAWL_DIR"]) == crawl.output_dir + assert Path(config["SNAP_DIR"]) == snapshot.output_dir + + def test_ensure_background_runner_skips_under_pytest_guard(): from archivebox.services.runner import ensure_background_runner diff --git a/etc/package.json b/etc/package.json index d4f601a5..532a8122 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.34rc1", + "version": "0.9.34rc2", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index 887c92e6..ca271777 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.34rc1" +version = "0.9.34rc2" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.8", # EventBus API - "abxpkg>=1.11.114", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.117", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.117", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.116", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.119", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.119", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]