From e1c84e7d26de58d70034d0e544a549b7d99245ec Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 1 Jun 2026 18:10:44 -0700 Subject: [PATCH] release: archivebox 0.9.34rc32 --- archivebox/config/common.py | 30 ++++++++++---- archivebox/personas/models.py | 36 ++++++++++++++++ archivebox/services/runner.py | 40 +++++++++++------- archivebox/tests/test_crawl_runner.py | 59 +++++++++++++++++++++++++-- etc/package.json | 2 +- pyproject.toml | 8 ++-- 6 files changed, 145 insertions(+), 30 deletions(-) diff --git a/archivebox/config/common.py b/archivebox/config/common.py index ff62c342..14e9d3f2 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -195,6 +195,11 @@ def build_crawl_config_snapshot( plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields) effective = get_config(persona=persona, base_config=base_config) frozen = effective.for_crawl_frozen(persona=persona) + if persona is not None: + persona_config = persona.get_derived_config() + for key in plugin_owned_keys - explicit_overrides: + if key in persona_config: + frozen.pop(key, None) if overrides: resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False) resolved_payload = normalize_runtime_config(resolved) @@ -202,6 +207,11 @@ def build_crawl_config_snapshot( for key in plugin_owned_keys & explicit_overrides: if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload: frozen[key] = resolved_payload[key] + if persona is not None: + persona_config = persona.get_derived_config() + for key in plugin_owned_keys - explicit_overrides: + if key in persona_config: + frozen.pop(key, None) return frozen @@ -1045,14 +1055,20 @@ def get_config( ) if persona is not None: - scope_overrides.update( - normalize_runtime_config( - persona.get_derived_config(), - only_crawl_execution=crawl_config_base, - exclude_runtime_derived=not crawl_config_base, - json_safe=False, - ), + persona_config = normalize_runtime_config( + persona.get_derived_config(), + exclude_runtime_derived=not crawl_config_base, + json_safe=False, ) + if crawl_config_base: + scope_by_key = ArchiveBoxConfig._scope_by_key() + crawl_keys = set(dict(crawl.config or {})) + persona_config = { + key: value + for key, value in persona_config.items() + if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION or key not in crawl_keys + } + scope_overrides.update(persona_config) if crawl is not None and crawl.config and not crawl_config_base: scope_overrides.update(normalize_runtime_config(crawl.config, exclude_crawl_execution=True, json_safe=False)) diff --git a/archivebox/personas/models.py b/archivebox/personas/models.py index fffca6dc..ba36a17f 100644 --- a/archivebox/personas/models.py +++ b/archivebox/personas/models.py @@ -237,6 +237,15 @@ class Persona(ModelWithConfig): def runtime_downloads_dir_for_crawl(self, crawl) -> Path: return self.runtime_root_for_crawl(crawl) / "chrome_downloads" + def runtime_root_for_snapshot(self, snapshot) -> Path: + return Path(snapshot.output_dir) / ".persona" / self.name + + def runtime_profile_dir_for_snapshot(self, snapshot) -> Path: + return self.runtime_root_for_snapshot(snapshot) / "chrome_profile" + + def runtime_downloads_dir_for_snapshot(self, snapshot) -> Path: + return self.runtime_root_for_snapshot(snapshot) / "chrome_downloads" + def copy_chrome_profile(self, source_dir: Path, destination_dir: Path) -> None: destination_dir.parent.mkdir(parents=True, exist_ok=True) shutil.rmtree(destination_dir, ignore_errors=True) @@ -289,6 +298,33 @@ class Persona(ModelWithConfig): "CHROME_DOWNLOADS_DIR": str(runtime_downloads_dir), } + def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]: + crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl) + template_dir = crawl_runtime_profile_dir if crawl_runtime_profile_dir.exists() else Path(self.CHROME_USER_DATA_DIR) + runtime_root = self.runtime_root_for_snapshot(snapshot) + runtime_profile_dir = self.runtime_profile_dir_for_snapshot(snapshot) + runtime_downloads_dir = self.runtime_downloads_dir_for_snapshot(snapshot) + + if runtime_root.exists(): + shutil.rmtree(runtime_root, ignore_errors=True) + if template_dir.exists() and any(template_dir.iterdir()): + self.copy_chrome_profile(template_dir, runtime_profile_dir) + else: + runtime_profile_dir.mkdir(parents=True, exist_ok=True) + + runtime_downloads_dir.mkdir(parents=True, exist_ok=True) + self.cleanup_chrome_profile(runtime_profile_dir) + + (runtime_root / "persona_name.txt").write_text(self.name) + (runtime_root / "template_dir.txt").write_text(str(template_dir)) + if chrome_binary: + (runtime_root / "chrome_binary.txt").write_text(chrome_binary) + + return { + "CHROME_USER_DATA_DIR": str(runtime_profile_dir), + "CHROME_DOWNLOADS_DIR": str(runtime_downloads_dir), + } + def cleanup_runtime_for_crawl(self, crawl) -> None: shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True) diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 1cf0bf43..d8fc9b19 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -760,21 +760,28 @@ class CrawlRunner: with _perf_span("runner.CrawlRunner.load_snapshot_payload.resolve_persona"): self.persona = snapshot.crawl.resolve_persona() with _perf_span("runner.CrawlRunner.load_snapshot_payload.get_config"): - self.base_config = get_config(crawl=snapshot.crawl) - if self.persona: - with _perf_span("runner.CrawlRunner.load_snapshot_payload.persona_runtime"): - self.base_config.update( - self.persona.prepare_runtime_for_crawl( - snapshot.crawl, - chrome_binary=self.base_config["CHROME_BINARY"], - ), - ) + self.base_config = get_config(crawl=snapshot.crawl, persona=self.persona) with _perf_span("runner.CrawlRunner.load_snapshot_payload.runtime_dirs"): self.base_config.update(self.config_overrides) self.crawl_output_dir = str(snapshot.crawl.output_dir) - runtime_chrome_overrides = { - key: self.base_config[key] for key in ("CHROME_USER_DATA_DIR", "CHROME_DOWNLOADS_DIR") if self.base_config.get(key) - } + runtime_chrome_overrides = {} + if self.persona: + if str(self.base_config.get("CHROME_ISOLATION") or "crawl").lower() == "snapshot": + runtime_chrome_overrides.update( + self.persona.prepare_runtime_for_snapshot( + snapshot, + chrome_binary=self.base_config["CHROME_BINARY"], + ), + ) + else: + crawl_downloads_dir = self.persona.runtime_downloads_dir_for_crawl(snapshot.crawl) + crawl_downloads_dir.mkdir(parents=True, exist_ok=True) + runtime_chrome_overrides.update( + { + "CHROME_USER_DATA_DIR": str(self.persona.runtime_profile_dir_for_crawl(snapshot.crawl)), + "CHROME_DOWNLOADS_DIR": str(crawl_downloads_dir), + }, + ) snapshot_output_dir = str(snapshot.output_dir) with _perf_span("runner.CrawlRunner.load_snapshot_payload.for_crawl_runtime"): config = self.base_config.for_crawl_runtime( @@ -824,10 +831,15 @@ class CrawlRunner: if parent_snapshot is None: return config = await sync_to_async( - lambda: get_config(crawl=self.crawl, snapshot=parent_snapshot), + lambda: get_config(crawl=self.crawl, snapshot=parent_snapshot).for_crawl_runtime( + crawl=self.crawl, + snapshot=parent_snapshot, + persona=self.crawl.resolve_persona(), + crawl_output_dir=self.crawl.output_dir, + snapshot_output_dir=parent_snapshot.output_dir, + ), thread_sensitive=True, )() - config = config.for_crawl_runtime(crawl=self.crawl, snapshot=parent_snapshot, persona=self.crawl.resolve_persona()) if CrawlLimitState.from_config(config).get_stop_reason() in ("crawl_max_size", "crawl_timeout"): return diff --git a/archivebox/tests/test_crawl_runner.py b/archivebox/tests/test_crawl_runner.py index 0e7e5f7b..fdb27f5a 100644 --- a/archivebox/tests/test_crawl_runner.py +++ b/archivebox/tests/test_crawl_runner.py @@ -105,35 +105,86 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path): @pytest.mark.django_db(transaction=True) -def test_snapshot_payload_uses_crawl_persona_runtime_dirs(): +def test_snapshot_payload_uses_crawl_chrome_dirs_by_default(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.personas.models import Persona from archivebox.services.runner import CrawlRunner - persona = Persona.objects.create(name="RuntimePersona") - crawl = Crawl.objects.create( + persona = Persona(name="RuntimePersona") + persona.save() + crawl = Crawl( urls="https://example.com", persona_id=persona.id, created_by_id=get_or_create_system_user_pk(), ) - snapshot = Snapshot.objects.create(url="https://example.com", crawl=crawl) + crawl.save() + snapshot = Snapshot(url="https://example.com", crawl=crawl) + snapshot.save() + other_snapshot = Snapshot(url="https://example.org", crawl=crawl) + other_snapshot.save() runner = CrawlRunner(crawl) runner.load_run_state() + crawl_downloads_sentinel = persona.runtime_downloads_dir_for_crawl(crawl) / "keep.txt" + crawl_downloads_sentinel.write_text("keep") payload = runner.load_snapshot_payload(str(snapshot.id)) + other_payload = runner.load_snapshot_payload(str(other_snapshot.id)) config = payload["config"] + other_config = other_payload["config"] assert Path(config["CHROME_USER_DATA_DIR"]).is_relative_to(crawl.output_dir) assert Path(config["CHROME_DOWNLOADS_DIR"]).is_relative_to(crawl.output_dir) assert Path(config["CHROME_USER_DATA_DIR"]).name == "chrome_profile" assert Path(config["CHROME_DOWNLOADS_DIR"]).name == "chrome_downloads" + assert Path(config["CHROME_USER_DATA_DIR"]) == Path(other_config["CHROME_USER_DATA_DIR"]) + assert Path(config["CHROME_DOWNLOADS_DIR"]) == Path(other_config["CHROME_DOWNLOADS_DIR"]) + assert crawl_downloads_sentinel.read_text() == "keep" assert config["ACTIVE_PERSONA"] == "RuntimePersona" assert Path(config["CRAWL_DIR"]) == crawl.output_dir assert Path(config["SNAP_DIR"]) == snapshot.output_dir +@pytest.mark.django_db(transaction=True) +def test_snapshot_payload_uses_snapshot_chrome_dirs_when_snapshot_isolated(): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.personas.models import Persona + from archivebox.services.runner import CrawlRunner + + persona = Persona(name="SnapshotRuntimePersona") + persona.save() + crawl = Crawl( + urls="https://example.com\nhttps://example.org", + persona_id=persona.id, + created_by_id=get_or_create_system_user_pk(), + config={"CHROME_ISOLATION": "snapshot"}, + ) + crawl.save() + snapshot = Snapshot(url="https://example.com", crawl=crawl) + snapshot.save() + other_snapshot = Snapshot(url="https://example.org", crawl=crawl) + other_snapshot.save() + + runner = CrawlRunner(crawl) + runner.load_run_state() + payload = runner.load_snapshot_payload(str(snapshot.id)) + other_payload = runner.load_snapshot_payload(str(other_snapshot.id)) + config = payload["config"] + other_config = other_payload["config"] + + assert Path(config["CHROME_USER_DATA_DIR"]).is_relative_to(snapshot.output_dir) + assert Path(config["CHROME_DOWNLOADS_DIR"]).is_relative_to(snapshot.output_dir) + assert Path(other_config["CHROME_USER_DATA_DIR"]).is_relative_to(other_snapshot.output_dir) + assert Path(other_config["CHROME_DOWNLOADS_DIR"]).is_relative_to(other_snapshot.output_dir) + assert Path(config["CHROME_USER_DATA_DIR"]) != Path(other_config["CHROME_USER_DATA_DIR"]) + assert Path(config["CHROME_DOWNLOADS_DIR"]) != Path(other_config["CHROME_DOWNLOADS_DIR"]) + assert Path(config["CRAWL_DIR"]) == crawl.output_dir + assert Path(config["SNAP_DIR"]) == snapshot.output_dir + + def test_ensure_background_runner_skips_under_pytest_guard(): from archivebox.services.runner import ensure_background_runner diff --git a/etc/package.json b/etc/package.json index f9783958..1ea10034 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.34rc31", + "version": "0.9.34rc32", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index 9d298277..e95e3607 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.34rc31" +version = "0.9.34rc32" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.9", # EventBus API - "abxpkg>=1.11.147", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.149", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.149", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.148", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.150", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.150", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]