The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

release: archivebox 0.9.34rc32

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-01 18:10:44 -07:00
parent 42fd16b820
commit e1c84e7d26
No known key found for this signature in database
6 changed files with 145 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
archivebox/config/common.py
View File

@ -195,6 +195,11 @@ def build_crawl_config_snapshot(
plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields)
effective = get_config(persona=persona, base_config=base_config)
frozen = effective.for_crawl_frozen(persona=persona)
if persona is not None:
persona_config = persona.get_derived_config()
for key in plugin_owned_keys - explicit_overrides:
if key in persona_config:
frozen.pop(key, None)
if overrides:
resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False)
resolved_payload = normalize_runtime_config(resolved)
@ -202,6 +207,11 @@ def build_crawl_config_snapshot(
for key in plugin_owned_keys & explicit_overrides:
if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload:
frozen[key] = resolved_payload[key]
if persona is not None:
persona_config = persona.get_derived_config()
for key in plugin_owned_keys - explicit_overrides:
if key in persona_config:
frozen.pop(key, None)
return frozen
@ -1045,14 +1055,20 @@ def get_config(
)
if persona is not None:
scope_overrides.update(
normalize_runtime_config(
persona.get_derived_config(),
only_crawl_execution=crawl_config_base,
exclude_runtime_derived=not crawl_config_base,
json_safe=False,
),
persona_config = normalize_runtime_config(
persona.get_derived_config(),
exclude_runtime_derived=not crawl_config_base,
json_safe=False,
)
if crawl_config_base:
scope_by_key = ArchiveBoxConfig._scope_by_key()
crawl_keys = set(dict(crawl.config or {}))
persona_config = {
key: value
for key, value in persona_config.items()
if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION or key not in crawl_keys
}
scope_overrides.update(persona_config)
if crawl is not None and crawl.config and not crawl_config_base:
scope_overrides.update(normalize_runtime_config(crawl.config, exclude_crawl_execution=True, json_safe=False))

36
archivebox/personas/models.py
View File

@ -237,6 +237,15 @@ class Persona(ModelWithConfig):
def runtime_downloads_dir_for_crawl(self, crawl) -> Path:
return self.runtime_root_for_crawl(crawl) / "chrome_downloads"
def runtime_root_for_snapshot(self, snapshot) -> Path:
return Path(snapshot.output_dir) / ".persona" / self.name
def runtime_profile_dir_for_snapshot(self, snapshot) -> Path:
return self.runtime_root_for_snapshot(snapshot) / "chrome_profile"
def runtime_downloads_dir_for_snapshot(self, snapshot) -> Path:
return self.runtime_root_for_snapshot(snapshot) / "chrome_downloads"
def copy_chrome_profile(self, source_dir: Path, destination_dir: Path) -> None:
destination_dir.parent.mkdir(parents=True, exist_ok=True)
shutil.rmtree(destination_dir, ignore_errors=True)
@ -289,6 +298,33 @@ class Persona(ModelWithConfig):
"CHROME_DOWNLOADS_DIR": str(runtime_downloads_dir),
}
def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
template_dir = crawl_runtime_profile_dir if crawl_runtime_profile_dir.exists() else Path(self.CHROME_USER_DATA_DIR)
runtime_root = self.runtime_root_for_snapshot(snapshot)
runtime_profile_dir = self.runtime_profile_dir_for_snapshot(snapshot)
runtime_downloads_dir = self.runtime_downloads_dir_for_snapshot(snapshot)
if runtime_root.exists():
shutil.rmtree(runtime_root, ignore_errors=True)
if template_dir.exists() and any(template_dir.iterdir()):
self.copy_chrome_profile(template_dir, runtime_profile_dir)
else:
runtime_profile_dir.mkdir(parents=True, exist_ok=True)
runtime_downloads_dir.mkdir(parents=True, exist_ok=True)
self.cleanup_chrome_profile(runtime_profile_dir)
(runtime_root / "persona_name.txt").write_text(self.name)
(runtime_root / "template_dir.txt").write_text(str(template_dir))
if chrome_binary:
(runtime_root / "chrome_binary.txt").write_text(chrome_binary)
return {
"CHROME_USER_DATA_DIR": str(runtime_profile_dir),
"CHROME_DOWNLOADS_DIR": str(runtime_downloads_dir),
}
def cleanup_runtime_for_crawl(self, crawl) -> None:
shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

40
archivebox/services/runner.py
View File

@ -760,21 +760,28 @@ class CrawlRunner:
with _perf_span("runner.CrawlRunner.load_snapshot_payload.resolve_persona"):
self.persona = snapshot.crawl.resolve_persona()
with _perf_span("runner.CrawlRunner.load_snapshot_payload.get_config"):
self.base_config = get_config(crawl=snapshot.crawl)
if self.persona:
with _perf_span("runner.CrawlRunner.load_snapshot_payload.persona_runtime"):
self.base_config.update(
self.persona.prepare_runtime_for_crawl(
snapshot.crawl,
chrome_binary=self.base_config["CHROME_BINARY"],
),
)
self.base_config = get_config(crawl=snapshot.crawl, persona=self.persona)
with _perf_span("runner.CrawlRunner.load_snapshot_payload.runtime_dirs"):
self.base_config.update(self.config_overrides)
self.crawl_output_dir = str(snapshot.crawl.output_dir)
runtime_chrome_overrides = {
key: self.base_config[key] for key in ("CHROME_USER_DATA_DIR", "CHROME_DOWNLOADS_DIR") if self.base_config.get(key)
}
runtime_chrome_overrides = {}
if self.persona:
if str(self.base_config.get("CHROME_ISOLATION") or "crawl").lower() == "snapshot":
runtime_chrome_overrides.update(
self.persona.prepare_runtime_for_snapshot(
snapshot,
chrome_binary=self.base_config["CHROME_BINARY"],
),
)
else:
crawl_downloads_dir = self.persona.runtime_downloads_dir_for_crawl(snapshot.crawl)
crawl_downloads_dir.mkdir(parents=True, exist_ok=True)
runtime_chrome_overrides.update(
{
"CHROME_USER_DATA_DIR": str(self.persona.runtime_profile_dir_for_crawl(snapshot.crawl)),
"CHROME_DOWNLOADS_DIR": str(crawl_downloads_dir),
},
)
snapshot_output_dir = str(snapshot.output_dir)
with _perf_span("runner.CrawlRunner.load_snapshot_payload.for_crawl_runtime"):
config = self.base_config.for_crawl_runtime(
@ -824,10 +831,15 @@ class CrawlRunner:
if parent_snapshot is None:
return
config = await sync_to_async(
lambda: get_config(crawl=self.crawl, snapshot=parent_snapshot),
lambda: get_config(crawl=self.crawl, snapshot=parent_snapshot).for_crawl_runtime(
crawl=self.crawl,
snapshot=parent_snapshot,
persona=self.crawl.resolve_persona(),
crawl_output_dir=self.crawl.output_dir,
snapshot_output_dir=parent_snapshot.output_dir,
),
thread_sensitive=True,
)()
config = config.for_crawl_runtime(crawl=self.crawl, snapshot=parent_snapshot, persona=self.crawl.resolve_persona())
if CrawlLimitState.from_config(config).get_stop_reason() in ("crawl_max_size", "crawl_timeout"):
return

59
archivebox/tests/test_crawl_runner.py
View File

@ -105,35 +105,86 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
@pytest.mark.django_db(transaction=True)
def test_snapshot_payload_uses_crawl_persona_runtime_dirs():
def test_snapshot_payload_uses_crawl_chrome_dirs_by_default():
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.personas.models import Persona
from archivebox.services.runner import CrawlRunner
persona = Persona.objects.create(name="RuntimePersona")
crawl = Crawl.objects.create(
persona = Persona(name="RuntimePersona")
persona.save()
crawl = Crawl(
urls="https://example.com",
persona_id=persona.id,
created_by_id=get_or_create_system_user_pk(),
)
snapshot = Snapshot.objects.create(url="https://example.com", crawl=crawl)
crawl.save()
snapshot = Snapshot(url="https://example.com", crawl=crawl)
snapshot.save()
other_snapshot = Snapshot(url="https://example.org", crawl=crawl)
other_snapshot.save()
runner = CrawlRunner(crawl)
runner.load_run_state()
crawl_downloads_sentinel = persona.runtime_downloads_dir_for_crawl(crawl) / "keep.txt"
crawl_downloads_sentinel.write_text("keep")
payload = runner.load_snapshot_payload(str(snapshot.id))
other_payload = runner.load_snapshot_payload(str(other_snapshot.id))
config = payload["config"]
other_config = other_payload["config"]
assert Path(config["CHROME_USER_DATA_DIR"]).is_relative_to(crawl.output_dir)
assert Path(config["CHROME_DOWNLOADS_DIR"]).is_relative_to(crawl.output_dir)
assert Path(config["CHROME_USER_DATA_DIR"]).name == "chrome_profile"
assert Path(config["CHROME_DOWNLOADS_DIR"]).name == "chrome_downloads"
assert Path(config["CHROME_USER_DATA_DIR"]) == Path(other_config["CHROME_USER_DATA_DIR"])
assert Path(config["CHROME_DOWNLOADS_DIR"]) == Path(other_config["CHROME_DOWNLOADS_DIR"])
assert crawl_downloads_sentinel.read_text() == "keep"
assert config["ACTIVE_PERSONA"] == "RuntimePersona"
assert Path(config["CRAWL_DIR"]) == crawl.output_dir
assert Path(config["SNAP_DIR"]) == snapshot.output_dir
@pytest.mark.django_db(transaction=True)
def test_snapshot_payload_uses_snapshot_chrome_dirs_when_snapshot_isolated():
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.personas.models import Persona
from archivebox.services.runner import CrawlRunner
persona = Persona(name="SnapshotRuntimePersona")
persona.save()
crawl = Crawl(
urls="https://example.com\nhttps://example.org",
persona_id=persona.id,
created_by_id=get_or_create_system_user_pk(),
config={"CHROME_ISOLATION": "snapshot"},
)
crawl.save()
snapshot = Snapshot(url="https://example.com", crawl=crawl)
snapshot.save()
other_snapshot = Snapshot(url="https://example.org", crawl=crawl)
other_snapshot.save()
runner = CrawlRunner(crawl)
runner.load_run_state()
payload = runner.load_snapshot_payload(str(snapshot.id))
other_payload = runner.load_snapshot_payload(str(other_snapshot.id))
config = payload["config"]
other_config = other_payload["config"]
assert Path(config["CHROME_USER_DATA_DIR"]).is_relative_to(snapshot.output_dir)
assert Path(config["CHROME_DOWNLOADS_DIR"]).is_relative_to(snapshot.output_dir)
assert Path(other_config["CHROME_USER_DATA_DIR"]).is_relative_to(other_snapshot.output_dir)
assert Path(other_config["CHROME_DOWNLOADS_DIR"]).is_relative_to(other_snapshot.output_dir)
assert Path(config["CHROME_USER_DATA_DIR"]) != Path(other_config["CHROME_USER_DATA_DIR"])
assert Path(config["CHROME_DOWNLOADS_DIR"]) != Path(other_config["CHROME_DOWNLOADS_DIR"])
assert Path(config["CRAWL_DIR"]) == crawl.output_dir
assert Path(config["SNAP_DIR"]) == snapshot.output_dir
def test_ensure_background_runner_skips_under_pytest_guard():
from archivebox.services.runner import ensure_background_runner

2
etc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.34rc31",
"version": "0.9.34rc32",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {

8
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.34rc31"
version = "0.9.34rc32"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -79,9 +79,9 @@ dependencies = [
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus==2.5.9", # EventBus API
"abxpkg>=1.11.147", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.149", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.149", # shared ArchiveBox downloader package with blocking install preflight
"abxpkg>=1.11.148", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.150", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.150", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]