The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Resolve binaries through abxpkg hook projections

Browse Source
This commit is contained in:
Nick Sweeting 2026-07-26 16:17:16 -07:00
parent 64d7da5cf4
commit 50939fdbcb
6 changed files with 166 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
archivebox/services/binary_service.py
View File

@ -12,6 +12,7 @@ from django.utils import timezone
from abxpkg import Binary as AbxBinary
from abxpkg import BinProvider, PROVIDER_CLASS_BY_NAME
from abxpkg.binary_service import BinaryEvent, BinaryRequestEvent
from abxpkg.config import load_derived_cache
from abxbus import BaseEvent, EventBus
from abx_dl.services.base import BaseService
@ -351,6 +352,139 @@ class ArchiveBoxBinaryService(BaseService):
f.write(json.dumps(process.to_json()) + "\n")
def project_abxpkg_derived_cache_to_db(lib_dir: Path | str | None) -> None:
"""Mirror abxpkg subprocess-resolved binaries into ArchiveBox's DB.
Hook shebangs resolve binaries through the abxpkg CLI in subprocesses, so
those resolutions cannot emit in-process BinaryRequestEvent/BinaryEvent
objects on the ArchiveBox runner bus. abxpkg's generic cross-process
projection point is ``LIB_DIR/env/derived.env``; ArchiveBox imports those
resolved records here after hook execution.
"""
if lib_dir is None:
return
lib_path = Path(lib_dir).expanduser()
derived_env_paths = sorted(lib_path.rglob("derived.env")) if lib_path.is_dir() else []
if not derived_env_paths:
return
from archivebox.machine.models import Binary, Machine, Process, _canonical_binary_name
machine = Machine.current()
for derived_env_path in derived_env_paths:
for record in load_derived_cache(derived_env_path).values():
if not isinstance(record, Mapping):
continue
binary_name = _canonical_binary_name(str(record.get("bin_name") or ""))
if not binary_name:
continue
abspath = str(record.get("abspath") or "").strip()
if not abspath:
continue
binary_path = Path(abspath).expanduser().resolve(strict=False)
if not binary_path.exists():
continue
version = str(record.get("loaded_version") or "")
sha256 = str(record.get("loaded_sha256") or "")
provider_name = str(record.get("provider_name") or "")
resolved_provider_name = str(record.get("resolved_provider_name") or provider_name)
installed_abspath = str(binary_path)
binary, _created = Binary.objects.get_or_create(
machine=machine,
name=binary_name,
defaults={
"status": Binary.StatusChoices.QUEUED,
"binproviders": provider_name or resolved_provider_name or "env",
},
)
previous_projection = (
binary.status,
binary.abspath,
binary.version,
binary.sha256,
binary.binprovider,
)
binary.abspath = installed_abspath
binary.version = version
binary.sha256 = sha256
binary.binproviders = provider_name or resolved_provider_name or binary.binproviders or "env"
binary.binprovider = resolved_provider_name or provider_name or binary.binprovider
binary.status = Binary.StatusChoices.INSTALLED
binary.retry_at = None
binary.save(
update_fields=[
"abspath",
"version",
"sha256",
"binproviders",
"binprovider",
"status",
"retry_at",
"modified_at",
],
)
current_projection = (
binary.status,
binary.abspath,
binary.version,
binary.sha256,
binary.binprovider,
)
if current_projection == previous_projection:
continue
output_dir = binary.output_dir.parent
output_dir.mkdir(parents=True, exist_ok=True)
now = timezone.now()
process = Process.objects.create(
machine=machine,
iface=None,
process_type=Process.TypeChoices.BINARY,
worker_type="",
pwd=str(output_dir),
cmd=[
"abxpkg",
"run",
"--script",
f"--name={binary_name}",
f"--binproviders={binary.binproviders}",
],
env={},
timeout=0,
pid=None,
url=None,
started_at=now,
ended_at=now,
stdout=json.dumps(
{
"type": "Binary",
"name": binary_name,
"binproviders": binary.binproviders,
"binprovider": binary.binprovider,
"abspath": binary.abspath,
"version": binary.version,
"sha256": binary.sha256,
"status": "installed",
},
)
+ "\n",
stderr="",
exit_code=0,
status=Process.StatusChoices.EXITED,
retry_at=None,
binary=binary,
)
index_path = output_dir / "index.jsonl"
with index_path.open("w", encoding="utf-8") as f:
f.write(json.dumps(binary.to_json()) + "\n")
f.write(json.dumps(process.to_json()) + "\n")
def _provider_names(binproviders: str | list[str] | None) -> list[str]:
if isinstance(binproviders, str):
raw_names = [part.strip() for part in binproviders.split(",")]

38
archivebox/services/runner.py
View File

@ -29,7 +29,6 @@ from abx_dl.events import (
CrawlEvent,
CrawlSetupEvent,
CrawlStartEvent,
InstallEvent,
MachineEvent,
ProcessCompletedEvent,
ProcessEvent,
@ -41,15 +40,13 @@ from abx_dl.heartbeat import CrawlHeartbeat
from abx_dl.limits import CrawlLimitState
from abx_dl.models import Plugin, Snapshot as AbxSnapshot, discover_plugins, filter_plugins
from abx_dl.orchestrator import (
compute_install_phase_timeout,
compute_phase_timeout,
create_bus,
get_install_plugins,
install_plugins as abx_install_plugins,
setup_services as setup_abx_services,
)
from abx_dl.services.process_service import ProcessService as HookProcessService
from abx_dl.services.binary_service import PluginBinariesService, split_abxpkg_binary_request_overrides
from abx_dl.services.binary_service import split_abxpkg_binary_request_overrides
from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService
from abx_dl.cli import LiveBusUI
from abxbus import BaseEvent
@ -68,7 +65,7 @@ from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS
from archivebox.crawls.locks import crawl_lifecycle_lock
from .archive_result_service import ArchiveResultService
from .binary_service import ArchiveBoxBinaryService, ArchiveBoxDBBinaryCacheBackend
from .binary_service import ArchiveBoxBinaryService, ArchiveBoxDBBinaryCacheBackend, project_abxpkg_derived_cache_to_db
from .crawl_service import CrawlService
from .machine_service import MachineService
from .process_service import ProcessService as PersistedProcessService
@ -393,6 +390,7 @@ class CrawlRunner:
except Exception:
pass
self._live_stream = None
await sync_to_async(project_abxpkg_derived_cache_to_db, thread_sensitive=True)(self.base_config.get("ABXPKG_LIB_DIR"))
await sync_to_async(self.finalize_run_state, thread_sensitive=True)()
async def enqueue_snapshot(self, snapshot_id: str, crawl_start_event: CrawlStartEvent | None = None) -> None:
@ -845,7 +843,6 @@ class CrawlRunner:
)
setup_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("CrawlSetup")]
crawl_setup_phase_timeout = compute_phase_timeout(setup_hooks, config)
install_phase_timeout = compute_install_phase_timeout(get_install_plugins(plugins), config)
snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")]
max_snapshot_count = max(1, int(config.get("CRAWL_MAX_URLS") or len(snapshot_ids) or 1))
snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config) + 120.0
@ -859,32 +856,13 @@ class CrawlRunner:
+ 30.0
)
await _emit_machine_config(self.bus, config=config, derived_config=derived_config)
install_cancel_watcher: asyncio.Task[None] | None = None
install_event = self.bus.emit(
InstallEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
output_dir=str(output_dir),
event_timeout=install_phase_timeout,
event_handler_slow_timeout=slow_warning_timeout(install_phase_timeout),
),
)
async def on_archivebox_InstallEvent(event: InstallEvent) -> None:
nonlocal install_cancel_watcher
if event.event_id != install_event.event_id:
return
install_cancel_watcher = asyncio.create_task(self.watch_for_cancelled_crawl(event))
on_archivebox_InstallEvent.__name__ = "on_archivebox_InstallEvent__cancel_watcher"
self.bus.on(InstallEvent, on_archivebox_InstallEvent)
setup_abx_services(
self.bus,
plugins=plugins,
url=snapshot["url"],
snapshot=abx_snapshot,
output_dir=output_dir,
install_enabled=True,
install_enabled=False,
crawl_setup_enabled=True,
crawl_event_enabled=False,
crawl_start_enabled=False,
@ -900,7 +878,7 @@ class CrawlRunner:
emit_jsonl=False,
abort_requested=self.crawl_is_cancelled,
MachineService=None,
PluginBinariesService=PluginBinariesService,
PluginBinariesService=None,
BinaryCacheService=None,
BinaryService=None,
ProcessService=None,
@ -908,12 +886,6 @@ class CrawlRunner:
TagService=None,
SnapshotService=None,
)
try:
await _run_event_now(install_event, install_phase_timeout)
finally:
if install_cancel_watcher is not None:
install_cancel_watcher.cancel()
await asyncio.gather(install_cancel_watcher, return_exceptions=True)
async def on_archivebox_CrawlStartEvent(event: CrawlStartEvent) -> None:
if event.event_id != self.root_crawl_start_event_id:

13
archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py
View File

@ -136,15 +136,18 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
hook_name="on_Snapshot__93_hashes.py",
lib_dir=lib_dir,
)
Snapshot.objects.filter(pk=snapshot.pk).update(url="http://127.0.0.1:1/")
Snapshot.objects.filter(pk=snapshot.pk).update(url="file:///nonexistent/archivebox-test-repository.git")
snapshot.refresh_from_db()
_failed_process, failed_result = _run_shipped_snapshot_hook(
snapshot,
plugin="title",
hook_name="on_Snapshot__54_title.js",
plugin="git",
hook_name="on_Snapshot__05_git.finite.bg.py",
event_hook_name=_snapshot_hook_name("git"),
lib_dir=lib_dir,
expected_exit_codes=(1,),
)
assert failed_result.status == ArchiveResult.StatusChoices.FAILED
assert failed_result.output_str == "git fetch failed (exit=128)"
now = timezone.now()
Snapshot.objects.filter(pk=snapshot.pk).update(
url=blocking_http_server.url,
@ -224,8 +227,8 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
assert finished_rows["hashes"][0] == ArchiveResult.StatusChoices.SUCCEEDED
assert finished_rows["hashes"][1] is None
assert finished_rows["hashes"][2] > 0
assert finished_rows["title"][0] == ArchiveResult.StatusChoices.FAILED
assert finished_rows["title"][1] is None
assert finished_rows["git"][0] == ArchiveResult.StatusChoices.FAILED
assert finished_rows["git"][1] is None
succeeded_row = ArchiveResult.objects.get(id=succeeded_result.id)
output_path = Path(snapshot.output_dir) / succeeded_row.plugin / next(iter(succeeded_row.output_files))

2
archivebox/tests/test_recursive_crawl.py
View File

@ -385,7 +385,7 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p
@pytest.mark.timeout(1200)
def test_add_archivewebpage_installs_required_chrome_dependency(initialized_archive):
"""archivebox add should install selected plugins' required_plugins and binaries before hooks run."""
"""archivebox add should resolve selected plugins' required plugins and persist binary projections."""
env = os.environ.copy()
env.pop("CHROME_BINARY", None)

10
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.35rc146"
version = "0.9.35rc147"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -81,9 +81,9 @@ dependencies = [
### Extractor dependencies (runtime binaries resolved through abxpkg)
### Binary/Package Management
"abxbus==2.5.40", # EventBus API
"abxpkg==1.12.4", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins==1.12.8", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl==1.12.9", # shared ArchiveBox downloader package with blocking install preflight
"abxpkg==1.12.11", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins==1.12.11", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl==1.12.11", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]
@ -348,7 +348,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
[tool.bumpver]
current_version = "v0.9.35rc146"
current_version = "v0.9.35rc147"
version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
commit_message = "bump version {old_version} -> {new_version}"
tag_message = "{new_version}"

26
uv.lock
View File

@ -24,7 +24,7 @@ abxpkg = "2100-01-01T00:00:00Z"
[[package]]
name = "abx-dl"
version = "1.12.9"
version = "1.12.11"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "abx-plugins" },
@ -39,14 +39,14 @@ dependencies = [
{ name = "rich" },
{ name = "rich-click" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7e/dc/a858ac0d7351d136d48dcfa7a4673d743d6513886b3878de943ba95a65af/abx_dl-1.12.9.tar.gz", hash = "sha256:bbc7928d816742f2b4a0ba2f57ddefd7b045349217d00946e4c480dcb4319f7c", size = 87455, upload-time = "2026-07-26T18:57:59.843Z" }
sdist = { url = "https://files.pythonhosted.org/packages/63/d0/410cb5f7d84c76147883037cad2fb68620aad38c6b9abbf55e1163675aed/abx_dl-1.12.11.tar.gz", hash = "sha256:73de416bde4adc9749c20a5ba1bcaec824fc75266dfd0f4d4d38b5b9848b9c7e", size = 87149, upload-time = "2026-07-26T23:13:44.164Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/2d/05/1d2eff92ca648ed8b517f132ba7029a46691206498ba795d008e95b5343e/abx_dl-1.12.9-py3-none-any.whl", hash = "sha256:eb048e69810db2d227c9daf1654f2e477069431a8fa976caecd7b3110596eec5", size = 91177, upload-time = "2026-07-26T18:57:58.755Z" },
{ url = "https://files.pythonhosted.org/packages/f7/be/93c0759bb5b0597c738e8b56d89ff4146388db798030c70c3636079531f9/abx_dl-1.12.11-py3-none-any.whl", hash = "sha256:8180f68264abacd16a0ac22f3c8264c1fa133117474b52ca3172342fe009662f", size = 90934, upload-time = "2026-07-26T23:13:42.919Z" },
]
[[package]]
name = "abx-plugins"
version = "1.12.8"
version = "1.12.11"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "abxbus" },
@ -56,9 +56,9 @@ dependencies = [
{ name = "rich-click" },
{ name = "uv" },
]
sdist = { url = "https://files.pythonhosted.org/packages/4a/9d/a5ac31cf4618d79167ec5da4e4def7f6cf0274340f76513955077d110a25/abx_plugins-1.12.8.tar.gz", hash = "sha256:fdc5d72efee660edc4936956f8bb406179798de17248916466eada5093557122", size = 257672, upload-time = "2026-07-26T18:26:06.572Z" }
sdist = { url = "https://files.pythonhosted.org/packages/e8/35/2bfd7e0e1cf2b5040df67da889180238610dec9deb47fafef7764daf7170/abx_plugins-1.12.11.tar.gz", hash = "sha256:fb40d3f5f69d50ae8e2763b8e85fbaed3dc818a4c33903b1d7616af9e7f611fa", size = 259524, upload-time = "2026-07-26T22:09:04.581Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/be/b8/5e4371a6321c222ab2c5537da8409b86c04bdbc520514a98d7d7d54d6bb1/abx_plugins-1.12.8-py3-none-any.whl", hash = "sha256:7ee4b140f2b9ed7a06cc457591ee043b72d80704387967acdfea50ffda757bf2", size = 405859, upload-time = "2026-07-26T18:26:08.124Z" },
{ url = "https://files.pythonhosted.org/packages/af/9e/2fd41b69217f176e6517f2931bb4b1ead4b7214d2429e861109d53ea4ef1/abx_plugins-1.12.11-py3-none-any.whl", hash = "sha256:0215cd8d219456cfab37724c85beb4246e91b62f535c2926cd96f336ee8409ff", size = 410510, upload-time = "2026-07-26T22:09:02.783Z" },
]
[[package]]
@ -77,7 +77,7 @@ wheels = [
[[package]]
name = "abxpkg"
version = "1.12.4"
version = "1.12.11"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pip" },
@ -86,9 +86,9 @@ dependencies = [
{ name = "rich-click" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b6/1f/55e39b430580e6307364f05cb70992355acdd772050d3543cfcaf27b07cb/abxpkg-1.12.4.tar.gz", hash = "sha256:3fd205162c6b3a1e29015f651dcef892191de17420f15bac2bf409cf9c98726d", size = 219648, upload-time = "2026-07-26T16:56:16.266Z" }
sdist = { url = "https://files.pythonhosted.org/packages/9b/1d/f7861f9afc0eb56942c19ed20cf208b2fa37279cc349636d79842e34e63e/abxpkg-1.12.11.tar.gz", hash = "sha256:0bee57aa0a65d7f1a2d5a415022a3bb45642f9ea8f2ff6bb60d0ae99278846e9", size = 219683, upload-time = "2026-07-26T21:45:16.052Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fa/44/b04277cd52b73fb7a0da99610f014a9847f0296bf366009cdaea6745b885/abxpkg-1.12.4-py3-none-any.whl", hash = "sha256:231f8f089edfe103588b53f3d367e2ad69191c522f930ea40bb980329144f11b", size = 233420, upload-time = "2026-07-26T16:56:17.638Z" },
{ url = "https://files.pythonhosted.org/packages/d7/ba/a13cabde5e3e227d82664442e1f16674e769ed8b182732426f968b9db8c0/abxpkg-1.12.11-py3-none-any.whl", hash = "sha256:e021e031ba3c0ffd83c1a76cbdbbb27f8ace62a34216aeeab7b40a42001afa74", size = 233462, upload-time = "2026-07-26T21:45:17.51Z" },
]
[[package]]
@ -123,7 +123,7 @@ wheels = [
[[package]]
name = "archivebox"
version = "0.9.35rc146"
version = "0.9.35rc147"
source = { editable = "." }
dependencies = [
{ name = "abx-dl" },
@ -223,10 +223,10 @@ dev = [
[package.metadata]
requires-dist = [
{ name = "abx-dl", specifier = "==1.12.9" },
{ name = "abx-plugins", specifier = "==1.12.8" },
{ name = "abx-dl", specifier = "==1.12.11" },
{ name = "abx-plugins", specifier = "==1.12.11" },
{ name = "abxbus", specifier = "==2.5.40" },
{ name = "abxpkg", specifier = "==1.12.4" },
{ name = "abxpkg", specifier = "==1.12.11" },
{ name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
{ name = "atomicwrites", specifier = "==1.4.1" },
{ name = "base32-crockford", specifier = ">=0.3.0" },