diff --git a/archivebox/services/binary_service.py b/archivebox/services/binary_service.py index 96b02d1c..96c376fc 100644 --- a/archivebox/services/binary_service.py +++ b/archivebox/services/binary_service.py @@ -12,6 +12,7 @@ from django.utils import timezone from abxpkg import Binary as AbxBinary from abxpkg import BinProvider, PROVIDER_CLASS_BY_NAME from abxpkg.binary_service import BinaryEvent, BinaryRequestEvent +from abxpkg.config import load_derived_cache from abxbus import BaseEvent, EventBus from abx_dl.services.base import BaseService @@ -351,6 +352,139 @@ class ArchiveBoxBinaryService(BaseService): f.write(json.dumps(process.to_json()) + "\n") +def project_abxpkg_derived_cache_to_db(lib_dir: Path | str | None) -> None: + """Mirror abxpkg subprocess-resolved binaries into ArchiveBox's DB. + + Hook shebangs resolve binaries through the abxpkg CLI in subprocesses, so + those resolutions cannot emit in-process BinaryRequestEvent/BinaryEvent + objects on the ArchiveBox runner bus. abxpkg's generic cross-process + projection point is ``LIB_DIR/env/derived.env``; ArchiveBox imports those + resolved records here after hook execution. + """ + + if lib_dir is None: + return + + lib_path = Path(lib_dir).expanduser() + derived_env_paths = sorted(lib_path.rglob("derived.env")) if lib_path.is_dir() else [] + if not derived_env_paths: + return + + from archivebox.machine.models import Binary, Machine, Process, _canonical_binary_name + + machine = Machine.current() + for derived_env_path in derived_env_paths: + for record in load_derived_cache(derived_env_path).values(): + if not isinstance(record, Mapping): + continue + binary_name = _canonical_binary_name(str(record.get("bin_name") or "")) + if not binary_name: + continue + abspath = str(record.get("abspath") or "").strip() + if not abspath: + continue + binary_path = Path(abspath).expanduser().resolve(strict=False) + if not binary_path.exists(): + continue + + version = str(record.get("loaded_version") or "") + sha256 = str(record.get("loaded_sha256") or "") + provider_name = str(record.get("provider_name") or "") + resolved_provider_name = str(record.get("resolved_provider_name") or provider_name) + installed_abspath = str(binary_path) + + binary, _created = Binary.objects.get_or_create( + machine=machine, + name=binary_name, + defaults={ + "status": Binary.StatusChoices.QUEUED, + "binproviders": provider_name or resolved_provider_name or "env", + }, + ) + previous_projection = ( + binary.status, + binary.abspath, + binary.version, + binary.sha256, + binary.binprovider, + ) + binary.abspath = installed_abspath + binary.version = version + binary.sha256 = sha256 + binary.binproviders = provider_name or resolved_provider_name or binary.binproviders or "env" + binary.binprovider = resolved_provider_name or provider_name or binary.binprovider + binary.status = Binary.StatusChoices.INSTALLED + binary.retry_at = None + binary.save( + update_fields=[ + "abspath", + "version", + "sha256", + "binproviders", + "binprovider", + "status", + "retry_at", + "modified_at", + ], + ) + + current_projection = ( + binary.status, + binary.abspath, + binary.version, + binary.sha256, + binary.binprovider, + ) + if current_projection == previous_projection: + continue + + output_dir = binary.output_dir.parent + output_dir.mkdir(parents=True, exist_ok=True) + now = timezone.now() + process = Process.objects.create( + machine=machine, + iface=None, + process_type=Process.TypeChoices.BINARY, + worker_type="", + pwd=str(output_dir), + cmd=[ + "abxpkg", + "run", + "--script", + f"--name={binary_name}", + f"--binproviders={binary.binproviders}", + ], + env={}, + timeout=0, + pid=None, + url=None, + started_at=now, + ended_at=now, + stdout=json.dumps( + { + "type": "Binary", + "name": binary_name, + "binproviders": binary.binproviders, + "binprovider": binary.binprovider, + "abspath": binary.abspath, + "version": binary.version, + "sha256": binary.sha256, + "status": "installed", + }, + ) + + "\n", + stderr="", + exit_code=0, + status=Process.StatusChoices.EXITED, + retry_at=None, + binary=binary, + ) + index_path = output_dir / "index.jsonl" + with index_path.open("w", encoding="utf-8") as f: + f.write(json.dumps(binary.to_json()) + "\n") + f.write(json.dumps(process.to_json()) + "\n") + + def _provider_names(binproviders: str | list[str] | None) -> list[str]: if isinstance(binproviders, str): raw_names = [part.strip() for part in binproviders.split(",")] diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 52adec7e..fdf2466d 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -29,7 +29,6 @@ from abx_dl.events import ( CrawlEvent, CrawlSetupEvent, CrawlStartEvent, - InstallEvent, MachineEvent, ProcessCompletedEvent, ProcessEvent, @@ -41,15 +40,13 @@ from abx_dl.heartbeat import CrawlHeartbeat from abx_dl.limits import CrawlLimitState from abx_dl.models import Plugin, Snapshot as AbxSnapshot, discover_plugins, filter_plugins from abx_dl.orchestrator import ( - compute_install_phase_timeout, compute_phase_timeout, create_bus, - get_install_plugins, install_plugins as abx_install_plugins, setup_services as setup_abx_services, ) from abx_dl.services.process_service import ProcessService as HookProcessService -from abx_dl.services.binary_service import PluginBinariesService, split_abxpkg_binary_request_overrides +from abx_dl.services.binary_service import split_abxpkg_binary_request_overrides from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService from abx_dl.cli import LiveBusUI from abxbus import BaseEvent @@ -68,7 +65,7 @@ from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS from archivebox.crawls.locks import crawl_lifecycle_lock from .archive_result_service import ArchiveResultService -from .binary_service import ArchiveBoxBinaryService, ArchiveBoxDBBinaryCacheBackend +from .binary_service import ArchiveBoxBinaryService, ArchiveBoxDBBinaryCacheBackend, project_abxpkg_derived_cache_to_db from .crawl_service import CrawlService from .machine_service import MachineService from .process_service import ProcessService as PersistedProcessService @@ -393,6 +390,7 @@ class CrawlRunner: except Exception: pass self._live_stream = None + await sync_to_async(project_abxpkg_derived_cache_to_db, thread_sensitive=True)(self.base_config.get("ABXPKG_LIB_DIR")) await sync_to_async(self.finalize_run_state, thread_sensitive=True)() async def enqueue_snapshot(self, snapshot_id: str, crawl_start_event: CrawlStartEvent | None = None) -> None: @@ -845,7 +843,6 @@ class CrawlRunner: ) setup_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("CrawlSetup")] crawl_setup_phase_timeout = compute_phase_timeout(setup_hooks, config) - install_phase_timeout = compute_install_phase_timeout(get_install_plugins(plugins), config) snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")] max_snapshot_count = max(1, int(config.get("CRAWL_MAX_URLS") or len(snapshot_ids) or 1)) snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config) + 120.0 @@ -859,32 +856,13 @@ class CrawlRunner: + 30.0 ) await _emit_machine_config(self.bus, config=config, derived_config=derived_config) - install_cancel_watcher: asyncio.Task[None] | None = None - install_event = self.bus.emit( - InstallEvent( - url=snapshot["url"], - snapshot_id=snapshot["id"], - output_dir=str(output_dir), - event_timeout=install_phase_timeout, - event_handler_slow_timeout=slow_warning_timeout(install_phase_timeout), - ), - ) - - async def on_archivebox_InstallEvent(event: InstallEvent) -> None: - nonlocal install_cancel_watcher - if event.event_id != install_event.event_id: - return - install_cancel_watcher = asyncio.create_task(self.watch_for_cancelled_crawl(event)) - - on_archivebox_InstallEvent.__name__ = "on_archivebox_InstallEvent__cancel_watcher" - self.bus.on(InstallEvent, on_archivebox_InstallEvent) setup_abx_services( self.bus, plugins=plugins, url=snapshot["url"], snapshot=abx_snapshot, output_dir=output_dir, - install_enabled=True, + install_enabled=False, crawl_setup_enabled=True, crawl_event_enabled=False, crawl_start_enabled=False, @@ -900,7 +878,7 @@ class CrawlRunner: emit_jsonl=False, abort_requested=self.crawl_is_cancelled, MachineService=None, - PluginBinariesService=PluginBinariesService, + PluginBinariesService=None, BinaryCacheService=None, BinaryService=None, ProcessService=None, @@ -908,12 +886,6 @@ class CrawlRunner: TagService=None, SnapshotService=None, ) - try: - await _run_event_now(install_event, install_phase_timeout) - finally: - if install_cancel_watcher is not None: - install_cancel_watcher.cancel() - await asyncio.gather(install_cancel_watcher, return_exceptions=True) async def on_archivebox_CrawlStartEvent(event: CrawlStartEvent) -> None: if event.event_id != self.root_crawl_start_event_id: diff --git a/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py b/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py index ac197e3e..9f7734a4 100644 --- a/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py +++ b/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py @@ -136,15 +136,18 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_ hook_name="on_Snapshot__93_hashes.py", lib_dir=lib_dir, ) - Snapshot.objects.filter(pk=snapshot.pk).update(url="http://127.0.0.1:1/") + Snapshot.objects.filter(pk=snapshot.pk).update(url="file:///nonexistent/archivebox-test-repository.git") snapshot.refresh_from_db() _failed_process, failed_result = _run_shipped_snapshot_hook( snapshot, - plugin="title", - hook_name="on_Snapshot__54_title.js", + plugin="git", + hook_name="on_Snapshot__05_git.finite.bg.py", + event_hook_name=_snapshot_hook_name("git"), lib_dir=lib_dir, expected_exit_codes=(1,), ) + assert failed_result.status == ArchiveResult.StatusChoices.FAILED + assert failed_result.output_str == "git fetch failed (exit=128)" now = timezone.now() Snapshot.objects.filter(pk=snapshot.pk).update( url=blocking_http_server.url, @@ -224,8 +227,8 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_ assert finished_rows["hashes"][0] == ArchiveResult.StatusChoices.SUCCEEDED assert finished_rows["hashes"][1] is None assert finished_rows["hashes"][2] > 0 - assert finished_rows["title"][0] == ArchiveResult.StatusChoices.FAILED - assert finished_rows["title"][1] is None + assert finished_rows["git"][0] == ArchiveResult.StatusChoices.FAILED + assert finished_rows["git"][1] is None succeeded_row = ArchiveResult.objects.get(id=succeeded_result.id) output_path = Path(snapshot.output_dir) / succeeded_row.plugin / next(iter(succeeded_row.output_files)) diff --git a/archivebox/tests/test_recursive_crawl.py b/archivebox/tests/test_recursive_crawl.py index 6dc00df2..921fa870 100644 --- a/archivebox/tests/test_recursive_crawl.py +++ b/archivebox/tests/test_recursive_crawl.py @@ -385,7 +385,7 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p @pytest.mark.timeout(1200) def test_add_archivewebpage_installs_required_chrome_dependency(initialized_archive): - """archivebox add should install selected plugins' required_plugins and binaries before hooks run.""" + """archivebox add should resolve selected plugins' required plugins and persist binary projections.""" env = os.environ.copy() env.pop("CHROME_BINARY", None) diff --git a/pyproject.toml b/pyproject.toml index 12ea1aff..52e6c126 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc146" +version = "0.9.35rc147" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -81,9 +81,9 @@ dependencies = [ ### Extractor dependencies (runtime binaries resolved through abxpkg) ### Binary/Package Management "abxbus==2.5.40", # EventBus API - "abxpkg==1.12.4", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins==1.12.8", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl==1.12.9", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg==1.12.11", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins==1.12.11", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl==1.12.11", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ] @@ -348,7 +348,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations" [tool.bumpver] -current_version = "v0.9.35rc146" +current_version = "v0.9.35rc147" version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" tag_message = "{new_version}" diff --git a/uv.lock b/uv.lock index 6585e0de..00425f35 100644 --- a/uv.lock +++ b/uv.lock @@ -24,7 +24,7 @@ abxpkg = "2100-01-01T00:00:00Z" [[package]] name = "abx-dl" -version = "1.12.9" +version = "1.12.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "abx-plugins" }, @@ -39,14 +39,14 @@ dependencies = [ { name = "rich" }, { name = "rich-click" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7e/dc/a858ac0d7351d136d48dcfa7a4673d743d6513886b3878de943ba95a65af/abx_dl-1.12.9.tar.gz", hash = "sha256:bbc7928d816742f2b4a0ba2f57ddefd7b045349217d00946e4c480dcb4319f7c", size = 87455, upload-time = "2026-07-26T18:57:59.843Z" } +sdist = { url = "https://files.pythonhosted.org/packages/63/d0/410cb5f7d84c76147883037cad2fb68620aad38c6b9abbf55e1163675aed/abx_dl-1.12.11.tar.gz", hash = "sha256:73de416bde4adc9749c20a5ba1bcaec824fc75266dfd0f4d4d38b5b9848b9c7e", size = 87149, upload-time = "2026-07-26T23:13:44.164Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/05/1d2eff92ca648ed8b517f132ba7029a46691206498ba795d008e95b5343e/abx_dl-1.12.9-py3-none-any.whl", hash = "sha256:eb048e69810db2d227c9daf1654f2e477069431a8fa976caecd7b3110596eec5", size = 91177, upload-time = "2026-07-26T18:57:58.755Z" }, + { url = "https://files.pythonhosted.org/packages/f7/be/93c0759bb5b0597c738e8b56d89ff4146388db798030c70c3636079531f9/abx_dl-1.12.11-py3-none-any.whl", hash = "sha256:8180f68264abacd16a0ac22f3c8264c1fa133117474b52ca3172342fe009662f", size = 90934, upload-time = "2026-07-26T23:13:42.919Z" }, ] [[package]] name = "abx-plugins" -version = "1.12.8" +version = "1.12.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "abxbus" }, @@ -56,9 +56,9 @@ dependencies = [ { name = "rich-click" }, { name = "uv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4a/9d/a5ac31cf4618d79167ec5da4e4def7f6cf0274340f76513955077d110a25/abx_plugins-1.12.8.tar.gz", hash = "sha256:fdc5d72efee660edc4936956f8bb406179798de17248916466eada5093557122", size = 257672, upload-time = "2026-07-26T18:26:06.572Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/35/2bfd7e0e1cf2b5040df67da889180238610dec9deb47fafef7764daf7170/abx_plugins-1.12.11.tar.gz", hash = "sha256:fb40d3f5f69d50ae8e2763b8e85fbaed3dc818a4c33903b1d7616af9e7f611fa", size = 259524, upload-time = "2026-07-26T22:09:04.581Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/b8/5e4371a6321c222ab2c5537da8409b86c04bdbc520514a98d7d7d54d6bb1/abx_plugins-1.12.8-py3-none-any.whl", hash = "sha256:7ee4b140f2b9ed7a06cc457591ee043b72d80704387967acdfea50ffda757bf2", size = 405859, upload-time = "2026-07-26T18:26:08.124Z" }, + { url = "https://files.pythonhosted.org/packages/af/9e/2fd41b69217f176e6517f2931bb4b1ead4b7214d2429e861109d53ea4ef1/abx_plugins-1.12.11-py3-none-any.whl", hash = "sha256:0215cd8d219456cfab37724c85beb4246e91b62f535c2926cd96f336ee8409ff", size = 410510, upload-time = "2026-07-26T22:09:02.783Z" }, ] [[package]] @@ -77,7 +77,7 @@ wheels = [ [[package]] name = "abxpkg" -version = "1.12.4" +version = "1.12.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pip" }, @@ -86,9 +86,9 @@ dependencies = [ { name = "rich-click" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b6/1f/55e39b430580e6307364f05cb70992355acdd772050d3543cfcaf27b07cb/abxpkg-1.12.4.tar.gz", hash = "sha256:3fd205162c6b3a1e29015f651dcef892191de17420f15bac2bf409cf9c98726d", size = 219648, upload-time = "2026-07-26T16:56:16.266Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9b/1d/f7861f9afc0eb56942c19ed20cf208b2fa37279cc349636d79842e34e63e/abxpkg-1.12.11.tar.gz", hash = "sha256:0bee57aa0a65d7f1a2d5a415022a3bb45642f9ea8f2ff6bb60d0ae99278846e9", size = 219683, upload-time = "2026-07-26T21:45:16.052Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/44/b04277cd52b73fb7a0da99610f014a9847f0296bf366009cdaea6745b885/abxpkg-1.12.4-py3-none-any.whl", hash = "sha256:231f8f089edfe103588b53f3d367e2ad69191c522f930ea40bb980329144f11b", size = 233420, upload-time = "2026-07-26T16:56:17.638Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ba/a13cabde5e3e227d82664442e1f16674e769ed8b182732426f968b9db8c0/abxpkg-1.12.11-py3-none-any.whl", hash = "sha256:e021e031ba3c0ffd83c1a76cbdbbb27f8ace62a34216aeeab7b40a42001afa74", size = 233462, upload-time = "2026-07-26T21:45:17.51Z" }, ] [[package]] @@ -123,7 +123,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.9.35rc146" +version = "0.9.35rc147" source = { editable = "." } dependencies = [ { name = "abx-dl" }, @@ -223,10 +223,10 @@ dev = [ [package.metadata] requires-dist = [ - { name = "abx-dl", specifier = "==1.12.9" }, - { name = "abx-plugins", specifier = "==1.12.8" }, + { name = "abx-dl", specifier = "==1.12.11" }, + { name = "abx-plugins", specifier = "==1.12.11" }, { name = "abxbus", specifier = "==2.5.40" }, - { name = "abxpkg", specifier = "==1.12.4" }, + { name = "abxpkg", specifier = "==1.12.11" }, { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" }, { name = "atomicwrites", specifier = "==1.4.1" }, { name = "base32-crockford", specifier = ">=0.3.0" },