From 0642d085beb37829761238f29ee4100287917432 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 8 Jun 2026 05:03:19 -0700 Subject: [PATCH] release: archivebox 0.9.35rc2 --- archivebox/services/binary_service.py | 158 ++++++++++++------------ archivebox/tests/test_binary_service.py | 32 ++++- archivebox/tests/test_crawl_pause.py | 10 ++ archivebox/workers/models.py | 2 +- etc/package.json | 2 +- pyproject.toml | 8 +- 6 files changed, 122 insertions(+), 90 deletions(-) diff --git a/archivebox/services/binary_service.py b/archivebox/services/binary_service.py index 41344834..92a2c148 100644 --- a/archivebox/services/binary_service.py +++ b/archivebox/services/binary_service.py @@ -16,21 +16,6 @@ from abxbus import BaseEvent, EventBus from abx_dl.services.base import BaseService -_LIB_DIR_MANAGED_PROVIDERS = { - "bash", - "cargo", - "deno", - "gem", - "goget", - "chromewebstore", - "nix", - "npm", - "pip", - "puppeteer", - "uv", -} - - class ArchiveBoxDBBinaryCacheBackend: """ArchiveBox machine.Binary projection backend for abxpkg BinaryCacheService.""" @@ -43,11 +28,62 @@ class ArchiveBoxDBBinaryCacheBackend: if not binary_name: return None - existing = await Binary.objects.filter(machine=machine, name=binary_name).afirst() persisted_overrides = _persisted_overrides_for_request(request) native_overrides = request.overrides or {} - cache_invalidated = False - if existing and existing.status == Binary.StatusChoices.INSTALLED: + requested_provider_names = _provider_names(request.binproviders) + await sync_to_async(get_config, thread_sensitive=True)() + + installed_qs = ( + Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED) + .exclude(abspath="") + .exclude(abspath__isnull=True) + .order_by("-modified_at") + ) + async for installed in installed_qs: + installed_path = Path(installed.abspath).expanduser().resolve(strict=False) + if not await sync_to_async(installed_path.exists, thread_sensitive=True)(): + await _mark_binary_queued(installed) + continue + if persisted_overrides and installed.overrides != persisted_overrides: + await _mark_binary_queued(installed) + continue + + provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip() + if provider_name and provider_name not in requested_provider_names: + await _mark_binary_queued(installed) + continue + + provider = _provider_for_name(provider_name, installed.name, native_overrides) + if await sync_to_async(_cached_provider_path_is_stale, thread_sensitive=True)(installed_path, provider, installed.name): + await _mark_binary_queued(installed) + continue + + binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {} + provider_names = _provider_names(installed.binproviders or request.binproviders or "env") + return AbxBinary.model_validate( + { + "name": request.name, + "description": request.description, + "binproviders": _providers_for_names(provider_names), + "overrides": native_overrides, + "loaded_binprovider": provider, + "loaded_abspath": installed.abspath, + "loaded_version": installed.version or None, + "loaded_sha256": installed.sha256 or None, + "env": binary_env, + }, + ) + + existing = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst() + if existing is None: + await Binary.objects.acreate( + machine=machine, + name=binary_name, + binproviders=_binproviders_to_str(request.binproviders), + overrides=persisted_overrides, + status=Binary.StatusChoices.QUEUED, + ) + else: changed = False requested_binproviders = _binproviders_to_str(request.binproviders) if requested_binproviders and existing.binproviders != requested_binproviders: @@ -56,72 +92,13 @@ class ArchiveBoxDBBinaryCacheBackend: if persisted_overrides and existing.overrides != persisted_overrides: existing.overrides = persisted_overrides changed = True - if changed: + if existing.status != Binary.StatusChoices.QUEUED: existing.status = Binary.StatusChoices.QUEUED existing.retry_at = None - cache_invalidated = True + changed = True + if changed: await existing.asave(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"]) - elif existing is None: - await Binary.objects.acreate( - machine=machine, - name=binary_name, - binproviders=_binproviders_to_str(request.binproviders), - overrides=persisted_overrides, - status=Binary.StatusChoices.QUEUED, - ) - - installed = None - if not cache_invalidated: - installed = ( - await Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED) - .exclude(abspath="") - .exclude(abspath__isnull=True) - .order_by("-modified_at") - .afirst() - ) - if installed is not None and not await sync_to_async(Path(installed.abspath).expanduser().exists, thread_sensitive=True)(): - installed.status = Binary.StatusChoices.QUEUED - installed.retry_at = None - await installed.asave(update_fields=["status", "retry_at", "modified_at"]) - installed = None - if installed is not None and persisted_overrides and installed.overrides != persisted_overrides: - installed.status = Binary.StatusChoices.QUEUED - installed.retry_at = None - await installed.asave(update_fields=["status", "retry_at", "modified_at"]) - installed = None - if installed is None: - return None - - installed_path = Path(installed.abspath).expanduser().resolve(strict=False) - active_lib_dir = ( - Path(str((await sync_to_async(get_config, thread_sensitive=True)()).get("LIB_DIR", ""))).expanduser().resolve(strict=False) - ) - provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip() - if active_lib_dir and provider_name in _LIB_DIR_MANAGED_PROVIDERS: - try: - installed_path.relative_to(active_lib_dir) - except ValueError: - installed.status = Binary.StatusChoices.QUEUED - installed.retry_at = None - await installed.asave(update_fields=["status", "retry_at", "modified_at"]) - return None - - provider = _provider_for_name(provider_name, installed.name, native_overrides) - binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {} - provider_names = _provider_names(installed.binproviders or request.binproviders or "env") - return AbxBinary.model_validate( - { - "name": request.name, - "description": request.description, - "binproviders": _providers_for_names(provider_names), - "overrides": native_overrides, - "loaded_binprovider": provider, - "loaded_abspath": installed.abspath, - "loaded_version": installed.version or None, - "loaded_sha256": installed.sha256 or None, - "env": binary_env, - }, - ) + return None async def set(self, request: BinaryRequestEvent | None, binary: AbxBinary) -> None: from archivebox.machine.models import Binary, Machine, _canonical_binary_name @@ -418,6 +395,25 @@ def _provider_for_name(provider_name: str, binary_name: str, overrides: dict[str return provider +async def _mark_binary_queued(binary) -> None: + from archivebox.machine.models import Binary + + if binary.status == Binary.StatusChoices.QUEUED: + return + binary.status = Binary.StatusChoices.QUEUED + binary.retry_at = None + await binary.asave(update_fields=["status", "retry_at", "modified_at"]) + + +def _cached_provider_path_is_stale(installed_path: Path, provider: BinProvider | None, binary_name: str) -> bool: + if provider is None: + return False + current_abspath = provider.get_abspath(binary_name, quiet=True, no_cache=True) + if not current_abspath: + return True + return Path(current_abspath).expanduser().resolve(strict=False) != installed_path + + def _persisted_overrides_for_request(request: BinaryRequestEvent | None) -> dict[str, Any]: if request is None: return {} diff --git a/archivebox/tests/test_binary_service.py b/archivebox/tests/test_binary_service.py index ac58ba7a..62dfad26 100644 --- a/archivebox/tests/test_binary_service.py +++ b/archivebox/tests/test_binary_service.py @@ -24,12 +24,13 @@ def _link_real_binary(bin_dir: Path, name: str, *, source: str | None = None) -> return link -def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]: +def _runtime_env(data_dir: Path, bin_dir: Path, *, lib_dir: Path | None = None) -> dict[str, str]: archivebox_bin = shutil.which("archivebox") assert archivebox_bin, "archivebox console script must be available for CLI tests" + lib_dir = lib_dir or data_dir / "lib" return { - "LIB_DIR": str(data_dir / "lib"), - "ABXPKG_LIB_DIR": str(data_dir / "lib"), + "LIB_DIR": str(lib_dir), + "ABXPKG_LIB_DIR": str(lib_dir), "PATH": os.pathsep.join([str(bin_dir), str(Path(archivebox_bin).parent), "/usr/bin", "/bin", "/usr/sbin", "/sbin"]), } @@ -173,6 +174,31 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized assert Path(recovered.abspath).resolve() == Path(shutil.which("rg") or "").resolve() assert process_count >= 2 + changed_lib_dir = tmp_path / "changed-lib" + changed_provider_bin_dir = changed_lib_dir / "env" / "bin" + _link_real_binary(changed_provider_bin_dir, name, source="rg") + + _cmd_result = run_archivebox_cmd( + ["run"], + cwd=initialized_archive, + stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n", + timeout=120, + env=_runtime_env(initialized_archive, bootstrap_bin_dir, lib_dir=changed_lib_dir), + default_cli_env=True, + disable_extractors=True, + ) + relib_stdout, relib_stderr, relib_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + + assert relib_code == 0, relib_stdout + relib_stderr + with use_archivebox_db(initialized_archive): + relibbed = Binary.objects.get(pk=first_binary_id) + + assert relibbed.status == Binary.StatusChoices.INSTALLED + assert relibbed.version == binary.version + assert Path(relibbed.abspath) == changed_provider_bin_dir / name + assert Path(relibbed.abspath).exists() + assert Path(relibbed.abspath).resolve() == Path(shutil.which("rg") or "").resolve() + def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive, tmp_path): name = f"abx-missing-rg-{uuid.uuid4().hex[:8]}" diff --git a/archivebox/tests/test_crawl_pause.py b/archivebox/tests/test_crawl_pause.py index c73cd25a..8c7b2e3f 100644 --- a/archivebox/tests/test_crawl_pause.py +++ b/archivebox/tests/test_crawl_pause.py @@ -1,2 +1,12 @@ +from django.utils import timezone + +from archivebox.workers.models import RETRY_AT_MAX + + +def test_retry_at_max_is_safe_for_admin_timezone_localization(): + with timezone.override("Pacific/Kiritimati"): + assert timezone.localtime(RETRY_AT_MAX).year == 9999 + + # test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume moved to test_api_v1_crawls_crawl_crawl_id.py. # test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl moved to test_api_v1_crawls_crawl_crawl_id.py. diff --git a/archivebox/workers/models.py b/archivebox/workers/models.py index 771d0331..f9bc2ee0 100644 --- a/archivebox/workers/models.py +++ b/archivebox/workers/models.py @@ -36,7 +36,7 @@ default_status_field: models.CharField = models.CharField( db_index=True, ) default_retry_at_field: models.DateTimeField = models.DateTimeField(default=timezone.now, null=True, blank=True, db_index=True) -RETRY_AT_MAX = datetime.max.replace(tzinfo=UTC) +RETRY_AT_MAX = datetime(9999, 1, 1, tzinfo=UTC) ACTIVE_STATE_LEASE_SECONDS = 60 logger = logging.getLogger(__name__) MODULE_PATH = Path(__file__).resolve() diff --git a/etc/package.json b/etc/package.json index 7b077546..ad1ada1e 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.34rc71", + "version": "0.9.35rc2", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index abdff31b..ce81dd50 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc1" +version = "0.9.35rc2" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.10", # EventBus API - "abxpkg>=1.11.186", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.190", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.190", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.187", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.191", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.191", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]