The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

release: archivebox 0.9.35rc2

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-08 05:03:19 -07:00
parent a99f3bfd49
commit 0642d085be
No known key found for this signature in database
6 changed files with 122 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
archivebox/services/binary_service.py
View File

@ -16,21 +16,6 @@ from abxbus import BaseEvent, EventBus
from abx_dl.services.base import BaseService
_LIB_DIR_MANAGED_PROVIDERS = {
"bash",
"cargo",
"deno",
"gem",
"goget",
"chromewebstore",
"nix",
"npm",
"pip",
"puppeteer",
"uv",
}
class ArchiveBoxDBBinaryCacheBackend:
"""ArchiveBox machine.Binary projection backend for abxpkg BinaryCacheService."""
@ -43,11 +28,62 @@ class ArchiveBoxDBBinaryCacheBackend:
if not binary_name:
return None
existing = await Binary.objects.filter(machine=machine, name=binary_name).afirst()
persisted_overrides = _persisted_overrides_for_request(request)
native_overrides = request.overrides or {}
cache_invalidated = False
if existing and existing.status == Binary.StatusChoices.INSTALLED:
requested_provider_names = _provider_names(request.binproviders)
await sync_to_async(get_config, thread_sensitive=True)()
installed_qs = (
Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED)
.exclude(abspath="")
.exclude(abspath__isnull=True)
.order_by("-modified_at")
)
async for installed in installed_qs:
installed_path = Path(installed.abspath).expanduser().resolve(strict=False)
if not await sync_to_async(installed_path.exists, thread_sensitive=True)():
await _mark_binary_queued(installed)
continue
if persisted_overrides and installed.overrides != persisted_overrides:
await _mark_binary_queued(installed)
continue
provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip()
if provider_name and provider_name not in requested_provider_names:
await _mark_binary_queued(installed)
continue
provider = _provider_for_name(provider_name, installed.name, native_overrides)
if await sync_to_async(_cached_provider_path_is_stale, thread_sensitive=True)(installed_path, provider, installed.name):
await _mark_binary_queued(installed)
continue
binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {}
provider_names = _provider_names(installed.binproviders or request.binproviders or "env")
return AbxBinary.model_validate(
{
"name": request.name,
"description": request.description,
"binproviders": _providers_for_names(provider_names),
"overrides": native_overrides,
"loaded_binprovider": provider,
"loaded_abspath": installed.abspath,
"loaded_version": installed.version or None,
"loaded_sha256": installed.sha256 or None,
"env": binary_env,
},
)
existing = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
if existing is None:
await Binary.objects.acreate(
machine=machine,
name=binary_name,
binproviders=_binproviders_to_str(request.binproviders),
overrides=persisted_overrides,
status=Binary.StatusChoices.QUEUED,
)
else:
changed = False
requested_binproviders = _binproviders_to_str(request.binproviders)
if requested_binproviders and existing.binproviders != requested_binproviders:
@ -56,72 +92,13 @@ class ArchiveBoxDBBinaryCacheBackend:
if persisted_overrides and existing.overrides != persisted_overrides:
existing.overrides = persisted_overrides
changed = True
if changed:
if existing.status != Binary.StatusChoices.QUEUED:
existing.status = Binary.StatusChoices.QUEUED
existing.retry_at = None
cache_invalidated = True
changed = True
if changed:
await existing.asave(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
elif existing is None:
await Binary.objects.acreate(
machine=machine,
name=binary_name,
binproviders=_binproviders_to_str(request.binproviders),
overrides=persisted_overrides,
status=Binary.StatusChoices.QUEUED,
)
installed = None
if not cache_invalidated:
installed = (
await Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED)
.exclude(abspath="")
.exclude(abspath__isnull=True)
.order_by("-modified_at")
.afirst()
)
if installed is not None and not await sync_to_async(Path(installed.abspath).expanduser().exists, thread_sensitive=True)():
installed.status = Binary.StatusChoices.QUEUED
installed.retry_at = None
await installed.asave(update_fields=["status", "retry_at", "modified_at"])
installed = None
if installed is not None and persisted_overrides and installed.overrides != persisted_overrides:
installed.status = Binary.StatusChoices.QUEUED
installed.retry_at = None
await installed.asave(update_fields=["status", "retry_at", "modified_at"])
installed = None
if installed is None:
return None
installed_path = Path(installed.abspath).expanduser().resolve(strict=False)
active_lib_dir = (
Path(str((await sync_to_async(get_config, thread_sensitive=True)()).get("LIB_DIR", ""))).expanduser().resolve(strict=False)
)
provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip()
if active_lib_dir and provider_name in _LIB_DIR_MANAGED_PROVIDERS:
try:
installed_path.relative_to(active_lib_dir)
except ValueError:
installed.status = Binary.StatusChoices.QUEUED
installed.retry_at = None
await installed.asave(update_fields=["status", "retry_at", "modified_at"])
return None
provider = _provider_for_name(provider_name, installed.name, native_overrides)
binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {}
provider_names = _provider_names(installed.binproviders or request.binproviders or "env")
return AbxBinary.model_validate(
{
"name": request.name,
"description": request.description,
"binproviders": _providers_for_names(provider_names),
"overrides": native_overrides,
"loaded_binprovider": provider,
"loaded_abspath": installed.abspath,
"loaded_version": installed.version or None,
"loaded_sha256": installed.sha256 or None,
"env": binary_env,
},
)
return None
async def set(self, request: BinaryRequestEvent | None, binary: AbxBinary) -> None:
from archivebox.machine.models import Binary, Machine, _canonical_binary_name
@ -418,6 +395,25 @@ def _provider_for_name(provider_name: str, binary_name: str, overrides: dict[str
return provider
async def _mark_binary_queued(binary) -> None:
from archivebox.machine.models import Binary
if binary.status == Binary.StatusChoices.QUEUED:
return
binary.status = Binary.StatusChoices.QUEUED
binary.retry_at = None
await binary.asave(update_fields=["status", "retry_at", "modified_at"])
def _cached_provider_path_is_stale(installed_path: Path, provider: BinProvider | None, binary_name: str) -> bool:
if provider is None:
return False
current_abspath = provider.get_abspath(binary_name, quiet=True, no_cache=True)
if not current_abspath:
return True
return Path(current_abspath).expanduser().resolve(strict=False) != installed_path
def _persisted_overrides_for_request(request: BinaryRequestEvent | None) -> dict[str, Any]:
if request is None:
return {}

32
archivebox/tests/test_binary_service.py
View File

@ -24,12 +24,13 @@ def _link_real_binary(bin_dir: Path, name: str, *, source: str | None = None) ->
return link
def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]:
def _runtime_env(data_dir: Path, bin_dir: Path, *, lib_dir: Path | None = None) -> dict[str, str]:
archivebox_bin = shutil.which("archivebox")
assert archivebox_bin, "archivebox console script must be available for CLI tests"
lib_dir = lib_dir or data_dir / "lib"
return {
"LIB_DIR": str(data_dir / "lib"),
"ABXPKG_LIB_DIR": str(data_dir / "lib"),
"LIB_DIR": str(lib_dir),
"ABXPKG_LIB_DIR": str(lib_dir),
"PATH": os.pathsep.join([str(bin_dir), str(Path(archivebox_bin).parent), "/usr/bin", "/bin", "/usr/sbin", "/sbin"]),
}
@ -173,6 +174,31 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
assert Path(recovered.abspath).resolve() == Path(shutil.which("rg") or "").resolve()
assert process_count >= 2
changed_lib_dir = tmp_path / "changed-lib"
changed_provider_bin_dir = changed_lib_dir / "env" / "bin"
_link_real_binary(changed_provider_bin_dir, name, source="rg")
_cmd_result = run_archivebox_cmd(
["run"],
cwd=initialized_archive,
stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n",
timeout=120,
env=_runtime_env(initialized_archive, bootstrap_bin_dir, lib_dir=changed_lib_dir),
default_cli_env=True,
disable_extractors=True,
)
relib_stdout, relib_stderr, relib_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
assert relib_code == 0, relib_stdout + relib_stderr
with use_archivebox_db(initialized_archive):
relibbed = Binary.objects.get(pk=first_binary_id)
assert relibbed.status == Binary.StatusChoices.INSTALLED
assert relibbed.version == binary.version
assert Path(relibbed.abspath) == changed_provider_bin_dir / name
assert Path(relibbed.abspath).exists()
assert Path(relibbed.abspath).resolve() == Path(shutil.which("rg") or "").resolve()
def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive, tmp_path):
name = f"abx-missing-rg-{uuid.uuid4().hex[:8]}"

10
archivebox/tests/test_crawl_pause.py
View File

@ -1,2 +1,12 @@
from django.utils import timezone
from archivebox.workers.models import RETRY_AT_MAX
def test_retry_at_max_is_safe_for_admin_timezone_localization():
with timezone.override("Pacific/Kiritimati"):
assert timezone.localtime(RETRY_AT_MAX).year == 9999
# test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume moved to test_api_v1_crawls_crawl_crawl_id.py.
# test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl moved to test_api_v1_crawls_crawl_crawl_id.py.

2
archivebox/workers/models.py
View File

@ -36,7 +36,7 @@ default_status_field: models.CharField = models.CharField(
db_index=True,
)
default_retry_at_field: models.DateTimeField = models.DateTimeField(default=timezone.now, null=True, blank=True, db_index=True)
RETRY_AT_MAX = datetime.max.replace(tzinfo=UTC)
RETRY_AT_MAX = datetime(9999, 1, 1, tzinfo=UTC)
ACTIVE_STATE_LEASE_SECONDS = 60
logger = logging.getLogger(__name__)
MODULE_PATH = Path(__file__).resolve()

2
etc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.34rc71",
"version": "0.9.35rc2",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {

8
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.35rc1"
version = "0.9.35rc2"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -79,9 +79,9 @@ dependencies = [
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus==2.5.10", # EventBus API
"abxpkg>=1.11.186", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.190", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.190", # shared ArchiveBox downloader package with blocking install preflight
"abxpkg>=1.11.187", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.191", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.191", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]