From c0fb8eb532e46587f159e082c130bf5f0d36617b Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 3 Jun 2026 17:19:48 -0700 Subject: [PATCH] release: archivebox 0.9.34rc39 --- Dockerfile | 4 +- archivebox/cli/archivebox_run.py | 7 +- archivebox/core/admin_archiveresults.py | 13 +- archivebox/core/admin_tags.py | 2 +- archivebox/core/settings.py | 1 + archivebox/core/tag_util.py | 4 +- archivebox/core/urls.py | 1 + archivebox/machine/admin.py | 27 +- .../0020_repair_process_binary_iface_links.py | 130 +++++++ archivebox/machine/models.py | 16 - archivebox/personas/importers.py | 2 +- archivebox/plugins/forms.py | 1 + archivebox/plugins/hooks.py | 10 +- archivebox/progressmonitor/views.py | 14 +- archivebox/services/archive_result_service.py | 5 +- archivebox/services/binary_service.py | 184 ++++++++-- archivebox/services/machine_service.py | 4 +- archivebox/services/runner.py | 22 +- .../templates/admin/core/tag/change_list.html | 8 +- archivebox/templates/core/navigation.html | 3 + archivebox/tests/conftest.py | 43 ++- .../tests/test_api_v1_core_tags_search.py | 20 + archivebox/tests/test_binary_service.py | 2 + archivebox/tests/test_cli_server.py | 1 + archivebox/tests/test_crawl_runner.py | 4 + archivebox/tests/test_hooks.py | 81 ++++- archivebox/tests/test_machine_models.py | 20 +- archivebox/tests/test_machine_service.py | 3 + archivebox/tests/test_migrations_08_to_09.py | 38 +- archivebox/tests/test_opencode_agent.py | 344 ++++++++++++++++++ archivebox/tests/test_recursive_crawl.py | 291 ++++++++++++++- .../tests/test_server_security_browser.py | 4 +- archivebox/tests/test_ui_admin_links.py | 13 +- archivebox/tests/test_ui_admin_machine.py | 6 + archivebox/tests/test_ui_live_progress.py | 37 +- archivebox/tests/test_urls.py | 45 ++- archivebox/workers/supervisord_util.py | 2 +- bin/deploy_dev_demo.sh | 8 + etc/package.json | 2 +- pyproject.toml | 8 +- 40 files changed, 1285 insertions(+), 145 deletions(-) create mode 100644 archivebox/machine/migrations/0020_repair_process_binary_iface_links.py create mode 100644 archivebox/tests/test_opencode_agent.py diff --git a/Dockerfile b/Dockerfile index fbb08fa8..704c7a95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -313,7 +313,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \ && export PERSONAS_DIR="$LIB_DIR/personas" \ && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \ - && export ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \ + && export ABX_RUNTIME=archivebox ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \ && mkdir -p "$LIB_DIR" \ && apt-get update -qq \ && apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \ @@ -327,7 +327,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T accessibility archivedotorg archivewebpage base chrome chrome_mhtml chrome_screencast \ claudechrome claudecode claudecodecleanup claudecodeextract consolelog defuddle dns dom \ favicon forumdl gallerydl git hashes headers htmltotext infiniscroll \ - istilldontcareaboutcookies liteparse media mercury modalcloser opendataloader papersdl \ + istilldontcareaboutcookies liteparse media mercury modalcloser opencode opendataloader papersdl \ parse_dom_outlinks parse_html_urls parse_jsonl_urls parse_netscape_urls parse_rss_urls \ parse_txt_urls pdf readability redirects responses screenshot search_backend_ripgrep \ search_backend_sonic search_backend_sqlite seo singlefile ssl sslcerts staticfile title \ diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py index e0908c1c..a3e1707a 100644 --- a/archivebox/cli/archivebox_run.py +++ b/archivebox/cli/archivebox_run.py @@ -320,7 +320,12 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on interactive_interrupts=interactive_interrupts, ) return 0 - except (KeyboardInterrupt, asyncio.CancelledError): + except KeyboardInterrupt: + return 0 + except asyncio.CancelledError as e: + if daemon: + rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr) + return 1 return 0 except Exception as e: rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr) diff --git a/archivebox/core/admin_archiveresults.py b/archivebox/core/admin_archiveresults.py index 82ec24e9..08e4e5fb 100644 --- a/archivebox/core/admin_archiveresults.py +++ b/archivebox/core/admin_archiveresults.py @@ -75,6 +75,12 @@ def get_plugin_admin_url(plugin_name: str) -> str: return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/" +def get_process_link_label(process) -> str: + if process.pid: + return str(process.pid) + return str(process.id)[-8:] + + def render_archiveresults_list(archiveresults_qs, limit=50, config=None): """Render a nice inline list view of archive results with status, plugin, output, and actions.""" @@ -139,7 +145,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None): process_display = f''' {process.pid or "-"} + title="View process">{get_process_link_label(process)} ''' machine_display = "-" @@ -659,7 +665,7 @@ class ArchiveResultAdmin(BaseModelAdmin): process = result.process_record if not process: return "-" - process_label = process.pid or "-" + process_label = get_process_link_label(process) return format_html( '{}', reverse("admin:machine_process_change", args=[process.id]), @@ -673,9 +679,8 @@ class ArchiveResultAdmin(BaseModelAdmin): return "-" machine = process.machine return format_html( - '{} {}', + '{}', reverse("admin:machine_machine_change", args=[machine.id]), - str(machine.id)[:8], machine.hostname, ) diff --git a/archivebox/core/admin_tags.py b/archivebox/core/admin_tags.py index db6c3dca..2ff8c94d 100644 --- a/archivebox/core/admin_tags.py +++ b/archivebox/core/admin_tags.py @@ -117,7 +117,7 @@ class TagAdmin(BaseModelAdmin): sort = normalize_tag_sort((request.GET.get("sort") or "created_desc").strip()) created_by = normalize_created_by_filter((request.GET.get("created_by") or "").strip()) year = normalize_created_year_filter((request.GET.get("year") or "").strip()) - has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "yes").strip()) + has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "all").strip()) context = { **self.admin_site.each_context(request), **(extra_context or {}), diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 403525d5..f902faf9 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -72,6 +72,7 @@ INSTALLED_APPS = [ "archivebox.crawls", # handles Crawl and CrawlSchedule models and management (depends on core) "archivebox.progressmonitor", # live progress endpoint and admin monitor template "archivebox.api", # Django-Ninja-based Rest API interfaces, config, APIToken model, etc. + "abx_plugins.plugins.opencode", # 3rd-party apps from PyPI that need to be loaded last "admin_data_views", # handles rendering some convenient automatic read-only views of data in Django admin "django_extensions", # provides Django Debug Toolbar (and other non-debug helpers) diff --git a/archivebox/core/tag_util.py b/archivebox/core/tag_util.py index 27289c05..04e7ef9b 100644 --- a/archivebox/core/tag_util.py +++ b/archivebox/core/tag_util.py @@ -114,9 +114,9 @@ def add_snapshot_counts(tags: list[Tag], snapshot_queryset: QuerySet[Snapshot] | queryset = SnapshotTag.objects.filter(tag_id__in=tag_ids) if snapshot_queryset is not None: queryset = queryset.filter(snapshot_id__in=snapshot_queryset.values("id")) - counts = {row["tag_id"]: row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))} + counts = {int(row["tag_id"]): row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))} for tag in tags: - tag.num_snapshots = counts.get(tag.pk, 0) + tag.num_snapshots = counts.get(int(tag.pk), 0) def get_tag_creator_choices() -> list[tuple[str, str]]: diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py index afc053ae..a06955c0 100644 --- a/archivebox/core/urls.py +++ b/archivebox/core/urls.py @@ -41,6 +41,7 @@ urlpatterns = [ path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}), path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}), path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"), + re_path(r"^admin/agent/?(?=$|opencode)", include("abx_plugins.plugins.opencode.urls")), path("public/", PublicIndexView.as_view(), name="public-index"), path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"), path("archive/", RedirectView.as_view(url="/")), diff --git a/archivebox/machine/admin.py b/archivebox/machine/admin.py index 4119b4c9..889f6769 100644 --- a/archivebox/machine/admin.py +++ b/archivebox/machine/admin.py @@ -434,6 +434,9 @@ class ProcessAdmin(BaseModelAdmin): "crawl_link", "cmd_display", "env_display", + "stdout_display", + "stderr_display", + "archiveresult_output_display", "timeout", "pid", "exit_code", @@ -475,7 +478,7 @@ class ProcessAdmin(BaseModelAdmin): ( "Output", { - "fields": ("stdout", "stderr"), + "fields": ("stdout_display", "stderr_display", "archiveresult_output_display"), "classes": ("card", "wide", "collapse"), }, ), @@ -740,6 +743,28 @@ class ProcessAdmin(BaseModelAdmin): return "-" return _render_copy_block(env_text, multiline=True) + @admin.display(description="Stdout") + def stdout_display(self, process): + if not process.stdout: + return "-" + return _render_copy_block(process.stdout, multiline=True) + + @admin.display(description="Stderr") + def stderr_display(self, process): + if not process.stderr: + return "-" + return _render_copy_block(process.stderr, multiline=True) + + @admin.display(description="ArchiveResult Output") + def archiveresult_output_display(self, process): + try: + output = process.archiveresult.output_str + except Process.archiveresult.RelatedObjectDoesNotExist: + return "-" + if not output: + return "-" + return _render_copy_block(output, multiline=True) + def register_admin(admin_site): admin_site.register(Machine, MachineAdmin) diff --git a/archivebox/machine/migrations/0020_repair_process_binary_iface_links.py b/archivebox/machine/migrations/0020_repair_process_binary_iface_links.py new file mode 100644 index 00000000..ac13b14b --- /dev/null +++ b/archivebox/machine/migrations/0020_repair_process_binary_iface_links.py @@ -0,0 +1,130 @@ +import json +from pathlib import Path + +from django.db import migrations +from django.utils import timezone + + +def _cmd_array(value): + if isinstance(value, list): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return [value] if value else [] + return parsed if isinstance(parsed, list) else [] + return [] + + +def _ensure_placeholder_iface(NetworkInterface, machine_id, hostname): + iface = NetworkInterface.objects.filter(machine_id=machine_id).order_by("-modified_at", "-created_at").first() + if iface is not None: + return iface + + now = timezone.now() + return NetworkInterface.objects.create( + machine_id=machine_id, + created_at=now, + modified_at=now, + mac_address="00:00:00:00:00:00", + ip_public="0.0.0.0", + ip_local="0.0.0.0", + dns_server="0.0.0.0", + hostname=(hostname or "unknown")[:63], + iface="unknown", + isp="", + city="", + region="", + country="", + ) + + +def _get_or_create_binary(Binary, machine_id, reference): + reference = str(reference or "").strip() + if not reference: + return None + + name = Path(reference).name or reference + qs = Binary.objects.filter(machine_id=machine_id) + binary = qs.filter(abspath=reference).order_by("-modified_at", "-created_at").first() + if binary is None: + binary = qs.filter(name=name).order_by("-modified_at", "-created_at").first() + if binary is not None: + return binary + + now = timezone.now() + return Binary.objects.create( + machine_id=machine_id, + created_at=now, + modified_at=now, + name=name[:63], + binproviders="env", + overrides={}, + binprovider="env", + abspath=reference[:255], + version="", + sha256="", + status="installed", + retry_at=None, + ) + + +def repair_process_binary_iface_links(apps, schema_editor): + Binary = apps.get_model("machine", "Binary") + Machine = apps.get_model("machine", "Machine") + NetworkInterface = apps.get_model("machine", "NetworkInterface") + Process = apps.get_model("machine", "Process") + + machines = {machine.id: machine for machine in Machine.objects.only("id", "hostname").iterator(chunk_size=100)} + iface_by_machine = {} + binary_by_key = {} + + qs = Process.objects.filter(machine_id__isnull=False).filter(binary_id__isnull=True) | Process.objects.filter( + machine_id__isnull=False, + iface_id__isnull=True, + ) + for process in qs.distinct().only("id", "machine_id", "binary_id", "iface_id", "cmd").iterator(chunk_size=500): + update_fields = [] + machine = machines.get(process.machine_id) + + if process.iface_id is None: + iface = iface_by_machine.get(process.machine_id) + if iface is None: + iface = _ensure_placeholder_iface( + NetworkInterface, + process.machine_id, + machine.hostname if machine is not None else "", + ) + iface_by_machine[process.machine_id] = iface + process.iface_id = iface.id + update_fields.append("iface_id") + + if process.binary_id is None: + cmd = _cmd_array(process.cmd) + reference = str(cmd[0]).strip() if cmd else "" + if reference: + key = (process.machine_id, reference) + binary = binary_by_key.get(key) + if binary is None: + binary = _get_or_create_binary(Binary, process.machine_id, reference) + binary_by_key[key] = binary + if binary is not None: + process.binary_id = binary.id + update_fields.append("binary_id") + + if update_fields: + process.modified_at = timezone.now() + process.save(update_fields=[*update_fields, "modified_at"]) + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("machine", "0019_single_active_runner_constraint"), + ] + + operations = [ + migrations.RunPython(repair_process_binary_iface_links, migrations.RunPython.noop), + ] diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py index 0530ed4e..56c1cb09 100755 --- a/archivebox/machine/models.py +++ b/archivebox/machine/models.py @@ -650,22 +650,6 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine): binary_overrides = record.get("overrides", {}) normalized_overrides = binary_overrides if isinstance(binary_overrides, dict) else {} - # abx-plugins currently emits a GitHub install URL for readability-extractor, - # but the package is published on npm. Prefer the registry package to avoid - # long git-based installs in CI while still using canonical install_args. - if ( - name == "readability-extractor" - and isinstance(normalized_overrides.get("npm"), dict) - and normalized_overrides["npm"].get("install_args") == ["https://github.com/ArchiveBox/readability-extractor"] - ): - normalized_overrides = { - **normalized_overrides, - "npm": { - **normalized_overrides["npm"], - "install_args": ["readability-extractor"], - }, - } - # Case 1: Already installed (from on_Crawl hooks) - has abspath AND binproviders # This happens when on_Crawl hooks detect already-installed binaries abspath = record.get("abspath") diff --git a/archivebox/personas/importers.py b/archivebox/personas/importers.py index e75278ad..f6b9a354 100644 --- a/archivebox/personas/importers.py +++ b/archivebox/personas/importers.py @@ -612,7 +612,7 @@ def export_browser_state( if not state_script.exists(): return False, None, f"Browser state export script not found at {state_script}" - node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules" + node_modules_dir = get_config().LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules" chrome_plugin_dir = Path(get_plugins_dir()).resolve() env = os.environ.copy() diff --git a/archivebox/plugins/forms.py b/archivebox/plugins/forms.py index 18f37b20..ba4d384f 100644 --- a/archivebox/plugins/forms.py +++ b/archivebox/plugins/forms.py @@ -130,6 +130,7 @@ HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = { "env", "media", "npm", + "opencode", "pip", "puppeteer", "search_backend_ripgrep", diff --git a/archivebox/plugins/hooks.py b/archivebox/plugins/hooks.py index 60df0512..921c7cc2 100644 --- a/archivebox/plugins/hooks.py +++ b/archivebox/plugins/hooks.py @@ -285,7 +285,6 @@ def run_hook( """ from archivebox.machine.models import Process, Machine, NetworkInterface from archivebox.config.common import get_config, normalize_runtime_config - import sys config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")} config_overrides = _config_to_overrides(config) @@ -333,12 +332,15 @@ def run_hook( ) return process - # Determine the interpreter based on file extension + # Python hooks carry their runtime contract in the shebang + # (usually `abxpkg run --script python3`), so execute them directly. + # For shell/JS hooks we still dispatch through the conventional + # interpreter because those hooks do not need per-script Python env setup. ext = script.suffix.lower() if ext == ".sh": cmd = ["bash", str(script)] elif ext == ".py": - cmd = [sys.executable, str(script)] + cmd = [str(script)] elif ext == ".js": cmd = ["node", str(script)] else: @@ -388,7 +390,7 @@ def run_hook( # NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory. node_modules_dir = hook_config.get("NODE_MODULES_DIR") if not node_modules_dir and lib_dir: - node_modules_dir = Path(lib_dir) / "npm" / "node_modules" + node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules" node_path_parts = [part for part in str(hook_config.get("NODE_PATH") or "").split(os.pathsep) if part] if node_modules_dir: diff --git a/archivebox/progressmonitor/views.py b/archivebox/progressmonitor/views.py index a73ba265..53ea7110 100644 --- a/archivebox/progressmonitor/views.py +++ b/archivebox/progressmonitor/views.py @@ -327,8 +327,20 @@ def live_progress_view(request): .order_by("-modified_at")[:max_active_crawls], ) paused_crawls = list( - crawl_scope.filter(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff) + crawl_scope.filter( + Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff) + | Q( + status=Crawl.StatusChoices.PAUSED, + snapshot_set__status__in=Snapshot.RUNNABLE_STATES, + snapshot_set__retry_at__lte=now, + ) + | Q( + status=Crawl.StatusChoices.PAUSED, + snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED, + ), + ) .values(*active_crawl_fields) + .distinct() .order_by("-modified_at")[:max_active_crawls], ) queued_crawls = list( diff --git a/archivebox/services/archive_result_service.py b/archivebox/services/archive_result_service.py index 255a7df0..520c803b 100644 --- a/archivebox/services/archive_result_service.py +++ b/archivebox/services/archive_result_service.py @@ -432,6 +432,9 @@ class ArchiveResultService(BaseService): ).now() return + # TODO: consider moving this fallback derivation into abx-dl itself. + # First try both patterns: if the whole abx-dl process crashes, restarting + # the snapshot may be enough, but don't guess before validating it. process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED) with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"): await event.emit( @@ -439,7 +442,7 @@ class ArchiveResultService(BaseService): snapshot_id=snapshot_event.snapshot_id, plugin=event.plugin_name, hook_name=event.hook_name, - status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "skipped"), + status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "noresult"), output_str=event.stderr if process_failed else "", output_files=event.output_files, start_ts=event.start_ts, diff --git a/archivebox/services/binary_service.py b/archivebox/services/binary_service.py index c2feaf3a..3c3f2103 100644 --- a/archivebox/services/binary_service.py +++ b/archivebox/services/binary_service.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import json from collections.abc import Mapping from pathlib import Path @@ -186,38 +187,109 @@ class ArchiveBoxDBBinaryCacheBackend: class ArchiveBoxBinaryService(BaseService): """Preserve ArchiveBox's legacy Binary Process rows around abxpkg requests.""" - LISTENS_TO = [BinaryRequestEvent] + LISTENS_TO = [BinaryRequestEvent, BinaryEvent] EMITS: list[type[BaseEvent]] = [] def __init__(self, bus: EventBus): super().__init__(bus) + self.process_ids_by_request_id: dict[str, str] = {} self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__project_process) + self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__schedule_missing_finalize) + self.bus.on(BinaryEvent, self.on_BinaryEvent__finalize_process) async def on_BinaryRequestEvent__project_process(self, request: BinaryRequestEvent) -> None: - from archivebox.machine.models import Binary, Machine, Process, _canonical_binary_name - from archivebox.services.process_service import current_network_interface_with_machine + from archivebox.machine.models import Machine, Process, _canonical_binary_name machine = await sync_to_async(Machine.current, thread_sensitive=True)() binary_name = _canonical_binary_name(request.name) if not binary_name: return - binary = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst() - if binary is None: - return + binary = await self._get_or_create_binary(machine, binary_name, request) + started_at = timezone.now() + output_dir = self._process_output_dir(binary, request) + await sync_to_async(output_dir.mkdir, thread_sensitive=True)(parents=True, exist_ok=True) + process = await Process.objects.acreate( + machine=machine, + iface=None, + process_type=Process.TypeChoices.BINARY, + worker_type="", + pwd=str(output_dir), + cmd=self._process_cmd(request), + env={}, + timeout=int(request.event_timeout or request.install_timeout or 600), + pid=None, + url=None, + started_at=started_at, + ended_at=None, + stdout="", + stderr="", + exit_code=None, + status=Process.StatusChoices.RUNNING, + retry_at=None, + binary=binary, + ) + self.process_ids_by_request_id[request.event_id] = str(process.id) - binary_event = await self.bus.find( - BinaryEvent, - child_of=request, + async def on_BinaryEvent__finalize_process(self, event: BinaryEvent) -> None: + from archivebox.machine.models import Binary, Process, _canonical_binary_name + + request = await self.bus.find( + BinaryRequestEvent, past=True, future=False, - name=request.name, - where=lambda candidate: bool(candidate.abspath), + where=lambda candidate: self.bus.event_is_child_of(event, candidate), ) - iface = await sync_to_async(current_network_interface_with_machine, thread_sensitive=True)() - now = timezone.now() - success = isinstance(binary_event, BinaryEvent) - output_dir = self._process_output_dir(binary, request) - output_dir.mkdir(parents=True, exist_ok=True) + request = request if isinstance(request, BinaryRequestEvent) else None + process_id = self.process_ids_by_request_id.pop(request.event_id, "") if request is not None else "" + if not process_id: + return + process = await Process.objects.filter(id=process_id).select_related("binary").afirst() + if process is None: + return + binary_name = _canonical_binary_name(event.name) + binary = process.binary + if binary is not None and binary_name: + binary.abspath = event.abspath + if event.version: + binary.version = str(event.version) + if event.sha256: + binary.sha256 = str(event.sha256) + binary.binproviders = event.binproviders or binary.binproviders + binary.binprovider = event.binprovider or binary.binprovider + binary.status = Binary.StatusChoices.INSTALLED + binary.retry_at = None + await binary.asave( + update_fields=["abspath", "version", "sha256", "binproviders", "binprovider", "status", "retry_at", "modified_at"], + ) + process.ended_at = timezone.now() + process.stdout = json.dumps(self._binary_event_json(event, binary)) + "\n" + process.stderr = "" + process.exit_code = 0 + process.status = Process.StatusChoices.EXITED + await process.asave(update_fields=["ended_at", "stdout", "stderr", "exit_code", "status", "modified_at"]) + if binary is not None: + await sync_to_async(self._write_binary_index, thread_sensitive=True)(binary, process, Path(process.pwd)) + + async def _get_or_create_binary(self, machine, binary_name: str, request: BinaryRequestEvent): + from archivebox.machine.models import Binary + + binary_id = str(request.extra_context.get("binary_id") or "") + if binary_id: + binary = await Binary.objects.filter(id=binary_id).afirst() + if binary is not None: + return binary + binary = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst() + if binary is not None: + return binary + return await Binary.objects.acreate( + machine=machine, + name=binary_name, + binproviders=_binproviders_to_str(request.binproviders), + overrides=_persisted_overrides_for_request(request), + status=Binary.StatusChoices.QUEUED, + ) + + def _process_cmd(self, request: BinaryRequestEvent) -> list[str]: cmd = [ "abxpkg", "install", @@ -226,29 +298,65 @@ class ArchiveBoxBinaryService(BaseService): ] if request.overrides: cmd.append(f"--overrides={json.dumps(request.overrides, sort_keys=True)}") - stdout = json.dumps(binary.to_json()) + "\n" if success else "" - stderr = "" if success else f"Binary request did not resolve: {request.name}" - process = await Process.objects.acreate( - machine=iface.machine, - iface=iface, - process_type=Process.TypeChoices.BINARY, - worker_type="", - pwd=str(output_dir), - cmd=cmd, - env={}, - timeout=int(request.event_timeout or request.install_timeout or 600), - pid=None, - url=None, - started_at=now, - ended_at=now, - stdout=stdout, - stderr=stderr, - exit_code=0 if success else 1, - status=Process.StatusChoices.EXITED, - retry_at=None, - binary=binary, + return cmd + + def _binary_event_json(self, event: BinaryEvent, binary) -> dict[str, Any]: + if binary is not None: + data = binary.to_json() + else: + data = {"type": "Binary", "name": event.name} + data.update( + { + "type": "Binary", + "name": event.name, + "binproviders": event.binproviders, + "binprovider": event.binprovider, + "abspath": event.abspath, + "version": str(event.version or ""), + "sha256": event.sha256 or "", + "status": "installed", + }, ) - self._write_binary_index(binary, process, output_dir) + return data + + async def _finalize_missing_process(self, request: BinaryRequestEvent) -> None: + from archivebox.machine.models import Process + + process_id = self.process_ids_by_request_id.pop(request.event_id, "") + if not process_id: + return + process = await Process.objects.filter(id=process_id).afirst() + if process is None or process.status == Process.StatusChoices.EXITED: + return + process.ended_at = timezone.now() + process.stderr = f"Binary request did not resolve: {request.name}" + process.exit_code = 1 + process.status = Process.StatusChoices.EXITED + await process.asave(update_fields=["ended_at", "stderr", "exit_code", "status", "modified_at"]) + + async def _finalize_request_when_done(self, request: BinaryRequestEvent) -> None: + try: + await request.wait(timeout=request.event_timeout) + except TimeoutError: + await self._finalize_missing_process(request) + return + binary_event = await self.bus.find( + BinaryEvent, + child_of=request, + past=True, + future=False, + name=request.name, + where=lambda candidate: bool(candidate.abspath), + ) + if not isinstance(binary_event, BinaryEvent): + await self._finalize_missing_process(request) + + def _schedule_missing_finalize(self, request: BinaryRequestEvent) -> None: + task = asyncio.create_task(self._finalize_request_when_done(request)) + task.add_done_callback(lambda done: None if done.cancelled() else done.exception()) + + async def on_BinaryRequestEvent__schedule_missing_finalize(self, request: BinaryRequestEvent) -> None: + self._schedule_missing_finalize(request) def _process_output_dir(self, binary, request: BinaryRequestEvent) -> Path: raw_output_dir = str(request.extra_context.get("output_dir") or "").strip() diff --git a/archivebox/services/machine_service.py b/archivebox/services/machine_service.py index f8a51567..602cd788 100644 --- a/archivebox/services/machine_service.py +++ b/archivebox/services/machine_service.py @@ -7,8 +7,6 @@ from asgiref.sync import sync_to_async from abx_dl.events import MachineEvent from abx_dl.services.base import BaseService -_BINARY_EVENT_ALLOWED_KEYS = frozenset({"ABX_INSTALL_CACHE"}) - def _is_binary_event_key(key: str) -> bool: """``MachineEvent`` projector only ever writes binary-related state. @@ -20,7 +18,7 @@ def _is_binary_event_key(key: str) -> bool: mirror is a security boundary), so the projector strips anything that isn't a binary path or the binary install cache. """ - if key in _BINARY_EVENT_ALLOWED_KEYS: + if key.startswith("ABX_") and key.endswith("CACHE"): return True return key.endswith("_BINARY") diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 5913e8dd..cd737749 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -121,6 +121,10 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name) +def _discover_archivebox_plugins() -> dict[str, Plugin]: + return discover_plugins(runtime="archivebox") + + def _runner_task_context() -> contextvars.Context: context = contextvars.copy_context() context.run(EventBus.current_event_context.set, None) @@ -213,13 +217,13 @@ class CrawlRunner: ): self.crawl = crawl self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0) - self.plugins = discover_plugins() + self.plugins = _discover_archivebox_plugins() HookProcessService(self.bus, emit_jsonl=False, interactive_tty=interactive_interrupts) register_sonic_daemon_event_handler(self.bus) PersistedProcessService(self.bus) + ArchiveBoxBinaryService(self.bus) BinaryCacheService(self.bus, backend=ArchiveBoxDBBinaryCacheBackend()) BinaryService(self.bus) - ArchiveBoxBinaryService(self.bus) TagService(self.bus) CrawlService(self.bus, crawl_id=str(crawl.id)) MachineService(self.bus) @@ -839,7 +843,7 @@ class CrawlRunner: url=snapshot["url"], snapshot=abx_snapshot, output_dir=output_dir, - install_enabled=False, + install_enabled=True, crawl_setup_enabled=True, crawl_event_enabled=False, crawl_start_enabled=False, @@ -1179,7 +1183,7 @@ async def _run_binary(binary_id: str) -> None: from archivebox.machine.models import Binary, Machine binary = await Binary.objects.aget(id=binary_id) - plugins = discover_plugins() + plugins = _discover_archivebox_plugins() config = get_config(include_machine=False) machine = await sync_to_async(Machine.current, thread_sensitive=True)() derived_config = normalize_runtime_config(machine.config) @@ -1187,9 +1191,9 @@ async def _run_binary(binary_id: str) -> None: config = normalize_runtime_config(config) bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0) process_service = PersistedProcessService(bus) + ArchiveBoxBinaryService(bus) BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend()) BinaryService(bus) - ArchiveBoxBinaryService(bus) TagService(bus) ArchiveResultService(bus) MachineService(bus) @@ -1238,7 +1242,9 @@ def run_binary(binary_id: str) -> None: @lru_cache(maxsize=1) def _snapshot_hook_names_by_plugin() -> dict[str, frozenset[str]]: - return {plugin.name: frozenset(hook.name for hook in plugin.filter_hooks("Snapshot")) for plugin in discover_plugins().values()} + return { + plugin.name: frozenset(hook.name for hook in plugin.filter_hooks("Snapshot")) for plugin in _discover_archivebox_plugins().values() + } def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None: @@ -1566,7 +1572,7 @@ async def _run_install(plugin_names: list[str] | None = None) -> None: from archivebox.config.common import get_config from archivebox.machine.models import Machine - plugins = discover_plugins() + plugins = _discover_archivebox_plugins() config = get_config(include_machine=False) machine = await sync_to_async(Machine.current, thread_sensitive=True)() derived_config = normalize_runtime_config(machine.config) @@ -1574,9 +1580,9 @@ async def _run_install(plugin_names: list[str] | None = None) -> None: config = normalize_runtime_config(config) bus = create_bus(name="ArchiveBox_install", total_timeout=3600.0) PersistedProcessService(bus) + ArchiveBoxBinaryService(bus) BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend()) BinaryService(bus) - ArchiveBoxBinaryService(bus) TagService(bus) ArchiveResultService(bus) MachineService(bus) diff --git a/archivebox/templates/admin/core/tag/change_list.html b/archivebox/templates/admin/core/tag/change_list.html index 7af180f3..ef3aa553 100644 --- a/archivebox/templates/admin/core/tag/change_list.html +++ b/archivebox/templates/admin/core/tag/change_list.html @@ -517,7 +517,7 @@ document.addEventListener('DOMContentLoaded', function () { sort: typeof next.sort === 'string' ? next.sort : (sortSelect?.value || 'created_desc'), created_by: typeof next.created_by === 'string' ? next.created_by : (createdBySelect?.value || ''), year: typeof next.year === 'string' ? next.year : (yearSelect?.value || ''), - has_snapshots: typeof next.has_snapshots === 'string' ? next.has_snapshots : (shell.dataset.initialHasSnapshots || 'yes'), + has_snapshots: typeof next.has_snapshots === 'string' ? next.has_snapshots : (shell.dataset.initialHasSnapshots || 'all'), }; } @@ -554,7 +554,7 @@ document.addEventListener('DOMContentLoaded', function () { url.searchParams.delete('year'); } - if (state.has_snapshots && state.has_snapshots !== 'yes') { + if (state.has_snapshots && state.has_snapshots !== 'all') { url.searchParams.set('has_snapshots', state.has_snapshots); } else { url.searchParams.delete('has_snapshots'); @@ -570,9 +570,7 @@ document.addEventListener('DOMContentLoaded', function () { } function renderCards(nextCards, state) { - cards = Array.isArray(nextCards) ? nextCards.filter(function (card) { - return Number(card && card.num_snapshots || 0) > 0; - }) : []; + cards = Array.isArray(nextCards) ? nextCards : []; setMeta(state || getCurrentState(), cards.length); if (!cards.length) { diff --git a/archivebox/templates/core/navigation.html b/archivebox/templates/core/navigation.html index aabcf069..0bc63c96 100644 --- a/archivebox/templates/core/navigation.html +++ b/archivebox/templates/core/navigation.html @@ -6,6 +6,9 @@ Snapshots | Log | Tags     + {% if user.is_authenticated and user.is_superuser %} + 💬 AI | + {% endif %} Docs | API | Admin diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py index d6481dde..d1c84b80 100644 --- a/archivebox/tests/conftest.py +++ b/archivebox/tests/conftest.py @@ -26,6 +26,7 @@ import requests from django.utils import timezone REPO_ROOT = Path(__file__).resolve().parents[2] +WORKSPACE_ROOT = REPO_ROOT.parent PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve() SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve() (SESSION_DATA_DIR / "tests").mkdir(parents=True, exist_ok=True) @@ -55,6 +56,29 @@ def _assert_safe_runtime_paths(*, cwd: Path | None = None, env: dict[str, str] | _assert_not_repo_path(Path(value), label=key) +def _test_source_pythonpath() -> str: + entries: list[str] = [] + for repo_name in ("abxpkg", "abx-plugins", "abx-dl"): + repo_path = WORKSPACE_ROOT / repo_name + if repo_path.exists(): + entries.append(str(repo_path.resolve(strict=False))) + return os.pathsep.join(entries) + + +def _set_test_source_pythonpath(env: dict[str, str]) -> None: + source_pythonpath = _test_source_pythonpath() + existing_entries = [ + str(Path(entry).expanduser().resolve(strict=False)) + for entry in (env.get("PYTHONPATH") or "").split(os.pathsep) + if entry and Path(entry).expanduser().is_absolute() + ] + entries = [entry for entry in [*source_pythonpath.split(os.pathsep), *existing_entries] if entry] + if entries: + env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(entries)) + else: + env.pop("PYTHONPATH", None) + + def _sync_archivebox_test_data_dir(data_dir: Path) -> None: from archivebox.config import constants as constants_mod from archivebox.config import paths as paths_mod @@ -170,9 +194,8 @@ def run_archivebox_cmd( _assert_not_repo_path(cwd, label="cwd") - run_env: dict[str, str] | None = None + run_env = {} if replace_env else os.environ.copy() if default_cli_env or disable_extractors or env is not None: - run_env = {} if replace_env else os.environ.copy() if default_cli_env: run_env["USE_COLOR"] = "False" run_env["SHOW_PROGRESS"] = "False" @@ -198,8 +221,9 @@ def run_archivebox_cmd( ) if env: run_env.update(env) + _set_test_source_pythonpath(run_env) - _assert_safe_runtime_paths(cwd=cwd, env=run_env or os.environ) + _assert_safe_runtime_paths(cwd=cwd, env=run_env) if stdin is not None: assert input is None, "pass either input or stdin, not both" @@ -562,6 +586,7 @@ def cli_env( **extra: str, ) -> dict[str, str]: env = {} if replace else os.environ.copy() + _set_test_source_pythonpath(env) env.update({"USE_COLOR": "False", "SHOW_PROGRESS": "False"}) if disable_extractors or live or server: @@ -1373,7 +1398,7 @@ def _find_cached_chrome(lib_dir: Path) -> Path | None: lib_dir / "puppeteer" / "chromium", lib_dir / "puppeteer", lib_dir / "ms-playwright", - lib_dir / "npm" / "node_modules" / "puppeteer" / ".local-chromium", + lib_dir / "pnpm" / "packages" / "chrome" / "node_modules" / "puppeteer" / ".local-chromium", ] for base in candidates: if not base.exists(): @@ -1401,17 +1426,17 @@ def _find_system_browser() -> Path | None: def _ensure_puppeteer(shared_lib: Path) -> None: - npm_prefix = shared_lib / "npm" - node_modules = npm_prefix / "node_modules" + pnpm_prefix = shared_lib / "pnpm" / "packages" / "chrome" + node_modules = pnpm_prefix / "node_modules" puppeteer_dir = node_modules / "puppeteer" if puppeteer_dir.exists(): return - npm_prefix.mkdir(parents=True, exist_ok=True) + pnpm_prefix.mkdir(parents=True, exist_ok=True) env = os.environ.copy() env["PUPPETEER_SKIP_DOWNLOAD"] = "1" subprocess.run( - ["npm", "install", "puppeteer"], - cwd=str(npm_prefix), + ["pnpm", "add", "--dir", str(pnpm_prefix), "puppeteer"], + cwd=str(pnpm_prefix), env=env, check=True, capture_output=True, diff --git a/archivebox/tests/test_api_v1_core_tags_search.py b/archivebox/tests/test_api_v1_core_tags_search.py index b489e90b..47fe97f7 100644 --- a/archivebox/tests/test_api_v1_core_tags_search.py +++ b/archivebox/tests/test_api_v1_core_tags_search.py @@ -41,6 +41,26 @@ def test_tag_search_api_returns_card_payload(client, api_token, tagged_data): assert {snap.url for snap in snapshots} == {"https://example.com/one", "https://example.com/two"} +def test_tag_search_api_default_includes_empty_tags_and_counts_linked_snapshots(client, api_token, tagged_data, api_admin_user): + linked_tag, _snapshots = tagged_data + empty_tag = Tag.objects.create(name="Empty Tag", created_by=api_admin_user) + + response = client.get( + "/api/v1/core/tags/search/", + {"api_key": api_token.token}, + HTTP_HOST=ADMIN_TEST_HOST, + ) + + assert response.status_code == 200 + payload = response.json() + cards_by_name = {tag["name"]: tag for tag in payload["tags"]} + assert payload["has_snapshots"] == "all" + assert cards_by_name["Alpha Research"]["id"] == linked_tag.id + assert cards_by_name["Alpha Research"]["num_snapshots"] == 2 + assert cards_by_name["Empty Tag"]["id"] == empty_tag.id + assert cards_by_name["Empty Tag"]["num_snapshots"] == 0 + + def test_tag_search_api_respects_sort_and_filters(client, api_token, admin_user, crawl, tagged_data): from datetime import datetime diff --git a/archivebox/tests/test_binary_service.py b/archivebox/tests/test_binary_service.py index 443d9d9a..b4aebc51 100644 --- a/archivebox/tests/test_binary_service.py +++ b/archivebox/tests/test_binary_service.py @@ -134,6 +134,8 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized assert binary_processes assert binary_processes[-1].status == Process.StatusChoices.EXITED assert binary_processes[-1].exit_code == 0 + assert binary_processes[-1].ended_at is not None + assert binary_processes[-1].started_at < binary_processes[-1].ended_at assert any(f"--name={name}" in arg for arg in binary_processes[-1].cmd) _cmd_result = run_archivebox_cmd( diff --git a/archivebox/tests/test_cli_server.py b/archivebox/tests/test_cli_server.py index d2d86633..ce02d2b1 100644 --- a/archivebox/tests/test_cli_server.py +++ b/archivebox/tests/test_cli_server.py @@ -139,6 +139,7 @@ def test_runner_worker_uses_current_interpreter(): from archivebox.workers.supervisord_util import RUNNER_WORKER assert RUNNER_WORKER["command"] == f"{sys.executable} -m archivebox run --daemon" + assert RUNNER_WORKER["autorestart"] == "true" assert 'ARCHIVEBOX_RUNNER_DAEMON="1"' in RUNNER_WORKER["environment"] diff --git a/archivebox/tests/test_crawl_runner.py b/archivebox/tests/test_crawl_runner.py index fea7d285..3d5504ae 100644 --- a/archivebox/tests/test_crawl_runner.py +++ b/archivebox/tests/test_crawl_runner.py @@ -435,6 +435,8 @@ def test_machine_service_persists_only_derived_config_events(tmp_path, hermetic_ config={ "WGET_BINARY": str(wget_binary), "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"}, + "ABX_UV_CACHE": "/tmp/uv-cache", + "ABX_PNPM_CACHE": "/tmp/pnpm-cache", "CHROME_USER_DATA_DIR": "/tmp/stale-derived-profile", }, config_type="derived", @@ -466,6 +468,8 @@ def test_machine_service_persists_only_derived_config_events(tmp_path, hermetic_ # LIB_DIR) then the unset removed it; ABX_INSTALL_CACHE survives. assert machine.config == { "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"}, + "ABX_PNPM_CACHE": "/tmp/pnpm-cache", + "ABX_UV_CACHE": "/tmp/uv-cache", } diff --git a/archivebox/tests/test_hooks.py b/archivebox/tests/test_hooks.py index 28e76aa2..6d2817a8 100755 --- a/archivebox/tests/test_hooks.py +++ b/archivebox/tests/test_hooks.py @@ -18,11 +18,13 @@ import textwrap from pathlib import Path import pytest +import rich_click as click # Set up Django before importing any Django-dependent modules os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.settings") REPO_ROOT = Path(__file__).resolve().parents[2] +WORKSPACE_ROOT = REPO_ROOT.parent RESULT_PREFIX = "__ARCHIVEBOX_TEST_RESULT__=" @@ -50,7 +52,12 @@ def run_plugin_discovery_subprocess(tmp_path: Path, plugins_dir: Path, script: s cwd_plugins_dir = data_dir / "custom_plugins" if plugins_dir != cwd_plugins_dir: shutil.copytree(plugins_dir, cwd_plugins_dir) - env["PYTHONPATH"] = str(REPO_ROOT) + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "") + existing_pythonpath = [ + str(Path(entry).expanduser().resolve(strict=False)) + for entry in env.get("PYTHONPATH", "").split(os.pathsep) + if entry and Path(entry).expanduser().is_absolute() + ] + env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys([str(REPO_ROOT), *existing_pythonpath])) subprocess_script = "\n".join( [ "import json", @@ -84,6 +91,28 @@ def run_plugin_discovery_subprocess(tmp_path: Path, plugins_dir: Path, script: s raise AssertionError(f"Subprocess did not emit a result line.\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}") +def test_cli_env_does_not_emit_relative_pythonpath_entries(): + from archivebox.tests.conftest import cli_env + + old_pythonpath = os.environ.get("PYTHONPATH") + try: + os.environ["PYTHONPATH"] = os.pathsep.join(["../abxpkg", "../abx-plugins", "../abx-dl"]) + + env = cli_env() + finally: + if old_pythonpath is None: + os.environ.pop("PYTHONPATH", None) + else: + os.environ["PYTHONPATH"] = old_pythonpath + + pythonpath_entries = env["PYTHONPATH"].split(os.pathsep) + assert str((WORKSPACE_ROOT / "abxpkg").resolve(strict=False)) in pythonpath_entries + assert str((WORKSPACE_ROOT / "abx-plugins").resolve(strict=False)) in pythonpath_entries + assert str((WORKSPACE_ROOT / "abx-dl").resolve(strict=False)) in pythonpath_entries + assert all(Path(entry).is_absolute() for entry in pythonpath_entries) + assert not any(entry.startswith("..") for entry in pythonpath_entries) + + class TestBackgroundHookDetection: """Test that background hooks are detected by .bg. suffix.""" @@ -704,13 +733,13 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat from archivebox.plugins.hooks import run_hook lib_dir = tmp_path / "lib" - node_modules_dir = lib_dir / "npm" / "node_modules" + node_modules_dir = lib_dir / "pnpm" / "packages" / "chrome" / "node_modules" configured_node_path = os.pathsep.join( [ - "/home/archivebox/.npm/lib/node_modules", + "/home/archivebox/.pnpm/packages/chrome/node_modules", "/usr/lib/node_modules", str(node_modules_dir), - "/usr/share/archivebox/lib/npm/node_modules", + "/usr/share/archivebox/lib/pnpm/packages/chrome/node_modules", ], ) @@ -750,3 +779,47 @@ print(json.dumps({ assert payload["NODE_MODULE_DIR"] == str(node_modules_dir) assert payload["NODE_PATH"].split(os.pathsep) == configured_node_path.split(os.pathsep) assert process.env["NODE_MODULES_DIR"] == str(node_modules_dir) + + +@pytest.mark.django_db(transaction=True) +def test_run_hook_executes_python_hooks_through_script_shebang(tmp_path): + """Python hooks must use their abxpkg script header instead of sys.executable.""" + from archivebox.plugins.hooks import run_hook + + plugin_dir = tmp_path / "plugins" / "shebangprobe" + plugin_dir.mkdir(parents=True) + hook_path = plugin_dir / "on_Snapshot__99_shebangprobe.py" + hook_path.write_text( + """#!/usr/bin/env -S abxpkg run --script python3 +# /// script +# requires-python = ">=3.12" +# /// +import json +import os +import rich_click + +print(json.dumps({ + "ABXPKG_FAST_SCRIPT": os.environ.get("ABXPKG_FAST_SCRIPT"), + "RICH_CLICK_FILE": rich_click.__file__, +})) +""", + encoding="utf-8", + ) + hook_path.chmod(0o755) + + output_dir = tmp_path / "archive" / "users" / "system" / "snapshots" / "20260603" / "example.com" / "test" / "shebangprobe" + process = run_hook( + hook_path, + output_dir, + config={ + "LIB_DIR": str(tmp_path / "lib"), + }, + timeout=10, + ) + process.refresh_from_db() + + assert process.cmd[0] == str(hook_path) + assert process.exit_code == 0, process.stderr + payload = json.loads(process.stdout.strip()) + assert payload["ABXPKG_FAST_SCRIPT"] == "1" + assert Path(payload["RICH_CLICK_FILE"]).resolve() == Path(click.__file__).resolve() diff --git a/archivebox/tests/test_machine_models.py b/archivebox/tests/test_machine_models.py index 49d5692a..fa013d17 100644 --- a/archivebox/tests/test_machine_models.py +++ b/archivebox/tests/test_machine_models.py @@ -382,14 +382,14 @@ class TestBinaryModel: """Binary.from_json() should persist provider overrides unchanged.""" overrides = { "apt": {"install_args": ["chromium"]}, - "npm": {"install_args": "puppeteer"}, + "pnpm": {"install_args": "puppeteer"}, "custom": {"install": "bash -lc 'echo ok'"}, } binary = Binary.from_json( { "name": "chrome", - "binproviders": "apt,npm,custom", + "binproviders": "apt,pnpm,custom", "overrides": overrides, }, ) @@ -414,13 +414,13 @@ class TestBinaryModel: """Binary.from_json() should no longer translate legacy non-dict provider overrides.""" overrides = { "apt": ["chromium"], - "npm": "puppeteer", + "pnpm": "puppeteer", } binary = Binary.from_json( { "name": "chrome", - "binproviders": "apt,npm", + "binproviders": "apt,pnpm", "overrides": overrides, }, ) @@ -428,15 +428,15 @@ class TestBinaryModel: assert binary is not None assert binary.overrides == overrides - def test_binary_from_json_prefers_published_readability_package(self): - """Binary.from_json() should rewrite readability's npm git URL to the published package.""" + def test_binary_from_json_preserves_readability_package_metadata(self): + """Binary.from_json() should preserve readability's pnpm package metadata.""" binary = Binary.from_json( { "name": "readability-extractor", - "binproviders": "env,npm", + "binproviders": "env,pnpm", "overrides": { - "npm": { - "install_args": ["https://github.com/ArchiveBox/readability-extractor"], + "pnpm": { + "install_args": ["readability-extractor"], }, }, }, @@ -444,7 +444,7 @@ class TestBinaryModel: assert binary is not None assert binary.overrides == { - "npm": { + "pnpm": { "install_args": ["readability-extractor"], }, } diff --git a/archivebox/tests/test_machine_service.py b/archivebox/tests/test_machine_service.py index 64db4591..301e90da 100644 --- a/archivebox/tests/test_machine_service.py +++ b/archivebox/tests/test_machine_service.py @@ -105,6 +105,7 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali "LITEPARSE_BINARY": {str(installed_liteparse_path)!r}, "NODE_BINARY": {str(external_tool)!r}, "ABX_INSTALL_CACHE": {{"lit": "cached"}}, + "ABX_UV_CACHE": "/tmp/uv-cache", "CHROME_USER_DATA_DIR": "/tmp/derived-profile", }}, config_type="derived")).now() await bus.emit(MachineEvent(method="unset", key="config/LITEPARSE_BINARY", config_type="derived")).now() @@ -137,6 +138,8 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali assert machine.config["LITEPARSE_BINARY"] == str(installed_liteparse_path) assert machine.config["LITEPARSE_BINARY"] != "/tmp/user-config-must-not-persist" + assert machine.config["ABX_INSTALL_CACHE"] == {"lit": "cached"} + assert machine.config["ABX_UV_CACHE"] == "/tmp/uv-cache" _cmd_result = run_archivebox_cmd( ["version"], diff --git a/archivebox/tests/test_migrations_08_to_09.py b/archivebox/tests/test_migrations_08_to_09.py index f045be55..9d156bda 100644 --- a/archivebox/tests/test_migrations_08_to_09.py +++ b/archivebox/tests/test_migrations_08_to_09.py @@ -404,7 +404,7 @@ def test_migration_creates_process_records(migration_08_data): def test_migration_creates_binary_records(migration_08_data): - """Migration should create Binary records from cmd_version data.""" + """Migration should create and link Binary/NetworkInterface records from migrated Process data.""" work_dir, db_path, original_data = migration_08_data result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" @@ -420,6 +420,31 @@ def test_migration_creates_binary_records(migration_08_data): extractors = {ar["extractor"] for ar in original_data["archiveresults"]} assert binary_count >= len(extractors), f"Expected at least {len(extractors)} Binaries, got {binary_count}" + cursor.execute(""" + SELECT COUNT(*) + FROM machine_process + WHERE cmd != '[]' AND binary_id IS NULL + """) + missing_binary_count = cursor.fetchone()[0] + assert missing_binary_count == 0 + + cursor.execute(""" + SELECT p.cmd, b.name, b.abspath + FROM machine_process p + JOIN machine_binary b ON p.binary_id = b.id + WHERE p.cmd != '[]' + """) + rows = cursor.fetchall() + assert rows + for cmd_raw, binary_name, binary_abspath in rows: + cmd = json.loads(cmd_raw) + assert binary_name == cmd[0] + assert binary_abspath == cmd[0] + + cursor.execute("SELECT COUNT(*) FROM machine_process WHERE iface_id IS NULL") + missing_iface_count = cursor.fetchone()[0] + assert missing_iface_count == 0 + conn.close() @@ -746,7 +771,12 @@ def test_archiveresult_files_preserved_after_migration(tmp_path): files_before.extend([f for f in d.rglob("*") if f.is_file()]) files_before_count = len(files_before) generated_metadata_names = {"index.html", "index.json", "index.jsonl"} - original_payloads = sorted(path.read_text() for path in files_before if path.name not in generated_metadata_names) + generated_search_backends = {"search_backend_sqlite", "search_backend_sonic"} + + def is_generated_file(path) -> bool: + return path.name in generated_metadata_names or any(part in generated_search_backends for part in path.parts) + + original_payloads = sorted(path.read_text() for path in files_before if not is_generated_file(path)) # Sample some specific files to check they're preserved sample_paths_before = {} @@ -881,9 +911,7 @@ def test_archiveresult_files_preserved_after_migration(tmp_path): # the hydrated DB row, so raw file counts are allowed to increase; compare # the legacy payload contents after excluding those generated metadata # files to keep the no-data-loss assertion strict. - migrated_payloads = sorted( - path.read_text() for path in [*files_new_structure, *old_files_remaining] if path.name not in generated_metadata_names - ) + migrated_payloads = sorted(path.read_text() for path in [*files_new_structure, *old_files_remaining] if not is_generated_file(path)) assert original_payloads == migrated_payloads, "Legacy payload files changed or were lost during reorganization" assert files_new_count >= files_before_count, "New 0.9 metadata should not replace legacy payload files" diff --git a/archivebox/tests/test_opencode_agent.py b/archivebox/tests/test_opencode_agent.py new file mode 100644 index 00000000..e0a2ce3a --- /dev/null +++ b/archivebox/tests/test_opencode_agent.py @@ -0,0 +1,344 @@ +import os +import sqlite3 + +import pytest +import requests +from asgiref.sync import async_to_sync + +from archivebox.tests.conftest import ADMIN_TEST_HOST + + +pytestmark = pytest.mark.django_db + + +def test_opencode_disabled_route_does_not_start_server(client, monkeypatch): + from abx_plugins.plugins.opencode import views + + monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": False}) + monkeypatch.setattr(views, "_ensure_opencode", lambda settings: pytest.fail("opencode should not start when disabled")) + + response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST) + + assert response.status_code == 404 + + +def test_opencode_agent_requires_superuser(client, db, monkeypatch, django_user_model): + from abx_plugins.plugins.opencode import views + + monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True}) + monkeypatch.setattr(views, "_ensure_opencode", lambda settings: pytest.fail("opencode should not start before auth passes")) + + response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST) + assert response.status_code == 302 + assert "/admin/login/" in response.headers["Location"] + + user = django_user_model.objects.create_user(username="regular", password="testpassword") + client.force_login(user) + response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST) + assert response.status_code == 403 + + +def test_opencode_agent_superuser_gets_wrapper(admin_client, db, monkeypatch): + from abx_plugins.plugins.opencode import views + + monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True}) + monkeypatch.setattr(views, "_ensure_opencode", lambda settings: (True, "")) + + response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST) + + assert response.status_code == 200 + assert b'