diff --git a/Dockerfile b/Dockerfile
index fbb08fa8..704c7a95 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -313,7 +313,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
&& export PERSONAS_DIR="$LIB_DIR/personas" \
&& export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
- && export ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \
+ && export ABX_RUNTIME=archivebox ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \
&& mkdir -p "$LIB_DIR" \
&& apt-get update -qq \
&& apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \
@@ -327,7 +327,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
accessibility archivedotorg archivewebpage base chrome chrome_mhtml chrome_screencast \
claudechrome claudecode claudecodecleanup claudecodeextract consolelog defuddle dns dom \
favicon forumdl gallerydl git hashes headers htmltotext infiniscroll \
- istilldontcareaboutcookies liteparse media mercury modalcloser opendataloader papersdl \
+ istilldontcareaboutcookies liteparse media mercury modalcloser opencode opendataloader papersdl \
parse_dom_outlinks parse_html_urls parse_jsonl_urls parse_netscape_urls parse_rss_urls \
parse_txt_urls pdf readability redirects responses screenshot search_backend_ripgrep \
search_backend_sonic search_backend_sqlite seo singlefile ssl sslcerts staticfile title \
diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py
index e0908c1c..a3e1707a 100644
--- a/archivebox/cli/archivebox_run.py
+++ b/archivebox/cli/archivebox_run.py
@@ -320,7 +320,12 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
interactive_interrupts=interactive_interrupts,
)
return 0
- except (KeyboardInterrupt, asyncio.CancelledError):
+ except KeyboardInterrupt:
+ return 0
+ except asyncio.CancelledError as e:
+ if daemon:
+ rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+ return 1
return 0
except Exception as e:
rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
diff --git a/archivebox/core/admin_archiveresults.py b/archivebox/core/admin_archiveresults.py
index 82ec24e9..08e4e5fb 100644
--- a/archivebox/core/admin_archiveresults.py
+++ b/archivebox/core/admin_archiveresults.py
@@ -75,6 +75,12 @@ def get_plugin_admin_url(plugin_name: str) -> str:
return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/"
+def get_process_link_label(process) -> str:
+ if process.pid:
+ return str(process.pid)
+ return str(process.id)[-8:]
+
+
def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
"""Render a nice inline list view of archive results with status, plugin, output, and actions."""
@@ -139,7 +145,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
process_display = f'''
{process.pid or "-"}
+ title="View process">{get_process_link_label(process)}
'''
machine_display = "-"
@@ -659,7 +665,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
process = result.process_record
if not process:
return "-"
- process_label = process.pid or "-"
+ process_label = get_process_link_label(process)
return format_html(
'{}',
reverse("admin:machine_process_change", args=[process.id]),
@@ -673,9 +679,8 @@ class ArchiveResultAdmin(BaseModelAdmin):
return "-"
machine = process.machine
return format_html(
- '{} {}',
+ '{}',
reverse("admin:machine_machine_change", args=[machine.id]),
- str(machine.id)[:8],
machine.hostname,
)
diff --git a/archivebox/core/admin_tags.py b/archivebox/core/admin_tags.py
index db6c3dca..2ff8c94d 100644
--- a/archivebox/core/admin_tags.py
+++ b/archivebox/core/admin_tags.py
@@ -117,7 +117,7 @@ class TagAdmin(BaseModelAdmin):
sort = normalize_tag_sort((request.GET.get("sort") or "created_desc").strip())
created_by = normalize_created_by_filter((request.GET.get("created_by") or "").strip())
year = normalize_created_year_filter((request.GET.get("year") or "").strip())
- has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "yes").strip())
+ has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "all").strip())
context = {
**self.admin_site.each_context(request),
**(extra_context or {}),
diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
index 403525d5..f902faf9 100644
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@@ -72,6 +72,7 @@ INSTALLED_APPS = [
"archivebox.crawls", # handles Crawl and CrawlSchedule models and management (depends on core)
"archivebox.progressmonitor", # live progress endpoint and admin monitor template
"archivebox.api", # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
+ "abx_plugins.plugins.opencode",
# 3rd-party apps from PyPI that need to be loaded last
"admin_data_views", # handles rendering some convenient automatic read-only views of data in Django admin
"django_extensions", # provides Django Debug Toolbar (and other non-debug helpers)
diff --git a/archivebox/core/tag_util.py b/archivebox/core/tag_util.py
index 27289c05..04e7ef9b 100644
--- a/archivebox/core/tag_util.py
+++ b/archivebox/core/tag_util.py
@@ -114,9 +114,9 @@ def add_snapshot_counts(tags: list[Tag], snapshot_queryset: QuerySet[Snapshot] |
queryset = SnapshotTag.objects.filter(tag_id__in=tag_ids)
if snapshot_queryset is not None:
queryset = queryset.filter(snapshot_id__in=snapshot_queryset.values("id"))
- counts = {row["tag_id"]: row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
+ counts = {int(row["tag_id"]): row["num_snapshots"] for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))}
for tag in tags:
- tag.num_snapshots = counts.get(tag.pk, 0)
+ tag.num_snapshots = counts.get(int(tag.pk), 0)
def get_tag_creator_choices() -> list[tuple[str, str]]:
diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py
index afc053ae..a06955c0 100644
--- a/archivebox/core/urls.py
+++ b/archivebox/core/urls.py
@@ -41,6 +41,7 @@ urlpatterns = [
path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
+ re_path(r"^admin/agent/?(?=$|opencode)", include("abx_plugins.plugins.opencode.urls")),
path("public/", PublicIndexView.as_view(), name="public-index"),
path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),
path("archive/", RedirectView.as_view(url="/")),
diff --git a/archivebox/machine/admin.py b/archivebox/machine/admin.py
index 4119b4c9..889f6769 100644
--- a/archivebox/machine/admin.py
+++ b/archivebox/machine/admin.py
@@ -434,6 +434,9 @@ class ProcessAdmin(BaseModelAdmin):
"crawl_link",
"cmd_display",
"env_display",
+ "stdout_display",
+ "stderr_display",
+ "archiveresult_output_display",
"timeout",
"pid",
"exit_code",
@@ -475,7 +478,7 @@ class ProcessAdmin(BaseModelAdmin):
(
"Output",
{
- "fields": ("stdout", "stderr"),
+ "fields": ("stdout_display", "stderr_display", "archiveresult_output_display"),
"classes": ("card", "wide", "collapse"),
},
),
@@ -740,6 +743,28 @@ class ProcessAdmin(BaseModelAdmin):
return "-"
return _render_copy_block(env_text, multiline=True)
+ @admin.display(description="Stdout")
+ def stdout_display(self, process):
+ if not process.stdout:
+ return "-"
+ return _render_copy_block(process.stdout, multiline=True)
+
+ @admin.display(description="Stderr")
+ def stderr_display(self, process):
+ if not process.stderr:
+ return "-"
+ return _render_copy_block(process.stderr, multiline=True)
+
+ @admin.display(description="ArchiveResult Output")
+ def archiveresult_output_display(self, process):
+ try:
+ output = process.archiveresult.output_str
+ except Process.archiveresult.RelatedObjectDoesNotExist:
+ return "-"
+ if not output:
+ return "-"
+ return _render_copy_block(output, multiline=True)
+
def register_admin(admin_site):
admin_site.register(Machine, MachineAdmin)
diff --git a/archivebox/machine/migrations/0020_repair_process_binary_iface_links.py b/archivebox/machine/migrations/0020_repair_process_binary_iface_links.py
new file mode 100644
index 00000000..ac13b14b
--- /dev/null
+++ b/archivebox/machine/migrations/0020_repair_process_binary_iface_links.py
@@ -0,0 +1,130 @@
+import json
+from pathlib import Path
+
+from django.db import migrations
+from django.utils import timezone
+
+
+def _cmd_array(value):
+ if isinstance(value, list):
+ return value
+ if isinstance(value, str):
+ try:
+ parsed = json.loads(value)
+ except json.JSONDecodeError:
+ return [value] if value else []
+ return parsed if isinstance(parsed, list) else []
+ return []
+
+
+def _ensure_placeholder_iface(NetworkInterface, machine_id, hostname):
+ iface = NetworkInterface.objects.filter(machine_id=machine_id).order_by("-modified_at", "-created_at").first()
+ if iface is not None:
+ return iface
+
+ now = timezone.now()
+ return NetworkInterface.objects.create(
+ machine_id=machine_id,
+ created_at=now,
+ modified_at=now,
+ mac_address="00:00:00:00:00:00",
+ ip_public="0.0.0.0",
+ ip_local="0.0.0.0",
+ dns_server="0.0.0.0",
+ hostname=(hostname or "unknown")[:63],
+ iface="unknown",
+ isp="",
+ city="",
+ region="",
+ country="",
+ )
+
+
+def _get_or_create_binary(Binary, machine_id, reference):
+ reference = str(reference or "").strip()
+ if not reference:
+ return None
+
+ name = Path(reference).name or reference
+ qs = Binary.objects.filter(machine_id=machine_id)
+ binary = qs.filter(abspath=reference).order_by("-modified_at", "-created_at").first()
+ if binary is None:
+ binary = qs.filter(name=name).order_by("-modified_at", "-created_at").first()
+ if binary is not None:
+ return binary
+
+ now = timezone.now()
+ return Binary.objects.create(
+ machine_id=machine_id,
+ created_at=now,
+ modified_at=now,
+ name=name[:63],
+ binproviders="env",
+ overrides={},
+ binprovider="env",
+ abspath=reference[:255],
+ version="",
+ sha256="",
+ status="installed",
+ retry_at=None,
+ )
+
+
+def repair_process_binary_iface_links(apps, schema_editor):
+ Binary = apps.get_model("machine", "Binary")
+ Machine = apps.get_model("machine", "Machine")
+ NetworkInterface = apps.get_model("machine", "NetworkInterface")
+ Process = apps.get_model("machine", "Process")
+
+ machines = {machine.id: machine for machine in Machine.objects.only("id", "hostname").iterator(chunk_size=100)}
+ iface_by_machine = {}
+ binary_by_key = {}
+
+ qs = Process.objects.filter(machine_id__isnull=False).filter(binary_id__isnull=True) | Process.objects.filter(
+ machine_id__isnull=False,
+ iface_id__isnull=True,
+ )
+ for process in qs.distinct().only("id", "machine_id", "binary_id", "iface_id", "cmd").iterator(chunk_size=500):
+ update_fields = []
+ machine = machines.get(process.machine_id)
+
+ if process.iface_id is None:
+ iface = iface_by_machine.get(process.machine_id)
+ if iface is None:
+ iface = _ensure_placeholder_iface(
+ NetworkInterface,
+ process.machine_id,
+ machine.hostname if machine is not None else "",
+ )
+ iface_by_machine[process.machine_id] = iface
+ process.iface_id = iface.id
+ update_fields.append("iface_id")
+
+ if process.binary_id is None:
+ cmd = _cmd_array(process.cmd)
+ reference = str(cmd[0]).strip() if cmd else ""
+ if reference:
+ key = (process.machine_id, reference)
+ binary = binary_by_key.get(key)
+ if binary is None:
+ binary = _get_or_create_binary(Binary, process.machine_id, reference)
+ binary_by_key[key] = binary
+ if binary is not None:
+ process.binary_id = binary.id
+ update_fields.append("binary_id")
+
+ if update_fields:
+ process.modified_at = timezone.now()
+ process.save(update_fields=[*update_fields, "modified_at"])
+
+
+class Migration(migrations.Migration):
+ atomic = False
+
+ dependencies = [
+ ("machine", "0019_single_active_runner_constraint"),
+ ]
+
+ operations = [
+ migrations.RunPython(repair_process_binary_iface_links, migrations.RunPython.noop),
+ ]
diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py
index 0530ed4e..56c1cb09 100755
--- a/archivebox/machine/models.py
+++ b/archivebox/machine/models.py
@@ -650,22 +650,6 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
binary_overrides = record.get("overrides", {})
normalized_overrides = binary_overrides if isinstance(binary_overrides, dict) else {}
- # abx-plugins currently emits a GitHub install URL for readability-extractor,
- # but the package is published on npm. Prefer the registry package to avoid
- # long git-based installs in CI while still using canonical install_args.
- if (
- name == "readability-extractor"
- and isinstance(normalized_overrides.get("npm"), dict)
- and normalized_overrides["npm"].get("install_args") == ["https://github.com/ArchiveBox/readability-extractor"]
- ):
- normalized_overrides = {
- **normalized_overrides,
- "npm": {
- **normalized_overrides["npm"],
- "install_args": ["readability-extractor"],
- },
- }
-
# Case 1: Already installed (from on_Crawl hooks) - has abspath AND binproviders
# This happens when on_Crawl hooks detect already-installed binaries
abspath = record.get("abspath")
diff --git a/archivebox/personas/importers.py b/archivebox/personas/importers.py
index e75278ad..f6b9a354 100644
--- a/archivebox/personas/importers.py
+++ b/archivebox/personas/importers.py
@@ -612,7 +612,7 @@ def export_browser_state(
if not state_script.exists():
return False, None, f"Browser state export script not found at {state_script}"
- node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules"
+ node_modules_dir = get_config().LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules"
chrome_plugin_dir = Path(get_plugins_dir()).resolve()
env = os.environ.copy()
diff --git a/archivebox/plugins/forms.py b/archivebox/plugins/forms.py
index 18f37b20..ba4d384f 100644
--- a/archivebox/plugins/forms.py
+++ b/archivebox/plugins/forms.py
@@ -130,6 +130,7 @@ HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
"env",
"media",
"npm",
+ "opencode",
"pip",
"puppeteer",
"search_backend_ripgrep",
diff --git a/archivebox/plugins/hooks.py b/archivebox/plugins/hooks.py
index 60df0512..921c7cc2 100644
--- a/archivebox/plugins/hooks.py
+++ b/archivebox/plugins/hooks.py
@@ -285,7 +285,6 @@ def run_hook(
"""
from archivebox.machine.models import Process, Machine, NetworkInterface
from archivebox.config.common import get_config, normalize_runtime_config
- import sys
config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
config_overrides = _config_to_overrides(config)
@@ -333,12 +332,15 @@ def run_hook(
)
return process
- # Determine the interpreter based on file extension
+ # Python hooks carry their runtime contract in the shebang
+ # (usually `abxpkg run --script python3`), so execute them directly.
+ # For shell/JS hooks we still dispatch through the conventional
+ # interpreter because those hooks do not need per-script Python env setup.
ext = script.suffix.lower()
if ext == ".sh":
cmd = ["bash", str(script)]
elif ext == ".py":
- cmd = [sys.executable, str(script)]
+ cmd = [str(script)]
elif ext == ".js":
cmd = ["node", str(script)]
else:
@@ -388,7 +390,7 @@ def run_hook(
# NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory.
node_modules_dir = hook_config.get("NODE_MODULES_DIR")
if not node_modules_dir and lib_dir:
- node_modules_dir = Path(lib_dir) / "npm" / "node_modules"
+ node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
node_path_parts = [part for part in str(hook_config.get("NODE_PATH") or "").split(os.pathsep) if part]
if node_modules_dir:
diff --git a/archivebox/progressmonitor/views.py b/archivebox/progressmonitor/views.py
index a73ba265..53ea7110 100644
--- a/archivebox/progressmonitor/views.py
+++ b/archivebox/progressmonitor/views.py
@@ -327,8 +327,20 @@ def live_progress_view(request):
.order_by("-modified_at")[:max_active_crawls],
)
paused_crawls = list(
- crawl_scope.filter(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+ crawl_scope.filter(
+ Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+ | Q(
+ status=Crawl.StatusChoices.PAUSED,
+ snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
+ snapshot_set__retry_at__lte=now,
+ )
+ | Q(
+ status=Crawl.StatusChoices.PAUSED,
+ snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
+ ),
+ )
.values(*active_crawl_fields)
+ .distinct()
.order_by("-modified_at")[:max_active_crawls],
)
queued_crawls = list(
diff --git a/archivebox/services/archive_result_service.py b/archivebox/services/archive_result_service.py
index 255a7df0..520c803b 100644
--- a/archivebox/services/archive_result_service.py
+++ b/archivebox/services/archive_result_service.py
@@ -432,6 +432,9 @@ class ArchiveResultService(BaseService):
).now()
return
+ # TODO: consider moving this fallback derivation into abx-dl itself.
+ # First try both patterns: if the whole abx-dl process crashes, restarting
+ # the snapshot may be enough, but don't guess before validating it.
process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED)
with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"):
await event.emit(
@@ -439,7 +442,7 @@ class ArchiveResultService(BaseService):
snapshot_id=snapshot_event.snapshot_id,
plugin=event.plugin_name,
hook_name=event.hook_name,
- status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "skipped"),
+ status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "noresult"),
output_str=event.stderr if process_failed else "",
output_files=event.output_files,
start_ts=event.start_ts,
diff --git a/archivebox/services/binary_service.py b/archivebox/services/binary_service.py
index c2feaf3a..3c3f2103 100644
--- a/archivebox/services/binary_service.py
+++ b/archivebox/services/binary_service.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import json
from collections.abc import Mapping
from pathlib import Path
@@ -186,38 +187,109 @@ class ArchiveBoxDBBinaryCacheBackend:
class ArchiveBoxBinaryService(BaseService):
"""Preserve ArchiveBox's legacy Binary Process rows around abxpkg requests."""
- LISTENS_TO = [BinaryRequestEvent]
+ LISTENS_TO = [BinaryRequestEvent, BinaryEvent]
EMITS: list[type[BaseEvent]] = []
def __init__(self, bus: EventBus):
super().__init__(bus)
+ self.process_ids_by_request_id: dict[str, str] = {}
self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__project_process)
+ self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__schedule_missing_finalize)
+ self.bus.on(BinaryEvent, self.on_BinaryEvent__finalize_process)
async def on_BinaryRequestEvent__project_process(self, request: BinaryRequestEvent) -> None:
- from archivebox.machine.models import Binary, Machine, Process, _canonical_binary_name
- from archivebox.services.process_service import current_network_interface_with_machine
+ from archivebox.machine.models import Machine, Process, _canonical_binary_name
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
binary_name = _canonical_binary_name(request.name)
if not binary_name:
return
- binary = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
- if binary is None:
- return
+ binary = await self._get_or_create_binary(machine, binary_name, request)
+ started_at = timezone.now()
+ output_dir = self._process_output_dir(binary, request)
+ await sync_to_async(output_dir.mkdir, thread_sensitive=True)(parents=True, exist_ok=True)
+ process = await Process.objects.acreate(
+ machine=machine,
+ iface=None,
+ process_type=Process.TypeChoices.BINARY,
+ worker_type="",
+ pwd=str(output_dir),
+ cmd=self._process_cmd(request),
+ env={},
+ timeout=int(request.event_timeout or request.install_timeout or 600),
+ pid=None,
+ url=None,
+ started_at=started_at,
+ ended_at=None,
+ stdout="",
+ stderr="",
+ exit_code=None,
+ status=Process.StatusChoices.RUNNING,
+ retry_at=None,
+ binary=binary,
+ )
+ self.process_ids_by_request_id[request.event_id] = str(process.id)
- binary_event = await self.bus.find(
- BinaryEvent,
- child_of=request,
+ async def on_BinaryEvent__finalize_process(self, event: BinaryEvent) -> None:
+ from archivebox.machine.models import Binary, Process, _canonical_binary_name
+
+ request = await self.bus.find(
+ BinaryRequestEvent,
past=True,
future=False,
- name=request.name,
- where=lambda candidate: bool(candidate.abspath),
+ where=lambda candidate: self.bus.event_is_child_of(event, candidate),
)
- iface = await sync_to_async(current_network_interface_with_machine, thread_sensitive=True)()
- now = timezone.now()
- success = isinstance(binary_event, BinaryEvent)
- output_dir = self._process_output_dir(binary, request)
- output_dir.mkdir(parents=True, exist_ok=True)
+ request = request if isinstance(request, BinaryRequestEvent) else None
+ process_id = self.process_ids_by_request_id.pop(request.event_id, "") if request is not None else ""
+ if not process_id:
+ return
+ process = await Process.objects.filter(id=process_id).select_related("binary").afirst()
+ if process is None:
+ return
+ binary_name = _canonical_binary_name(event.name)
+ binary = process.binary
+ if binary is not None and binary_name:
+ binary.abspath = event.abspath
+ if event.version:
+ binary.version = str(event.version)
+ if event.sha256:
+ binary.sha256 = str(event.sha256)
+ binary.binproviders = event.binproviders or binary.binproviders
+ binary.binprovider = event.binprovider or binary.binprovider
+ binary.status = Binary.StatusChoices.INSTALLED
+ binary.retry_at = None
+ await binary.asave(
+ update_fields=["abspath", "version", "sha256", "binproviders", "binprovider", "status", "retry_at", "modified_at"],
+ )
+ process.ended_at = timezone.now()
+ process.stdout = json.dumps(self._binary_event_json(event, binary)) + "\n"
+ process.stderr = ""
+ process.exit_code = 0
+ process.status = Process.StatusChoices.EXITED
+ await process.asave(update_fields=["ended_at", "stdout", "stderr", "exit_code", "status", "modified_at"])
+ if binary is not None:
+ await sync_to_async(self._write_binary_index, thread_sensitive=True)(binary, process, Path(process.pwd))
+
+ async def _get_or_create_binary(self, machine, binary_name: str, request: BinaryRequestEvent):
+ from archivebox.machine.models import Binary
+
+ binary_id = str(request.extra_context.get("binary_id") or "")
+ if binary_id:
+ binary = await Binary.objects.filter(id=binary_id).afirst()
+ if binary is not None:
+ return binary
+ binary = await Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").afirst()
+ if binary is not None:
+ return binary
+ return await Binary.objects.acreate(
+ machine=machine,
+ name=binary_name,
+ binproviders=_binproviders_to_str(request.binproviders),
+ overrides=_persisted_overrides_for_request(request),
+ status=Binary.StatusChoices.QUEUED,
+ )
+
+ def _process_cmd(self, request: BinaryRequestEvent) -> list[str]:
cmd = [
"abxpkg",
"install",
@@ -226,29 +298,65 @@ class ArchiveBoxBinaryService(BaseService):
]
if request.overrides:
cmd.append(f"--overrides={json.dumps(request.overrides, sort_keys=True)}")
- stdout = json.dumps(binary.to_json()) + "\n" if success else ""
- stderr = "" if success else f"Binary request did not resolve: {request.name}"
- process = await Process.objects.acreate(
- machine=iface.machine,
- iface=iface,
- process_type=Process.TypeChoices.BINARY,
- worker_type="",
- pwd=str(output_dir),
- cmd=cmd,
- env={},
- timeout=int(request.event_timeout or request.install_timeout or 600),
- pid=None,
- url=None,
- started_at=now,
- ended_at=now,
- stdout=stdout,
- stderr=stderr,
- exit_code=0 if success else 1,
- status=Process.StatusChoices.EXITED,
- retry_at=None,
- binary=binary,
+ return cmd
+
+ def _binary_event_json(self, event: BinaryEvent, binary) -> dict[str, Any]:
+ if binary is not None:
+ data = binary.to_json()
+ else:
+ data = {"type": "Binary", "name": event.name}
+ data.update(
+ {
+ "type": "Binary",
+ "name": event.name,
+ "binproviders": event.binproviders,
+ "binprovider": event.binprovider,
+ "abspath": event.abspath,
+ "version": str(event.version or ""),
+ "sha256": event.sha256 or "",
+ "status": "installed",
+ },
)
- self._write_binary_index(binary, process, output_dir)
+ return data
+
+ async def _finalize_missing_process(self, request: BinaryRequestEvent) -> None:
+ from archivebox.machine.models import Process
+
+ process_id = self.process_ids_by_request_id.pop(request.event_id, "")
+ if not process_id:
+ return
+ process = await Process.objects.filter(id=process_id).afirst()
+ if process is None or process.status == Process.StatusChoices.EXITED:
+ return
+ process.ended_at = timezone.now()
+ process.stderr = f"Binary request did not resolve: {request.name}"
+ process.exit_code = 1
+ process.status = Process.StatusChoices.EXITED
+ await process.asave(update_fields=["ended_at", "stderr", "exit_code", "status", "modified_at"])
+
+ async def _finalize_request_when_done(self, request: BinaryRequestEvent) -> None:
+ try:
+ await request.wait(timeout=request.event_timeout)
+ except TimeoutError:
+ await self._finalize_missing_process(request)
+ return
+ binary_event = await self.bus.find(
+ BinaryEvent,
+ child_of=request,
+ past=True,
+ future=False,
+ name=request.name,
+ where=lambda candidate: bool(candidate.abspath),
+ )
+ if not isinstance(binary_event, BinaryEvent):
+ await self._finalize_missing_process(request)
+
+ def _schedule_missing_finalize(self, request: BinaryRequestEvent) -> None:
+ task = asyncio.create_task(self._finalize_request_when_done(request))
+ task.add_done_callback(lambda done: None if done.cancelled() else done.exception())
+
+ async def on_BinaryRequestEvent__schedule_missing_finalize(self, request: BinaryRequestEvent) -> None:
+ self._schedule_missing_finalize(request)
def _process_output_dir(self, binary, request: BinaryRequestEvent) -> Path:
raw_output_dir = str(request.extra_context.get("output_dir") or "").strip()
diff --git a/archivebox/services/machine_service.py b/archivebox/services/machine_service.py
index f8a51567..602cd788 100644
--- a/archivebox/services/machine_service.py
+++ b/archivebox/services/machine_service.py
@@ -7,8 +7,6 @@ from asgiref.sync import sync_to_async
from abx_dl.events import MachineEvent
from abx_dl.services.base import BaseService
-_BINARY_EVENT_ALLOWED_KEYS = frozenset({"ABX_INSTALL_CACHE"})
-
def _is_binary_event_key(key: str) -> bool:
"""``MachineEvent`` projector only ever writes binary-related state.
@@ -20,7 +18,7 @@ def _is_binary_event_key(key: str) -> bool:
mirror is a security boundary), so the projector strips anything that
isn't a binary path or the binary install cache.
"""
- if key in _BINARY_EVENT_ALLOWED_KEYS:
+ if key.startswith("ABX_") and key.endswith("CACHE"):
return True
return key.endswith("_BINARY")
diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py
index 5913e8dd..cd737749 100644
--- a/archivebox/services/runner.py
+++ b/archivebox/services/runner.py
@@ -121,6 +121,10 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str
return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
+def _discover_archivebox_plugins() -> dict[str, Plugin]:
+ return discover_plugins(runtime="archivebox")
+
+
def _runner_task_context() -> contextvars.Context:
context = contextvars.copy_context()
context.run(EventBus.current_event_context.set, None)
@@ -213,13 +217,13 @@ class CrawlRunner:
):
self.crawl = crawl
self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0)
- self.plugins = discover_plugins()
+ self.plugins = _discover_archivebox_plugins()
HookProcessService(self.bus, emit_jsonl=False, interactive_tty=interactive_interrupts)
register_sonic_daemon_event_handler(self.bus)
PersistedProcessService(self.bus)
+ ArchiveBoxBinaryService(self.bus)
BinaryCacheService(self.bus, backend=ArchiveBoxDBBinaryCacheBackend())
BinaryService(self.bus)
- ArchiveBoxBinaryService(self.bus)
TagService(self.bus)
CrawlService(self.bus, crawl_id=str(crawl.id))
MachineService(self.bus)
@@ -839,7 +843,7 @@ class CrawlRunner:
url=snapshot["url"],
snapshot=abx_snapshot,
output_dir=output_dir,
- install_enabled=False,
+ install_enabled=True,
crawl_setup_enabled=True,
crawl_event_enabled=False,
crawl_start_enabled=False,
@@ -1179,7 +1183,7 @@ async def _run_binary(binary_id: str) -> None:
from archivebox.machine.models import Binary, Machine
binary = await Binary.objects.aget(id=binary_id)
- plugins = discover_plugins()
+ plugins = _discover_archivebox_plugins()
config = get_config(include_machine=False)
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
derived_config = normalize_runtime_config(machine.config)
@@ -1187,9 +1191,9 @@ async def _run_binary(binary_id: str) -> None:
config = normalize_runtime_config(config)
bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0)
process_service = PersistedProcessService(bus)
+ ArchiveBoxBinaryService(bus)
BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
BinaryService(bus)
- ArchiveBoxBinaryService(bus)
TagService(bus)
ArchiveResultService(bus)
MachineService(bus)
@@ -1238,7 +1242,9 @@ def run_binary(binary_id: str) -> None:
@lru_cache(maxsize=1)
def _snapshot_hook_names_by_plugin() -> dict[str, frozenset[str]]:
- return {plugin.name: frozenset(hook.name for hook in plugin.filter_hooks("Snapshot")) for plugin in discover_plugins().values()}
+ return {
+ plugin.name: frozenset(hook.name for hook in plugin.filter_hooks("Snapshot")) for plugin in _discover_archivebox_plugins().values()
+ }
def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
@@ -1566,7 +1572,7 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
from archivebox.config.common import get_config
from archivebox.machine.models import Machine
- plugins = discover_plugins()
+ plugins = _discover_archivebox_plugins()
config = get_config(include_machine=False)
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
derived_config = normalize_runtime_config(machine.config)
@@ -1574,9 +1580,9 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
config = normalize_runtime_config(config)
bus = create_bus(name="ArchiveBox_install", total_timeout=3600.0)
PersistedProcessService(bus)
+ ArchiveBoxBinaryService(bus)
BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
BinaryService(bus)
- ArchiveBoxBinaryService(bus)
TagService(bus)
ArchiveResultService(bus)
MachineService(bus)
diff --git a/archivebox/templates/admin/core/tag/change_list.html b/archivebox/templates/admin/core/tag/change_list.html
index 7af180f3..ef3aa553 100644
--- a/archivebox/templates/admin/core/tag/change_list.html
+++ b/archivebox/templates/admin/core/tag/change_list.html
@@ -517,7 +517,7 @@ document.addEventListener('DOMContentLoaded', function () {
sort: typeof next.sort === 'string' ? next.sort : (sortSelect?.value || 'created_desc'),
created_by: typeof next.created_by === 'string' ? next.created_by : (createdBySelect?.value || ''),
year: typeof next.year === 'string' ? next.year : (yearSelect?.value || ''),
- has_snapshots: typeof next.has_snapshots === 'string' ? next.has_snapshots : (shell.dataset.initialHasSnapshots || 'yes'),
+ has_snapshots: typeof next.has_snapshots === 'string' ? next.has_snapshots : (shell.dataset.initialHasSnapshots || 'all'),
};
}
@@ -554,7 +554,7 @@ document.addEventListener('DOMContentLoaded', function () {
url.searchParams.delete('year');
}
- if (state.has_snapshots && state.has_snapshots !== 'yes') {
+ if (state.has_snapshots && state.has_snapshots !== 'all') {
url.searchParams.set('has_snapshots', state.has_snapshots);
} else {
url.searchParams.delete('has_snapshots');
@@ -570,9 +570,7 @@ document.addEventListener('DOMContentLoaded', function () {
}
function renderCards(nextCards, state) {
- cards = Array.isArray(nextCards) ? nextCards.filter(function (card) {
- return Number(card && card.num_snapshots || 0) > 0;
- }) : [];
+ cards = Array.isArray(nextCards) ? nextCards : [];
setMeta(state || getCurrentState(), cards.length);
if (!cards.length) {
diff --git a/archivebox/templates/core/navigation.html b/archivebox/templates/core/navigation.html
index aabcf069..0bc63c96 100644
--- a/archivebox/templates/core/navigation.html
+++ b/archivebox/templates/core/navigation.html
@@ -6,6 +6,9 @@
Snapshots |
Log |
Tags
+ {% if user.is_authenticated and user.is_superuser %}
+ 💬 AI |
+ {% endif %}
Docs |
API |
Admin
diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py
index d6481dde..d1c84b80 100644
--- a/archivebox/tests/conftest.py
+++ b/archivebox/tests/conftest.py
@@ -26,6 +26,7 @@ import requests
from django.utils import timezone
REPO_ROOT = Path(__file__).resolve().parents[2]
+WORKSPACE_ROOT = REPO_ROOT.parent
PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve()
(SESSION_DATA_DIR / "tests").mkdir(parents=True, exist_ok=True)
@@ -55,6 +56,29 @@ def _assert_safe_runtime_paths(*, cwd: Path | None = None, env: dict[str, str] |
_assert_not_repo_path(Path(value), label=key)
+def _test_source_pythonpath() -> str:
+ entries: list[str] = []
+ for repo_name in ("abxpkg", "abx-plugins", "abx-dl"):
+ repo_path = WORKSPACE_ROOT / repo_name
+ if repo_path.exists():
+ entries.append(str(repo_path.resolve(strict=False)))
+ return os.pathsep.join(entries)
+
+
+def _set_test_source_pythonpath(env: dict[str, str]) -> None:
+ source_pythonpath = _test_source_pythonpath()
+ existing_entries = [
+ str(Path(entry).expanduser().resolve(strict=False))
+ for entry in (env.get("PYTHONPATH") or "").split(os.pathsep)
+ if entry and Path(entry).expanduser().is_absolute()
+ ]
+ entries = [entry for entry in [*source_pythonpath.split(os.pathsep), *existing_entries] if entry]
+ if entries:
+ env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(entries))
+ else:
+ env.pop("PYTHONPATH", None)
+
+
def _sync_archivebox_test_data_dir(data_dir: Path) -> None:
from archivebox.config import constants as constants_mod
from archivebox.config import paths as paths_mod
@@ -170,9 +194,8 @@ def run_archivebox_cmd(
_assert_not_repo_path(cwd, label="cwd")
- run_env: dict[str, str] | None = None
+ run_env = {} if replace_env else os.environ.copy()
if default_cli_env or disable_extractors or env is not None:
- run_env = {} if replace_env else os.environ.copy()
if default_cli_env:
run_env["USE_COLOR"] = "False"
run_env["SHOW_PROGRESS"] = "False"
@@ -198,8 +221,9 @@ def run_archivebox_cmd(
)
if env:
run_env.update(env)
+ _set_test_source_pythonpath(run_env)
- _assert_safe_runtime_paths(cwd=cwd, env=run_env or os.environ)
+ _assert_safe_runtime_paths(cwd=cwd, env=run_env)
if stdin is not None:
assert input is None, "pass either input or stdin, not both"
@@ -562,6 +586,7 @@ def cli_env(
**extra: str,
) -> dict[str, str]:
env = {} if replace else os.environ.copy()
+ _set_test_source_pythonpath(env)
env.update({"USE_COLOR": "False", "SHOW_PROGRESS": "False"})
if disable_extractors or live or server:
@@ -1373,7 +1398,7 @@ def _find_cached_chrome(lib_dir: Path) -> Path | None:
lib_dir / "puppeteer" / "chromium",
lib_dir / "puppeteer",
lib_dir / "ms-playwright",
- lib_dir / "npm" / "node_modules" / "puppeteer" / ".local-chromium",
+ lib_dir / "pnpm" / "packages" / "chrome" / "node_modules" / "puppeteer" / ".local-chromium",
]
for base in candidates:
if not base.exists():
@@ -1401,17 +1426,17 @@ def _find_system_browser() -> Path | None:
def _ensure_puppeteer(shared_lib: Path) -> None:
- npm_prefix = shared_lib / "npm"
- node_modules = npm_prefix / "node_modules"
+ pnpm_prefix = shared_lib / "pnpm" / "packages" / "chrome"
+ node_modules = pnpm_prefix / "node_modules"
puppeteer_dir = node_modules / "puppeteer"
if puppeteer_dir.exists():
return
- npm_prefix.mkdir(parents=True, exist_ok=True)
+ pnpm_prefix.mkdir(parents=True, exist_ok=True)
env = os.environ.copy()
env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
subprocess.run(
- ["npm", "install", "puppeteer"],
- cwd=str(npm_prefix),
+ ["pnpm", "add", "--dir", str(pnpm_prefix), "puppeteer"],
+ cwd=str(pnpm_prefix),
env=env,
check=True,
capture_output=True,
diff --git a/archivebox/tests/test_api_v1_core_tags_search.py b/archivebox/tests/test_api_v1_core_tags_search.py
index b489e90b..47fe97f7 100644
--- a/archivebox/tests/test_api_v1_core_tags_search.py
+++ b/archivebox/tests/test_api_v1_core_tags_search.py
@@ -41,6 +41,26 @@ def test_tag_search_api_returns_card_payload(client, api_token, tagged_data):
assert {snap.url for snap in snapshots} == {"https://example.com/one", "https://example.com/two"}
+def test_tag_search_api_default_includes_empty_tags_and_counts_linked_snapshots(client, api_token, tagged_data, api_admin_user):
+ linked_tag, _snapshots = tagged_data
+ empty_tag = Tag.objects.create(name="Empty Tag", created_by=api_admin_user)
+
+ response = client.get(
+ "/api/v1/core/tags/search/",
+ {"api_key": api_token.token},
+ HTTP_HOST=ADMIN_TEST_HOST,
+ )
+
+ assert response.status_code == 200
+ payload = response.json()
+ cards_by_name = {tag["name"]: tag for tag in payload["tags"]}
+ assert payload["has_snapshots"] == "all"
+ assert cards_by_name["Alpha Research"]["id"] == linked_tag.id
+ assert cards_by_name["Alpha Research"]["num_snapshots"] == 2
+ assert cards_by_name["Empty Tag"]["id"] == empty_tag.id
+ assert cards_by_name["Empty Tag"]["num_snapshots"] == 0
+
+
def test_tag_search_api_respects_sort_and_filters(client, api_token, admin_user, crawl, tagged_data):
from datetime import datetime
diff --git a/archivebox/tests/test_binary_service.py b/archivebox/tests/test_binary_service.py
index 443d9d9a..b4aebc51 100644
--- a/archivebox/tests/test_binary_service.py
+++ b/archivebox/tests/test_binary_service.py
@@ -134,6 +134,8 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
assert binary_processes
assert binary_processes[-1].status == Process.StatusChoices.EXITED
assert binary_processes[-1].exit_code == 0
+ assert binary_processes[-1].ended_at is not None
+ assert binary_processes[-1].started_at < binary_processes[-1].ended_at
assert any(f"--name={name}" in arg for arg in binary_processes[-1].cmd)
_cmd_result = run_archivebox_cmd(
diff --git a/archivebox/tests/test_cli_server.py b/archivebox/tests/test_cli_server.py
index d2d86633..ce02d2b1 100644
--- a/archivebox/tests/test_cli_server.py
+++ b/archivebox/tests/test_cli_server.py
@@ -139,6 +139,7 @@ def test_runner_worker_uses_current_interpreter():
from archivebox.workers.supervisord_util import RUNNER_WORKER
assert RUNNER_WORKER["command"] == f"{sys.executable} -m archivebox run --daemon"
+ assert RUNNER_WORKER["autorestart"] == "true"
assert 'ARCHIVEBOX_RUNNER_DAEMON="1"' in RUNNER_WORKER["environment"]
diff --git a/archivebox/tests/test_crawl_runner.py b/archivebox/tests/test_crawl_runner.py
index fea7d285..3d5504ae 100644
--- a/archivebox/tests/test_crawl_runner.py
+++ b/archivebox/tests/test_crawl_runner.py
@@ -435,6 +435,8 @@ def test_machine_service_persists_only_derived_config_events(tmp_path, hermetic_
config={
"WGET_BINARY": str(wget_binary),
"ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
+ "ABX_UV_CACHE": "/tmp/uv-cache",
+ "ABX_PNPM_CACHE": "/tmp/pnpm-cache",
"CHROME_USER_DATA_DIR": "/tmp/stale-derived-profile",
},
config_type="derived",
@@ -466,6 +468,8 @@ def test_machine_service_persists_only_derived_config_events(tmp_path, hermetic_
# LIB_DIR) then the unset removed it; ABX_INSTALL_CACHE survives.
assert machine.config == {
"ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
+ "ABX_PNPM_CACHE": "/tmp/pnpm-cache",
+ "ABX_UV_CACHE": "/tmp/uv-cache",
}
diff --git a/archivebox/tests/test_hooks.py b/archivebox/tests/test_hooks.py
index 28e76aa2..6d2817a8 100755
--- a/archivebox/tests/test_hooks.py
+++ b/archivebox/tests/test_hooks.py
@@ -18,11 +18,13 @@ import textwrap
from pathlib import Path
import pytest
+import rich_click as click
# Set up Django before importing any Django-dependent modules
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.settings")
REPO_ROOT = Path(__file__).resolve().parents[2]
+WORKSPACE_ROOT = REPO_ROOT.parent
RESULT_PREFIX = "__ARCHIVEBOX_TEST_RESULT__="
@@ -50,7 +52,12 @@ def run_plugin_discovery_subprocess(tmp_path: Path, plugins_dir: Path, script: s
cwd_plugins_dir = data_dir / "custom_plugins"
if plugins_dir != cwd_plugins_dir:
shutil.copytree(plugins_dir, cwd_plugins_dir)
- env["PYTHONPATH"] = str(REPO_ROOT) + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "")
+ existing_pythonpath = [
+ str(Path(entry).expanduser().resolve(strict=False))
+ for entry in env.get("PYTHONPATH", "").split(os.pathsep)
+ if entry and Path(entry).expanduser().is_absolute()
+ ]
+ env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys([str(REPO_ROOT), *existing_pythonpath]))
subprocess_script = "\n".join(
[
"import json",
@@ -84,6 +91,28 @@ def run_plugin_discovery_subprocess(tmp_path: Path, plugins_dir: Path, script: s
raise AssertionError(f"Subprocess did not emit a result line.\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}")
+def test_cli_env_does_not_emit_relative_pythonpath_entries():
+ from archivebox.tests.conftest import cli_env
+
+ old_pythonpath = os.environ.get("PYTHONPATH")
+ try:
+ os.environ["PYTHONPATH"] = os.pathsep.join(["../abxpkg", "../abx-plugins", "../abx-dl"])
+
+ env = cli_env()
+ finally:
+ if old_pythonpath is None:
+ os.environ.pop("PYTHONPATH", None)
+ else:
+ os.environ["PYTHONPATH"] = old_pythonpath
+
+ pythonpath_entries = env["PYTHONPATH"].split(os.pathsep)
+ assert str((WORKSPACE_ROOT / "abxpkg").resolve(strict=False)) in pythonpath_entries
+ assert str((WORKSPACE_ROOT / "abx-plugins").resolve(strict=False)) in pythonpath_entries
+ assert str((WORKSPACE_ROOT / "abx-dl").resolve(strict=False)) in pythonpath_entries
+ assert all(Path(entry).is_absolute() for entry in pythonpath_entries)
+ assert not any(entry.startswith("..") for entry in pythonpath_entries)
+
+
class TestBackgroundHookDetection:
"""Test that background hooks are detected by .bg. suffix."""
@@ -704,13 +733,13 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat
from archivebox.plugins.hooks import run_hook
lib_dir = tmp_path / "lib"
- node_modules_dir = lib_dir / "npm" / "node_modules"
+ node_modules_dir = lib_dir / "pnpm" / "packages" / "chrome" / "node_modules"
configured_node_path = os.pathsep.join(
[
- "/home/archivebox/.npm/lib/node_modules",
+ "/home/archivebox/.pnpm/packages/chrome/node_modules",
"/usr/lib/node_modules",
str(node_modules_dir),
- "/usr/share/archivebox/lib/npm/node_modules",
+ "/usr/share/archivebox/lib/pnpm/packages/chrome/node_modules",
],
)
@@ -750,3 +779,47 @@ print(json.dumps({
assert payload["NODE_MODULE_DIR"] == str(node_modules_dir)
assert payload["NODE_PATH"].split(os.pathsep) == configured_node_path.split(os.pathsep)
assert process.env["NODE_MODULES_DIR"] == str(node_modules_dir)
+
+
+@pytest.mark.django_db(transaction=True)
+def test_run_hook_executes_python_hooks_through_script_shebang(tmp_path):
+ """Python hooks must use their abxpkg script header instead of sys.executable."""
+ from archivebox.plugins.hooks import run_hook
+
+ plugin_dir = tmp_path / "plugins" / "shebangprobe"
+ plugin_dir.mkdir(parents=True)
+ hook_path = plugin_dir / "on_Snapshot__99_shebangprobe.py"
+ hook_path.write_text(
+ """#!/usr/bin/env -S abxpkg run --script python3
+# /// script
+# requires-python = ">=3.12"
+# ///
+import json
+import os
+import rich_click
+
+print(json.dumps({
+ "ABXPKG_FAST_SCRIPT": os.environ.get("ABXPKG_FAST_SCRIPT"),
+ "RICH_CLICK_FILE": rich_click.__file__,
+}))
+""",
+ encoding="utf-8",
+ )
+ hook_path.chmod(0o755)
+
+ output_dir = tmp_path / "archive" / "users" / "system" / "snapshots" / "20260603" / "example.com" / "test" / "shebangprobe"
+ process = run_hook(
+ hook_path,
+ output_dir,
+ config={
+ "LIB_DIR": str(tmp_path / "lib"),
+ },
+ timeout=10,
+ )
+ process.refresh_from_db()
+
+ assert process.cmd[0] == str(hook_path)
+ assert process.exit_code == 0, process.stderr
+ payload = json.loads(process.stdout.strip())
+ assert payload["ABXPKG_FAST_SCRIPT"] == "1"
+ assert Path(payload["RICH_CLICK_FILE"]).resolve() == Path(click.__file__).resolve()
diff --git a/archivebox/tests/test_machine_models.py b/archivebox/tests/test_machine_models.py
index 49d5692a..fa013d17 100644
--- a/archivebox/tests/test_machine_models.py
+++ b/archivebox/tests/test_machine_models.py
@@ -382,14 +382,14 @@ class TestBinaryModel:
"""Binary.from_json() should persist provider overrides unchanged."""
overrides = {
"apt": {"install_args": ["chromium"]},
- "npm": {"install_args": "puppeteer"},
+ "pnpm": {"install_args": "puppeteer"},
"custom": {"install": "bash -lc 'echo ok'"},
}
binary = Binary.from_json(
{
"name": "chrome",
- "binproviders": "apt,npm,custom",
+ "binproviders": "apt,pnpm,custom",
"overrides": overrides,
},
)
@@ -414,13 +414,13 @@ class TestBinaryModel:
"""Binary.from_json() should no longer translate legacy non-dict provider overrides."""
overrides = {
"apt": ["chromium"],
- "npm": "puppeteer",
+ "pnpm": "puppeteer",
}
binary = Binary.from_json(
{
"name": "chrome",
- "binproviders": "apt,npm",
+ "binproviders": "apt,pnpm",
"overrides": overrides,
},
)
@@ -428,15 +428,15 @@ class TestBinaryModel:
assert binary is not None
assert binary.overrides == overrides
- def test_binary_from_json_prefers_published_readability_package(self):
- """Binary.from_json() should rewrite readability's npm git URL to the published package."""
+ def test_binary_from_json_preserves_readability_package_metadata(self):
+ """Binary.from_json() should preserve readability's pnpm package metadata."""
binary = Binary.from_json(
{
"name": "readability-extractor",
- "binproviders": "env,npm",
+ "binproviders": "env,pnpm",
"overrides": {
- "npm": {
- "install_args": ["https://github.com/ArchiveBox/readability-extractor"],
+ "pnpm": {
+ "install_args": ["readability-extractor"],
},
},
},
@@ -444,7 +444,7 @@ class TestBinaryModel:
assert binary is not None
assert binary.overrides == {
- "npm": {
+ "pnpm": {
"install_args": ["readability-extractor"],
},
}
diff --git a/archivebox/tests/test_machine_service.py b/archivebox/tests/test_machine_service.py
index 64db4591..301e90da 100644
--- a/archivebox/tests/test_machine_service.py
+++ b/archivebox/tests/test_machine_service.py
@@ -105,6 +105,7 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
"LITEPARSE_BINARY": {str(installed_liteparse_path)!r},
"NODE_BINARY": {str(external_tool)!r},
"ABX_INSTALL_CACHE": {{"lit": "cached"}},
+ "ABX_UV_CACHE": "/tmp/uv-cache",
"CHROME_USER_DATA_DIR": "/tmp/derived-profile",
}}, config_type="derived")).now()
await bus.emit(MachineEvent(method="unset", key="config/LITEPARSE_BINARY", config_type="derived")).now()
@@ -137,6 +138,8 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
assert machine.config["LITEPARSE_BINARY"] == str(installed_liteparse_path)
assert machine.config["LITEPARSE_BINARY"] != "/tmp/user-config-must-not-persist"
+ assert machine.config["ABX_INSTALL_CACHE"] == {"lit": "cached"}
+ assert machine.config["ABX_UV_CACHE"] == "/tmp/uv-cache"
_cmd_result = run_archivebox_cmd(
["version"],
diff --git a/archivebox/tests/test_migrations_08_to_09.py b/archivebox/tests/test_migrations_08_to_09.py
index f045be55..9d156bda 100644
--- a/archivebox/tests/test_migrations_08_to_09.py
+++ b/archivebox/tests/test_migrations_08_to_09.py
@@ -404,7 +404,7 @@ def test_migration_creates_process_records(migration_08_data):
def test_migration_creates_binary_records(migration_08_data):
- """Migration should create Binary records from cmd_version data."""
+ """Migration should create and link Binary/NetworkInterface records from migrated Process data."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
@@ -420,6 +420,31 @@ def test_migration_creates_binary_records(migration_08_data):
extractors = {ar["extractor"] for ar in original_data["archiveresults"]}
assert binary_count >= len(extractors), f"Expected at least {len(extractors)} Binaries, got {binary_count}"
+ cursor.execute("""
+ SELECT COUNT(*)
+ FROM machine_process
+ WHERE cmd != '[]' AND binary_id IS NULL
+ """)
+ missing_binary_count = cursor.fetchone()[0]
+ assert missing_binary_count == 0
+
+ cursor.execute("""
+ SELECT p.cmd, b.name, b.abspath
+ FROM machine_process p
+ JOIN machine_binary b ON p.binary_id = b.id
+ WHERE p.cmd != '[]'
+ """)
+ rows = cursor.fetchall()
+ assert rows
+ for cmd_raw, binary_name, binary_abspath in rows:
+ cmd = json.loads(cmd_raw)
+ assert binary_name == cmd[0]
+ assert binary_abspath == cmd[0]
+
+ cursor.execute("SELECT COUNT(*) FROM machine_process WHERE iface_id IS NULL")
+ missing_iface_count = cursor.fetchone()[0]
+ assert missing_iface_count == 0
+
conn.close()
@@ -746,7 +771,12 @@ def test_archiveresult_files_preserved_after_migration(tmp_path):
files_before.extend([f for f in d.rglob("*") if f.is_file()])
files_before_count = len(files_before)
generated_metadata_names = {"index.html", "index.json", "index.jsonl"}
- original_payloads = sorted(path.read_text() for path in files_before if path.name not in generated_metadata_names)
+ generated_search_backends = {"search_backend_sqlite", "search_backend_sonic"}
+
+ def is_generated_file(path) -> bool:
+ return path.name in generated_metadata_names or any(part in generated_search_backends for part in path.parts)
+
+ original_payloads = sorted(path.read_text() for path in files_before if not is_generated_file(path))
# Sample some specific files to check they're preserved
sample_paths_before = {}
@@ -881,9 +911,7 @@ def test_archiveresult_files_preserved_after_migration(tmp_path):
# the hydrated DB row, so raw file counts are allowed to increase; compare
# the legacy payload contents after excluding those generated metadata
# files to keep the no-data-loss assertion strict.
- migrated_payloads = sorted(
- path.read_text() for path in [*files_new_structure, *old_files_remaining] if path.name not in generated_metadata_names
- )
+ migrated_payloads = sorted(path.read_text() for path in [*files_new_structure, *old_files_remaining] if not is_generated_file(path))
assert original_payloads == migrated_payloads, "Legacy payload files changed or were lost during reorganization"
assert files_new_count >= files_before_count, "New 0.9 metadata should not replace legacy payload files"
diff --git a/archivebox/tests/test_opencode_agent.py b/archivebox/tests/test_opencode_agent.py
new file mode 100644
index 00000000..e0a2ce3a
--- /dev/null
+++ b/archivebox/tests/test_opencode_agent.py
@@ -0,0 +1,344 @@
+import os
+import sqlite3
+
+import pytest
+import requests
+from asgiref.sync import async_to_sync
+
+from archivebox.tests.conftest import ADMIN_TEST_HOST
+
+
+pytestmark = pytest.mark.django_db
+
+
+def test_opencode_disabled_route_does_not_start_server(client, monkeypatch):
+ from abx_plugins.plugins.opencode import views
+
+ monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": False})
+ monkeypatch.setattr(views, "_ensure_opencode", lambda settings: pytest.fail("opencode should not start when disabled"))
+
+ response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+
+ assert response.status_code == 404
+
+
+def test_opencode_agent_requires_superuser(client, db, monkeypatch, django_user_model):
+ from abx_plugins.plugins.opencode import views
+
+ monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True})
+ monkeypatch.setattr(views, "_ensure_opencode", lambda settings: pytest.fail("opencode should not start before auth passes"))
+
+ response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+ assert response.status_code == 302
+ assert "/admin/login/" in response.headers["Location"]
+
+ user = django_user_model.objects.create_user(username="regular", password="testpassword")
+ client.force_login(user)
+ response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+ assert response.status_code == 403
+
+
+def test_opencode_agent_superuser_gets_wrapper(admin_client, db, monkeypatch):
+ from abx_plugins.plugins.opencode import views
+
+ monkeypatch.setattr(views, "_machine_config", lambda: {"OPENCODE_ENABLED": True})
+ monkeypatch.setattr(views, "_ensure_opencode", lambda settings: (True, ""))
+
+ response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
+
+ assert response.status_code == 200
+ assert b'