From ea4ce5b641df33e4941922ac352900d8ff4198b6 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 18 May 2026 01:51:55 -0700 Subject: [PATCH] Prepare ArchiveBox 0.9.30rc81 demo release --- .dockerignore | 1 + Dockerfile | 16 +- archivebox/api/urls.py | 14 ++ archivebox/api/v1_api.py | 1 + archivebox/api/v1_personas.py | 157 ++++++++++++++++++ archivebox/cli/archivebox_status.py | 85 ++++++---- archivebox/cli/archivebox_update.py | 21 ++- archivebox/config/constants.py | 2 +- archivebox/core/admin_archiveresults.py | 92 +++++++--- archivebox/core/admin_snapshots.py | 50 ++++-- archivebox/core/models.py | 121 ++++++++++++-- archivebox/core/settings.py | 2 +- archivebox/core/views.py | 93 ++++------- archivebox/machine/models.py | 20 +++ archivebox/search/__init__.py | 12 +- archivebox/services/runner.py | 21 ++- archivebox/templates/admin/base.html | 8 +- archivebox/templates/static/admin.css | 22 +++ archivebox/tests/test_add_view.py | 41 +++++ archivebox/tests/test_admin_views.py | 51 ++++++ .../tests/test_archive_result_service.py | 48 ++++++ archivebox/tests/test_cli_run.py | 36 ++++ archivebox/tests/test_cli_server.py | 17 ++ archivebox/tests/test_config.py | 6 + archivebox/tests/test_machine_models.py | 41 ++++- archivebox/tests/test_runner.py | 75 ++++++++- archivebox/tests/test_search.py | 34 ++++ archivebox/tests/test_urls.py | 26 +++ archivebox/workers/supervisord_util.py | 7 +- pyproject.toml | 6 +- 30 files changed, 951 insertions(+), 175 deletions(-) create mode 100644 archivebox/api/v1_personas.py create mode 100644 archivebox/tests/test_search.py diff --git a/.dockerignore b/.dockerignore index f1fd1558..0c12c372 100644 --- a/.dockerignore +++ b/.dockerignore @@ -51,6 +51,7 @@ website/ typings/ tmp/ +.tmp/ data/ data*/ - diff --git a/Dockerfile b/Dockerfile index b241ac6d..49e1cba1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,6 +53,8 @@ ARG TARGETPLATFORM ARG TARGETOS ARG TARGETARCH ARG TARGETVARIANT +ARG ABX_PLUGINS_REF=f576a7c81a17b0e49adf4789ac7f7143ece1282f +ARG ABX_DL_REF=3cddb1bf4d8a37c49f2a70810e0da840acc96c17 ######### Environment Variables ################################# # Global build-time and runtime environment constants + default pkg manager config @@ -348,8 +350,20 @@ RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ && rm -rf /var/lib/apt/lists/* # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies +# Install unreleased dev abx package revisions used by the ArchiveBox dev branch. +RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ + echo "[+] Installing dev abx package revisions..." \ + && uv pip install --reinstall --no-deps \ + "git+https://github.com/ArchiveBox/abx-plugins.git@${ABX_PLUGINS_REF}" \ + "git+https://github.com/ArchiveBox/abx-dl.git@${ABX_DL_REF}" \ + && ( \ + pip show abx-plugins \ + && pip show abx-dl \ + && echo -e '\n\n' \ + ) | tee -a /VERSION.txt + # Install ArchiveBox Python package from the checked-out source. -# Sibling abx-* packages are installed from PyPI inside the container, not copied from the local checkout. +# Sibling abx-* packages are installed from pinned upstream refs inside the container. COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \ diff --git a/archivebox/api/urls.py b/archivebox/api/urls.py index 5324e217..5ec44357 100644 --- a/archivebox/api/urls.py +++ b/archivebox/api/urls.py @@ -1,12 +1,26 @@ __package__ = "archivebox.api" +from urllib.parse import quote + +from django.http import HttpRequest, HttpResponseRedirect +from django.shortcuts import redirect from django.urls import path from django.views.generic.base import RedirectView +from archivebox.core.host_utils import build_web_url + from .v1_api import urls as v1_api_urls + +def archive_redirect_view(request: HttpRequest, url: str) -> HttpResponseRedirect: + if request.META.get("QUERY_STRING"): + url = f"{url}?{request.META['QUERY_STRING']}" + return redirect(build_web_url(f"/web/{quote(url, safe=':/')}", request=request)) + + urlpatterns = [ path("", RedirectView.as_view(url="/api/v1/docs")), + path("archive/", archive_redirect_view, name="api-archive-redirect"), path("v1/", RedirectView.as_view(url="/api/v1/docs")), path("v1/", v1_api_urls), path("v1", RedirectView.as_view(url="/api/v1/docs")), diff --git a/archivebox/api/v1_api.py b/archivebox/api/v1_api.py index e45b8125..18cea9f0 100644 --- a/archivebox/api/v1_api.py +++ b/archivebox/api/v1_api.py @@ -44,6 +44,7 @@ def register_urls(api: NinjaAPI) -> NinjaAPI: api.add_router("/crawls/", "archivebox.api.v1_crawls.router") api.add_router("/cli/", "archivebox.api.v1_cli.router") api.add_router("/machine/", "archivebox.api.v1_machine.router") + api.add_router("/personas/", "archivebox.api.v1_personas.router") return api diff --git a/archivebox/api/v1_personas.py b/archivebox/api/v1_personas.py new file mode 100644 index 00000000..c0a19cdb --- /dev/null +++ b/archivebox/api/v1_personas.py @@ -0,0 +1,157 @@ +__package__ = "archivebox.api" + +import json +from datetime import datetime +from typing import Any +from uuid import UUID + +from django.db.models import Q +from django.http import HttpRequest +from ninja import Router, Schema +from pydantic import Field + +from archivebox.personas.importers import validate_persona_name +from archivebox.personas.models import Persona + + +router = Router(tags=["Personas"]) + + +class PersonaBrowserSettingsSchema(Schema): + user_agent: str = "" + viewport_size: str = "" + viewport_device_scale_factor: float | None = None + language: str = "" + timezone: str = "" + geolocation: dict[str, Any] | None = None + + +class PersonaSyncSchema(Schema): + extension_persona_id: str + name: str + settings: PersonaBrowserSettingsSchema = Field(default_factory=PersonaBrowserSettingsSchema) + cookies_txt: str = "" + auth_json: dict[str, Any] = Field(default_factory=dict) + + +class PersonaSchema(Schema): + TYPE: str = "personas.models.Persona" + id: UUID + name: str + created_at: datetime + created_by_id: str + created_by_username: str + config: dict[str, Any] | None + + @staticmethod + def resolve_created_by_id(obj): + return str(obj.created_by.pk) + + @staticmethod + def resolve_created_by_username(obj) -> str: + return obj.created_by.username + + +class PersonaSyncResponseSchema(Schema): + success: bool + created: bool + persona: PersonaSchema + cookies_file_written: bool + auth_file_written: bool + + +def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrowserSettingsSchema) -> dict[str, Any]: + config: dict[str, Any] = { + "BROWSER_EXTENSION_PERSONA_ID": extension_persona_id, + "BROWSER_EXTENSION_SYNCED_AT": datetime.utcnow().isoformat() + "Z", + } + + if settings.user_agent: + config.update( + { + "USER_AGENT": settings.user_agent, + "CHROME_USER_AGENT": settings.user_agent, + "WGET_USER_AGENT": settings.user_agent, + "CURL_USER_AGENT": settings.user_agent, + }, + ) + if settings.viewport_size: + config.update( + { + "RESOLUTION": settings.viewport_size, + "CHROME_RESOLUTION": settings.viewport_size, + }, + ) + if settings.viewport_device_scale_factor is not None: + config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor + if settings.language: + config["BROWSER_LANGUAGE"] = settings.language + if settings.timezone: + config["BROWSER_TIMEZONE"] = settings.timezone + if settings.geolocation: + config["BROWSER_GEOLOCATION"] = settings.geolocation + + return config + + +def find_persona(extension_persona_id: str, name: str) -> Persona | None: + return ( + Persona.objects.filter( + Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name), + ) + .order_by("created_at") + .first() + ) + + +@router.get("/personas", response=list[PersonaSchema], url_name="get_personas") +def get_personas(request: HttpRequest): + """List personas available on this ArchiveBox server.""" + return Persona.objects.all().order_by("name") + + +@router.post("/sync", response=PersonaSyncResponseSchema, url_name="sync_persona") +def sync_persona(request: HttpRequest, payload: PersonaSyncSchema): + """ + Create or update a Persona from a browser extension profile export. + + The extension sends browser settings plus portable auth artifacts. The server + keeps browser override settings in Persona.config and writes cookies.txt / + auth.json into the persona directory for extractors to consume. + """ + name = payload.name.strip() + is_valid, error_message = validate_persona_name(name) + if not is_valid: + raise ValueError(error_message) + + persona = find_persona(payload.extension_persona_id, name) + created = persona is None + if persona is None: + persona = Persona(name=name) + if getattr(request.user, "is_authenticated", False): + persona.created_by = request.user + + persona.config = { + **(persona.config or {}), + **browser_settings_to_config(payload.extension_persona_id, payload.settings), + } + persona.save() + persona.ensure_dirs() + + cookies_written = False + if payload.cookies_txt.strip(): + (persona.path / "cookies.txt").write_text(payload.cookies_txt) + cookies_written = True + + auth_written = False + if payload.auth_json: + (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n") + auth_written = True + + return { + "success": True, + "created": created, + "persona": persona, + "cookies_file_written": cookies_written, + "auth_file_written": auth_written, + } diff --git a/archivebox/cli/archivebox_status.py b/archivebox/cli/archivebox_status.py index d0e5e2e5..9cbd38ef 100644 --- a/archivebox/cli/archivebox_status.py +++ b/archivebox/cli/archivebox_status.py @@ -10,11 +10,13 @@ from rich import print from archivebox.misc.util import enforce_types, docstring from archivebox.config import DATA_DIR, CONSTANTS from archivebox.config.common import get_config -from archivebox.misc.legacy import parse_json_links_details from archivebox.misc.system import get_dir_size from archivebox.misc.logging_util import printable_filesize +MAX_STATUS_FS_DIR_SCAN = 5000 + + @enforce_types def status(out_dir: Path = DATA_DIR) -> None: """Print out some info and statistics about the archive collection""" @@ -22,7 +24,7 @@ def status(out_dir: Path = DATA_DIR) -> None: from django.contrib.auth import get_user_model from django.db.models import Sum from django.db.models.functions import Coalesce - from archivebox.core.models import Snapshot + from archivebox.core.models import ArchiveResult, Snapshot config = get_config() User = get_user_model() @@ -34,54 +36,66 @@ def status(out_dir: Path = DATA_DIR) -> None: print(f" Index size: {size} across {num_files} files") print() - links = list(Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0))) - num_sql_links = len(links) - num_link_details = sum(1 for link in parse_json_links_details(out_dir=out_dir)) + snapshots_qs = Snapshot.objects.all() + num_sql_links = snapshots_qs.count() archive_dir = config.ARCHIVE_DIR + legacy_snapshot_dirs = [] + if archive_dir.exists(): + legacy_snapshot_dirs = [ + entry for entry in archive_dir.iterdir() if entry.is_dir() and not entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry) + ] print(f" > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {CONSTANTS.SQL_INDEX_FILENAME})") - print(f" > JSON Link Details: {num_link_details} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)") + print(f" > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)") print() print("[green]\\[*] Scanning archive data directories...[/green]") users_dir = config.USERS_DIR scan_roots = [root for root in (archive_dir, users_dir) if root.exists()] scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir) print(f"[yellow] {scan_roots_display}[/yellow]") - num_bytes = num_dirs = num_files = 0 - for root in scan_roots: - root_bytes, root_dirs, root_files = get_dir_size(root) - num_bytes += root_bytes - num_dirs += root_dirs - num_files += root_files + do_precise_fs_scan = num_sql_links <= MAX_STATUS_FS_DIR_SCAN + if do_precise_fs_scan: + num_bytes = num_dirs = num_files = 0 + for root in scan_roots: + root_bytes, root_dirs, root_files = get_dir_size(root) + num_bytes += root_bytes + num_dirs += root_dirs + num_files += root_files + else: + num_bytes = ArchiveResult.objects.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0 + num_dirs = 0 + num_files = ArchiveResult.objects.exclude(output_files__in=["", "{}"]).count() size = printable_filesize(num_bytes) - print(f" Size: {size} across {num_files} files in {num_dirs} directories") + if do_precise_fs_scan: + print(f" Size: {size} across {num_files} files in {num_dirs} directories") + else: + print(f" Size: {size} across {num_files} DB-tracked output records") # Use DB as source of truth for snapshot status - num_indexed = len(links) - num_archived = sum(1 for snapshot in links if snapshot.is_archived) + num_indexed = num_sql_links + num_archived = snapshots_qs.filter(status=Snapshot.StatusChoices.SEALED).count() num_unarchived = max(num_indexed - num_archived, 0) print(f" > indexed: {num_indexed}".ljust(36), "(total snapshots in DB)") print(f" > archived: {num_archived}".ljust(36), "(snapshots with archived content)") print(f" > unarchived: {num_unarchived}".ljust(36), "(snapshots pending archiving)") # Count snapshot directories on filesystem across both legacy and current layouts. - expected_snapshot_dirs = {str(Path(snapshot.output_dir).resolve()) for snapshot in links if Path(snapshot.output_dir).exists()} - discovered_snapshot_dirs = set() + if do_precise_fs_scan: + links = list(snapshots_qs) + expected_snapshot_dirs = {str(Path(snapshot.output_dir).resolve()) for snapshot in links if Path(snapshot.output_dir).exists()} + discovered_snapshot_dirs = {str(entry.resolve()) for entry in legacy_snapshot_dirs} - if archive_dir.exists(): - discovered_snapshot_dirs.update( - str(entry.resolve()) - for entry in archive_dir.iterdir() - if entry.is_dir() and not entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry) - ) + if users_dir.exists(): + discovered_snapshot_dirs.update( + str(entry.resolve()) for entry in users_dir.glob(f"*/{CONSTANTS.SNAPSHOTS_DIR_NAME}/*/*/*") if entry.is_dir() + ) - if users_dir.exists(): - discovered_snapshot_dirs.update( - str(entry.resolve()) for entry in users_dir.glob(f"*/{CONSTANTS.SNAPSHOTS_DIR_NAME}/*/*/*") if entry.is_dir() - ) - - orphaned_dirs = sorted(discovered_snapshot_dirs - expected_snapshot_dirs) - num_present = len(discovered_snapshot_dirs) - num_valid = len(discovered_snapshot_dirs & expected_snapshot_dirs) + orphaned_dirs = sorted(discovered_snapshot_dirs - expected_snapshot_dirs) + num_present = len(discovered_snapshot_dirs) + num_valid = len(discovered_snapshot_dirs & expected_snapshot_dirs) + else: + orphaned_dirs = [] + num_present = num_archived + num_valid = num_archived print() print(f" > present: {num_present}".ljust(36), "(snapshot directories on disk)") print(f" > [green]valid:[/green] {num_valid}".ljust(36), " (directories with matching DB entry)") @@ -116,10 +130,9 @@ def status(out_dir: Path = DATA_DIR) -> None: print(" [green]archivebox manage createsuperuser[/green]") print() - recent_snapshots = sorted( - links, - key=lambda snapshot: snapshot.downloaded_at or snapshot.modified_at or snapshot.created_at, - reverse=True, + recent_snapshots = snapshots_qs.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).order_by( + "-downloaded_at", + "-modified_at", )[:10] for snapshot in recent_snapshots: if not snapshot.downloaded_at: @@ -128,7 +141,7 @@ def status(out_dir: Path = DATA_DIR) -> None: ( "[grey53] " f" > {str(snapshot.downloaded_at)[:16]} " - f"[{snapshot.num_outputs} {('X', '√')[snapshot.is_archived]} {printable_filesize(snapshot.archive_size)}] " + f"[{snapshot.num_outputs} {('X', '√')[snapshot.status == Snapshot.StatusChoices.SEALED]} {printable_filesize(snapshot.output_size_sum or 0)}] " f'"{snapshot.title}": {snapshot.url}' "[/grey53]" )[: config.TERM_WIDTH], diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index c9300af1..136a5456 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -448,7 +448,7 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) - No orphan detection needed - we trust 1:1 mapping between DB and filesystem after Phase 1 has drained all old archive/ directories. """ - from archivebox.core.models import Snapshot + from archivebox.core.models import ArchiveResult, Snapshot from archivebox.config.common import get_config from archivebox.crawls.models import Crawl from django.db import transaction @@ -487,6 +487,25 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) - if has_directory: old_title = snapshot.title snapshot.reconcile_with_index_json(output_dir=output_dir, update_existing_archive_results=False) + metadata_updates = [] + for archiveresult in ArchiveResult.objects.filter(snapshot=snapshot).only( + "id", + "snapshot_id", + "plugin", + "output_str", + "output_files", + "output_size", + "output_mimetypes", + "modified_at", + ): + if archiveresult.update_output_metadata_from_filesystem(snapshot_dir=output_dir, save=False): + metadata_updates.append(archiveresult) + if metadata_updates: + ArchiveResult.objects.bulk_update( + metadata_updates, + ["output_files", "output_size", "output_mimetypes", "modified_at"], + batch_size=batch_size, + ) if snapshot.title != old_title: update_values["title"] = snapshot.title update_values["modified_at"] = timezone.now() diff --git a/archivebox/config/constants.py b/archivebox/config/constants.py index 74f60844..15bf8ba2 100644 --- a/archivebox/config/constants.py +++ b/archivebox/config/constants.py @@ -249,7 +249,7 @@ class ConstantsDict: "user_plugins", # old name for USER_PLUGINS_DIR (now 'plugins') "user_templates", # old name for CUSTOM_TEMPLATES_DIR (now 'templates') "static", # created by old static exports str: return f"abx-dl --plugins={plugin_name} {_quote_shell_string(source_url)}" -def build_abx_dl_replay_command(result: ArchiveResult) -> str: +def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str: display_command = build_abx_dl_display_command(result) process = getattr(result, "process", None) env_items = env_to_shell_exports(getattr(process, "env", None) or {}) - snapshot_dir = shlex.quote(str(result.snapshot_dir)) + if config is not None: + result.snapshot._runtime_config = config + snapshot_dir = shlex.quote(str(result.pwd or result.snapshot_dir)) if env_items: return f"cd {snapshot_dir}; env {env_items} {display_command}" return f"cd {snapshot_dir}; {display_command}" @@ -78,7 +80,7 @@ def get_plugin_admin_url(plugin_name: str) -> str: return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/" -def render_archiveresults_list(archiveresults_qs, limit=50): +def render_archiveresults_list(archiveresults_qs, limit=50, config=None): """Render a nice inline list view of archive results with status, plugin, output, and actions.""" result_ids = list(archiveresults_qs.order_by("plugin").values_list("pk", flat=True)[:limit]) @@ -87,7 +89,13 @@ def render_archiveresults_list(archiveresults_qs, limit=50): results_by_id = { result.pk: result - for result in ArchiveResult.objects.filter(pk__in=result_ids).select_related("snapshot", "process", "process__machine") + for result in ArchiveResult.objects.filter(pk__in=result_ids).select_related( + "snapshot", + "snapshot__crawl", + "snapshot__crawl__created_by", + "process", + "process__machine", + ) } results = [results_by_id[result_id] for result_id in result_ids if result_id in results_by_id] @@ -152,7 +160,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50): output_display += "..." display_cmd = build_abx_dl_display_command(result) - replay_cmd = build_abx_dl_replay_command(result) + replay_cmd = build_abx_dl_replay_command(result, config=config) cmd_str_escaped = html.escape(display_cmd) cmd_attr = html.escape(replay_cmd, quote=True) @@ -160,9 +168,9 @@ def render_archiveresults_list(archiveresults_qs, limit=50): embed_path = result.embed_path() if hasattr(result, "embed_path") else None snapshot_id = str(getattr(result, "snapshot_id", "")) if embed_path and result.status == "succeeded": - output_link = build_snapshot_url(snapshot_id, embed_path) + output_link = build_snapshot_url(snapshot_id, embed_path, config=config) else: - output_link = build_snapshot_url(snapshot_id, "") + output_link = build_snapshot_url(snapshot_id, "", config=config) # Get version - try cmd_version field version = result.cmd_version if result.cmd_version else "-" @@ -469,7 +477,7 @@ class ArchiveResultAdmin(BaseModelAdmin): list_filter = ("status", "plugin", "start_ts") ordering = ["-start_ts"] - list_per_page = get_config().SNAPSHOTS_PER_PAGE + list_per_page = 50 paginator = AcceleratedPaginator save_on_top = True @@ -482,20 +490,54 @@ class ArchiveResultAdmin(BaseModelAdmin): def change_view(self, request, object_id, form_url="", extra_context=None): self.request = request + request.archivebox_config = get_config() return super().change_view(request, object_id, form_url, extra_context) def changelist_view(self, request, extra_context=None): self.request = request - return super().changelist_view(request, extra_context) + request.archivebox_config = get_config() + saved_list_per_page = self.list_per_page + self.list_per_page = min(max(5, request.archivebox_config.SNAPSHOTS_PER_PAGE), 5000) + try: + return super().changelist_view(request, extra_context) + finally: + self.list_per_page = saved_list_per_page def get_queryset(self, request): - return ( + ordering = request.GET.get("o") + ordering_fields = set() + if ordering: + for part in ordering.split("."): + if not part: + continue + try: + idx = abs(int(part)) - 1 + except ValueError: + continue + if 0 <= idx < len(self.list_display): + ordering_fields.add(self.list_display[idx]) + + qs = ( super() .get_queryset(request) - .select_related("snapshot", "process") + .select_related("snapshot", "snapshot__crawl", "snapshot__crawl__created_by", "process", "process__machine") + .defer( + "config", + "notes", + "output_json", + "process__stdout", + "process__stderr", + "snapshot__config", + "snapshot__notes", + "snapshot__crawl__config", + "snapshot__crawl__notes", + "snapshot__crawl__urls", + ) .prefetch_related("snapshot__tags") - .annotate(snapshot_first_tag=Min("snapshot__tags__name")) ) + if "tags_inline" in ordering_fields: + qs = qs.annotate(snapshot_first_tag=Min("snapshot__tags__name")) + return qs def get_search_results(self, request, queryset, search_term): if not search_term: @@ -532,16 +574,20 @@ class ArchiveResultAdmin(BaseModelAdmin): return queryset.filter(reduce(and_, filters)).distinct(), True def get_snapshot_view_url(self, result: ArchiveResult) -> str: - return build_snapshot_url(str(result.snapshot_id), request=getattr(self, "request", None)) + request = getattr(self, "request", None) + return build_snapshot_url(str(result.snapshot_id), request=request, config=getattr(request, "archivebox_config", None)) def get_output_view_url(self, result: ArchiveResult) -> str: + request = getattr(self, "request", None) + config = getattr(request, "archivebox_config", None) output_path = result.embed_path() if hasattr(result, "embed_path") else None if not output_path: output_path = result.plugin or "" - return build_snapshot_url(str(result.snapshot_id), output_path, request=getattr(self, "request", None)) + return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config) def get_output_files_url(self, result: ArchiveResult) -> str: - return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=getattr(self, 'request', None))}/?files=1" + request = getattr(self, "request", None) + return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=getattr(request, 'archivebox_config', None))}/?files=1" def get_output_zip_url(self, result: ArchiveResult) -> str: return f"{self.get_output_files_url(result)}&download=zip" @@ -567,9 +613,10 @@ class ArchiveResultAdmin(BaseModelAdmin): ) def snapshot_info(self, result): snapshot_id = str(result.snapshot_id) + request = getattr(self, "request", None) return format_html( '[{}]   {}   {}
', - build_snapshot_url(snapshot_id, "index.html"), + build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)), snapshot_id[:8], result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"), result.snapshot.url[:128], @@ -639,8 +686,9 @@ class ArchiveResultAdmin(BaseModelAdmin): @admin.display(description="Command") def cmd_str(self, result): + request = getattr(self, "request", None) display_cmd = build_abx_dl_display_command(result) - replay_cmd = build_abx_dl_replay_command(result) + replay_cmd = build_abx_dl_replay_command(result, config=getattr(request, "archivebox_config", None)) return format_html( """
@@ -661,13 +709,15 @@ class ArchiveResultAdmin(BaseModelAdmin): ) def output_display(self, result): + request = getattr(self, "request", None) + config = getattr(request, "archivebox_config", None) # Determine output link path - use embed_path() which checks output_files embed_path = result.embed_path() if hasattr(result, "embed_path") else None output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html" snapshot_id = str(result.snapshot_id) return format_html( '↗️
{}
', - build_snapshot_url(snapshot_id, output_path), + build_snapshot_url(snapshot_id, output_path, request=request, config=config), result.output_str, ) @@ -677,11 +727,12 @@ class ArchiveResultAdmin(BaseModelAdmin): if not output_text: return "-" + request = getattr(self, "request", None) live_path = result.embed_path() if hasattr(result, "embed_path") else None if live_path: return format_html( '{}', - build_snapshot_url(str(result.snapshot_id), live_path), + build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=getattr(request, "archivebox_config", None)), output_text, output_text, ) @@ -739,9 +790,10 @@ class ArchiveResultAdmin(BaseModelAdmin): result.output_str, ) snapshot_id = str(result.snapshot_id) + request = getattr(self, "request", None) output_html += format_html( 'See result files ...
',
-            build_snapshot_url(snapshot_id, "index.html"),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
         )
         embed_path = result.embed_path() if hasattr(result, "embed_path") else ""
         path_from_embed = snapshot_dir / (embed_path or "")
diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py
index b6978f37..89b82c6b 100644
--- a/archivebox/core/admin_snapshots.py
+++ b/archivebox/core/admin_snapshots.py
@@ -1,6 +1,7 @@
 __package__ = "archivebox.core"
 
 from functools import lru_cache
+import json
 
 from django.contrib import admin, messages
 from django.urls import path
@@ -13,7 +14,6 @@ from django import forms
 from django.template import Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
 
-from archivebox.config import DATA_DIR
 from archivebox.config.common import get_config
 from archivebox.misc.util import htmldecode, urldecode
 from archivebox.misc.paginators import AcceleratedPaginator
@@ -333,7 +333,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             super()
             .get_queryset(request)
             .select_related("crawl__created_by")
-            .defer("config", "notes")
+            .defer("notes")
             .prefetch_related("tags")
             .prefetch_related(Prefetch("archiveresult_set", queryset=prefetch_qs))
         )
@@ -499,7 +499,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Archive Results")
     def archiveresults_list(self, obj):
-        return render_archiveresults_list(obj.archiveresult_set.all())
+        request = getattr(self, "request", None)
+        return render_archiveresults_list(obj.archiveresult_set.all(), config=getattr(request, "archivebox_config", None))
 
     @admin.display(
         description="Title",
@@ -916,8 +917,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def _get_expected_hook_total(self, obj) -> int:
         try:
             request = getattr(self, "request", None)
+            if getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_changelist":
+                return 0
+
             crawl = getattr(obj, "crawl", None)
-            has_scoped_config = bool(getattr(obj, "config", None) or getattr(crawl, "config", None) or getattr(crawl, "persona_id", None))
+            snapshot_config = getattr(obj, "config", None) or {}
+            crawl_config = getattr(crawl, "config", None) or {}
+            crawl_persona_id = getattr(crawl, "persona_id", None)
+            has_scoped_config = bool(snapshot_config or crawl_config or crawl_persona_id)
 
             if request is not None and not has_scoped_config:
                 cached_total = getattr(request, "archivebox_expected_snapshot_hook_total", None)
@@ -927,7 +934,23 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                     request.archivebox_expected_snapshot_hook_total = cached_total
                 return cached_total
 
-            return len(discover_hooks("Snapshot", config=get_config(crawl=crawl, snapshot=obj)))
+            if request is not None:
+                scoped_cache = getattr(request, "archivebox_expected_snapshot_hook_totals_by_scope", None)
+                if scoped_cache is None:
+                    scoped_cache = {}
+                    request.archivebox_expected_snapshot_hook_totals_by_scope = scoped_cache
+                if snapshot_config:
+                    cache_key = ("snapshot", json.dumps(snapshot_config, sort_keys=True, default=str))
+                else:
+                    cache_key = ("crawl", json.dumps(crawl_config, sort_keys=True, default=str), crawl_persona_id)
+                cached_total = scoped_cache.get(cache_key)
+                if cached_total is None:
+                    config = get_config(crawl=crawl, snapshot=obj if snapshot_config else None)
+                    cached_total = len(discover_hooks("Snapshot", config=config))
+                    scoped_cache[cache_key] = cached_total
+                return cached_total
+
+            return len(discover_hooks("Snapshot", config=get_config(crawl=crawl, snapshot=obj if snapshot_config else None)))
         except Exception:
             return 0
 
@@ -1038,12 +1061,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         description="🔁 Redo Failed",
     )
     def update_snapshots(self, request, queryset):
-        queued = bg_archive_snapshots(queryset, kwargs={"overwrite": False, "out_dir": DATA_DIR})
+        queued = 0
+        for snapshot in queryset:
+            queued += snapshot.retry_failed_archiveresults()
 
-        messages.success(
-            request,
-            f"Queued {queued} snapshots for re-archiving. The background runner will process them.",
-        )
+        if queued:
+            messages.success(
+                request,
+                f"Queued {queued} failed/skipped extractors for retry. The background runner will process them.",
+            )
+        else:
+            messages.info(request, "No failed/skipped extractors were found in the selected snapshots.")
 
     @admin.action(
         description="🆕 Archive Now",
@@ -1070,7 +1098,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         description="🔄 Redo",
     )
     def overwrite_snapshots(self, request, queryset):
-        queued = bg_archive_snapshots(queryset, kwargs={"overwrite": True, "out_dir": DATA_DIR})
+        queued = bg_archive_snapshots(queryset, kwargs={"overwrite": True})
 
         messages.success(
             request,
diff --git a/archivebox/core/models.py b/archivebox/core/models.py
index 68b0dd44..0a31f65d 100755
--- a/archivebox/core/models.py
+++ b/archivebox/core/models.py
@@ -1037,13 +1037,14 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
         # Merge ArchiveResults
         self._merge_archive_results_from_index(index_data, update_existing=update_existing_archive_results)
         if not self._normalize_title_candidate(self.title, snapshot_url=self.url):
-            title_result = (
-                self.archiveresult_set.filter(plugin="title").exclude(output_str="").order_by("-start_ts", "-end_ts", "-created_at").first()
+            title_results = (
+                self.archiveresult_set.filter(plugin="title").exclude(output_str="").order_by("-start_ts", "-end_ts", "-created_at")
             )
-            if title_result:
+            for title_result in title_results.only("output_str"):
                 result_title = self._normalize_title_candidate(title_result.output_str, snapshot_url=self.url)
                 if result_title:
                     self.title = result_title
+                    break
 
         # Write back in JSONL format
         self.write_index_jsonl(output_dir=output_dir)
@@ -1054,14 +1055,16 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
 
     def _merge_title_from_index(self, index_data: dict):
         """Merge title - prefer longest non-URL title."""
-        index_title = (index_data.get("title") or "").strip()
-        db_title = self.title or ""
+        index_title = self._normalize_title_candidate(index_data.get("title"), snapshot_url=self.url)
+        db_title = self._normalize_title_candidate(self.title, snapshot_url=self.url)
 
-        candidates = [t for t in [index_title, db_title] if t and t != self.url]
+        candidates = [t for t in [index_title, db_title] if t]
         if candidates:
             best_title = max(candidates, key=len)
             if self.title != best_title:
                 self.title = best_title
+        elif self.title:
+            self.title = ""
 
     def _merge_tags_from_index(self, index_data: dict):
         """Merge tags - union of both sources."""
@@ -1639,7 +1642,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
         title = " ".join(line.strip() for line in str(candidate or "").splitlines() if line.strip()).strip()
         if not title:
             return ""
-        if title.lower() in {"pending...", "no title found"}:
+        if title.lower() in {"pending...", "no title found", "unable to detect page title"}:
             return ""
         if title == snapshot_url:
             return ""
@@ -1655,10 +1658,8 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
         if stored_title:
             return stored_title
 
-        title_result = (
-            self.archiveresult_set.filter(plugin="title").exclude(output_str="").order_by("-start_ts", "-end_ts", "-created_at").first()
-        )
-        if title_result:
+        title_results = self.archiveresult_set.filter(plugin="title").exclude(output_str="").order_by("-start_ts", "-end_ts", "-created_at")
+        for title_result in title_results.only("output_str"):
             result_title = self._normalize_title_candidate(title_result.output_str, snapshot_url=self.url)
             if result_title:
                 return result_title
@@ -2283,13 +2284,16 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
 
         Returns count of ArchiveResults reset.
         """
-        count = self.archiveresult_set.filter(
+        retryable_results = ArchiveResult.objects.filter(
+            snapshot=self,
             status__in=[
                 ArchiveResult.StatusChoices.FAILED,
                 ArchiveResult.StatusChoices.SKIPPED,
                 ArchiveResult.StatusChoices.NORESULTS,
             ],
-        ).update(
+        )
+        legacy_result_count = retryable_results.filter(hook_name="").count()
+        count = retryable_results.exclude(hook_name="").update(
             status=ArchiveResult.StatusChoices.QUEUED,
             output_str="",
             output_json=None,
@@ -2300,13 +2304,19 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
             end_ts=None,
         )
 
-        if count > 0:
+        if count + legacy_result_count > 0:
             self.status = self.StatusChoices.QUEUED
             self.retry_at = timezone.now()
             self.current_step = 0  # Reset to step 0 for retry
             self.save(update_fields=["status", "retry_at", "current_step", "modified_at"])
 
-        return count
+            crawl = self.crawl
+            if crawl.status != crawl.StatusChoices.STARTED:
+                crawl.status = crawl.StatusChoices.QUEUED
+            crawl.retry_at = timezone.now()
+            crawl.save(update_fields=["status", "retry_at", "modified_at"])
+
+        return count + legacy_result_count
 
     # =========================================================================
     # URL Helper Properties (migrated from Link schema)
@@ -2347,6 +2357,10 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
 
     @cached_property
     def is_archived(self) -> bool:
+        cached_is_archived = getattr(self, "_is_archived_cached", None)
+        if cached_is_archived is not None:
+            return bool(cached_is_archived)
+
         if self.downloaded_at or self.status == self.StatusChoices.SEALED:
             return True
 
@@ -2362,7 +2376,8 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
             "media",
             "git",
         )
-        return any((Path(self.output_dir) / path).exists() for path in output_paths)
+        output_dir = Path(self.output_dir)
+        return any((output_dir / path).exists() for path in output_paths)
 
     # =========================================================================
     # Date/Time Properties (migrated from Link schema)
@@ -3218,6 +3233,80 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
     def output_size_from_files(self) -> int:
         return sum(self._coerce_output_file_size(metadata.get("size")) for metadata in self.output_file_map().values())
 
+    def update_output_metadata_from_filesystem(self, snapshot_dir: Path | None = None, save: bool = True) -> bool:
+        from collections import defaultdict
+        from abx_dl.output_files import guess_mimetype
+
+        if self.plugin == "title":
+            return False
+
+        snapshot_dir = Path(snapshot_dir or self.snapshot.output_dir)
+        exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid", "cmd.sh"}
+        output_files: dict[str, dict[str, Any]] = {}
+        mime_sizes: dict[str, int] = defaultdict(int)
+        total_size = 0
+
+        def add_file(file_path: Path, rel_path: str, *, root_relative: bool = False) -> None:
+            nonlocal total_size
+            try:
+                if not file_path.is_file() or file_path.name in exclude_names:
+                    return
+                stat = file_path.stat()
+            except OSError:
+                return
+            mime_type = guess_mimetype(file_path) or "application/octet-stream"
+            metadata = {
+                "extension": file_path.suffix.lower().lstrip("."),
+                "mimetype": mime_type,
+                "size": stat.st_size,
+            }
+            if root_relative:
+                metadata["root_relative"] = True
+            output_files[rel_path] = metadata
+            mime_sizes[mime_type] += stat.st_size
+            total_size += stat.st_size
+
+        for raw_line in str(self.output_str or "").splitlines():
+            raw_output = raw_line.strip().lstrip("/")
+            if not raw_output or raw_output in {".", "./", "/"} or "://" in raw_output or raw_output.startswith("/"):
+                continue
+            if not self._looks_like_output_path(raw_output, self.plugin):
+                continue
+
+            raw_path = Path(raw_output)
+            if raw_output.startswith(f"{self.plugin}/"):
+                plugin_relative = raw_output.removeprefix(f"{self.plugin}/")
+                add_file(snapshot_dir / raw_output, plugin_relative)
+            elif len(raw_path.parts) == 1:
+                add_file(snapshot_dir / self.plugin / raw_output, raw_output)
+                add_file(snapshot_dir / raw_output, raw_output, root_relative=True)
+            else:
+                add_file(snapshot_dir / self.plugin / raw_output, raw_output)
+                add_file(snapshot_dir / raw_output, raw_output, root_relative=True)
+
+        plugin_dir = snapshot_dir / self.plugin
+        if not output_files and plugin_dir.is_dir():
+            for file_path in plugin_dir.rglob("*"):
+                if not file_path.is_file() or ".hooks" in file_path.parts:
+                    continue
+                add_file(file_path, str(file_path.relative_to(plugin_dir)))
+
+        if not output_files:
+            return False
+
+        sorted_mimes = sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True)
+        output_mimetypes = ",".join(mime for mime, _ in sorted_mimes)
+        if self.output_files == output_files and self.output_size == total_size and self.output_mimetypes == output_mimetypes:
+            return False
+
+        self.output_files = output_files
+        self.output_size = total_size
+        self.output_mimetypes = output_mimetypes
+        self.modified_at = timezone.now()
+        if save:
+            self.save(update_fields=["output_files", "output_size", "output_mimetypes", "modified_at"])
+        return True
+
     def output_exists(self) -> bool:
         return os.path.exists(Path(self.snapshot_dir) / self.plugin)
 
diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
index 067b6b23..1c9cb027 100644
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@@ -226,7 +226,7 @@ TEMPLATES = [
 # CACHE_DB_TABLE = 'django_cache'
 
 DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(CONSTANTS.DATABASE_FILE))
-SQLITE_JOURNAL_MODE = os.environ.get("ARCHIVEBOX_SQLITE_JOURNAL_MODE", "TRUNCATE" if CONSTANTS.IN_DOCKER else "WAL")
+SQLITE_JOURNAL_MODE = os.environ.get("ARCHIVEBOX_SQLITE_JOURNAL_MODE", "WAL")
 SQLITE_MMAP_SIZE = os.environ.get("ARCHIVEBOX_SQLITE_MMAP_SIZE", "0" if CONSTANTS.IN_DOCKER else "134217728")
 
 SQLITE_CONNECTION_OPTIONS = {
diff --git a/archivebox/core/views.py b/archivebox/core/views.py
index 964eec34..09ce0f66 100644
--- a/archivebox/core/views.py
+++ b/archivebox/core/views.py
@@ -893,6 +893,7 @@ class PublicIndexView(ListView):
         }
         for snapshot in context.get("object_list") or ():
             snapshot._icons_compact = True
+            snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
         return context
 
     def get_queryset(self, **kwargs):
@@ -1044,6 +1045,9 @@ class AddView(UserPassesTestMixin, FormView):
         notes = form.cleaned_data.get("notes", "")
         url_filters = form.cleaned_data.get("url_filters") or {}
         custom_config = self._get_custom_config_overrides(form)
+        persona_name = persona.name if persona else "Default"
+        if persona:
+            persona.ensure_dirs()
 
         from archivebox.config.permissions import HOSTNAME
 
@@ -1070,11 +1074,12 @@ class AddView(UserPassesTestMixin, FormView):
             "INDEX_ONLY": index_only,
             "DEPTH": depth,
             "PLUGINS": plugins or "",
-            "DEFAULT_PERSONA": (persona.name if persona else "Default"),
+            "DEFAULT_PERSONA": persona_name,
         }
 
         # Merge custom config overrides
         config.update(custom_config)
+        config["DEFAULT_PERSONA"] = persona_name
         if url_filters.get("allowlist"):
             config["URL_ALLOWLIST"] = url_filters["allowlist"]
         if url_filters.get("denylist"):
@@ -1090,6 +1095,7 @@ class AddView(UserPassesTestMixin, FormView):
             label=f"{created_by_name}@{HOSTNAME}{self.request.path} {timestamp}",
             created_by_id=created_by_id,
             config=config,
+            persona_id=persona.id if persona else None,
         )
 
         # 3. create a CrawlSchedule if schedule is provided
@@ -1321,46 +1327,11 @@ def live_progress_view(request):
         archiveresults_succeeded = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.SUCCEEDED).count()
         archiveresults_failed = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.FAILED).count()
 
-        # Get recently completed ArchiveResults with thumbnails (last 20 succeeded results)
-        recent_thumbnails = []
-        recent_results = (
-            ArchiveResult.objects.filter(
-                status=ArchiveResult.StatusChoices.SUCCEEDED,
-            )
-            .select_related("snapshot")
-            .order_by("-end_ts")[:20]
-        )
-
-        for ar in recent_results:
-            embed = ar.embed_path()
-            if embed:
-                # Only include results with embeddable image/media files
-                ext = embed.lower().split(".")[-1] if "." in embed else ""
-                is_embeddable = ext in ("png", "jpg", "jpeg", "gif", "webp", "svg", "ico", "pdf", "html")
-                if is_embeddable or ar.plugin in ("screenshot", "favicon", "dom"):
-                    archive_path = embed or ""
-                    recent_thumbnails.append(
-                        {
-                            "id": str(ar.id),
-                            "plugin": ar.plugin,
-                            "snapshot_id": str(ar.snapshot_id),
-                            "snapshot_url": ar.snapshot.url[:60] if ar.snapshot else "",
-                            "embed_path": embed,
-                            "archive_path": archive_path,
-                            "archive_url": build_snapshot_url(str(ar.snapshot_id), archive_path, request=request) if archive_path else "",
-                            "end_ts": ar.end_ts.isoformat() if ar.end_ts else None,
-                        },
-                    )
-
         # Build hierarchical active crawls with nested snapshots and archive results
 
         active_crawls_qs = (
             Crawl.objects.filter(status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED])
-            .prefetch_related(
-                "snapshot_set",
-                "snapshot_set__archiveresult_set",
-                "snapshot_set__archiveresult_set__process",
-            )
+            .prefetch_related("snapshot_set")
             .distinct()
             .order_by("-modified_at")[:10]
         )
@@ -1386,19 +1357,23 @@ def live_progress_view(request):
         process_records_by_crawl: dict[str, list[tuple[dict[str, object], object | None]]] = {}
         process_records_by_snapshot: dict[str, list[tuple[dict[str, object], object | None]]] = {}
         seen_process_records: set[str] = set()
-        snapshots = [snapshot for crawl in active_crawls_qs for snapshot in crawl.snapshot_set.all()]
+        active_snapshot_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED}
+        snapshots = [
+            snapshot for crawl in active_crawls_qs for snapshot in crawl.snapshot_set.all() if snapshot.status in active_snapshot_statuses
+        ]
+        snapshots_by_id = {str(snapshot.id): snapshot for snapshot in snapshots}
+
+        def find_snapshot_for_process(proc_pwd: Path) -> Snapshot | None:
+            for path_part in reversed(proc_pwd.parts):
+                snapshot = snapshots_by_id.get(path_part)
+                if snapshot:
+                    return snapshot
+            return None
+
         for proc in running_processes:
             if not proc.pwd:
                 continue
-            proc_pwd = Path(proc.pwd)
-            matched_snapshot = None
-            for snapshot in snapshots:
-                try:
-                    proc_pwd.relative_to(snapshot.output_dir)
-                    matched_snapshot = snapshot
-                    break
-                except ValueError:
-                    continue
+            matched_snapshot = find_snapshot_for_process(Path(proc.pwd))
             if matched_snapshot is None:
                 continue
             crawl_id = str(matched_snapshot.crawl_id)
@@ -1412,15 +1387,7 @@ def live_progress_view(request):
         for proc in recent_processes:
             if not proc.pwd:
                 continue
-            proc_pwd = Path(proc.pwd)
-            matched_snapshot = None
-            for snapshot in snapshots:
-                try:
-                    proc_pwd.relative_to(snapshot.output_dir)
-                    matched_snapshot = snapshot
-                    break
-                except ValueError:
-                    continue
+            matched_snapshot = find_snapshot_for_process(Path(proc.pwd))
             if matched_snapshot is None:
                 continue
             crawl_id = str(matched_snapshot.crawl_id)
@@ -1470,9 +1437,7 @@ def live_progress_view(request):
             pending_snapshots = sum(1 for s in all_crawl_snapshots if s.status == Snapshot.StatusChoices.QUEUED)
 
             # Get only ACTIVE snapshots to display (limit to 5 most recent)
-            active_crawl_snapshots = [
-                s for s in all_crawl_snapshots if s.status in [Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED]
-            ][:5]
+            active_crawl_snapshots = [s for s in all_crawl_snapshots if s.status in active_snapshot_statuses][:5]
 
             # Count URLs in the crawl (for when snapshots haven't been created yet)
             urls_count = 0
@@ -1497,9 +1462,13 @@ def live_progress_view(request):
             active_snapshots_for_crawl = []
             for snapshot in active_crawl_snapshots:
                 snapshot_run_started_at = snapshot.downloaded_at or snapshot.created_at
-                # Get archive results for this snapshot (already prefetched)
+                # Get archive results only for displayed active snapshots. Large crawls can
+                # contain thousands of sealed snapshots, and prefetching all their results
+                # makes the progress endpoint compete with the runner.
                 snapshot_results = [
-                    ar for ar in snapshot.archiveresult_set.all() if archiveresult_matches_current_run(ar, snapshot_run_started_at)
+                    ar
+                    for ar in snapshot.archiveresult_set.select_related("process").all()
+                    if archiveresult_matches_current_run(ar, snapshot_run_started_at)
                 ]
 
                 now = timezone.now()
@@ -1653,7 +1622,7 @@ def live_progress_view(request):
                 "archiveresults_succeeded": archiveresults_succeeded,
                 "archiveresults_failed": archiveresults_failed,
                 "active_crawls": active_crawls,
-                "recent_thumbnails": recent_thumbnails,
+                "recent_thumbnails": [],
                 "server_time": timezone.now().isoformat(),
             },
         )
diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py
index d732e4cf..aa387f94 100755
--- a/archivebox/machine/models.py
+++ b/archivebox/machine/models.py
@@ -128,6 +128,21 @@ def _sanitize_machine_config(config: dict[str, Any] | None) -> dict[str, Any]:
     sanitized = {key: value for key, value in config.items() if key in MACHINE_CONFIG_ALWAYS_ALLOWED_KEYS or str(key).endswith("_BINARY")}
     for key in LEGACY_MACHINE_CONFIG_KEYS:
         sanitized.pop(key, None)
+    for key, value in list(sanitized.items()):
+        if not str(key).endswith("_BINARY"):
+            continue
+        if not isinstance(value, str):
+            continue
+        value = value.strip()
+        if not value:
+            sanitized.pop(key, None)
+            continue
+        if "/" in value or value.startswith("~"):
+            try:
+                if not Path(value).expanduser().exists():
+                    sanitized.pop(key, None)
+            except OSError:
+                sanitized.pop(key, None)
     return sanitized
 
 
@@ -2380,6 +2395,11 @@ class Process(models.Model):
 
         for proc in running_children:
             if not proc.is_running:
+                proc.status = cls.StatusChoices.EXITED
+                proc.ended_at = proc.ended_at or timezone.now()
+                proc.exit_code = proc.exit_code if proc.exit_code is not None else 0
+                proc.save(update_fields=["status", "ended_at", "exit_code"])
+                cleaned += 1
                 continue
 
             root = proc.root
diff --git a/archivebox/search/__init__.py b/archivebox/search/__init__.py
index a30f0b16..6d89b66a 100644
--- a/archivebox/search/__init__.py
+++ b/archivebox/search/__init__.py
@@ -34,10 +34,14 @@ SEARCH_MODES = ("meta", "contents", "deep")
 def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
     """Expose ArchiveBox collection roots to in-process search backends."""
     config = config or get_config(**config_kwargs)
-    updates = {
-        "DATA_DIR": str(config.DATA_DIR),
-        "SNAP_DIR": str(config.USERS_DIR),
-    }
+    updates = {}
+    for key, value in config.items():
+        if value is None:
+            continue
+        if isinstance(value, (str, int, float, bool, os.PathLike)):
+            updates[str(key)] = str(value)
+    updates["DATA_DIR"] = str(config.DATA_DIR)
+    updates["SNAP_DIR"] = str(config.USERS_DIR)
     previous = {key: os.environ.get(key) for key in updates}
     os.environ.update(updates)
     try:
diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py
index 649687c7..1c8fa1cc 100644
--- a/archivebox/services/runner.py
+++ b/archivebox/services/runner.py
@@ -137,6 +137,12 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
 
     from archivebox.config import CONSTANTS
     from archivebox.machine.models import Machine, Process
+    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+
+    supervisor = get_existing_supervisord_process()
+    runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
+    if runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"):
+        return False
 
     Process.cleanup_stale_running()
     Process.cleanup_orphaned_workers()
@@ -1024,12 +1030,17 @@ def recover_orphaned_snapshots() -> int:
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.machine.models import Process
+    from django.db.models import Q
 
     active_snapshot_ids: set[str] = set()
     orphaned_snapshots = list(
-        Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED, retry_at__isnull=True)
+        Snapshot.objects.filter(
+            Q(status=Snapshot.StatusChoices.STARTED, retry_at__isnull=True)
+            | Q(status=Snapshot.StatusChoices.SEALED, archiveresult__status=ArchiveResult.StatusChoices.QUEUED),
+        )
         .select_related("crawl")
-        .prefetch_related("archiveresult_set"),
+        .prefetch_related("archiveresult_set")
+        .distinct(),
     )
     running_processes = Process.objects.filter(
         status=Process.StatusChoices.RUNNING,
@@ -1157,6 +1168,12 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) ->
                 .first()
             )
             if binary is not None:
+                binary_name = str(binary.name or "")
+                binary_path = Path(binary_name).expanduser()
+                if (binary_path.is_absolute() or binary_name.startswith("~")) and not binary_path.exists():
+                    binary.retry_at = None
+                    binary.save(update_fields=["retry_at", "modified_at"])
+                    continue
                 if not binary.claim_processing_lock(lock_seconds=60):
                     continue
                 run_binary(str(binary.id))
diff --git a/archivebox/templates/admin/base.html b/archivebox/templates/admin/base.html
index 8654c2cd..be7a4503 100644
--- a/archivebox/templates/admin/base.html
+++ b/archivebox/templates/admin/base.html
@@ -1684,7 +1684,7 @@
                 const buttons = $('
')
                     .insertAfter('div.actions button[type=submit]')
                     .css('display', 'inline')
-                    .addClass('class', 'action-buttons');
+                    .addClass('action-buttons');
 
                 // for each action in the dropdown, turn it into a button instead
                 container.find('select[name=action] option:gt(0)').each(function () {
@@ -1715,10 +1715,10 @@
                                 if (!window.confirm(message)) return false
                             }
 
-                            // select the action button from the dropdown
+                            // select the action from the original Django admin dropdown
                             container.find('select[name=action]')
-                                .find('[selected]').removeAttr('selected').end()
-                                .find('[value=' + action_type + ']').attr('selected', 'selected').click()
+                                .val(action_type)
+                                .trigger('change')
                             
                             // click submit & replace the archivebox logo with a spinner
                             $('#changelist-form button[name="index"]').click()
diff --git a/archivebox/templates/static/admin.css b/archivebox/templates/static/admin.css
index ca0cfffc..408f178f 100755
--- a/archivebox/templates/static/admin.css
+++ b/archivebox/templates/static/admin.css
@@ -145,6 +145,26 @@ body.change-list #content .object-tools {
     margin-bottom: 10px;
     border-top: 0px; 
     border-bottom: 0px;
+    display: inline-flex;
+    width: auto;
+    max-width: 100%;
+    box-sizing: border-box;
+    position: relative;
+    z-index: 1;
+}
+
+#content #changelist #toolbar #changelist-search,
+#content #changelist #toolbar #changelist-search > div {
+    width: auto;
+    max-width: 100%;
+    flex: 0 1 auto;
+    box-sizing: border-box;
+}
+
+#content #changelist #toolbar #searchbar {
+    width: clamp(180px, 32vw, 420px);
+    max-width: 100%;
+    flex: 0 1 auto;
 }
 
 #content #changelist #toolbar form input[type="submit"] {
@@ -170,6 +190,8 @@ body.change-list #content .object-tools {
     width: auto;
     max-height: 40px;
     display: block;
+    position: relative;
+    z-index: 2;
 }
 @media (max-width: 1000px) {
     #content #changelist .actions {
diff --git a/archivebox/tests/test_add_view.py b/archivebox/tests/test_add_view.py
index 839044a0..d0793af7 100644
--- a/archivebox/tests/test_add_view.py
+++ b/archivebox/tests/test_add_view.py
@@ -4,8 +4,10 @@ import pytest
 from django.contrib.auth import get_user_model
 from django.urls import reverse
 
+from archivebox.config.common import get_config
 from archivebox.core.models import Tag
 from archivebox.crawls.models import Crawl
+from archivebox.personas.models import Persona
 
 
 pytestmark = pytest.mark.django_db
@@ -105,6 +107,45 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
     assert "ONLY_NEW" not in crawl.config
 
 
+def test_add_view_selected_persona_wins_over_stale_config_override(client, admin_user, monkeypatch):
+    monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
+    client.force_login(admin_user)
+    private_persona = Persona.objects.create(name="Private", created_by=admin_user)
+    private_persona.ensure_dirs()
+    private_cookies_file = private_persona.path / "cookies.txt"
+    private_cookies_file.write_text("# Private cookies\n", encoding="utf-8")
+
+    response = client.post(
+        reverse("add"),
+        data={
+            "url": "https://example.com/private",
+            "tag": "",
+            "depth": "0",
+            "max_urls": "0",
+            "max_size": "0",
+            "url_filters_allowlist": "",
+            "url_filters_denylist": "",
+            "notes": "",
+            "schedule": "",
+            "persona": "Private",
+            "index_only": "",
+            "config": '{"DEFAULT_PERSONA": "Default"}',
+        },
+        HTTP_HOST=WEB_HOST,
+    )
+
+    assert response.status_code == 302
+
+    crawl = Crawl.objects.order_by("-created_at").first()
+    assert crawl is not None
+    assert crawl.persona_id == private_persona.id
+    assert crawl.config.get("DEFAULT_PERSONA") == "Private"
+    assert crawl.resolve_persona() == private_persona
+    runtime_config = get_config(crawl=crawl)
+    assert runtime_config.ACTIVE_PERSONA == "Private"
+    assert runtime_config.COOKIES_FILE == private_cookies_file
+
+
 def test_add_view_starts_background_runner_after_creating_crawl(client, admin_user, monkeypatch):
     monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
     client.force_login(admin_user)
diff --git a/archivebox/tests/test_admin_views.py b/archivebox/tests/test_admin_views.py
index 2b0cfd9c..d31d834d 100644
--- a/archivebox/tests/test_admin_views.py
+++ b/archivebox/tests/test_admin_views.py
@@ -987,6 +987,57 @@ class TestAdminSnapshotListView:
         assert response["Location"].endswith(f"/admin/core/snapshot/{snapshot.pk}/change/")
         assert snapshot.archiveresult_set.get(plugin="title").status == ArchiveResult.StatusChoices.QUEUED
 
+    def test_list_redo_failed_action_requeues_failed_archiveresults_only(self, client, admin_user, snapshot, monkeypatch):
+        import archivebox.core.admin_snapshots as admin_snapshots
+        from archivebox.core.models import ArchiveResult
+
+        def bg_archive_snapshots_should_not_run(*args, **kwargs):
+            raise AssertionError("Redo Failed should reset failed ArchiveResults directly")
+
+        monkeypatch.setattr(admin_snapshots, "bg_archive_snapshots", bg_archive_snapshots_should_not_run)
+
+        failed = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="wget",
+            hook_name="on_Snapshot__50_wget",
+            status=ArchiveResult.StatusChoices.FAILED,
+            output_str="boom",
+            output_files={"index.html": {"path": "index.html", "size": 123}},
+            output_size=123,
+            output_mimetypes="text/html",
+        )
+        succeeded = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="title",
+            hook_name="on_Snapshot__54_title",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+            output_str="Example Domain",
+        )
+
+        client.login(username="testadmin", password="testpassword")
+        response = client.post(
+            reverse("admin:core_snapshot_changelist"),
+            {
+                "action": "update_snapshots",
+                "_selected_action": [str(snapshot.pk)],
+                "index": "0",
+            },
+            HTTP_HOST=ADMIN_HOST,
+        )
+
+        assert response.status_code == 302
+        failed.refresh_from_db()
+        succeeded.refresh_from_db()
+        snapshot.refresh_from_db()
+        assert failed.status == ArchiveResult.StatusChoices.QUEUED
+        assert failed.output_str == ""
+        assert failed.output_files == {}
+        assert failed.output_size == 0
+        assert failed.output_mimetypes == ""
+        assert succeeded.status == ArchiveResult.StatusChoices.SUCCEEDED
+        assert succeeded.output_str == "Example Domain"
+        assert snapshot.status == snapshot.StatusChoices.QUEUED
+
     def test_archive_now_action_uses_original_snapshot_url_without_timestamp_suffix(self, client, admin_user, snapshot, monkeypatch):
         import archivebox.core.admin_snapshots as admin_snapshots
 
diff --git a/archivebox/tests/test_archive_result_service.py b/archivebox/tests/test_archive_result_service.py
index 5f8be141..e6ba3688 100644
--- a/archivebox/tests/test_archive_result_service.py
+++ b/archivebox/tests/test_archive_result_service.py
@@ -250,6 +250,54 @@ def test_retry_failed_archiveresults_requeues_snapshot_in_queued_state():
     _cleanup_machine_process_rows()
 
 
+def test_retry_failed_archiveresults_preserves_legacy_plugin_rows_without_hook_name():
+    from archivebox.core.models import ArchiveResult, Snapshot
+
+    snapshot = _create_snapshot()
+    legacy_result = ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="wget",
+        hook_name="",
+        status=ArchiveResult.StatusChoices.FAILED,
+        output_str="legacy failure",
+        output_files={"index.html": {"size": 123}},
+        output_size=123,
+        output_mimetypes="text/html",
+    )
+    hook_result = ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="wget",
+        hook_name="on_Snapshot__06_wget.finite.bg",
+        status=ArchiveResult.StatusChoices.FAILED,
+        output_str="hook failure",
+        output_files={"stderr.log": {}},
+        output_size=10,
+        output_mimetypes="text/plain",
+    )
+
+    reset_count = snapshot.retry_failed_archiveresults()
+
+    snapshot.refresh_from_db()
+    snapshot.crawl.refresh_from_db()
+    legacy_result.refresh_from_db()
+    hook_result.refresh_from_db()
+
+    assert reset_count == 2
+    assert snapshot.status == Snapshot.StatusChoices.QUEUED
+    assert snapshot.retry_at is not None
+    assert snapshot.crawl.status == snapshot.crawl.StatusChoices.QUEUED
+    assert snapshot.crawl.retry_at is not None
+    assert legacy_result.status == ArchiveResult.StatusChoices.FAILED
+    assert legacy_result.output_str == "legacy failure"
+    assert legacy_result.output_files == {"index.html": {"size": 123}}
+    assert legacy_result.output_size == 123
+    assert hook_result.status == ArchiveResult.StatusChoices.QUEUED
+    assert hook_result.output_str == ""
+    assert hook_result.output_files == {}
+    assert hook_result.output_size == 0
+    _cleanup_machine_process_rows()
+
+
 def test_process_completed_projects_snapshot_title_from_output_str():
     from archivebox.services.archive_result_service import ArchiveResultService
     import asyncio
diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py
index 1d539ea1..8bc9d8d2 100644
--- a/archivebox/tests/test_cli_run.py
+++ b/archivebox/tests/test_cli_run.py
@@ -427,3 +427,39 @@ class TestRecoverOrphanedSnapshots:
         assert snapshot.retry_at is not None
         assert crawl.status == Crawl.StatusChoices.QUEUED
         assert crawl.retry_at is not None
+
+    def test_recover_orphaned_snapshot_requeues_sealed_snapshot_with_queued_results(self):
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import ArchiveResult, Snapshot
+        from archivebox.services.runner import recover_orphaned_snapshots
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=None,
+        )
+        ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="title",
+            hook_name="on_Snapshot__01_title",
+            status=ArchiveResult.StatusChoices.QUEUED,
+        )
+
+        recovered = recover_orphaned_snapshots()
+
+        snapshot.refresh_from_db()
+        crawl.refresh_from_db()
+
+        assert recovered == 1
+        assert snapshot.status == Snapshot.StatusChoices.QUEUED
+        assert snapshot.retry_at is not None
+        assert crawl.status == Crawl.StatusChoices.QUEUED
+        assert crawl.retry_at is not None
diff --git a/archivebox/tests/test_cli_server.py b/archivebox/tests/test_cli_server.py
index f4b73c27..7faa51ea 100644
--- a/archivebox/tests/test_cli_server.py
+++ b/archivebox/tests/test_cli_server.py
@@ -6,6 +6,7 @@ Verify server can start (basic smoke tests only, no full server testing).
 
 import os
 import asyncio
+import builtins
 import json
 import subprocess
 import sys
@@ -18,6 +19,7 @@ def test_sqlite_connections_use_explicit_30_second_busy_timeout():
 
     assert SQLITE_CONNECTION_OPTIONS["OPTIONS"]["timeout"] == 30
     assert "PRAGMA busy_timeout = 30000;" in SQLITE_CONNECTION_OPTIONS["OPTIONS"]["init_command"]
+    assert "PRAGMA journal_mode = WAL;" in SQLITE_CONNECTION_OPTIONS["OPTIONS"]["init_command"]
 
 
 def test_server_shows_usage_info(tmp_path, process):
@@ -110,6 +112,21 @@ def test_start_server_workers_starts_plugin_owned_sonic_worker(monkeypatch):
     ]
 
 
+def test_missing_plugin_owned_sonic_worker_is_optional(monkeypatch):
+    from archivebox.workers.supervisord_util import get_sonic_supervisord_worker_from_plugin
+
+    original_import = builtins.__import__
+
+    def fake_import(name, *args, **kwargs):
+        if name == "abx_plugins.plugins.search_backend_sonic.daemon":
+            raise ModuleNotFoundError("No module named 'abx_plugins.plugins.search_backend_sonic.daemon'", name=name)
+        return original_import(name, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "__import__", fake_import)
+
+    assert get_sonic_supervisord_worker_from_plugin(SimpleNamespace()) is None
+
+
 def test_sonic_daemon_event_handler_requires_running_supervised_worker(monkeypatch):
     from abx_dl.events import ProcessStdoutEvent
     from abx_dl.orchestrator import create_bus
diff --git a/archivebox/tests/test_config.py b/archivebox/tests/test_config.py
index 58add203..eec85da2 100644
--- a/archivebox/tests/test_config.py
+++ b/archivebox/tests/test_config.py
@@ -130,6 +130,12 @@ SEARCH_BACKEND_PASSWORD = "SecretPassword"
     assert "Extra inputs are not permitted" not in result.stderr
 
 
+def test_sonic_dir_is_allowed_inside_data_dir():
+    from archivebox.config import CONSTANTS
+
+    assert "sonic" in CONSTANTS.ALLOWED_IN_DATA_DIR
+
+
 def test_config_set_requires_equals_sign(tmp_path, process):
     """Test that --set requires KEY=VALUE format."""
     os.chdir(tmp_path)
diff --git a/archivebox/tests/test_machine_models.py b/archivebox/tests/test_machine_models.py
index a91f3e82..33ee4229 100644
--- a/archivebox/tests/test_machine_models.py
+++ b/archivebox/tests/test_machine_models.py
@@ -133,35 +133,46 @@ class TestMachineModel(TestCase):
         """Machine.current() should keep derived cache entries, not runtime config."""
         import archivebox.machine.models as models
 
+        chrome_path = "/tmp/archivebox-test-chromium"
+        node_path = "/tmp/archivebox-test-node"
+        open(chrome_path, "a").close()
+        open(node_path, "a").close()
+        self.addCleanup(lambda: os.path.exists(chrome_path) and os.remove(chrome_path))
+        self.addCleanup(lambda: os.path.exists(node_path) and os.remove(node_path))
         machine = Machine.current()
         machine.config = {
-            "CHROME_BINARY": "/tmp/chromium",
-            "NODE_BINARY": "/tmp/node",
+            "CHROME_BINARY": chrome_path,
+            "NODE_BINARY": node_path,
             "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
             "CHROME_ISOLATION": "snapshot",
             "CHROME_USER_DATA_DIR": "/tmp/profile",
             "CHROMIUM_VERSION": "123.4.5",
+            "WGET_BINARY": "/tmp/archivebox-test-missing-wget",
         }
         machine.save(update_fields=["config"])
         models._CURRENT_MACHINE = machine
 
         refreshed = Machine.current()
 
-        self.assertEqual(refreshed.config.get("CHROME_BINARY"), "/tmp/chromium")
-        self.assertEqual(refreshed.config.get("NODE_BINARY"), "/tmp/node")
+        self.assertEqual(refreshed.config.get("CHROME_BINARY"), chrome_path)
+        self.assertEqual(refreshed.config.get("NODE_BINARY"), node_path)
         self.assertEqual(refreshed.config.get("ABX_INSTALL_CACHE"), {"wget": "2026-03-24T00:00:00+00:00"})
         self.assertNotIn("CHROME_ISOLATION", refreshed.config)
         self.assertNotIn("CHROME_USER_DATA_DIR", refreshed.config)
         self.assertNotIn("CHROMIUM_VERSION", refreshed.config)
+        self.assertNotIn("WGET_BINARY", refreshed.config)
 
     def test_get_config_auto_applies_current_machine_config(self):
         """get_config() should include sanitized Machine.current() config by default."""
         import archivebox.machine.models as models
         from archivebox.config.common import get_config
 
+        chrome_path = "/tmp/archivebox-test-chromium"
+        open(chrome_path, "a").close()
+        self.addCleanup(lambda: os.path.exists(chrome_path) and os.remove(chrome_path))
         machine = Machine.current()
         machine.config = {
-            "CHROME_BINARY": "/tmp/chromium",
+            "CHROME_BINARY": chrome_path,
             "ABX_INSTALL_CACHE": {"chrome": "2026-03-24T00:00:00+00:00"},
             "CHROME_ISOLATION": "snapshot",
         }
@@ -170,7 +181,7 @@ class TestMachineModel(TestCase):
 
         config = get_config()
 
-        self.assertEqual(config.CHROME_BINARY, "/tmp/chromium")
+        self.assertEqual(config.CHROME_BINARY, chrome_path)
         self.assertEqual(config["ABX_INSTALL_CACHE"], {"chrome": "2026-03-24T00:00:00+00:00"})
         self.assertEqual(config.CHROME_ISOLATION, "crawl")
 
@@ -822,6 +833,24 @@ class TestProcessClassMethods(TestCase):
         kill_tree.assert_not_called()
         terminate.assert_not_called()
 
+    def test_cleanup_orphaned_workers_marks_non_running_children_exited(self):
+        """cleanup_orphaned_workers should retire child rows whose OS process is already gone."""
+        child = Process.objects.create(
+            machine=self.machine,
+            process_type=Process.TypeChoices.HOOK,
+            status=Process.StatusChoices.RUNNING,
+            pid=999997,
+            started_at=timezone.now() - timedelta(minutes=5),
+        )
+
+        cleaned = Process.cleanup_orphaned_workers()
+
+        self.assertEqual(cleaned, 1)
+        child.refresh_from_db()
+        self.assertEqual(child.status, Process.StatusChoices.EXITED)
+        self.assertIsNotNone(child.ended_at)
+        self.assertEqual(child.exit_code, 0)
+
 
 class TestProcessStateMachine(TestCase):
     """Test the ProcessMachine state machine."""
diff --git a/archivebox/tests/test_runner.py b/archivebox/tests/test_runner.py
index c42bd683..44f373c9 100644
--- a/archivebox/tests/test_runner.py
+++ b/archivebox/tests/test_runner.py
@@ -302,6 +302,7 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
 
 def test_ensure_background_runner_starts_when_none_running(monkeypatch):
     import archivebox.machine.models as machine_models
+    import archivebox.workers.supervisord_util as supervisord_util
     from archivebox.services import runner as runner_module
 
     popen_calls = []
@@ -310,6 +311,7 @@ def test_ensure_background_runner_starts_when_none_running(monkeypatch):
         def __init__(self, args, **kwargs):
             popen_calls.append((args, kwargs))
 
+    monkeypatch.setattr(supervisord_util, "get_existing_supervisord_process", lambda: None)
     monkeypatch.setattr(machine_models.Process, "cleanup_stale_running", classmethod(lambda cls, machine=None: 0))
     monkeypatch.setattr(machine_models.Process, "cleanup_orphaned_workers", classmethod(lambda cls: 0))
     monkeypatch.setattr(machine_models.Machine, "current", classmethod(lambda cls: SimpleNamespace(id="machine-1")))
@@ -330,8 +332,10 @@ def test_ensure_background_runner_starts_when_none_running(monkeypatch):
 
 def test_ensure_background_runner_skips_when_orchestrator_running(monkeypatch):
     import archivebox.machine.models as machine_models
+    import archivebox.workers.supervisord_util as supervisord_util
     from archivebox.services import runner as runner_module
 
+    monkeypatch.setattr(supervisord_util, "get_existing_supervisord_process", lambda: None)
     monkeypatch.setattr(machine_models.Process, "cleanup_stale_running", classmethod(lambda cls, machine=None: 0))
     monkeypatch.setattr(machine_models.Process, "cleanup_orphaned_workers", classmethod(lambda cls: 0))
     monkeypatch.setattr(machine_models.Machine, "current", classmethod(lambda cls: SimpleNamespace(id="machine-1")))
@@ -351,6 +355,31 @@ def test_ensure_background_runner_skips_when_orchestrator_running(monkeypatch):
     assert started is False
 
 
+def test_ensure_background_runner_skips_when_supervisord_runner_running(monkeypatch):
+    import archivebox.machine.models as machine_models
+    import archivebox.workers.supervisord_util as supervisord_util
+    from archivebox.services import runner as runner_module
+
+    supervisor = object()
+
+    monkeypatch.setattr(supervisord_util, "get_existing_supervisord_process", lambda: supervisor)
+    monkeypatch.setattr(supervisord_util, "get_worker", lambda supervisor_arg, name: {"statename": "RUNNING"})
+    monkeypatch.setattr(
+        machine_models.Process,
+        "cleanup_stale_running",
+        classmethod(lambda cls, machine=None: (_ for _ in ()).throw(AssertionError("db process cleanup should not run"))),
+    )
+    monkeypatch.setattr(
+        runner_module.subprocess,
+        "Popen",
+        lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("runner should not be spawned")),
+    )
+
+    started = runner_module.ensure_background_runner(allow_under_pytest=True)
+
+    assert started is False
+
+
 def test_runner_task_context_clears_inherited_abxbus_handler_context(tmp_path):
     from abx_dl.events import CrawlEvent, MachineEvent
     from abx_dl.orchestrator import create_bus
@@ -395,7 +424,7 @@ def test_runner_task_context_clears_inherited_abxbus_handler_context(tmp_path):
 
 
 @pytest.mark.django_db(transaction=True)
-def test_machine_service_persists_only_derived_config_events():
+def test_machine_service_persists_only_derived_config_events(tmp_path):
     from abx_dl.events import MachineEvent
     from abx_dl.orchestrator import create_bus
     from archivebox.machine.models import Machine
@@ -404,6 +433,9 @@ def test_machine_service_persists_only_derived_config_events():
     machine = Machine.current()
     machine.config = {}
     machine.save(update_fields=["config"])
+    wget_binary = tmp_path / "wget"
+    wget_binary.write_text("#!/bin/sh\n")
+    wget_binary.chmod(0o755)
 
     async def run_test():
         bus = create_bus(name="test_machine_service_persists_only_derived_config_events")
@@ -424,7 +456,7 @@ def test_machine_service_persists_only_derived_config_events():
             derived_event = bus.emit(
                 MachineEvent(
                     config={
-                        "WGET_BINARY": "/tmp/wget",
+                        "WGET_BINARY": str(wget_binary),
                         "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
                         "CHROME_USER_DATA_DIR": "/tmp/stale-derived-profile",
                     },
@@ -441,7 +473,7 @@ def test_machine_service_persists_only_derived_config_events():
 
     machine.refresh_from_db()
     assert machine.config == {
-        "WGET_BINARY": "/tmp/wget",
+        "WGET_BINARY": str(wget_binary),
         "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
     }
 
@@ -508,13 +540,16 @@ def test_runner_prepare_refreshes_network_interface_and_attaches_current_process
     assert saved_updates == [("iface", "machine", "modified_at")]
 
 
-def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
+def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch, tmp_path):
     from archivebox.machine.models import Machine, NetworkInterface, Process
     from archivebox.services import runner as runner_module
     from archivebox.config import common as config_common
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
 
+    wget_binary = tmp_path / "wget"
+    wget_binary.write_text("#!/bin/sh\n")
+    wget_binary.chmod(0o755)
     machine = Machine.objects.create(
         guid="test-guid-runner-overrides",
         hostname="runner-host",
@@ -530,7 +565,7 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
         os_kernel="Darwin",
         stats={},
         config={
-            "WGET_BINARY": "/tmp/wget",
+            "WGET_BINARY": str(wget_binary),
             "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
             "CHROME_ISOLATION": "snapshot",
             "CHROME_USER_DATA_DIR": "/tmp/stale-profile",
@@ -560,7 +595,7 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
     crawl_runner.load_run_state()
 
     assert crawl_runner.derived_config == {
-        "WGET_BINARY": "/tmp/wget",
+        "WGET_BINARY": str(wget_binary),
         "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
     }
 
@@ -1239,6 +1274,34 @@ def test_run_pending_crawls_prioritizes_queued_crawl_before_unrelated_binary_bac
     assert binary_calls == []
 
 
+def test_run_pending_crawls_disables_missing_absolute_binary_backlog(monkeypatch, tmp_path):
+    from archivebox.machine.models import Binary, Machine
+    from archivebox.services import runner as runner_module
+
+    missing_binary = tmp_path / "missing-node"
+    binary = Binary.objects.create(
+        machine=Machine.current(),
+        name=str(missing_binary),
+        status=Binary.StatusChoices.QUEUED,
+        retry_at=runner_module.timezone.now(),
+        binproviders="env,apt",
+        overrides={"apt": {"install_args": ["nodejs"]}},
+    )
+
+    monkeypatch.setattr(
+        runner_module,
+        "run_binary",
+        lambda binary_id: (_ for _ in ()).throw(AssertionError("missing absolute binary should not be retried")),
+    )
+
+    result = runner_module.run_pending_crawls(daemon=False)
+
+    binary.refresh_from_db()
+    assert result == 0
+    assert binary.status == Binary.StatusChoices.QUEUED
+    assert binary.retry_at is None
+
+
 @pytest.mark.django_db(transaction=True)
 def test_crawl_completed_event_requeues_active_snapshots():
     from archivebox.base_models.models import get_or_create_system_user_pk
diff --git a/archivebox/tests/test_search.py b/archivebox/tests/test_search.py
new file mode 100644
index 00000000..18805e77
--- /dev/null
+++ b/archivebox/tests/test_search.py
@@ -0,0 +1,34 @@
+import os
+from pathlib import Path
+
+from benedict import benedict
+
+
+def test_search_backend_env_exposes_resolved_runtime_config(monkeypatch, tmp_path):
+    from archivebox.search import search_backend_env
+
+    monkeypatch.setenv("SEARCH_BACKEND_SONIC_HOST_NAME", "old-host")
+    config = benedict(
+        {
+            "DATA_DIR": tmp_path,
+            "USERS_DIR": tmp_path / "archive" / "users",
+            "SEARCH_BACKEND_ENGINE": "sonic",
+            "SEARCH_BACKEND_SONIC_HOST_NAME": "sonic",
+            "SEARCH_BACKEND_SONIC_PORT": 1491,
+            "SEARCH_BACKEND_SONIC_PASSWORD": "SecretPassword",
+            "USE_INDEXING_BACKEND": True,
+            "IGNORED_NONE_VALUE": None,
+        },
+    )
+
+    with search_backend_env(config=config):
+        assert os.environ["DATA_DIR"] == str(tmp_path)
+        assert os.environ["SNAP_DIR"] == str(Path(tmp_path) / "archive" / "users")
+        assert os.environ["SEARCH_BACKEND_ENGINE"] == "sonic"
+        assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "sonic"
+        assert os.environ["SEARCH_BACKEND_SONIC_PORT"] == "1491"
+        assert os.environ["SEARCH_BACKEND_SONIC_PASSWORD"] == "SecretPassword"
+        assert os.environ["USE_INDEXING_BACKEND"] == "True"
+        assert "IGNORED_NONE_VALUE" not in os.environ
+
+    assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "old-host"
diff --git a/archivebox/tests/test_urls.py b/archivebox/tests/test_urls.py
index c1f76c0a..ecac4556 100644
--- a/archivebox/tests/test_urls.py
+++ b/archivebox/tests/test_urls.py
@@ -235,10 +235,36 @@ class TestUrlRouting:
             assert resp.status_code in (301, 302)
             assert resp["Location"].startswith("/api/")
 
+            resp = client.get("/api/archive/https://example.com/", HTTP_HOST=api_host)
+            assert resp.status_code in (301, 302)
+            assert resp["Location"] == f"http://{web_host}/web/https://example.com/"
+
             print("OK")
             """,
         )
 
+    def test_api_archive_redirect_uses_public_web_base_url(self) -> None:
+        self._run(
+            """
+            client = Client()
+
+            resp = client.get(
+                "/api/archive/https://example.com/",
+                HTTP_HOST="api.archivebox.io",
+                secure=True,
+            )
+
+            assert resp.status_code in (301, 302)
+            assert resp["Location"] == "https://web.archivebox.io/web/https://example.com/"
+
+            print("OK")
+            """,
+            mode="safe-subdomains-fullreplay",
+            env_overrides={
+                "LISTEN_HOST": "archivebox.io",
+            },
+        )
+
     def test_web_admin_routing(self) -> None:
         self._run(
             """
diff --git a/archivebox/workers/supervisord_util.py b/archivebox/workers/supervisord_util.py
index 56b8a55d..83ecbbdc 100644
--- a/archivebox/workers/supervisord_util.py
+++ b/archivebox/workers/supervisord_util.py
@@ -577,7 +577,12 @@ def watch_worker(supervisor, daemon_name, interval=5):
 
 
 def get_sonic_supervisord_worker_from_plugin(config) -> dict[str, str] | None:
-    from abx_plugins.plugins.search_backend_sonic.daemon import get_sonic_supervisord_worker
+    try:
+        from abx_plugins.plugins.search_backend_sonic.daemon import get_sonic_supervisord_worker
+    except ModuleNotFoundError as err:
+        if err.name != "abx_plugins.plugins.search_backend_sonic.daemon":
+            raise
+        return None
 
     worker = get_sonic_supervisord_worker(config)
     return cast(dict[str, str] | None, worker)
diff --git a/pyproject.toml b/pyproject.toml
index 0422bc55..c58886d6 100755
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.30rc57"
+version = "0.9.30rc81"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -80,8 +80,8 @@ dependencies = [
     ### Binary/Package Management
     "abxbus>=2.5.4",          # EventBus API
     "abxpkg>=1.10.7",         # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins>=1.10.56",   # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl>=1.10.51",        # shared ArchiveBox downloader package with blocking install preflight
+    "abx-plugins>=1.10.57",   # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl>=1.10.56",        # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]