Prepare ArchiveBox 0.9.30rc81 demo release

.dockerignore

@@ -51,6 +51,7 @@ website/
 typings/
 
 tmp/
+.tmp/
 data/
 data*/
 -

Dockerfile

@@ -53,6 +53,8 @@ ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
+ARG ABX_PLUGINS_REF=f576a7c81a17b0e49adf4789ac7f7143ece1282f
+ARG ABX_DL_REF=3cddb1bf4d8a37c49f2a70810e0da840acc96c17
 ######### Environment Variables #################################
 
 # Global build-time and runtime environment constants + default pkg manager config
@@ -348,8 +350,20 @@ RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
     && rm -rf /var/lib/apt/lists/*
     # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
 
+# Install unreleased dev abx package revisions used by the ArchiveBox dev branch.
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing dev abx package revisions..." \
+    && uv pip install --reinstall --no-deps \
+        "git+https://github.com/ArchiveBox/abx-plugins.git@${ABX_PLUGINS_REF}" \
+        "git+https://github.com/ArchiveBox/abx-dl.git@${ABX_DL_REF}" \
+    && ( \
+        pip show abx-plugins \
+        && pip show abx-dl \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
 # Install ArchiveBox Python package from the checked-out source.
-# Sibling abx-* packages are installed from PyPI inside the container, not copied from the local checkout.
+# Sibling abx-* packages are installed from pinned upstream refs inside the container.
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
 RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
     echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \

archivebox/api/urls.py

@@ -1,12 +1,26 @@
 __package__ = "archivebox.api"
 
+from urllib.parse import quote
+
+from django.http import HttpRequest, HttpResponseRedirect
+from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView
 
+from archivebox.core.host_utils import build_web_url
+
 from .v1_api import urls as v1_api_urls
 
+
+def archive_redirect_view(request: HttpRequest, url: str) -> HttpResponseRedirect:
+    if request.META.get("QUERY_STRING"):
+        url = f"{url}?{request.META['QUERY_STRING']}"
+    return redirect(build_web_url(f"/web/{quote(url, safe=':/')}", request=request))
+
+
 urlpatterns = [
     path("", RedirectView.as_view(url="/api/v1/docs")),
+    path("archive/<path:url>", archive_redirect_view, name="api-archive-redirect"),
     path("v1/", RedirectView.as_view(url="/api/v1/docs")),
     path("v1/", v1_api_urls),
     path("v1", RedirectView.as_view(url="/api/v1/docs")),

archivebox/api/v1_api.py

@@ -44,6 +44,7 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
     api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
     api.add_router("/cli/", "archivebox.api.v1_cli.router")
     api.add_router("/machine/", "archivebox.api.v1_machine.router")
+    api.add_router("/personas/", "archivebox.api.v1_personas.router")
     return api
 
 

archivebox/api/v1_personas.py

@@ -0,0 +1,157 @@
+__package__ = "archivebox.api"
+
+import json
+from datetime import datetime
+from typing import Any
+from uuid import UUID
+
+from django.db.models import Q
+from django.http import HttpRequest
+from ninja import Router, Schema
+from pydantic import Field
+
+from archivebox.personas.importers import validate_persona_name
+from archivebox.personas.models import Persona
+
+
+router = Router(tags=["Personas"])
+
+
+class PersonaBrowserSettingsSchema(Schema):
+    user_agent: str = ""
+    viewport_size: str = ""
+    viewport_device_scale_factor: float | None = None
+    language: str = ""
+    timezone: str = ""
+    geolocation: dict[str, Any] | None = None
+
+
+class PersonaSyncSchema(Schema):
+    extension_persona_id: str
+    name: str
+    settings: PersonaBrowserSettingsSchema = Field(default_factory=PersonaBrowserSettingsSchema)
+    cookies_txt: str = ""
+    auth_json: dict[str, Any] = Field(default_factory=dict)
+
+
+class PersonaSchema(Schema):
+    TYPE: str = "personas.models.Persona"
+    id: UUID
+    name: str
+    created_at: datetime
+    created_by_id: str
+    created_by_username: str
+    config: dict[str, Any] | None
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by.pk)
+
+    @staticmethod
+    def resolve_created_by_username(obj) -> str:
+        return obj.created_by.username
+
+
+class PersonaSyncResponseSchema(Schema):
+    success: bool
+    created: bool
+    persona: PersonaSchema
+    cookies_file_written: bool
+    auth_file_written: bool
+
+
+def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrowserSettingsSchema) -> dict[str, Any]:
+    config: dict[str, Any] = {
+        "BROWSER_EXTENSION_PERSONA_ID": extension_persona_id,
+        "BROWSER_EXTENSION_SYNCED_AT": datetime.utcnow().isoformat() + "Z",
+    }
+
+    if settings.user_agent:
+        config.update(
+            {
+                "USER_AGENT": settings.user_agent,
+                "CHROME_USER_AGENT": settings.user_agent,
+                "WGET_USER_AGENT": settings.user_agent,
+                "CURL_USER_AGENT": settings.user_agent,
+            },
+        )
+    if settings.viewport_size:
+        config.update(
+            {
+                "RESOLUTION": settings.viewport_size,
+                "CHROME_RESOLUTION": settings.viewport_size,
+            },
+        )
+    if settings.viewport_device_scale_factor is not None:
+        config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
+    if settings.language:
+        config["BROWSER_LANGUAGE"] = settings.language
+    if settings.timezone:
+        config["BROWSER_TIMEZONE"] = settings.timezone
+    if settings.geolocation:
+        config["BROWSER_GEOLOCATION"] = settings.geolocation
+
+    return config
+
+
+def find_persona(extension_persona_id: str, name: str) -> Persona | None:
+    return (
+        Persona.objects.filter(
+            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
+        )
+        .order_by("created_at")
+        .first()
+    )
+
+
+@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
+def get_personas(request: HttpRequest):
+    """List personas available on this ArchiveBox server."""
+    return Persona.objects.all().order_by("name")
+
+
+@router.post("/sync", response=PersonaSyncResponseSchema, url_name="sync_persona")
+def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
+    """
+    Create or update a Persona from a browser extension profile export.
+
+    The extension sends browser settings plus portable auth artifacts. The server
+    keeps browser override settings in Persona.config and writes cookies.txt /
+    auth.json into the persona directory for extractors to consume.
+    """
+    name = payload.name.strip()
+    is_valid, error_message = validate_persona_name(name)
+    if not is_valid:
+        raise ValueError(error_message)
+
+    persona = find_persona(payload.extension_persona_id, name)
+    created = persona is None
+    if persona is None:
+        persona = Persona(name=name)
+        if getattr(request.user, "is_authenticated", False):
+            persona.created_by = request.user
+
+    persona.config = {
+        **(persona.config or {}),
+        **browser_settings_to_config(payload.extension_persona_id, payload.settings),
+    }
+    persona.save()
+    persona.ensure_dirs()
+
+    cookies_written = False
+    if payload.cookies_txt.strip():
+        (persona.path / "cookies.txt").write_text(payload.cookies_txt)
+        cookies_written = True
+
+    auth_written = False
+    if payload.auth_json:
+        (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
+        auth_written = True
+
+    return {
+        "success": True,
+        "created": created,
+        "persona": persona,
+        "cookies_file_written": cookies_written,
+        "auth_file_written": auth_written,
+    }

archivebox/cli/archivebox_status.py

@@ -10,11 +10,13 @@ from rich import print
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.config import DATA_DIR, CONSTANTS
 from archivebox.config.common import get_config
-from archivebox.misc.legacy import parse_json_links_details
 from archivebox.misc.system import get_dir_size
 from archivebox.misc.logging_util import printable_filesize
 
 
+MAX_STATUS_FS_DIR_SCAN = 5000
+
+
 @enforce_types
 def status(out_dir: Path = DATA_DIR) -> None:
     """Print out some info and statistics about the archive collection"""
@@ -22,7 +24,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
     from django.contrib.auth import get_user_model
     from django.db.models import Sum
     from django.db.models.functions import Coalesce
-    from archivebox.core.models import Snapshot
+    from archivebox.core.models import ArchiveResult, Snapshot
 
     config = get_config()
     User = get_user_model()
@@ -34,54 +36,66 @@ def status(out_dir: Path = DATA_DIR) -> None:
     print(f"    Index size: {size} across {num_files} files")
     print()
 
-    links = list(Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)))
-    num_sql_links = len(links)
-    num_link_details = sum(1 for link in parse_json_links_details(out_dir=out_dir))
+    snapshots_qs = Snapshot.objects.all()
+    num_sql_links = snapshots_qs.count()
     archive_dir = config.ARCHIVE_DIR
+    legacy_snapshot_dirs = []
+    if archive_dir.exists():
+        legacy_snapshot_dirs = [
+            entry for entry in archive_dir.iterdir() if entry.is_dir() and not entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry)
+        ]
     print(f"    > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {CONSTANTS.SQL_INDEX_FILENAME})")
-    print(f"    > JSON Link Details: {num_link_details} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
+    print(f"    > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
     print()
     print("[green]\\[*] Scanning archive data directories...[/green]")
     users_dir = config.USERS_DIR
     scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
     scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
     print(f"[yellow]   {scan_roots_display}[/yellow]")
-    num_bytes = num_dirs = num_files = 0
-    for root in scan_roots:
-        root_bytes, root_dirs, root_files = get_dir_size(root)
-        num_bytes += root_bytes
-        num_dirs += root_dirs
-        num_files += root_files
+    do_precise_fs_scan = num_sql_links <= MAX_STATUS_FS_DIR_SCAN
+    if do_precise_fs_scan:
+        num_bytes = num_dirs = num_files = 0
+        for root in scan_roots:
+            root_bytes, root_dirs, root_files = get_dir_size(root)
+            num_bytes += root_bytes
+            num_dirs += root_dirs
+            num_files += root_files
+    else:
+        num_bytes = ArchiveResult.objects.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
+        num_dirs = 0
+        num_files = ArchiveResult.objects.exclude(output_files__in=["", "{}"]).count()
     size = printable_filesize(num_bytes)
-    print(f"    Size: {size} across {num_files} files in {num_dirs} directories")
+    if do_precise_fs_scan:
+        print(f"    Size: {size} across {num_files} files in {num_dirs} directories")
+    else:
+        print(f"    Size: {size} across {num_files} DB-tracked output records")
 
     # Use DB as source of truth for snapshot status
-    num_indexed = len(links)
-    num_archived = sum(1 for snapshot in links if snapshot.is_archived)
+    num_indexed = num_sql_links
+    num_archived = snapshots_qs.filter(status=Snapshot.StatusChoices.SEALED).count()
     num_unarchived = max(num_indexed - num_archived, 0)
     print(f"    > indexed: {num_indexed}".ljust(36), "(total snapshots in DB)")
     print(f"      > archived: {num_archived}".ljust(36), "(snapshots with archived content)")
     print(f"      > unarchived: {num_unarchived}".ljust(36), "(snapshots pending archiving)")
 
     # Count snapshot directories on filesystem across both legacy and current layouts.
-    expected_snapshot_dirs = {str(Path(snapshot.output_dir).resolve()) for snapshot in links if Path(snapshot.output_dir).exists()}
-    discovered_snapshot_dirs = set()
+    if do_precise_fs_scan:
+        links = list(snapshots_qs)
+        expected_snapshot_dirs = {str(Path(snapshot.output_dir).resolve()) for snapshot in links if Path(snapshot.output_dir).exists()}
+        discovered_snapshot_dirs = {str(entry.resolve()) for entry in legacy_snapshot_dirs}
 
-    if archive_dir.exists():
-        discovered_snapshot_dirs.update(
-            str(entry.resolve())
-            for entry in archive_dir.iterdir()
-            if entry.is_dir() and not entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry)
-        )
+        if users_dir.exists():
+            discovered_snapshot_dirs.update(
+                str(entry.resolve()) for entry in users_dir.glob(f"*/{CONSTANTS.SNAPSHOTS_DIR_NAME}/*/*/*") if entry.is_dir()
+            )
 
-    if users_dir.exists():
-        discovered_snapshot_dirs.update(
-            str(entry.resolve()) for entry in users_dir.glob(f"*/{CONSTANTS.SNAPSHOTS_DIR_NAME}/*/*/*") if entry.is_dir()
-        )
-
-    orphaned_dirs = sorted(discovered_snapshot_dirs - expected_snapshot_dirs)
-    num_present = len(discovered_snapshot_dirs)
-    num_valid = len(discovered_snapshot_dirs & expected_snapshot_dirs)
+        orphaned_dirs = sorted(discovered_snapshot_dirs - expected_snapshot_dirs)
+        num_present = len(discovered_snapshot_dirs)
+        num_valid = len(discovered_snapshot_dirs & expected_snapshot_dirs)
+    else:
+        orphaned_dirs = []
+        num_present = num_archived
+        num_valid = num_archived
     print()
     print(f"    > present: {num_present}".ljust(36), "(snapshot directories on disk)")
     print(f"      > [green]valid:[/green] {num_valid}".ljust(36), "               (directories with matching DB entry)")
@@ -116,10 +130,9 @@ def status(out_dir: Path = DATA_DIR) -> None:
         print("        [green]archivebox manage createsuperuser[/green]")
 
     print()
-    recent_snapshots = sorted(
-        links,
-        key=lambda snapshot: snapshot.downloaded_at or snapshot.modified_at or snapshot.created_at,
-        reverse=True,
+    recent_snapshots = snapshots_qs.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).order_by(
+        "-downloaded_at",
+        "-modified_at",
     )[:10]
     for snapshot in recent_snapshots:
         if not snapshot.downloaded_at:
@@ -128,7 +141,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
             (
                 "[grey53] "
                 f"   > {str(snapshot.downloaded_at)[:16]} "
-                f"[{snapshot.num_outputs} {('X', '√')[snapshot.is_archived]} {printable_filesize(snapshot.archive_size)}] "
+                f"[{snapshot.num_outputs} {('X', '√')[snapshot.status == Snapshot.StatusChoices.SEALED]} {printable_filesize(snapshot.output_size_sum or 0)}] "
                 f'"{snapshot.title}": {snapshot.url}'
                 "[/grey53]"
             )[: config.TERM_WIDTH],

archivebox/cli/archivebox_update.py

@@ -448,7 +448,7 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -
     No orphan detection needed - we trust 1:1 mapping between DB and filesystem
     after Phase 1 has drained all old archive/ directories.
     """
-    from archivebox.core.models import Snapshot
+    from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.config.common import get_config
     from archivebox.crawls.models import Crawl
     from django.db import transaction
@@ -487,6 +487,25 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -
             if has_directory:
                 old_title = snapshot.title
                 snapshot.reconcile_with_index_json(output_dir=output_dir, update_existing_archive_results=False)
+                metadata_updates = []
+                for archiveresult in ArchiveResult.objects.filter(snapshot=snapshot).only(
+                    "id",
+                    "snapshot_id",
+                    "plugin",
+                    "output_str",
+                    "output_files",
+                    "output_size",
+                    "output_mimetypes",
+                    "modified_at",
+                ):
+                    if archiveresult.update_output_metadata_from_filesystem(snapshot_dir=output_dir, save=False):
+                        metadata_updates.append(archiveresult)
+                if metadata_updates:
+                    ArchiveResult.objects.bulk_update(
+                        metadata_updates,
+                        ["output_files", "output_size", "output_mimetypes", "modified_at"],
+                        batch_size=batch_size,
+                    )
                 if snapshot.title != old_title:
                     update_values["title"] = snapshot.title
                     update_values["modified_at"] = timezone.now()

archivebox/config/constants.py

@@ -249,7 +249,7 @@ class ConstantsDict:
             "user_plugins",  # old name for USER_PLUGINS_DIR (now 'plugins')
             "user_templates",  # old name for CUSTOM_TEMPLATES_DIR (now 'templates')
             "static",  # created by old static exports <v0.6.0
-            "sonic",  # created by docker bind mount / sonic FTS process
+            "sonic",  # created by plugin-managed Sonic FTS worker
             ".git",
             ".svn",
             ### Files:

archivebox/core/admin_archiveresults.py

@@ -52,11 +52,13 @@ def build_abx_dl_display_command(result: ArchiveResult) -> str:
     return f"abx-dl --plugins={plugin_name} {_quote_shell_string(source_url)}"
 
 
-def build_abx_dl_replay_command(result: ArchiveResult) -> str:
+def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
     display_command = build_abx_dl_display_command(result)
     process = getattr(result, "process", None)
     env_items = env_to_shell_exports(getattr(process, "env", None) or {})
-    snapshot_dir = shlex.quote(str(result.snapshot_dir))
+    if config is not None:
+        result.snapshot._runtime_config = config
+    snapshot_dir = shlex.quote(str(result.pwd or result.snapshot_dir))
     if env_items:
         return f"cd {snapshot_dir}; env {env_items} {display_command}"
     return f"cd {snapshot_dir}; {display_command}"
@@ -78,7 +80,7 @@ def get_plugin_admin_url(plugin_name: str) -> str:
     return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/"
 
 
-def render_archiveresults_list(archiveresults_qs, limit=50):
+def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
     """Render a nice inline list view of archive results with status, plugin, output, and actions."""
 
     result_ids = list(archiveresults_qs.order_by("plugin").values_list("pk", flat=True)[:limit])
@@ -87,7 +89,13 @@ def render_archiveresults_list(archiveresults_qs, limit=50):
 
     results_by_id = {
         result.pk: result
-        for result in ArchiveResult.objects.filter(pk__in=result_ids).select_related("snapshot", "process", "process__machine")
+        for result in ArchiveResult.objects.filter(pk__in=result_ids).select_related(
+            "snapshot",
+            "snapshot__crawl",
+            "snapshot__crawl__created_by",
+            "process",
+            "process__machine",
+        )
     }
     results = [results_by_id[result_id] for result_id in result_ids if result_id in results_by_id]
 
@@ -152,7 +160,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50):
             output_display += "..."
 
         display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result)
+        replay_cmd = build_abx_dl_replay_command(result, config=config)
         cmd_str_escaped = html.escape(display_cmd)
         cmd_attr = html.escape(replay_cmd, quote=True)
 
@@ -160,9 +168,9 @@ def render_archiveresults_list(archiveresults_qs, limit=50):
         embed_path = result.embed_path() if hasattr(result, "embed_path") else None
         snapshot_id = str(getattr(result, "snapshot_id", ""))
         if embed_path and result.status == "succeeded":
-            output_link = build_snapshot_url(snapshot_id, embed_path)
+            output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
         else:
-            output_link = build_snapshot_url(snapshot_id, "")
+            output_link = build_snapshot_url(snapshot_id, "", config=config)
 
         # Get version - try cmd_version field
         version = result.cmd_version if result.cmd_version else "-"
@@ -469,7 +477,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     list_filter = ("status", "plugin", "start_ts")
     ordering = ["-start_ts"]
-    list_per_page = get_config().SNAPSHOTS_PER_PAGE
+    list_per_page = 50
 
     paginator = AcceleratedPaginator
     save_on_top = True
@@ -482,20 +490,54 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
         self.request = request
+        request.archivebox_config = get_config()
         return super().change_view(request, object_id, form_url, extra_context)
 
     def changelist_view(self, request, extra_context=None):
         self.request = request
-        return super().changelist_view(request, extra_context)
+        request.archivebox_config = get_config()
+        saved_list_per_page = self.list_per_page
+        self.list_per_page = min(max(5, request.archivebox_config.SNAPSHOTS_PER_PAGE), 5000)
+        try:
+            return super().changelist_view(request, extra_context)
+        finally:
+            self.list_per_page = saved_list_per_page
 
     def get_queryset(self, request):
-        return (
+        ordering = request.GET.get("o")
+        ordering_fields = set()
+        if ordering:
+            for part in ordering.split("."):
+                if not part:
+                    continue
+                try:
+                    idx = abs(int(part)) - 1
+                except ValueError:
+                    continue
+                if 0 <= idx < len(self.list_display):
+                    ordering_fields.add(self.list_display[idx])
+
+        qs = (
             super()
             .get_queryset(request)
-            .select_related("snapshot", "process")
+            .select_related("snapshot", "snapshot__crawl", "snapshot__crawl__created_by", "process", "process__machine")
+            .defer(
+                "config",
+                "notes",
+                "output_json",
+                "process__stdout",
+                "process__stderr",
+                "snapshot__config",
+                "snapshot__notes",
+                "snapshot__crawl__config",
+                "snapshot__crawl__notes",
+                "snapshot__crawl__urls",
+            )
             .prefetch_related("snapshot__tags")
-            .annotate(snapshot_first_tag=Min("snapshot__tags__name"))
         )
+        if "tags_inline" in ordering_fields:
+            qs = qs.annotate(snapshot_first_tag=Min("snapshot__tags__name"))
+        return qs
 
     def get_search_results(self, request, queryset, search_term):
         if not search_term:
@@ -532,16 +574,20 @@ class ArchiveResultAdmin(BaseModelAdmin):
         return queryset.filter(reduce(and_, filters)).distinct(), True
 
     def get_snapshot_view_url(self, result: ArchiveResult) -> str:
-        return build_snapshot_url(str(result.snapshot_id), request=getattr(self, "request", None))
+        request = getattr(self, "request", None)
+        return build_snapshot_url(str(result.snapshot_id), request=request, config=getattr(request, "archivebox_config", None))
 
     def get_output_view_url(self, result: ArchiveResult) -> str:
+        request = getattr(self, "request", None)
+        config = getattr(request, "archivebox_config", None)
         output_path = result.embed_path() if hasattr(result, "embed_path") else None
         if not output_path:
             output_path = result.plugin or ""
-        return build_snapshot_url(str(result.snapshot_id), output_path, request=getattr(self, "request", None))
+        return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config)
 
     def get_output_files_url(self, result: ArchiveResult) -> str:
-        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=getattr(self, 'request', None))}/?files=1"
+        request = getattr(self, "request", None)
+        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
 
     def get_output_zip_url(self, result: ArchiveResult) -> str:
         return f"{self.get_output_files_url(result)}&download=zip"
@@ -567,9 +613,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
     )
     def snapshot_info(self, result):
         snapshot_id = str(result.snapshot_id)
+        request = getattr(self, "request", None)
         return format_html(
             '<a href="{}"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            build_snapshot_url(snapshot_id, "index.html"),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
             snapshot_id[:8],
             result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"),
             result.snapshot.url[:128],
@@ -639,8 +686,9 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     @admin.display(description="Command")
     def cmd_str(self, result):
+        request = getattr(self, "request", None)
         display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result)
+        replay_cmd = build_abx_dl_replay_command(result, config=getattr(request, "archivebox_config", None))
         return format_html(
             """
             <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
@@ -661,13 +709,15 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_display(self, result):
+        request = getattr(self, "request", None)
+        config = getattr(request, "archivebox_config", None)
         # Determine output link path - use embed_path() which checks output_files
         embed_path = result.embed_path() if hasattr(result, "embed_path") else None
         output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
         snapshot_id = str(result.snapshot_id)
         return format_html(
             '<a href="{}" class="output-link">↗️</a><pre>{}</pre>',
-            build_snapshot_url(snapshot_id, output_path),
+            build_snapshot_url(snapshot_id, output_path, request=request, config=config),
             result.output_str,
         )
 
@@ -677,11 +727,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
         if not output_text:
             return "-"
 
+        request = getattr(self, "request", None)
         live_path = result.embed_path() if hasattr(result, "embed_path") else None
         if live_path:
             return format_html(
                 '<a href="{}" title="{}"><code>{}</code></a>',
-                build_snapshot_url(str(result.snapshot_id), live_path),
+                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=getattr(request, "archivebox_config", None)),
                 output_text,
                 output_text,
             )
@@ -739,9 +790,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
             result.output_str,
         )
         snapshot_id = str(result.snapshot_id)
+        request = getattr(self, "request", None)
         output_html += format_html(
             '<a href="{}#all">See result files ...</a><br/><pre><code>',
-            build_snapshot_url(snapshot_id, "index.html"),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
         )
         embed_path = result.embed_path() if hasattr(result, "embed_path") else ""
         path_from_embed = snapshot_dir / (embed_path or "")

archivebox/core/admin_snapshots.py

@@ -1,6 +1,7 @@
 __package__ = "archivebox.core"
 
 from functools import lru_cache
+import json
 
 from django.contrib import admin, messages
 from django.urls import path
@@ -13,7 +14,6 @@ from django import forms
 from django.template import Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
 
-from archivebox.config import DATA_DIR
 from archivebox.config.common import get_config
 from archivebox.misc.util import htmldecode, urldecode
 from archivebox.misc.paginators import AcceleratedPaginator
@@ -333,7 +333,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             super()
             .get_queryset(request)
             .select_related("crawl__created_by")
-            .defer("config", "notes")
+            .defer("notes")
             .prefetch_related("tags")
             .prefetch_related(Prefetch("archiveresult_set", queryset=prefetch_qs))
         )
@@ -499,7 +499,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Archive Results")
     def archiveresults_list(self, obj):
-        return render_archiveresults_list(obj.archiveresult_set.all())
+        request = getattr(self, "request", None)
+        return render_archiveresults_list(obj.archiveresult_set.all(), config=getattr(request, "archivebox_config", None))
 
     @admin.display(
         description="Title",
@@ -916,8 +917,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def _get_expected_hook_total(self, obj) -> int:
         try:
             request = getattr(self, "request", None)
+            if getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_changelist":
+                return 0
+
             crawl = getattr(obj, "crawl", None)
-            has_scoped_config = bool(getattr(obj, "config", None) or getattr(crawl, "config", None) or getattr(crawl, "persona_id", None))
+            snapshot_config = getattr(obj, "config", None) or {}
+            crawl_config = getattr(crawl, "config", None) or {}
+            crawl_persona_id = getattr(crawl, "persona_id", None)
+            has_scoped_config = bool(snapshot_config or crawl_config or crawl_persona_id)
 
             if request is not None and not has_scoped_config:
                 cached_total = getattr(request, "archivebox_expected_snapshot_hook_total", None)
@@ -927,7 +934,23 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                     request.archivebox_expected_snapshot_hook_total = cached_total
                 return cached_total
 
-            return len(discover_hooks("Snapshot", config=get_config(crawl=crawl, snapshot=obj)))
+            if request is not None:
+                scoped_cache = getattr(request, "archivebox_expected_snapshot_hook_totals_by_scope", None)
+                if scoped_cache is None:
+                    scoped_cache = {}
+                    request.archivebox_expected_snapshot_hook_totals_by_scope = scoped_cache
+                if snapshot_config:
+                    cache_key = ("snapshot", json.dumps(snapshot_config, sort_keys=True, default=str))
+                else:
+                    cache_key = ("crawl", json.dumps(crawl_config, sort_keys=True, default=str), crawl_persona_id)
+                cached_total = scoped_cache.get(cache_key)
+                if cached_total is None:
+                    config = get_config(crawl=crawl, snapshot=obj if snapshot_config else None)
+                    cached_total = len(discover_hooks("Snapshot", config=config))
+                    scoped_cache[cache_key] = cached_total
+                return cached_total
+
+            return len(discover_hooks("Snapshot", config=get_config(crawl=crawl, snapshot=obj if snapshot_config else None)))
         except Exception:
             return 0
 
@@ -1038,12 +1061,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         description="🔁 Redo Failed",
     )
     def update_snapshots(self, request, queryset):
-        queued = bg_archive_snapshots(queryset, kwargs={"overwrite": False, "out_dir": DATA_DIR})
+        queued = 0
+        for snapshot in queryset:
+            queued += snapshot.retry_failed_archiveresults()
 
-        messages.success(
-            request,
-            f"Queued {queued} snapshots for re-archiving. The background runner will process them.",
-        )
+        if queued:
+            messages.success(
+                request,
+                f"Queued {queued} failed/skipped extractors for retry. The background runner will process them.",
+            )
+        else:
+            messages.info(request, "No failed/skipped extractors were found in the selected snapshots.")
 
     @admin.action(
         description="🆕 Archive Now",
@@ -1070,7 +1098,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         description="🔄 Redo",
     )
     def overwrite_snapshots(self, request, queryset):
-        queued = bg_archive_snapshots(queryset, kwargs={"overwrite": True, "out_dir": DATA_DIR})
+        queued = bg_archive_snapshots(queryset, kwargs={"overwrite": True})
 
         messages.success(
             request,

archivebox/core/models.py

@@ -1037,13 +1037,14 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
         # Merge ArchiveResults
         self._merge_archive_results_from_index(index_data, update_existing=update_existing_archive_results)
         if not self._normalize_title_candidate(self.title, snapshot_url=self.url):
-            title_result = (
-                self.archiveresult_set.filter(plugin="title").exclude(output_str="").order_by("-start_ts", "-end_ts", "-created_at").first()
+            title_results = (
+                self.archiveresult_set.filter(plugin="title").exclude(output_str="").order_by("-start_ts", "-end_ts", "-created_at")
             )
-            if title_result:
+            for title_result in title_results.only("output_str"):
                 result_title = self._normalize_title_candidate(title_result.output_str, snapshot_url=self.url)
                 if result_title:
                     self.title = result_title
+                    break
 
         # Write back in JSONL format
         self.write_index_jsonl(output_dir=output_dir)
@@ -1054,14 +1055,16 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
 
     def _merge_title_from_index(self, index_data: dict):
         """Merge title - prefer longest non-URL title."""
-        index_title = (index_data.get("title") or "").strip()
-        db_title = self.title or ""
+        index_title = self._normalize_title_candidate(index_data.get("title"), snapshot_url=self.url)
+        db_title = self._normalize_title_candidate(self.title, snapshot_url=self.url)
 
-        candidates = [t for t in [index_title, db_title] if t and t != self.url]
+        candidates = [t for t in [index_title, db_title] if t]
         if candidates:
             best_title = max(candidates, key=len)
             if self.title != best_title:
                 self.title = best_title
+        elif self.title:
+            self.title = ""
 
     def _merge_tags_from_index(self, index_data: dict):
         """Merge tags - union of both sources."""
@@ -1639,7 +1642,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
         title = " ".join(line.strip() for line in str(candidate or "").splitlines() if line.strip()).strip()
         if not title:
             return ""
-        if title.lower() in {"pending...", "no title found"}:
+        if title.lower() in {"pending...", "no title found", "unable to detect page title"}:
             return ""
         if title == snapshot_url:
             return ""
@@ -1655,10 +1658,8 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
         if stored_title:
             return stored_title
 
-        title_result = (
-            self.archiveresult_set.filter(plugin="title").exclude(output_str="").order_by("-start_ts", "-end_ts", "-created_at").first()
-        )
-        if title_result:
+        title_results = self.archiveresult_set.filter(plugin="title").exclude(output_str="").order_by("-start_ts", "-end_ts", "-created_at")
+        for title_result in title_results.only("output_str"):
             result_title = self._normalize_title_candidate(title_result.output_str, snapshot_url=self.url)
             if result_title:
                 return result_title
@@ -2283,13 +2284,16 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
 
         Returns count of ArchiveResults reset.
         """
-        count = self.archiveresult_set.filter(
+        retryable_results = ArchiveResult.objects.filter(
+            snapshot=self,
             status__in=[
                 ArchiveResult.StatusChoices.FAILED,
                 ArchiveResult.StatusChoices.SKIPPED,
                 ArchiveResult.StatusChoices.NORESULTS,
             ],
-        ).update(
+        )
+        legacy_result_count = retryable_results.filter(hook_name="").count()
+        count = retryable_results.exclude(hook_name="").update(
             status=ArchiveResult.StatusChoices.QUEUED,
             output_str="",
             output_json=None,
@@ -2300,13 +2304,19 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
             end_ts=None,
         )
 
-        if count > 0:
+        if count + legacy_result_count > 0:
             self.status = self.StatusChoices.QUEUED
             self.retry_at = timezone.now()
             self.current_step = 0  # Reset to step 0 for retry
             self.save(update_fields=["status", "retry_at", "current_step", "modified_at"])
 
-        return count
+            crawl = self.crawl
+            if crawl.status != crawl.StatusChoices.STARTED:
+                crawl.status = crawl.StatusChoices.QUEUED
+            crawl.retry_at = timezone.now()
+            crawl.save(update_fields=["status", "retry_at", "modified_at"])
+
+        return count + legacy_result_count
 
     # =========================================================================
     # URL Helper Properties (migrated from Link schema)
@@ -2347,6 +2357,10 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
 
     @cached_property
     def is_archived(self) -> bool:
+        cached_is_archived = getattr(self, "_is_archived_cached", None)
+        if cached_is_archived is not None:
+            return bool(cached_is_archived)
+
         if self.downloaded_at or self.status == self.StatusChoices.SEALED:
             return True
 
@@ -2362,7 +2376,8 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
             "media",
             "git",
         )
-        return any((Path(self.output_dir) / path).exists() for path in output_paths)
+        output_dir = Path(self.output_dir)
+        return any((output_dir / path).exists() for path in output_paths)
 
     # =========================================================================
     # Date/Time Properties (migrated from Link schema)
@@ -3218,6 +3233,80 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
     def output_size_from_files(self) -> int:
         return sum(self._coerce_output_file_size(metadata.get("size")) for metadata in self.output_file_map().values())
 
+    def update_output_metadata_from_filesystem(self, snapshot_dir: Path | None = None, save: bool = True) -> bool:
+        from collections import defaultdict
+        from abx_dl.output_files import guess_mimetype
+
+        if self.plugin == "title":
+            return False
+
+        snapshot_dir = Path(snapshot_dir or self.snapshot.output_dir)
+        exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid", "cmd.sh"}
+        output_files: dict[str, dict[str, Any]] = {}
+        mime_sizes: dict[str, int] = defaultdict(int)
+        total_size = 0
+
+        def add_file(file_path: Path, rel_path: str, *, root_relative: bool = False) -> None:
+            nonlocal total_size
+            try:
+                if not file_path.is_file() or file_path.name in exclude_names:
+                    return
+                stat = file_path.stat()
+            except OSError:
+                return
+            mime_type = guess_mimetype(file_path) or "application/octet-stream"
+            metadata = {
+                "extension": file_path.suffix.lower().lstrip("."),
+                "mimetype": mime_type,
+                "size": stat.st_size,
+            }
+            if root_relative:
+                metadata["root_relative"] = True
+            output_files[rel_path] = metadata
+            mime_sizes[mime_type] += stat.st_size
+            total_size += stat.st_size
+
+        for raw_line in str(self.output_str or "").splitlines():
+            raw_output = raw_line.strip().lstrip("/")
+            if not raw_output or raw_output in {".", "./", "/"} or "://" in raw_output or raw_output.startswith("/"):
+                continue
+            if not self._looks_like_output_path(raw_output, self.plugin):
+                continue
+
+            raw_path = Path(raw_output)
+            if raw_output.startswith(f"{self.plugin}/"):
+                plugin_relative = raw_output.removeprefix(f"{self.plugin}/")
+                add_file(snapshot_dir / raw_output, plugin_relative)
+            elif len(raw_path.parts) == 1:
+                add_file(snapshot_dir / self.plugin / raw_output, raw_output)
+                add_file(snapshot_dir / raw_output, raw_output, root_relative=True)
+            else:
+                add_file(snapshot_dir / self.plugin / raw_output, raw_output)
+                add_file(snapshot_dir / raw_output, raw_output, root_relative=True)
+
+        plugin_dir = snapshot_dir / self.plugin
+        if not output_files and plugin_dir.is_dir():
+            for file_path in plugin_dir.rglob("*"):
+                if not file_path.is_file() or ".hooks" in file_path.parts:
+                    continue
+                add_file(file_path, str(file_path.relative_to(plugin_dir)))
+
+        if not output_files:
+            return False
+
+        sorted_mimes = sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True)
+        output_mimetypes = ",".join(mime for mime, _ in sorted_mimes)
+        if self.output_files == output_files and self.output_size == total_size and self.output_mimetypes == output_mimetypes:
+            return False
+
+        self.output_files = output_files
+        self.output_size = total_size
+        self.output_mimetypes = output_mimetypes
+        self.modified_at = timezone.now()
+        if save:
+            self.save(update_fields=["output_files", "output_size", "output_mimetypes", "modified_at"])
+        return True
+
     def output_exists(self) -> bool:
         return os.path.exists(Path(self.snapshot_dir) / self.plugin)
 

archivebox/core/settings.py

@@ -226,7 +226,7 @@ TEMPLATES = [
 # CACHE_DB_TABLE = 'django_cache'
 
 DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(CONSTANTS.DATABASE_FILE))
-SQLITE_JOURNAL_MODE = os.environ.get("ARCHIVEBOX_SQLITE_JOURNAL_MODE", "TRUNCATE" if CONSTANTS.IN_DOCKER else "WAL")
+SQLITE_JOURNAL_MODE = os.environ.get("ARCHIVEBOX_SQLITE_JOURNAL_MODE", "WAL")
 SQLITE_MMAP_SIZE = os.environ.get("ARCHIVEBOX_SQLITE_MMAP_SIZE", "0" if CONSTANTS.IN_DOCKER else "134217728")
 
 SQLITE_CONNECTION_OPTIONS = {

archivebox/core/views.py

@@ -893,6 +893,7 @@ class PublicIndexView(ListView):
         }
         for snapshot in context.get("object_list") or ():
             snapshot._icons_compact = True
+            snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
         return context
 
     def get_queryset(self, **kwargs):
@@ -1044,6 +1045,9 @@ class AddView(UserPassesTestMixin, FormView):
         notes = form.cleaned_data.get("notes", "")
         url_filters = form.cleaned_data.get("url_filters") or {}
         custom_config = self._get_custom_config_overrides(form)
+        persona_name = persona.name if persona else "Default"
+        if persona:
+            persona.ensure_dirs()
 
         from archivebox.config.permissions import HOSTNAME
 
@@ -1070,11 +1074,12 @@ class AddView(UserPassesTestMixin, FormView):
             "INDEX_ONLY": index_only,
             "DEPTH": depth,
             "PLUGINS": plugins or "",
-            "DEFAULT_PERSONA": (persona.name if persona else "Default"),
+            "DEFAULT_PERSONA": persona_name,
         }
 
         # Merge custom config overrides
         config.update(custom_config)
+        config["DEFAULT_PERSONA"] = persona_name
         if url_filters.get("allowlist"):
             config["URL_ALLOWLIST"] = url_filters["allowlist"]
         if url_filters.get("denylist"):
@@ -1090,6 +1095,7 @@ class AddView(UserPassesTestMixin, FormView):
             label=f"{created_by_name}@{HOSTNAME}{self.request.path} {timestamp}",
             created_by_id=created_by_id,
             config=config,
+            persona_id=persona.id if persona else None,
         )
 
         # 3. create a CrawlSchedule if schedule is provided
@@ -1321,46 +1327,11 @@ def live_progress_view(request):
         archiveresults_succeeded = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.SUCCEEDED).count()
         archiveresults_failed = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.FAILED).count()
 
-        # Get recently completed ArchiveResults with thumbnails (last 20 succeeded results)
-        recent_thumbnails = []
-        recent_results = (
-            ArchiveResult.objects.filter(
-                status=ArchiveResult.StatusChoices.SUCCEEDED,
-            )
-            .select_related("snapshot")
-            .order_by("-end_ts")[:20]
-        )
-
-        for ar in recent_results:
-            embed = ar.embed_path()
-            if embed:
-                # Only include results with embeddable image/media files
-                ext = embed.lower().split(".")[-1] if "." in embed else ""
-                is_embeddable = ext in ("png", "jpg", "jpeg", "gif", "webp", "svg", "ico", "pdf", "html")
-                if is_embeddable or ar.plugin in ("screenshot", "favicon", "dom"):
-                    archive_path = embed or ""
-                    recent_thumbnails.append(
-                        {
-                            "id": str(ar.id),
-                            "plugin": ar.plugin,
-                            "snapshot_id": str(ar.snapshot_id),
-                            "snapshot_url": ar.snapshot.url[:60] if ar.snapshot else "",
-                            "embed_path": embed,
-                            "archive_path": archive_path,
-                            "archive_url": build_snapshot_url(str(ar.snapshot_id), archive_path, request=request) if archive_path else "",
-                            "end_ts": ar.end_ts.isoformat() if ar.end_ts else None,
-                        },
-                    )
-
         # Build hierarchical active crawls with nested snapshots and archive results
 
         active_crawls_qs = (
             Crawl.objects.filter(status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED])
-            .prefetch_related(
-                "snapshot_set",
-                "snapshot_set__archiveresult_set",
-                "snapshot_set__archiveresult_set__process",
-            )
+            .prefetch_related("snapshot_set")
             .distinct()
             .order_by("-modified_at")[:10]
         )
@@ -1386,19 +1357,23 @@ def live_progress_view(request):
         process_records_by_crawl: dict[str, list[tuple[dict[str, object], object | None]]] = {}
         process_records_by_snapshot: dict[str, list[tuple[dict[str, object], object | None]]] = {}
         seen_process_records: set[str] = set()
-        snapshots = [snapshot for crawl in active_crawls_qs for snapshot in crawl.snapshot_set.all()]
+        active_snapshot_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED}
+        snapshots = [
+            snapshot for crawl in active_crawls_qs for snapshot in crawl.snapshot_set.all() if snapshot.status in active_snapshot_statuses
+        ]
+        snapshots_by_id = {str(snapshot.id): snapshot for snapshot in snapshots}
+
+        def find_snapshot_for_process(proc_pwd: Path) -> Snapshot | None:
+            for path_part in reversed(proc_pwd.parts):
+                snapshot = snapshots_by_id.get(path_part)
+                if snapshot:
+                    return snapshot
+            return None
+
         for proc in running_processes:
             if not proc.pwd:
                 continue
-            proc_pwd = Path(proc.pwd)
-            matched_snapshot = None
-            for snapshot in snapshots:
-                try:
-                    proc_pwd.relative_to(snapshot.output_dir)
-                    matched_snapshot = snapshot
-                    break
-                except ValueError:
-                    continue
+            matched_snapshot = find_snapshot_for_process(Path(proc.pwd))
             if matched_snapshot is None:
                 continue
             crawl_id = str(matched_snapshot.crawl_id)
@@ -1412,15 +1387,7 @@ def live_progress_view(request):
         for proc in recent_processes:
             if not proc.pwd:
                 continue
-            proc_pwd = Path(proc.pwd)
-            matched_snapshot = None
-            for snapshot in snapshots:
-                try:
-                    proc_pwd.relative_to(snapshot.output_dir)
-                    matched_snapshot = snapshot
-                    break
-                except ValueError:
-                    continue
+            matched_snapshot = find_snapshot_for_process(Path(proc.pwd))
             if matched_snapshot is None:
                 continue
             crawl_id = str(matched_snapshot.crawl_id)
@@ -1470,9 +1437,7 @@ def live_progress_view(request):
             pending_snapshots = sum(1 for s in all_crawl_snapshots if s.status == Snapshot.StatusChoices.QUEUED)
 
             # Get only ACTIVE snapshots to display (limit to 5 most recent)
-            active_crawl_snapshots = [
-                s for s in all_crawl_snapshots if s.status in [Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED]
-            ][:5]
+            active_crawl_snapshots = [s for s in all_crawl_snapshots if s.status in active_snapshot_statuses][:5]
 
             # Count URLs in the crawl (for when snapshots haven't been created yet)
             urls_count = 0
@@ -1497,9 +1462,13 @@ def live_progress_view(request):
             active_snapshots_for_crawl = []
             for snapshot in active_crawl_snapshots:
                 snapshot_run_started_at = snapshot.downloaded_at or snapshot.created_at
-                # Get archive results for this snapshot (already prefetched)
+                # Get archive results only for displayed active snapshots. Large crawls can
+                # contain thousands of sealed snapshots, and prefetching all their results
+                # makes the progress endpoint compete with the runner.
                 snapshot_results = [
-                    ar for ar in snapshot.archiveresult_set.all() if archiveresult_matches_current_run(ar, snapshot_run_started_at)
+                    ar
+                    for ar in snapshot.archiveresult_set.select_related("process").all()
+                    if archiveresult_matches_current_run(ar, snapshot_run_started_at)
                 ]
 
                 now = timezone.now()
@@ -1653,7 +1622,7 @@ def live_progress_view(request):
                 "archiveresults_succeeded": archiveresults_succeeded,
                 "archiveresults_failed": archiveresults_failed,
                 "active_crawls": active_crawls,
-                "recent_thumbnails": recent_thumbnails,
+                "recent_thumbnails": [],
                 "server_time": timezone.now().isoformat(),
             },
         )

archivebox/machine/models.py

@@ -128,6 +128,21 @@ def _sanitize_machine_config(config: dict[str, Any] | None) -> dict[str, Any]:
     sanitized = {key: value for key, value in config.items() if key in MACHINE_CONFIG_ALWAYS_ALLOWED_KEYS or str(key).endswith("_BINARY")}
     for key in LEGACY_MACHINE_CONFIG_KEYS:
         sanitized.pop(key, None)
+    for key, value in list(sanitized.items()):
+        if not str(key).endswith("_BINARY"):
+            continue
+        if not isinstance(value, str):
+            continue
+        value = value.strip()
+        if not value:
+            sanitized.pop(key, None)
+            continue
+        if "/" in value or value.startswith("~"):
+            try:
+                if not Path(value).expanduser().exists():
+                    sanitized.pop(key, None)
+            except OSError:
+                sanitized.pop(key, None)
     return sanitized
 
 
@@ -2380,6 +2395,11 @@ class Process(models.Model):
 
         for proc in running_children:
             if not proc.is_running:
+                proc.status = cls.StatusChoices.EXITED
+                proc.ended_at = proc.ended_at or timezone.now()
+                proc.exit_code = proc.exit_code if proc.exit_code is not None else 0
+                proc.save(update_fields=["status", "ended_at", "exit_code"])
+                cleaned += 1
                 continue
 
             root = proc.root

archivebox/search/__init__.py

@@ -34,10 +34,14 @@ SEARCH_MODES = ("meta", "contents", "deep")
 def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
     """Expose ArchiveBox collection roots to in-process search backends."""
     config = config or get_config(**config_kwargs)
-    updates = {
-        "DATA_DIR": str(config.DATA_DIR),
-        "SNAP_DIR": str(config.USERS_DIR),
-    }
+    updates = {}
+    for key, value in config.items():
+        if value is None:
+            continue
+        if isinstance(value, (str, int, float, bool, os.PathLike)):
+            updates[str(key)] = str(value)
+    updates["DATA_DIR"] = str(config.DATA_DIR)
+    updates["SNAP_DIR"] = str(config.USERS_DIR)
     previous = {key: os.environ.get(key) for key in updates}
     os.environ.update(updates)
     try:

archivebox/services/runner.py

@@ -137,6 +137,12 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
 
     from archivebox.config import CONSTANTS
     from archivebox.machine.models import Machine, Process
+    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+
+    supervisor = get_existing_supervisord_process()
+    runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
+    if runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"):
+        return False
 
     Process.cleanup_stale_running()
     Process.cleanup_orphaned_workers()
@@ -1024,12 +1030,17 @@ def recover_orphaned_snapshots() -> int:
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.machine.models import Process
+    from django.db.models import Q
 
     active_snapshot_ids: set[str] = set()
     orphaned_snapshots = list(
-        Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED, retry_at__isnull=True)
+        Snapshot.objects.filter(
+            Q(status=Snapshot.StatusChoices.STARTED, retry_at__isnull=True)
+            | Q(status=Snapshot.StatusChoices.SEALED, archiveresult__status=ArchiveResult.StatusChoices.QUEUED),
+        )
         .select_related("crawl")
-        .prefetch_related("archiveresult_set"),
+        .prefetch_related("archiveresult_set")
+        .distinct(),
     )
     running_processes = Process.objects.filter(
         status=Process.StatusChoices.RUNNING,
@@ -1157,6 +1168,12 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) ->
                 .first()
             )
             if binary is not None:
+                binary_name = str(binary.name or "")
+                binary_path = Path(binary_name).expanduser()
+                if (binary_path.is_absolute() or binary_name.startswith("~")) and not binary_path.exists():
+                    binary.retry_at = None
+                    binary.save(update_fields=["retry_at", "modified_at"])
+                    continue
                 if not binary.claim_processing_lock(lock_seconds=60):
                     continue
                 run_binary(str(binary.id))

archivebox/templates/admin/base.html

@@ -1684,7 +1684,7 @@
                 const buttons = $('<div></div>')
                     .insertAfter('div.actions button[type=submit]')
                     .css('display', 'inline')
-                    .addClass('class', 'action-buttons');
+                    .addClass('action-buttons');
 
                 // for each action in the dropdown, turn it into a button instead
                 container.find('select[name=action] option:gt(0)').each(function () {
@@ -1715,10 +1715,10 @@
                                 if (!window.confirm(message)) return false
                             }
 
-                            // select the action button from the dropdown
+                            // select the action from the original Django admin dropdown
                             container.find('select[name=action]')
-                                .find('[selected]').removeAttr('selected').end()
-                                .find('[value=' + action_type + ']').attr('selected', 'selected').click()
+                                .val(action_type)
+                                .trigger('change')
                             
                             // click submit & replace the archivebox logo with a spinner
                             $('#changelist-form button[name="index"]').click()

archivebox/templates/static/admin.css

@@ -145,6 +145,26 @@ body.change-list #content .object-tools {
     margin-bottom: 10px;
     border-top: 0px; 
     border-bottom: 0px;
+    display: inline-flex;
+    width: auto;
+    max-width: 100%;
+    box-sizing: border-box;
+    position: relative;
+    z-index: 1;
+}
+
+#content #changelist #toolbar #changelist-search,
+#content #changelist #toolbar #changelist-search > div {
+    width: auto;
+    max-width: 100%;
+    flex: 0 1 auto;
+    box-sizing: border-box;
+}
+
+#content #changelist #toolbar #searchbar {
+    width: clamp(180px, 32vw, 420px);
+    max-width: 100%;
+    flex: 0 1 auto;
 }
 
 #content #changelist #toolbar form input[type="submit"] {
@@ -170,6 +190,8 @@ body.change-list #content .object-tools {
     width: auto;
     max-height: 40px;
     display: block;
+    position: relative;
+    z-index: 2;
 }
 @media (max-width: 1000px) {
     #content #changelist .actions {

archivebox/tests/test_add_view.py

@@ -4,8 +4,10 @@ import pytest
 from django.contrib.auth import get_user_model
 from django.urls import reverse
 
+from archivebox.config.common import get_config
 from archivebox.core.models import Tag
 from archivebox.crawls.models import Crawl
+from archivebox.personas.models import Persona
 
 
 pytestmark = pytest.mark.django_db
@@ -105,6 +107,45 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
     assert "ONLY_NEW" not in crawl.config
 
 
+def test_add_view_selected_persona_wins_over_stale_config_override(client, admin_user, monkeypatch):
+    monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
+    client.force_login(admin_user)
+    private_persona = Persona.objects.create(name="Private", created_by=admin_user)
+    private_persona.ensure_dirs()
+    private_cookies_file = private_persona.path / "cookies.txt"
+    private_cookies_file.write_text("# Private cookies\n", encoding="utf-8")
+
+    response = client.post(
+        reverse("add"),
+        data={
+            "url": "https://example.com/private",
+            "tag": "",
+            "depth": "0",
+            "max_urls": "0",
+            "max_size": "0",
+            "url_filters_allowlist": "",
+            "url_filters_denylist": "",
+            "notes": "",
+            "schedule": "",
+            "persona": "Private",
+            "index_only": "",
+            "config": '{"DEFAULT_PERSONA": "Default"}',
+        },
+        HTTP_HOST=WEB_HOST,
+    )
+
+    assert response.status_code == 302
+
+    crawl = Crawl.objects.order_by("-created_at").first()
+    assert crawl is not None
+    assert crawl.persona_id == private_persona.id
+    assert crawl.config.get("DEFAULT_PERSONA") == "Private"
+    assert crawl.resolve_persona() == private_persona
+    runtime_config = get_config(crawl=crawl)
+    assert runtime_config.ACTIVE_PERSONA == "Private"
+    assert runtime_config.COOKIES_FILE == private_cookies_file
+
+
 def test_add_view_starts_background_runner_after_creating_crawl(client, admin_user, monkeypatch):
     monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
     client.force_login(admin_user)

archivebox/tests/test_admin_views.py

@@ -987,6 +987,57 @@ class TestAdminSnapshotListView:
         assert response["Location"].endswith(f"/admin/core/snapshot/{snapshot.pk}/change/")
         assert snapshot.archiveresult_set.get(plugin="title").status == ArchiveResult.StatusChoices.QUEUED
 
+    def test_list_redo_failed_action_requeues_failed_archiveresults_only(self, client, admin_user, snapshot, monkeypatch):
+        import archivebox.core.admin_snapshots as admin_snapshots
+        from archivebox.core.models import ArchiveResult
+
+        def bg_archive_snapshots_should_not_run(*args, **kwargs):
+            raise AssertionError("Redo Failed should reset failed ArchiveResults directly")
+
+        monkeypatch.setattr(admin_snapshots, "bg_archive_snapshots", bg_archive_snapshots_should_not_run)
+
+        failed = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="wget",
+            hook_name="on_Snapshot__50_wget",
+            status=ArchiveResult.StatusChoices.FAILED,
+            output_str="boom",
+            output_files={"index.html": {"path": "index.html", "size": 123}},
+            output_size=123,
+            output_mimetypes="text/html",
+        )
+        succeeded = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="title",
+            hook_name="on_Snapshot__54_title",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+            output_str="Example Domain",
+        )
+
+        client.login(username="testadmin", password="testpassword")
+        response = client.post(
+            reverse("admin:core_snapshot_changelist"),
+            {
+                "action": "update_snapshots",
+                "_selected_action": [str(snapshot.pk)],
+                "index": "0",
+            },
+            HTTP_HOST=ADMIN_HOST,
+        )
+
+        assert response.status_code == 302
+        failed.refresh_from_db()
+        succeeded.refresh_from_db()
+        snapshot.refresh_from_db()
+        assert failed.status == ArchiveResult.StatusChoices.QUEUED
+        assert failed.output_str == ""
+        assert failed.output_files == {}
+        assert failed.output_size == 0
+        assert failed.output_mimetypes == ""
+        assert succeeded.status == ArchiveResult.StatusChoices.SUCCEEDED
+        assert succeeded.output_str == "Example Domain"
+        assert snapshot.status == snapshot.StatusChoices.QUEUED
+
     def test_archive_now_action_uses_original_snapshot_url_without_timestamp_suffix(self, client, admin_user, snapshot, monkeypatch):
         import archivebox.core.admin_snapshots as admin_snapshots
 

archivebox/tests/test_archive_result_service.py

@@ -250,6 +250,54 @@ def test_retry_failed_archiveresults_requeues_snapshot_in_queued_state():
     _cleanup_machine_process_rows()
 
 
+def test_retry_failed_archiveresults_preserves_legacy_plugin_rows_without_hook_name():
+    from archivebox.core.models import ArchiveResult, Snapshot
+
+    snapshot = _create_snapshot()
+    legacy_result = ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="wget",
+        hook_name="",
+        status=ArchiveResult.StatusChoices.FAILED,
+        output_str="legacy failure",
+        output_files={"index.html": {"size": 123}},
+        output_size=123,
+        output_mimetypes="text/html",
+    )
+    hook_result = ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="wget",
+        hook_name="on_Snapshot__06_wget.finite.bg",
+        status=ArchiveResult.StatusChoices.FAILED,
+        output_str="hook failure",
+        output_files={"stderr.log": {}},
+        output_size=10,
+        output_mimetypes="text/plain",
+    )
+
+    reset_count = snapshot.retry_failed_archiveresults()
+
+    snapshot.refresh_from_db()
+    snapshot.crawl.refresh_from_db()
+    legacy_result.refresh_from_db()
+    hook_result.refresh_from_db()
+
+    assert reset_count == 2
+    assert snapshot.status == Snapshot.StatusChoices.QUEUED
+    assert snapshot.retry_at is not None
+    assert snapshot.crawl.status == snapshot.crawl.StatusChoices.QUEUED
+    assert snapshot.crawl.retry_at is not None
+    assert legacy_result.status == ArchiveResult.StatusChoices.FAILED
+    assert legacy_result.output_str == "legacy failure"
+    assert legacy_result.output_files == {"index.html": {"size": 123}}
+    assert legacy_result.output_size == 123
+    assert hook_result.status == ArchiveResult.StatusChoices.QUEUED
+    assert hook_result.output_str == ""
+    assert hook_result.output_files == {}
+    assert hook_result.output_size == 0
+    _cleanup_machine_process_rows()
+
+
 def test_process_completed_projects_snapshot_title_from_output_str():
     from archivebox.services.archive_result_service import ArchiveResultService
     import asyncio

archivebox/tests/test_cli_run.py

@@ -427,3 +427,39 @@ class TestRecoverOrphanedSnapshots:
         assert snapshot.retry_at is not None
         assert crawl.status == Crawl.StatusChoices.QUEUED
         assert crawl.retry_at is not None
+
+    def test_recover_orphaned_snapshot_requeues_sealed_snapshot_with_queued_results(self):
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import ArchiveResult, Snapshot
+        from archivebox.services.runner import recover_orphaned_snapshots
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=None,
+        )
+        ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="title",
+            hook_name="on_Snapshot__01_title",
+            status=ArchiveResult.StatusChoices.QUEUED,
+        )
+
+        recovered = recover_orphaned_snapshots()
+
+        snapshot.refresh_from_db()
+        crawl.refresh_from_db()
+
+        assert recovered == 1
+        assert snapshot.status == Snapshot.StatusChoices.QUEUED
+        assert snapshot.retry_at is not None
+        assert crawl.status == Crawl.StatusChoices.QUEUED
+        assert crawl.retry_at is not None

archivebox/tests/test_cli_server.py

@@ -6,6 +6,7 @@ Verify server can start (basic smoke tests only, no full server testing).
 
 import os
 import asyncio
+import builtins
 import json
 import subprocess
 import sys
@@ -18,6 +19,7 @@ def test_sqlite_connections_use_explicit_30_second_busy_timeout():
 
     assert SQLITE_CONNECTION_OPTIONS["OPTIONS"]["timeout"] == 30
     assert "PRAGMA busy_timeout = 30000;" in SQLITE_CONNECTION_OPTIONS["OPTIONS"]["init_command"]
+    assert "PRAGMA journal_mode = WAL;" in SQLITE_CONNECTION_OPTIONS["OPTIONS"]["init_command"]
 
 
 def test_server_shows_usage_info(tmp_path, process):
@@ -110,6 +112,21 @@ def test_start_server_workers_starts_plugin_owned_sonic_worker(monkeypatch):
     ]
 
 
+def test_missing_plugin_owned_sonic_worker_is_optional(monkeypatch):
+    from archivebox.workers.supervisord_util import get_sonic_supervisord_worker_from_plugin
+
+    original_import = builtins.__import__
+
+    def fake_import(name, *args, **kwargs):
+        if name == "abx_plugins.plugins.search_backend_sonic.daemon":
+            raise ModuleNotFoundError("No module named 'abx_plugins.plugins.search_backend_sonic.daemon'", name=name)
+        return original_import(name, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "__import__", fake_import)
+
+    assert get_sonic_supervisord_worker_from_plugin(SimpleNamespace()) is None
+
+
 def test_sonic_daemon_event_handler_requires_running_supervised_worker(monkeypatch):
     from abx_dl.events import ProcessStdoutEvent
     from abx_dl.orchestrator import create_bus

archivebox/tests/test_config.py

@@ -130,6 +130,12 @@ SEARCH_BACKEND_PASSWORD = "SecretPassword"
     assert "Extra inputs are not permitted" not in result.stderr
 
 
+def test_sonic_dir_is_allowed_inside_data_dir():
+    from archivebox.config import CONSTANTS
+
+    assert "sonic" in CONSTANTS.ALLOWED_IN_DATA_DIR
+
+
 def test_config_set_requires_equals_sign(tmp_path, process):
     """Test that --set requires KEY=VALUE format."""
     os.chdir(tmp_path)

archivebox/tests/test_machine_models.py

@@ -133,35 +133,46 @@ class TestMachineModel(TestCase):
         """Machine.current() should keep derived cache entries, not runtime config."""
         import archivebox.machine.models as models
 
+        chrome_path = "/tmp/archivebox-test-chromium"
+        node_path = "/tmp/archivebox-test-node"
+        open(chrome_path, "a").close()
+        open(node_path, "a").close()
+        self.addCleanup(lambda: os.path.exists(chrome_path) and os.remove(chrome_path))
+        self.addCleanup(lambda: os.path.exists(node_path) and os.remove(node_path))
         machine = Machine.current()
         machine.config = {
-            "CHROME_BINARY": "/tmp/chromium",
-            "NODE_BINARY": "/tmp/node",
+            "CHROME_BINARY": chrome_path,
+            "NODE_BINARY": node_path,
             "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
             "CHROME_ISOLATION": "snapshot",
             "CHROME_USER_DATA_DIR": "/tmp/profile",
             "CHROMIUM_VERSION": "123.4.5",
+            "WGET_BINARY": "/tmp/archivebox-test-missing-wget",
         }
         machine.save(update_fields=["config"])
         models._CURRENT_MACHINE = machine
 
         refreshed = Machine.current()
 
-        self.assertEqual(refreshed.config.get("CHROME_BINARY"), "/tmp/chromium")
-        self.assertEqual(refreshed.config.get("NODE_BINARY"), "/tmp/node")
+        self.assertEqual(refreshed.config.get("CHROME_BINARY"), chrome_path)
+        self.assertEqual(refreshed.config.get("NODE_BINARY"), node_path)
         self.assertEqual(refreshed.config.get("ABX_INSTALL_CACHE"), {"wget": "2026-03-24T00:00:00+00:00"})
         self.assertNotIn("CHROME_ISOLATION", refreshed.config)
         self.assertNotIn("CHROME_USER_DATA_DIR", refreshed.config)
         self.assertNotIn("CHROMIUM_VERSION", refreshed.config)
+        self.assertNotIn("WGET_BINARY", refreshed.config)
 
     def test_get_config_auto_applies_current_machine_config(self):
         """get_config() should include sanitized Machine.current() config by default."""
         import archivebox.machine.models as models
         from archivebox.config.common import get_config
 
+        chrome_path = "/tmp/archivebox-test-chromium"
+        open(chrome_path, "a").close()
+        self.addCleanup(lambda: os.path.exists(chrome_path) and os.remove(chrome_path))
         machine = Machine.current()
         machine.config = {
-            "CHROME_BINARY": "/tmp/chromium",
+            "CHROME_BINARY": chrome_path,
             "ABX_INSTALL_CACHE": {"chrome": "2026-03-24T00:00:00+00:00"},
             "CHROME_ISOLATION": "snapshot",
         }
@@ -170,7 +181,7 @@ class TestMachineModel(TestCase):
 
         config = get_config()
 
-        self.assertEqual(config.CHROME_BINARY, "/tmp/chromium")
+        self.assertEqual(config.CHROME_BINARY, chrome_path)
         self.assertEqual(config["ABX_INSTALL_CACHE"], {"chrome": "2026-03-24T00:00:00+00:00"})
         self.assertEqual(config.CHROME_ISOLATION, "crawl")
 
@@ -822,6 +833,24 @@ class TestProcessClassMethods(TestCase):
         kill_tree.assert_not_called()
         terminate.assert_not_called()
 
+    def test_cleanup_orphaned_workers_marks_non_running_children_exited(self):
+        """cleanup_orphaned_workers should retire child rows whose OS process is already gone."""
+        child = Process.objects.create(
+            machine=self.machine,
+            process_type=Process.TypeChoices.HOOK,
+            status=Process.StatusChoices.RUNNING,
+            pid=999997,
+            started_at=timezone.now() - timedelta(minutes=5),
+        )
+
+        cleaned = Process.cleanup_orphaned_workers()
+
+        self.assertEqual(cleaned, 1)
+        child.refresh_from_db()
+        self.assertEqual(child.status, Process.StatusChoices.EXITED)
+        self.assertIsNotNone(child.ended_at)
+        self.assertEqual(child.exit_code, 0)
+
 
 class TestProcessStateMachine(TestCase):
     """Test the ProcessMachine state machine."""

archivebox/tests/test_runner.py

@@ -302,6 +302,7 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
 
 def test_ensure_background_runner_starts_when_none_running(monkeypatch):
     import archivebox.machine.models as machine_models
+    import archivebox.workers.supervisord_util as supervisord_util
     from archivebox.services import runner as runner_module
 
     popen_calls = []
@@ -310,6 +311,7 @@ def test_ensure_background_runner_starts_when_none_running(monkeypatch):
         def __init__(self, args, **kwargs):
             popen_calls.append((args, kwargs))
 
+    monkeypatch.setattr(supervisord_util, "get_existing_supervisord_process", lambda: None)
     monkeypatch.setattr(machine_models.Process, "cleanup_stale_running", classmethod(lambda cls, machine=None: 0))
     monkeypatch.setattr(machine_models.Process, "cleanup_orphaned_workers", classmethod(lambda cls: 0))
     monkeypatch.setattr(machine_models.Machine, "current", classmethod(lambda cls: SimpleNamespace(id="machine-1")))
@@ -330,8 +332,10 @@ def test_ensure_background_runner_starts_when_none_running(monkeypatch):
 
 def test_ensure_background_runner_skips_when_orchestrator_running(monkeypatch):
     import archivebox.machine.models as machine_models
+    import archivebox.workers.supervisord_util as supervisord_util
     from archivebox.services import runner as runner_module
 
+    monkeypatch.setattr(supervisord_util, "get_existing_supervisord_process", lambda: None)
     monkeypatch.setattr(machine_models.Process, "cleanup_stale_running", classmethod(lambda cls, machine=None: 0))
     monkeypatch.setattr(machine_models.Process, "cleanup_orphaned_workers", classmethod(lambda cls: 0))
     monkeypatch.setattr(machine_models.Machine, "current", classmethod(lambda cls: SimpleNamespace(id="machine-1")))
@@ -351,6 +355,31 @@ def test_ensure_background_runner_skips_when_orchestrator_running(monkeypatch):
     assert started is False
 
 
+def test_ensure_background_runner_skips_when_supervisord_runner_running(monkeypatch):
+    import archivebox.machine.models as machine_models
+    import archivebox.workers.supervisord_util as supervisord_util
+    from archivebox.services import runner as runner_module
+
+    supervisor = object()
+
+    monkeypatch.setattr(supervisord_util, "get_existing_supervisord_process", lambda: supervisor)
+    monkeypatch.setattr(supervisord_util, "get_worker", lambda supervisor_arg, name: {"statename": "RUNNING"})
+    monkeypatch.setattr(
+        machine_models.Process,
+        "cleanup_stale_running",
+        classmethod(lambda cls, machine=None: (_ for _ in ()).throw(AssertionError("db process cleanup should not run"))),
+    )
+    monkeypatch.setattr(
+        runner_module.subprocess,
+        "Popen",
+        lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("runner should not be spawned")),
+    )
+
+    started = runner_module.ensure_background_runner(allow_under_pytest=True)
+
+    assert started is False
+
+
 def test_runner_task_context_clears_inherited_abxbus_handler_context(tmp_path):
     from abx_dl.events import CrawlEvent, MachineEvent
     from abx_dl.orchestrator import create_bus
@@ -395,7 +424,7 @@ def test_runner_task_context_clears_inherited_abxbus_handler_context(tmp_path):
 
 
 @pytest.mark.django_db(transaction=True)
-def test_machine_service_persists_only_derived_config_events():
+def test_machine_service_persists_only_derived_config_events(tmp_path):
     from abx_dl.events import MachineEvent
     from abx_dl.orchestrator import create_bus
     from archivebox.machine.models import Machine
@@ -404,6 +433,9 @@ def test_machine_service_persists_only_derived_config_events():
     machine = Machine.current()
     machine.config = {}
     machine.save(update_fields=["config"])
+    wget_binary = tmp_path / "wget"
+    wget_binary.write_text("#!/bin/sh\n")
+    wget_binary.chmod(0o755)
 
     async def run_test():
         bus = create_bus(name="test_machine_service_persists_only_derived_config_events")
@@ -424,7 +456,7 @@ def test_machine_service_persists_only_derived_config_events():
             derived_event = bus.emit(
                 MachineEvent(
                     config={
-                        "WGET_BINARY": "/tmp/wget",
+                        "WGET_BINARY": str(wget_binary),
                         "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
                         "CHROME_USER_DATA_DIR": "/tmp/stale-derived-profile",
                     },
@@ -441,7 +473,7 @@ def test_machine_service_persists_only_derived_config_events():
 
     machine.refresh_from_db()
     assert machine.config == {
-        "WGET_BINARY": "/tmp/wget",
+        "WGET_BINARY": str(wget_binary),
         "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
     }
 
@@ -508,13 +540,16 @@ def test_runner_prepare_refreshes_network_interface_and_attaches_current_process
     assert saved_updates == [("iface", "machine", "modified_at")]
 
 
-def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
+def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch, tmp_path):
     from archivebox.machine.models import Machine, NetworkInterface, Process
     from archivebox.services import runner as runner_module
     from archivebox.config import common as config_common
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
 
+    wget_binary = tmp_path / "wget"
+    wget_binary.write_text("#!/bin/sh\n")
+    wget_binary.chmod(0o755)
     machine = Machine.objects.create(
         guid="test-guid-runner-overrides",
         hostname="runner-host",
@@ -530,7 +565,7 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
         os_kernel="Darwin",
         stats={},
         config={
-            "WGET_BINARY": "/tmp/wget",
+            "WGET_BINARY": str(wget_binary),
             "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
             "CHROME_ISOLATION": "snapshot",
             "CHROME_USER_DATA_DIR": "/tmp/stale-profile",
@@ -560,7 +595,7 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
     crawl_runner.load_run_state()
 
     assert crawl_runner.derived_config == {
-        "WGET_BINARY": "/tmp/wget",
+        "WGET_BINARY": str(wget_binary),
         "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
     }
 
@@ -1239,6 +1274,34 @@ def test_run_pending_crawls_prioritizes_queued_crawl_before_unrelated_binary_bac
     assert binary_calls == []
 
 
+def test_run_pending_crawls_disables_missing_absolute_binary_backlog(monkeypatch, tmp_path):
+    from archivebox.machine.models import Binary, Machine
+    from archivebox.services import runner as runner_module
+
+    missing_binary = tmp_path / "missing-node"
+    binary = Binary.objects.create(
+        machine=Machine.current(),
+        name=str(missing_binary),
+        status=Binary.StatusChoices.QUEUED,
+        retry_at=runner_module.timezone.now(),
+        binproviders="env,apt",
+        overrides={"apt": {"install_args": ["nodejs"]}},
+    )
+
+    monkeypatch.setattr(
+        runner_module,
+        "run_binary",
+        lambda binary_id: (_ for _ in ()).throw(AssertionError("missing absolute binary should not be retried")),
+    )
+
+    result = runner_module.run_pending_crawls(daemon=False)
+
+    binary.refresh_from_db()
+    assert result == 0
+    assert binary.status == Binary.StatusChoices.QUEUED
+    assert binary.retry_at is None
+
+
 @pytest.mark.django_db(transaction=True)
 def test_crawl_completed_event_requeues_active_snapshots():
     from archivebox.base_models.models import get_or_create_system_user_pk

archivebox/tests/test_search.py

@@ -0,0 +1,34 @@
+import os
+from pathlib import Path
+
+from benedict import benedict
+
+
+def test_search_backend_env_exposes_resolved_runtime_config(monkeypatch, tmp_path):
+    from archivebox.search import search_backend_env
+
+    monkeypatch.setenv("SEARCH_BACKEND_SONIC_HOST_NAME", "old-host")
+    config = benedict(
+        {
+            "DATA_DIR": tmp_path,
+            "USERS_DIR": tmp_path / "archive" / "users",
+            "SEARCH_BACKEND_ENGINE": "sonic",
+            "SEARCH_BACKEND_SONIC_HOST_NAME": "sonic",
+            "SEARCH_BACKEND_SONIC_PORT": 1491,
+            "SEARCH_BACKEND_SONIC_PASSWORD": "SecretPassword",
+            "USE_INDEXING_BACKEND": True,
+            "IGNORED_NONE_VALUE": None,
+        },
+    )
+
+    with search_backend_env(config=config):
+        assert os.environ["DATA_DIR"] == str(tmp_path)
+        assert os.environ["SNAP_DIR"] == str(Path(tmp_path) / "archive" / "users")
+        assert os.environ["SEARCH_BACKEND_ENGINE"] == "sonic"
+        assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "sonic"
+        assert os.environ["SEARCH_BACKEND_SONIC_PORT"] == "1491"
+        assert os.environ["SEARCH_BACKEND_SONIC_PASSWORD"] == "SecretPassword"
+        assert os.environ["USE_INDEXING_BACKEND"] == "True"
+        assert "IGNORED_NONE_VALUE" not in os.environ
+
+    assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "old-host"

archivebox/tests/test_urls.py

@@ -235,10 +235,36 @@ class TestUrlRouting:
             assert resp.status_code in (301, 302)
             assert resp["Location"].startswith("/api/")
 
+            resp = client.get("/api/archive/https://example.com/", HTTP_HOST=api_host)
+            assert resp.status_code in (301, 302)
+            assert resp["Location"] == f"http://{web_host}/web/https://example.com/"
+
             print("OK")
             """,
         )
 
+    def test_api_archive_redirect_uses_public_web_base_url(self) -> None:
+        self._run(
+            """
+            client = Client()
+
+            resp = client.get(
+                "/api/archive/https://example.com/",
+                HTTP_HOST="api.archivebox.io",
+                secure=True,
+            )
+
+            assert resp.status_code in (301, 302)
+            assert resp["Location"] == "https://web.archivebox.io/web/https://example.com/"
+
+            print("OK")
+            """,
+            mode="safe-subdomains-fullreplay",
+            env_overrides={
+                "LISTEN_HOST": "archivebox.io",
+            },
+        )
+
     def test_web_admin_routing(self) -> None:
         self._run(
             """

archivebox/workers/supervisord_util.py

@@ -577,7 +577,12 @@ def watch_worker(supervisor, daemon_name, interval=5):
 
 
 def get_sonic_supervisord_worker_from_plugin(config) -> dict[str, str] | None:
-    from abx_plugins.plugins.search_backend_sonic.daemon import get_sonic_supervisord_worker
+    try:
+        from abx_plugins.plugins.search_backend_sonic.daemon import get_sonic_supervisord_worker
+    except ModuleNotFoundError as err:
+        if err.name != "abx_plugins.plugins.search_backend_sonic.daemon":
+            raise
+        return None
 
     worker = get_sonic_supervisord_worker(config)
     return cast(dict[str, str] | None, worker)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.30rc57"
+version = "0.9.30rc81"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -80,8 +80,8 @@ dependencies = [
     ### Binary/Package Management
     "abxbus>=2.5.4",          # EventBus API
     "abxpkg>=1.10.7",         # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins>=1.10.56",   # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl>=1.10.51",        # shared ArchiveBox downloader package with blocking install preflight
+    "abx-plugins>=1.10.57",   # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl>=1.10.56",        # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]