Refactor plugins search progress and config flows

Dockerfile

@@ -313,11 +313,10 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
     && mkdir -p "$LIB_DIR" \
     && apt-get update -qq \
-    && abxpkg install --no-cache --install-timeout=900 --binproviders=playwright chrome \
-    && CHROME_BINARY="$(abxpkg load --binproviders=playwright chromium | awk 'NF {print $2; exit}')" \
-    && export CHROME_BINARY \
-    && test -x "$CHROME_BINARY" \
-    && "$CHROME_BINARY" --version | tee -a /VERSION.txt \
+    && if [ "$TARGETARCH" = "arm64" ]; then \
+        abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
+        abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
+    fi \
     && TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
     && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
     && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
@@ -371,11 +370,8 @@ RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary
 # Verify ArchiveBox is installed and write full version/dependency info.
 RUN chmod +x "$CODE_DIR"/bin/*.sh \
     && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && CHROME_BINARY="$(abxpkg load --binproviders=playwright chromium | awk 'NF {print $2; exit}')" \
-    && export CHROME_BINARY \
-    && test -x "$CHROME_BINARY" \
-    && "$CHROME_BINARY" --version | tee -a /VERSION.txt \
+    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
+    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
     && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
     && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
     && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \

Dockerfile.multistage

@@ -52,7 +52,7 @@ ENV CODE_DIR=/app \
     DATA_DIR=/data \
     LIB_DIR=/opt/archivebox/lib \
     ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/browsers \
+    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
     PERSONAS_DIR=/data/personas \
     CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
     CHROME_HEADLESS=true \

archivebox/api/models.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.api"
 
 import secrets
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 
 from django.conf import settings
 from django.db import models
@@ -17,7 +17,7 @@ def generate_secret_token() -> str:
 
 
 class APIToken(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
@@ -41,7 +41,7 @@ class APIToken(models.Model):
 
 
 class OutboundWebhook(WebhookBase):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)

archivebox/api/urls.py

@@ -7,7 +7,7 @@ from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView
 
-from archivebox.core.host_util import build_web_url
+from archivebox.core.routes_util import build_web_url
 from .v1_api import urls as v1_api_urls
 
 

archivebox/api/v1_cli.py

@@ -66,6 +66,8 @@ class AddCommandSchema(Schema):
     parser: str = "auto"
     plugins: str = ""
     only_new: bool | None = None
+    update: bool = False
+    overwrite: bool = False
     index_only: bool = False
 
 
@@ -90,6 +92,8 @@ class ScheduleCommandSchema(Schema):
     tag: str = ""
     depth: int = 0
     only_new: bool | None = None
+    update: bool = False
+    overwrite: bool = False
     clear: bool = False
 
 
@@ -120,6 +124,8 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
     config_overrides: dict[str, object] = {}
     if args.only_new is not None:
         config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
     crawl, snapshots = add(
         urls=args.urls,
         snapshot_ids=args.snapshot_ids,
@@ -189,6 +195,8 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
     config_overrides: dict[str, object] = {}
     if args.only_new is not None:
         config_overrides["ONLY_NEW"] = bool(args.only_new)
+    if args.update or args.overwrite:
+        config_overrides["ONLY_NEW"] = False
     result = schedule(
         import_path=args.import_path,
         add=args.add,

archivebox/api/v1_core.py

@@ -31,7 +31,7 @@ from archivebox.core.models import Snapshot, ArchiveResult, Tag
 from archivebox.core.permissions import public_snapshots_queryset
 from archivebox.api.auth import auth_using_token
 from archivebox.config.common import get_config
-from archivebox.core.host_util import build_web_url
+from archivebox.core.routes_util import build_web_url
 from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
 from archivebox.core.tag_util import (
     add_snapshot_counts,
@@ -50,6 +50,8 @@ from archivebox.core.tag_util import (
 )
 from archivebox.crawls.models import Crawl
 from archivebox.api.v1_crawls import CrawlSchema
+from archivebox.search.config import get_search_mode, get_search_mode_backend
+from archivebox.search.query import apply_snapshot_search
 
 
 router = Router(tags=["Core Models"])
@@ -855,12 +857,8 @@ class SnapshotFilterSchema(FilterSchema):
     modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
     modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
     modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
-    search: Annotated[
-        str | None,
-        FilterLookup(
-            ["url__icontains", "title__icontains", "tags__name__icontains", "id__istartswith", "id__iendswith", "timestamp__startswith"],
-        ),
-    ] = None
+    search: str | None = None
+    search_mode: str | None = None
     url: Annotated[str | None, FilterLookup("url")] = None
     tag: Annotated[str | None, FilterLookup("tags__name")] = None
     title: Annotated[str | None, FilterLookup("title__icontains")] = None
@@ -868,14 +866,37 @@ class SnapshotFilterSchema(FilterSchema):
     bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
     bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
 
+    def filter_search(self, value: str | None) -> Q:
+        return Q()
+
+    def filter_search_mode(self, value: str | None) -> Q:
+        return Q()
+
 
 @router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
 @paginate(CustomPagination)
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
     """List all Snapshot entries matching these filters."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.all()
-    return filters.filter(queryset).distinct()
+    queryset = filters.filter(Snapshot.objects.all()).distinct()
+    query = (filters.search or "").strip()
+    if not query:
+        return queryset
+
+    runtime_config = getattr(request, "archivebox_config", None)
+    search_mode = get_search_mode(filters.search_mode, config=runtime_config)
+    try:
+        return apply_snapshot_search(
+            queryset,
+            query,
+            search_mode=search_mode,
+            config=runtime_config,
+            include_id_matches=True,
+        )
+    except Exception:
+        if get_search_mode_backend(search_mode, config=runtime_config):
+            return queryset.none()
+        return apply_snapshot_search(queryset, query, search_mode="meta", config=runtime_config, include_id_matches=True)
 
 
 @router.get("/snapshots.rss", url_name="get_snapshots_rss")
@@ -1212,7 +1233,7 @@ def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
     is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
     if is_full_uuid:
         try:
-            return snapshot_qs.get(pk=snapshot_ref)
+            return snapshot_qs.get(pk=snapshot_ref.replace("-", ""))
         except (Snapshot.DoesNotExist, ValueError):
             pass
 
@@ -1295,7 +1316,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
         raise HttpError(401, "Authentication required")
 
     public_only = not getattr(request.user, "is_authenticated", False) and not getattr(request, "_api_token", None)
-    queryset = get_matching_tags(q, with_snapshot_counts=False)
+    queryset = get_matching_tags(q)
     public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
     if public_only:
         queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()

archivebox/api/v1_crawls.py

@@ -122,7 +122,8 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
-    config.setdefault("PERMISSIONS", str(get_config(user=request.user).PERMISSIONS))
+    request_user = request.user if request.user.is_authenticated else None
+    config.setdefault("PERMISSIONS", str(get_config(user=request_user).PERMISSIONS))
     crawl = Crawl.objects.create(
         urls="\n".join(urls),
         max_depth=data.max_depth,

archivebox/base_models/admin.py

@@ -77,7 +77,7 @@ class KeyValueWidget(forms.Widget):
         """Get available config options from plugins."""
         try:
             from archivebox.config.common import ArchiveBoxConfig
-            from archivebox.hooks import discover_plugin_configs
+            from archivebox.plugins.discovery import discover_plugin_configs
 
             options: dict[str, ConfigOption] = {}
             skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "SNAP_DIR"}

archivebox/base_models/models.py

@@ -6,7 +6,7 @@ import json
 import shutil
 from typing import Any
 
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
 from django.db import models
@@ -65,7 +65,7 @@ class AutoDateTimeField(models.DateTimeField):
 
 
 class ModelWithUUID(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
     created_by = models.ForeignKey(

archivebox/cli/archivebox_add.py

@@ -185,7 +185,7 @@ def add(
         label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
         created_by_id=created_by_id,
         status=Crawl.StatusChoices.QUEUED,
-        retry_at=None if index_only else timezone.now(),
+        retry_at=None if (index_only or bg) else timezone.now(),
         config=crawl_config,
     )
 
@@ -198,15 +198,8 @@ def add(
     #    Discovered URLs become child Snapshots (depth+1)
 
     if index_only:
-        # ``--index-only`` means "add the URLs to the index without archiving
-        # them now". That only holds if we actually materialize the Snapshot
-        # rows here — otherwise the CLI returns success with nothing in the
-        # index, which broke ``test_add_url_after_init`` & friends. Create
-        # the Snapshots synchronously (the same step the runner would do)
-        # but skip starting any worker so extractors don't run.
-        crawl.create_snapshots_from_urls()
-        print("[yellow]\\[*] Index-only mode - URLs indexed, runner not started[/yellow]")
-        return crawl, crawl.snapshot_set.all()
+        print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
+        return crawl, crawl.snapshot_set.none()
 
     # 5. Start the crawl runner to process the queue
     #    The runner will:
@@ -257,13 +250,14 @@ def add(
         # Print summary for foreground runs
         try:
             crawl.refresh_from_db()
-            snapshots_count = crawl.snapshot_set.count()
             try:
                 from django.db.models import Count, Sum
 
-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("archiveresult__output_size"))
-                total_bytes = int(totals["total_bytes"] or 0) if totals["snapshot_count"] else 0
+                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("output_size"))
+                snapshots_count = int(totals["snapshot_count"] or 0)
+                total_bytes = int(totals["total_bytes"] or 0)
             except Exception:
+                snapshots_count = crawl.snapshot_set.count()
                 total_bytes, _, _ = get_dir_size(crawl.output_dir)
             total_size = printable_filesize(total_bytes)
             total_time = timezone.now() - started_at
@@ -284,9 +278,9 @@ def add(
             except Exception:
                 rel_output_str = str(crawl.output_dir)
 
-            from archivebox.core.host_util import build_admin_url
+            from archivebox.core.routes_util import build_admin_url
 
-            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id.hex}/change/", config=config)
+            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
 
             print("\n[bold]crawl output saved to:[/bold]")
             print(f"  {rel_output_str}")
@@ -330,6 +324,8 @@ def add(
     "Pass --no-only-new to force re-archive of URLs that already exist.",
 )
 @click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
+@click.option("--overwrite", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
+@click.option("--update", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
 @click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
 @click.argument("urls", nargs=-1, type=click.Path())
 @docstring(add.__doc__)
@@ -360,7 +356,11 @@ def main(**kwargs):
 
         # Translate --only-new/--no-only-new into a crawl config override.
         # add() takes config overrides as a dict; no per-flag kwargs.
+        overwrite = kwargs.pop("overwrite", False)
+        update = kwargs.pop("update", False)
         only_new = kwargs.pop("only_new", None)
+        if overwrite or update:
+            only_new = False
         if only_new is not None:
             kwargs["config"] = {"ONLY_NEW": bool(only_new)}
 

archivebox/cli/archivebox_archiveresult.py

@@ -74,7 +74,7 @@ def create_archiveresults(
         1: Failure
     """
     from archivebox.config.common import get_config
-    from archivebox.hooks import discover_hooks
+    from archivebox.plugins.hooks import discover_hooks
     from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
     from archivebox.core.models import Snapshot
 

archivebox/cli/archivebox_config.py

@@ -31,7 +31,7 @@ def config(
     from abx_plugins.plugins.base.utils import resolve_alias
     from archivebox.config.collection import write_config_file
     from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
 
     check_data_folder()
 

archivebox/cli/archivebox_init.py

@@ -73,6 +73,8 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
     config.USERS_DIR.mkdir(parents=True, exist_ok=True)
     Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
+    for path in (Path(CONSTANTS.SOURCES_DIR), config.ARCHIVE_DIR, config.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
+        path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
 
     print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
 

archivebox/cli/archivebox_install.py

@@ -28,6 +28,11 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
 
     config = get_config()
     archive_dir = config.ARCHIVE_DIR
+
+    if dry_run:
+        print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")
+        return
+
     if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()):
         init()  # must init full index because we need a db to store Binary entries in
 
@@ -47,10 +52,6 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
         print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
         print()
 
-    if dry_run:
-        print("[dim]Dry run - would run the abx-dl install flow[/dim]")
-        return
-
     # Set up Django
     from archivebox.config.django import setup_django
 

archivebox/cli/archivebox_list.py

@@ -20,7 +20,7 @@ from archivebox.cli.archivebox_snapshot import list_snapshots
 @click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
 @click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
 @click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
+@click.option("--search", help="Search mode to use for the query")
 @click.argument("query", nargs=-1)
 def main(
     status: str | None,

archivebox/cli/archivebox_pluginmap.py

@@ -56,13 +56,15 @@ def pluginmap(
     from rich.panel import Panel
     from rich import box
 
-    from archivebox.hooks import (
-        BUILTIN_PLUGINS_DIR,
-        USER_PLUGINS_DIR,
+    from archivebox.plugins.hooks import (
         discover_hooks,
         is_background_hook,
         normalize_hook_event_name,
     )
+    from archivebox.plugins.discovery import (
+        BUILTIN_PLUGINS_DIR,
+        USER_PLUGINS_DIR,
+    )
 
     console = Console()
     prnt = console.print

archivebox/cli/archivebox_remove.py

@@ -65,7 +65,7 @@ def remove(
     log_removal_started(snapshots, yes=yes)
 
     from archivebox.core.models import Snapshot
-    from archivebox.search import flush_search_index
+    from archivebox.search.query import flush_search_index
 
     # Freeze the target set up-front so a concurrent daemon writing new
     # snapshots can't extend the deletion under us, and so the cursor isn't

archivebox/cli/archivebox_run.py

@@ -264,7 +264,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
         from django.utils import timezone
         from archivebox.crawls.models import Crawl
 
-        crawl = Crawl.objects.filter(id=crawl_id, status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED]).first()
+        crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
         now = timezone.now()
         # Only re-lease when the row is unscheduled (retry_at IS NULL) or its
         # existing lease has already expired. A future retry_at means another

archivebox/cli/archivebox_server.py

@@ -162,7 +162,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
     # CSRF_TRUSTED_ORIGINS set, get_base_url() will silently use that as the
     # implicit BASE_URL. Surface what we picked so the user knows where their
     # links / redirects are going — and tell them how to make it explicit.
-    from archivebox.core.host_util import derive_base_url_from_csrf
+    from archivebox.core.routes_util import derive_base_url_from_csrf
 
     csrf_derived = derive_base_url_from_csrf(config)
     if csrf_derived:
@@ -178,7 +178,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
         print()
         return
 
-    # BASE_URL was not set explicitly. The host_util derivation gives one of
+    # BASE_URL was not set explicitly. The routes_util derivation gives one of
     # three results, with very different risk profiles — show a tailored hint
     # so new users coming from the 0.7.x single-domain world know whether the
     # default is fine for them or needs attention.
@@ -203,7 +203,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
             )
         print()
     else:
-        # Loopback / wildcard bind. The host_util default of
+        # Loopback / wildcard bind. The routes_util default of
         # http://archivebox.localhost:PORT works in a browser on the same
         # machine, but anything else (reverse proxy, k8s ingress, LAN client)
         # needs BASE_URL set. (Real hostnames can't reach this branch — the
@@ -298,7 +298,7 @@ def server(
         return
 
     os.environ["BIND_ADDR"] = f"{host}:{port}"
-    from archivebox.core.host_util import get_base_url
+    from archivebox.core.routes_util import get_base_url
 
     base_url = get_base_url().rstrip("/")
     admin_url = f"{base_url}/admin/"

archivebox/cli/archivebox_snapshot.py

@@ -35,7 +35,7 @@ from collections.abc import Iterable
 
 import rich_click as click
 from rich import print as rprint
-from django.db.models import Case, IntegerField, Q, QuerySet, When
+from django.db.models import Case, IntegerField, QuerySet, When
 
 from archivebox.cli.cli_util import apply_filters
 
@@ -191,12 +191,7 @@ def build_snapshot_queryset(
     limit: int | None = None,
 ) -> QuerySet:
     from archivebox.core.models import Snapshot
-    from archivebox.search import (
-        get_default_search_mode,
-        get_search_mode,
-        prioritize_metadata_matches,
-        query_search_index,
-    )
+    from archivebox.search.query import apply_snapshot_search
 
     queryset = Snapshot.objects.order_by("-created_at")
     queryset = apply_filters(
@@ -214,39 +209,22 @@ def build_snapshot_queryset(
 
     query = (query or "").strip()
     if query:
-        metadata_qs = queryset.filter(
-            Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
-        )
-        requested_search_mode = (search or "").strip().lower()
-        if requested_search_mode == "content":
-            requested_search_mode = "contents"
-        search_mode = get_default_search_mode() if not requested_search_mode else get_search_mode(requested_search_mode)
-
-        if search_mode == "meta":
-            queryset = metadata_qs
-        elif limit and len(list(metadata_qs.values_list("pk", flat=True).distinct()[:limit])) >= limit:
-            queryset = metadata_qs
-        else:
-            try:
-                deep_qsearch = None
-                if search_mode == "deep":
-                    qsearch = query_search_index(query, search_mode="contents", max_results=limit)
-                    deep_qsearch = query_search_index(query, search_mode="deep", max_results=limit)
-                else:
-                    qsearch = query_search_index(query, search_mode=search_mode, max_results=limit)
-                queryset = prioritize_metadata_matches(
-                    queryset,
-                    metadata_qs,
-                    qsearch,
-                    deep_queryset=deep_qsearch,
-                    ordering=("-created_at",) if not sort else None,
-                )
-            except Exception as err:
-                rprint(
-                    f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                    file=sys.stderr,
-                )
-                queryset = metadata_qs
+        try:
+            queryset = apply_snapshot_search(
+                queryset,
+                query,
+                search_mode=search,
+                ordering=("-created_at",) if not sort else None,
+                max_results=limit,
+                skip_backend_when_metadata_satisfies_limit=True,
+                include_metadata_for_forced_backend=True,
+            )
+        except Exception as err:
+            rprint(
+                f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
+                file=sys.stderr,
+            )
+            queryset = apply_snapshot_search(queryset, query, search_mode="meta")
 
     if sort:
         queryset = queryset.order_by(sort)
@@ -531,7 +509,7 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int):
 @click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
 @click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
 @click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
+@click.option("--search", help="Search mode to use for the query")
 @click.argument("query", nargs=-1)
 def list_cmd(
     status: str | None,

archivebox/cli/archivebox_status.py

@@ -61,7 +61,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
             num_dirs += root_dirs
             num_files += root_files
     else:
-        num_bytes = ArchiveResult.objects.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
+        num_bytes = snapshots_qs.aggregate(total=Coalesce(Sum("output_size"), 0))["total"] or 0
         num_dirs = 0
         num_files = ArchiveResult.objects.exclude(output_files__in=["", "{}"]).count()
     size = printable_filesize(num_bytes)

archivebox/cli/archivebox_update.py

@@ -33,7 +33,7 @@ def _get_snapshot_crawl(snapshot: Snapshot) -> Crawl | None:
 
 def _get_search_indexing_plugins() -> list[str]:
     from abx_dl.models import discover_plugins
-    from archivebox.hooks import get_search_backends
+    from archivebox.plugins.discovery import get_search_backends
 
     available_backends = set(get_search_backends())
     plugins = discover_plugins()
@@ -629,6 +629,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     """
     from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
+    from django.db.models import Q
     from django.utils import timezone
 
     stats = {
@@ -694,7 +695,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     stats["sealed"] += updated_rows
     stats["updated_db"] += updated_rows
 
-    fs_version_rows = queryset.exclude(fs_version=current_fs_version)
+    fs_version_rows = queryset.exclude(fs_version=current_fs_version).filter(Q(retry_at__isnull=True) | Q(retry_at__gt=now))
     stale_batch = []
 
     def queue_stale_fs_batch() -> None:
@@ -746,14 +747,10 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     now = timezone.now()
     stats["crawls_queued"] = (
         Crawl.objects.filter(
-            status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED],
+            status__in=Crawl.RUNNABLE_STATES,
         )
         .exclude(
-            snapshot_set__status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-                Snapshot.StatusChoices.PAUSED,
-            ],
+            snapshot_set__status__in=Snapshot.OPEN_STATES,
         )
         .update(
             retry_at=now,
@@ -921,7 +918,7 @@ def print_index_stats(stats: dict[str, Any]) -> None:
 @click.option("--crawl-id", help="Filter by crawl ID")
 @click.option("--limit", "-n", type=int, help="Limit number of snapshots to update")
 @click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for positional query")
+@click.option("--search", help="Search mode to use for positional query")
 @click.option("--before", type=float, help="Only snapshots before timestamp")
 @click.option("--after", type=float, help="Only snapshots after timestamp")
 @click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")

archivebox/config/collection.py

@@ -98,7 +98,7 @@ def _resolve_section_for_key(key: str, config_sections, plugin_configs) -> str:
 def _render_config_file_content(config: dict[str, str]) -> str:
     """Render a flat config dict to INI text, grouped by inferred section."""
     from archivebox.config.common import get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
 
     config_sections = get_all_configs()
     plugin_configs = discover_plugin_configs()
@@ -284,7 +284,7 @@ def write_config_file(config: dict[str, str]) -> AttrDict:
     """
 
     from archivebox.config.common import get_all_configs
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
     from archivebox.misc.system import atomic_write
 
     config_path = CONSTANTS.CONFIG_FILE

archivebox/config/common.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 __package__ = "archivebox.config"
 
 import json
@@ -13,7 +15,7 @@ from typing import Any, ClassVar, cast
 from pathlib import Path
 
 from rich.console import Console
-from pydantic import BaseModel, Field, create_model, field_validator, model_validator
+from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
 from pydantic_settings import SettingsConfigDict
 from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
 
@@ -64,6 +66,18 @@ def permissions_from_legacy_public_flags(raw_config: Mapping[str, object]) -> st
 
 _SENSITIVE_CONFIG_KEY_NEEDLES = ("TOKEN", "SECRET", "API_KEY", "APIKEY", "PASSWORD")
 SENSITIVE_CONFIG_VALUE_REDACTED = "********"
+_SCOPE_CRAWL_FROZEN = "crawl_frozen"
+_SCOPE_CRAWL_EXECUTION = "crawl_execution"
+_SCOPE_SERVER = "server"
+
+
+@lru_cache(maxsize=1)
+def _plugin_sensitive_config_keys() -> frozenset[str]:
+    sensitive_keys: set[str] = set()
+    for prop_key, prop_schema in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).items():
+        if isinstance(prop_schema, Mapping) and prop_schema.get("x-sensitive"):
+            sensitive_keys.add(str(prop_key))
+    return frozenset(sensitive_keys)
 
 
 def is_sensitive_config_key(key: str) -> bool:
@@ -77,8 +91,9 @@ def is_sensitive_config_key(key: str) -> bool:
     REST API responses, and any future surface that round-trips raw config
     values all agree on which keys to redact.
     """
-    upper = (key or "").upper()
-    return any(needle in upper for needle in _SENSITIVE_CONFIG_KEY_NEEDLES)
+    key = str(key or "")
+    upper = key.upper()
+    return key in _plugin_sensitive_config_keys() or any(needle in upper for needle in _SENSITIVE_CONFIG_KEY_NEEDLES)
 
 
 def redact_sensitive_config(config: Mapping[str, Any] | None) -> dict[str, Any]:
@@ -103,6 +118,34 @@ def redact_sensitive_config(config: Mapping[str, Any] | None) -> dict[str, Any]:
     return redacted
 
 
+def normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | None) -> dict[str, Any]:
+    """Return a JSON-safe config dict suitable for storage or event payloads."""
+    if config is None:
+        return {}
+    if isinstance(config, BaseConfigSet):
+        config = config.model_dump(mode="json")
+    elif isinstance(config, str):
+        config = json.loads(config)
+    else:
+        config = dict(config)
+    return {key: value for key, value in json.loads(json.dumps(config, default=str)).items() if value is not None}
+
+
+def build_crawl_config_snapshot(
+    *,
+    user: Any = None,
+    persona: Any = None,
+    overrides: Mapping[str, Any] | None = None,
+    base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
+) -> dict[str, Any]:
+    """Build the frozen runtime config stored on Crawl.config at creation time."""
+    effective = get_config(user=user, persona=persona, base_config=base_config)
+    frozen = effective.for_crawl_frozen()
+    if overrides:
+        frozen = get_config(base_config=frozen, overrides=overrides, include_machine=False).for_crawl_frozen()
+    return frozen
+
+
 def rprint(*args, file=None, **kwargs):
     console = _STDERR_CONSOLE if file is sys.stderr else _STDOUT_CONSOLE
     console.print(*args, **kwargs)
@@ -110,6 +153,7 @@ def rprint(*args, file=None, **kwargs):
 
 class ShellConfig(BaseConfigSet):
     toml_section_header: str = "SHELL_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_CRAWL_EXECUTION)
 
     DEBUG: bool = Field(default="--debug" in sys.argv)
 
@@ -141,6 +185,7 @@ class ShellConfig(BaseConfigSet):
 
 class StorageConfig(BaseConfigSet):
     toml_section_header: str = "STORAGE_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     # ARCHIVE_DIR / USERS_DIR are resolved dynamically via get_config().
     ARCHIVE_DIR: Path = Field(default=CONSTANTS.ARCHIVE_DIR)
@@ -150,16 +195,16 @@ class StorageConfig(BaseConfigSet):
     # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
     # must be a short path due to unix path length restrictions for socket files (<100 chars)
     # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
-    TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR)
+    TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
     # LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
     # must be able to contain executable binaries (up to 5GB size)
     # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
-    LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR)
+    LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
     # LIB_BIN_DIR is an optional human-facing symlink convenience directory.
     # Runtime lookup must use provider-specific paths under LIB_DIR instead.
-    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR)
+    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
     # CUSTOM_TEMPLATES_DIR allows users to override default templates
     # defaults to DATA_DIR / 'user_templates' but can be configured
@@ -172,12 +217,14 @@ class StorageConfig(BaseConfigSet):
 
 class GeneralConfig(BaseConfigSet):
     toml_section_header: str = "GENERAL_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     TAG_SEPARATOR_PATTERN: str = Field(default=r"[,]")
 
 
 class ServerConfig(BaseConfigSet):
     toml_section_header: str = "SERVER_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     SERVER_SECURITY_MODES: ClassVar[tuple[str, ...]] = (
         "safe-subdomains-fullreplay",
@@ -258,6 +305,7 @@ class ServerConfig(BaseConfigSet):
 
 class DatabaseConfig(BaseConfigSet):
     toml_section_header: str = "DATABASE_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     DATABASE_NAME: str = Field(default=str(CONSTANTS.DATABASE_FILE), alias="ARCHIVEBOX_DATABASE_NAME")
     SQLITE_JOURNAL_MODE: str = Field(
@@ -277,6 +325,7 @@ class DatabaseConfig(BaseConfigSet):
 
 class ArchivingConfig(BaseConfigSet):
     toml_section_header: str = "ARCHIVING_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_CRAWL_FROZEN)
 
     PLUGINS: str = Field(
         default="",
@@ -284,6 +333,7 @@ class ArchivingConfig(BaseConfigSet):
     )
 
     ONLY_NEW: bool = Field(default=True)
+    INDEX_ONLY: bool = Field(default=False)
 
     TIMEOUT: int = Field(default=60)
     CRAWL_MAX_URLS: int = Field(default=0)
@@ -397,6 +447,7 @@ def parse_delete_after(value) -> timedelta | None:
 
 class SearchBackendConfig(BaseConfigSet):
     toml_section_header: str = "SEARCH_BACKEND_CONFIG"
+    _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep")
 
@@ -414,7 +465,7 @@ def _plugin_user_config(config: Mapping[str, object]) -> dict[str, str]:
 
 
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
 
     schemas: PluginSchemaDocuments = {}
     if BASE_CONFIG_PATH.exists():
@@ -473,12 +524,78 @@ class ArchiveBoxBaseConfig(
         populate_by_name=True,
     )
 
-    DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR)
-    ABX_RUNTIME: str = Field(default="archivebox")
-    CRAWL_DIR: Path | None = Field(default=None)
-    SNAP_DIR: Path | None = Field(default=None)
+    DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    ABX_RUNTIME: str = Field(default="archivebox", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    CRAWL_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    SNAP_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     computed_config_keys: ClassVar[tuple[str, ...]] = COMPUTED_CONFIG_KEYS
 
+    @classmethod
+    def _core_config_classes(cls) -> tuple[type[BaseConfigSet], ...]:
+        return (
+            ShellConfig,
+            StorageConfig,
+            GeneralConfig,
+            ServerConfig,
+            DatabaseConfig,
+            ArchivingConfig,
+            SearchBackendConfig,
+            LDAPConfig,
+        )
+
+    @classmethod
+    def _core_field_scope(cls, key: str) -> str | None:
+        if key == "toml_section_header":
+            return _SCOPE_SERVER
+        for config_cls in cls._core_config_classes():
+            field = config_cls.model_fields.get(key)
+            if field is None:
+                continue
+            default_scope = str(config_cls.__private_attributes__["_scope"].default)
+            extra = field.json_schema_extra
+            if isinstance(extra, dict) and "scope" in extra:
+                return str(extra["scope"])
+            return default_scope
+        if key in ArchiveBoxBaseConfig.model_fields:
+            field = ArchiveBoxBaseConfig.model_fields[key]
+            extra = field.json_schema_extra
+            if isinstance(extra, dict) and "scope" in extra:
+                return str(extra["scope"])
+            return _SCOPE_SERVER
+        return None
+
+    @classmethod
+    def _plugin_field_scope(cls, key: str) -> str | None:
+        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+            properties = schema.get("properties") if isinstance(schema, dict) else None
+            if not isinstance(properties, dict) or key not in properties:
+                continue
+            prop_schema = properties.get(key) or {}
+            if isinstance(prop_schema, Mapping) and prop_schema.get("x-scope"):
+                return str(prop_schema["x-scope"])
+            if str(plugin_name).startswith("search_backend_"):
+                return _SCOPE_SERVER
+            return _SCOPE_CRAWL_FROZEN
+        return None
+
+    @classmethod
+    def scope_for_key(cls, key: str) -> str:
+        return cls._core_field_scope(key) or cls._plugin_field_scope(key) or _SCOPE_SERVER
+
+    def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
+        allowed_scopes = {_SCOPE_CRAWL_FROZEN}
+        if include_execution:
+            allowed_scopes.add(_SCOPE_CRAWL_EXECUTION)
+        return {key: value for key, value in normalize_runtime_config(self).items() if type(self).scope_for_key(key) in allowed_scopes}
+
+    def for_crawl_execution(self) -> dict[str, Any]:
+        """Config safe to pass to crawl/snapshot hook execution."""
+        return self._scoped_config(include_execution=True)
+
+    def for_crawl_frozen(self) -> dict[str, Any]:
+        """Config safe to persist permanently on Crawl.config."""
+        return self._scoped_config(include_execution=False)
+
     @model_validator(mode="after")
     def resolve_runtime_paths(self):
         self.DATA_DIR = self.DATA_DIR.expanduser().resolve()
@@ -544,6 +661,7 @@ def get_config(
     machine: Any = None,
     include_machine: bool = True,
     resolve_plugins: bool = True,
+    redact_sensitive: bool = False,
 ) -> ArchiveBoxBaseConfig:
     """
     Get merged config from all sources.
@@ -552,12 +670,12 @@ def get_config(
     1. Explicit overrides
     2. Per-ArchiveResult config
     3. Per-snapshot config and output path
-    4. Per-crawl config and output path
-    5. Per-user config
-    6. Per-persona derived config
-    7. Current machine derived config
-    8. Environment variables
-    9. Config file (ArchiveBox.conf)
+    4. Frozen per-crawl config and output path
+    5. Per-user config (only when resolving outside a crawl)
+    6. Per-persona derived config (only when resolving outside a crawl)
+    7. Current machine derived config (only when resolving outside a crawl)
+    8. Environment variables (only when resolving outside a crawl)
+    9. Config file (ArchiveBox.conf, only when resolving outside a crawl)
     10. Plugin schema defaults
     11. Core config defaults
     """
@@ -567,7 +685,9 @@ def get_config(
     if crawl is None and snapshot is not None:
         crawl = snapshot.crawl
 
-    if include_machine and machine is None:
+    crawl_config_base = crawl is not None and base_config is None
+
+    if include_machine and machine is None and not crawl_config_base:
         try:
             from django.apps import apps
 
@@ -578,15 +698,19 @@ def get_config(
         except Exception:
             machine = None
 
-    if persona is None and crawl is not None:
+    if persona is None and crawl is not None and not crawl_config_base:
         persona = crawl.resolve_persona()
 
     config_data: ConfigPayload = dict(defaults or {})
-    if base_config is not None:
+    base_config_payload: ConfigPayload = {}
+    if crawl_config_base:
+        config_data.update(dict(crawl.config or {}))
+    elif base_config is not None:
         if isinstance(base_config, ArchiveBoxBaseConfig):
-            config_data.update(base_config.model_dump(mode="json"))
+            base_config_payload.update(base_config.model_dump(mode="json"))
         else:
-            config_data.update(dict(base_config))
+            base_config_payload.update(dict(base_config))
+        config_data.update(base_config_payload)
     else:
         config_data.update(ArchiveBoxConfig().model_dump(mode="json"))
         legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
@@ -595,19 +719,20 @@ def get_config(
 
     scope_overrides: ConfigPayload = {}
 
-    if include_machine and machine is not None and machine.config:
-        from archivebox.machine.models import _sanitize_machine_config
+    if not crawl_config_base:
+        if include_machine and machine is not None and machine.config:
+            from archivebox.machine.models import _sanitize_machine_config
 
-        scope_overrides.update(_sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")))
+            scope_overrides.update(_sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")))
 
-    if persona is not None:
-        scope_overrides.update(persona.get_derived_config())
+        if persona is not None:
+            scope_overrides.update(persona.get_derived_config())
 
-    user_config = getattr(user, "config", None)
-    if user_config:
-        scope_overrides.update(user_config)
+        user_config = getattr(user, "config", None)
+        if user_config:
+            scope_overrides.update(user_config)
 
-    if crawl is not None and crawl.config:
+    if crawl is not None and crawl.config and not crawl_config_base:
         scope_overrides.update(crawl.config)
 
     if crawl is not None:
@@ -638,13 +763,20 @@ def get_config(
             plugin_name: schema.get("properties", {}) for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)
         }
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
+        plugin_user_config = _plugin_user_config(scope_overrides)
+        if not crawl_config_base:
+            plugin_user_config = {**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **plugin_user_config}
         plugin_sections = resolve_plugin_configs(
             plugin_schemas,
             global_config=plugin_global_config,
-            user_config={**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **_plugin_user_config(scope_overrides)},
+            user_config=plugin_user_config,
         )
         for plugin_config in plugin_sections.values():
             config_data.update(plugin_config)
+        if base_config_payload:
+            config_data.update({key: value for key, value in base_config_payload.items() if key in _archivebox_config_input_names()})
+        if crawl_config_base:
+            config_data.update(dict(crawl.config or {}))
         config_data.update(archivebox_scope_overrides)
 
     config_data["ABX_RUNTIME"] = "archivebox"
@@ -673,6 +805,11 @@ def get_config(
             )
 
     config = ArchiveBoxConfig.model_validate(config_data)
+    if redact_sensitive:
+        for key in type(config).model_fields:
+            value = getattr(config, key, None)
+            if is_sensitive_config_key(key) and value not in (None, ""):
+                setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
     os.environ["LIB_DIR"] = str(config.LIB_DIR)
     os.environ["LIB_BIN_DIR"] = str(config.LIB_BIN_DIR)
     os.environ["ABXPKG_LIB_DIR"] = str(config.LIB_DIR)

archivebox/config/ldap.py

@@ -1,6 +1,6 @@
 __package__ = "archivebox.config"
 
-from pydantic import Field
+from pydantic import Field, PrivateAttr
 
 from archivebox.config.configset import BaseConfigSet
 
@@ -14,6 +14,7 @@ class LDAPConfig(BaseConfigSet):
     """
 
     toml_section_header: str = "LDAP_CONFIG"
+    _scope: str = PrivateAttr(default="server")
 
     LDAP_ENABLED: bool = Field(default=False)
     LDAP_SERVER_URI: str | None = Field(default=None)

archivebox/config/views.py

@@ -1,13 +1,9 @@
 __package__ = "archivebox.config"
 
-import html
-import json
 import os
 import inspect
-import re
 from pathlib import Path
 from typing import Any
-from collections.abc import Callable
 from urllib.parse import quote, urlencode
 from django.http import HttpRequest
 from django.utils import timezone
@@ -22,8 +18,6 @@ from archivebox.misc.util import parse_date
 
 from archivebox.machine.models import Binary
 
-ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
-ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
@@ -38,55 +32,6 @@ def format_parsed_datetime(value: object) -> str:
     return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""
 
 
-JSON_TOKEN_RE = re.compile(
-    r'(?P<key>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")(?=\s*:)'
-    r'|(?P<string>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")'
-    r"|(?P<boolean>\btrue\b|\bfalse\b)"
-    r"|(?P<null>\bnull\b)"
-    r"|(?P<number>-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)",
-)
-
-
-def render_code_block(text: str, *, highlighted: bool = False) -> str:
-    code = html.escape(text, quote=False)
-
-    if highlighted:
-
-        def _wrap_token(match: re.Match[str]) -> str:
-            styles = {
-                "key": "color: #0550ae;",
-                "string": "color: #0a7f45;",
-                "boolean": "color: #8250df; font-weight: 600;",
-                "null": "color: #6e7781; font-style: italic;",
-                "number": "color: #b35900;",
-            }
-            token_type = next(name for name, value in match.groupdict().items() if value is not None)
-            return f'<span style="{styles[token_type]}">{match.group(0)}</span>'
-
-        code = JSON_TOKEN_RE.sub(_wrap_token, code)
-
-    return (
-        '<pre style="max-height: 600px; overflow: auto; background: #f6f8fa; '
-        'border: 1px solid #d0d7de; border-radius: 6px; padding: 12px; margin: 0;">'
-        '<code style="font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, '
-        "'Liberation Mono', monospace; white-space: pre; line-height: 1.5;\">"
-        f"{code}"
-        "</code></pre>"
-    )
-
-
-def render_highlighted_json_block(value: Any) -> str:
-    return render_code_block(json.dumps(value, indent=2, ensure_ascii=False), highlighted=True)
-
-
-def get_plugin_docs_url(plugin_name: str) -> str:
-    return f"{ABX_PLUGINS_DOCS_BASE_URL}#{plugin_name}"
-
-
-def get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str:
-    return f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/{quote(hook_name)}"
-
-
 def get_live_config_url(key: str) -> str:
     return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
 
@@ -104,195 +49,6 @@ def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | N
     return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}"
 
 
-def get_machine_admin_url() -> str | None:
-    try:
-        from archivebox.machine.models import Machine
-
-        machine = Machine.current()
-        return getattr(machine, "admin_change_url", None) or f"/admin/machine/machine/{machine.id.hex}/change/"
-    except Exception:
-        return None
-
-
-def render_code_tag_list(values: list[str]) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = "".join(
-        str(
-            format_html(
-                '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                value,
-            ),
-        )
-        for value in values
-    )
-    return f'<div style="display: flex; flex-wrap: wrap;">{tags}</div>'
-
-
-def render_plugin_metadata_html(config: dict[str, Any]) -> str:
-    required_binaries = [
-        str(item.get("name")) for item in (config.get("required_binaries") or []) if isinstance(item, dict) and item.get("name")
-    ]
-    rows = (
-        ("Title", config.get("title") or "(none)"),
-        ("Description", config.get("description") or "(none)"),
-        ("Required Plugins", mark_safe(render_link_tag_list(config.get("required_plugins") or [], get_plugin_docs_url))),
-        ("Required Binaries", mark_safe(render_link_tag_list(required_binaries, get_environment_binary_url))),
-        ("Output MIME Types", mark_safe(render_code_tag_list(config.get("output_mimetypes") or []))),
-    )
-
-    rendered_rows = "".join(
-        str(
-            format_html(
-                '<div style="margin: 0 0 14px 0;"><div style="font-weight: 600; margin-bottom: 4px;">{}</div><div>{}</div></div>',
-                label,
-                value,
-            ),
-        )
-        for label, value in rows
-    )
-    return f'<div style="margin: 4px 0 0 0;">{rendered_rows}</div>'
-
-
-def render_link_tag_list(values: list[str], url_resolver: Callable[[str], str] | None = None) -> str:
-    if not values:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    tags = []
-    for value in values:
-        if url_resolver is None:
-            tags.append(
-                str(
-                    format_html(
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
-                        value,
-                    ),
-                ),
-            )
-        else:
-            tags.append(
-                str(
-                    format_html(
-                        '<a href="{}" style="text-decoration: none;">'
-                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
-                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>'
-                        "</a>",
-                        url_resolver(value),
-                        value,
-                    ),
-                ),
-            )
-    return f'<div style="display: flex; flex-wrap: wrap;">{"".join(tags)}</div>'
-
-
-def render_property_links(prop_name: str, prop_info: dict[str, Any], machine_admin_url: str | None) -> str:
-    links = [
-        str(format_html('<a href="{}">Computed value</a>', get_live_config_url(prop_name))),
-    ]
-    if machine_admin_url:
-        links.append(str(format_html('<a href="{}">Edit override</a>', machine_admin_url)))
-
-    fallback = prop_info.get("x-fallback")
-    if isinstance(fallback, str) and fallback:
-        links.append(str(format_html('<a href="{}">Fallback: <code>{}</code></a>', get_live_config_url(fallback), fallback)))
-
-    aliases = prop_info.get("x-aliases") or []
-    if isinstance(aliases, list):
-        for alias in aliases:
-            if isinstance(alias, str) and alias:
-                links.append(str(format_html('<a href="{}">Alias: <code>{}</code></a>', get_live_config_url(alias), alias)))
-
-    default = prop_info.get("default")
-    if prop_name.endswith("_BINARY") and isinstance(default, str) and default:
-        links.append(str(format_html('<a href="{}">Binary: <code>{}</code></a>', get_environment_binary_url(default), default)))
-
-    return " &nbsp; ".join(links)
-
-
-def render_config_properties_html(properties: dict[str, Any], machine_admin_url: str | None) -> str:
-    header_links = [
-        str(format_html('<a href="{}">Dependencies</a>', ENVIRONMENT_BINARIES_BASE_URL)),
-        str(format_html('<a href="{}">Installed Binaries</a>', INSTALLED_BINARIES_BASE_URL)),
-    ]
-    if machine_admin_url:
-        header_links.insert(0, str(format_html('<a href="{}">Machine Config Editor</a>', machine_admin_url)))
-
-    cards = [
-        f'<div style="margin: 0 0 16px 0;">{" &nbsp; | &nbsp; ".join(header_links)}</div>',
-    ]
-
-    for prop_name, prop_info in properties.items():
-        prop_type = prop_info.get("type", "unknown")
-        if isinstance(prop_type, list):
-            prop_type = " | ".join(str(type_name) for type_name in prop_type)
-        prop_desc = prop_info.get("description", "")
-
-        default_html = ""
-        if "default" in prop_info:
-            default_html = str(
-                format_html(
-                    '<div style="margin-top: 6px;"><b>Default:</b> <code>{}</code></div>',
-                    prop_info["default"],
-                ),
-            )
-
-        description_html = prop_desc or mark_safe('<span style="color: #6e7781;">(no description)</span>')
-        cards.append(
-            str(
-                format_html(
-                    '<div style="margin: 0 0 14px 0; padding: 12px; background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 6px;">'
-                    '<div style="margin-bottom: 6px;">'
-                    '<a href="{}" style="font-weight: 600;"><code>{}</code></a>'
-                    ' <span style="color: #6e7781;">({})</span>'
-                    "</div>"
-                    '<div style="margin-bottom: 6px;">{}</div>'
-                    '<div style="font-size: 0.95em;">{}</div>'
-                    "{}"
-                    "</div>",
-                    get_live_config_url(prop_name),
-                    prop_name,
-                    prop_type,
-                    description_html,
-                    mark_safe(render_property_links(prop_name, prop_info, machine_admin_url)),
-                    mark_safe(default_html),
-                ),
-            ),
-        )
-
-    return "".join(cards)
-
-
-def render_hook_links_html(plugin_name: str, hooks: list[str], source: str) -> str:
-    if not hooks:
-        return '<span style="color: #6e7781;">(none)</span>'
-
-    items = []
-    for hook_name in hooks:
-        if source == "builtin":
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><a href="{}" target="_blank" rel="noopener noreferrer"><code>{}</code></a></div>',
-                        get_plugin_hook_source_url(plugin_name, hook_name),
-                        hook_name,
-                    ),
-                ),
-            )
-        else:
-            items.append(
-                str(
-                    format_html(
-                        '<div style="margin: 0 0 8px 0;"><code>{}</code></div>',
-                        hook_name,
-                    ),
-                ),
-            )
-    return "".join(items)
-
-
 def render_binary_detail_description(name: str, merged: dict[str, Any], db_binary: Any) -> str:
     installed_binary_url = get_installed_binary_change_url(name, db_binary)
 
@@ -386,48 +142,6 @@ def get_db_binaries_by_name() -> dict[str, Binary]:
     return {name: max(records, key=_binary_sort_key) for name, records in grouped.items()}
 
 
-def get_filesystem_plugins() -> dict[str, dict[str, Any]]:
-    """Discover plugins from filesystem directories."""
-    import json
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR
-
-    plugins = {}
-
-    for base_dir, source in [(BUILTIN_PLUGINS_DIR, "builtin"), (USER_PLUGINS_DIR, "user")]:
-        if not base_dir.exists():
-            continue
-
-        for plugin_dir in base_dir.iterdir():
-            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
-                plugin_id = f"{source}.{plugin_dir.name}"
-
-                # Find hook scripts
-                hooks = []
-                for ext in ("sh", "py", "js"):
-                    hooks.extend(plugin_dir.glob(f"on_*__*.{ext}"))
-
-                # Load config.json if it exists
-                config_file = plugin_dir / "config.json"
-                config_data = None
-                if config_file.exists():
-                    try:
-                        with open(config_file) as f:
-                            config_data = json.load(f)
-                    except (json.JSONDecodeError, OSError):
-                        config_data = None
-
-                plugins[plugin_id] = {
-                    "id": plugin_id,
-                    "name": plugin_dir.name,
-                    "path": str(plugin_dir),
-                    "source": source,
-                    "hooks": [str(h.name) for h in hooks],
-                    "config": config_data,
-                }
-
-    return plugins
-
-
 @render_with_table_view
 def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
@@ -512,131 +226,6 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     )
 
 
-@render_with_table_view
-def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    rows = {
-        "Name": [],
-        "Source": [],
-        "Path": [],
-        "Hooks": [],
-        "Config": [],
-    }
-
-    plugins = get_filesystem_plugins()
-
-    for plugin_id, plugin in plugins.items():
-        rows["Name"].append(ItemLink(plugin["name"], key=plugin_id))
-        rows["Source"].append(plugin["source"])
-        rows["Path"].append(format_html("<code>{}</code>", plugin["path"]))
-        rows["Hooks"].append(", ".join(plugin["hooks"]) or "(none)")
-
-        # Show config status
-        if plugin.get("config"):
-            config_properties = plugin["config"].get("properties", {})
-            config_count = len(config_properties)
-            rows["Config"].append(f"✅ {config_count} properties" if config_count > 0 else "✅ present")
-        else:
-            rows["Config"].append("❌ none")
-
-    if not plugins:
-        # Show a helpful message when no plugins found
-        rows["Name"].append("(no plugins found)")
-        rows["Source"].append("-")
-        rows["Path"].append(mark_safe("<code>abx_plugins/plugins/</code> or <code>data/custom_plugins/</code>"))
-        rows["Hooks"].append("-")
-        rows["Config"].append("-")
-
-    return TableContext(
-        title="Installed plugins",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert is_superuser(request), "Must be a superuser to view configuration settings."
-
-    plugins = get_filesystem_plugins()
-
-    plugin = plugins.get(key)
-    if not plugin:
-        return ItemContext(
-            slug=key,
-            title=f"Plugin not found: {key}",
-            data=[],
-        )
-
-    # Base fields that all plugins have
-    docs_url = get_plugin_docs_url(plugin["name"])
-    machine_admin_url = get_machine_admin_url()
-    fields = {
-        "id": plugin["id"],
-        "name": plugin["name"],
-        "source": plugin["source"],
-    }
-
-    sections: list[SectionData] = [
-        {
-            "name": plugin["name"],
-            "description": format_html(
-                '<code>{}</code><br/><a href="{}" target="_blank" rel="noopener noreferrer">ABX Plugin Docs</a>',
-                plugin["path"],
-                docs_url,
-            ),
-            "fields": fields,
-            "help_texts": {},
-        },
-    ]
-
-    if plugin["hooks"]:
-        sections.append(
-            {
-                "name": "Hooks",
-                "description": mark_safe(render_hook_links_html(plugin["name"], plugin["hooks"], plugin["source"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-    if plugin.get("config"):
-        sections.append(
-            {
-                "name": "Plugin Metadata",
-                "description": mark_safe(render_plugin_metadata_html(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        sections.append(
-            {
-                "name": "config.json",
-                "description": mark_safe(render_highlighted_json_block(plugin["config"])),
-                "fields": {},
-                "help_texts": {},
-            },
-        )
-
-        config_properties = plugin["config"].get("properties", {})
-        if config_properties:
-            sections.append(
-                {
-                    "name": "Config Properties",
-                    "description": mark_safe(render_config_properties_html(config_properties, machine_admin_url)),
-                    "fields": {},
-                    "help_texts": {},
-                },
-            )
-
-    return ItemContext(
-        slug=key,
-        title=plugin["name"],
-        data=sections,
-    )
-
-
 @render_with_table_view
 def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."

archivebox/core/admin_archiveresults.py

@@ -23,10 +23,10 @@ from archivebox.config import DATA_DIR
 from archivebox.config.common import get_config
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.base_models.admin import BaseModelAdmin
-from archivebox.hooks import get_plugin_icon
-from archivebox.core.host_util import build_snapshot_url
+from archivebox.plugins.discovery import get_plugin_icon
+from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL
+from archivebox.core.routes_util import build_snapshot_url
 from archivebox.core.widgets import InlineTagEditorWidget
-from archivebox.core.views import LIVE_PLUGIN_BASE_URL
 from archivebox.machine.env_util import env_to_shell_exports
 
 
@@ -62,7 +62,7 @@ def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
 
 
 def get_plugin_admin_url(plugin_name: str) -> str:
-    from archivebox.hooks import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs
+    from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, iter_plugin_dirs
 
     plugin_dir = next((path.resolve() for path in iter_plugin_dirs() if path.name == plugin_name), None)
     if plugin_dir:

archivebox/core/admin_snapshots.py

@@ -1,43 +1,37 @@
 __package__ = "archivebox.core"
 
-import asyncio
 import json
-import threading
-from copy import copy
 from functools import lru_cache
-from queue import Full, Queue
 from types import SimpleNamespace
-from urllib.parse import urlsplit
-from uuid import UUID
 
 from django.contrib import admin, messages
-from django.contrib.admin.views.main import IncorrectLookupParameters
 from django.urls import path, reverse
 from django.shortcuts import get_object_or_404, redirect
-from django.core.cache import cache
-from django.core.paginator import InvalidPage
-from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed, QueryDict, StreamingHttpResponse
+from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed
 from django.utils import timezone
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
-from django.db.models import Q, Count, Exists, F, IntegerField, OuterRef, Prefetch, Subquery
+from django.db.models import Q, Count, Exists, OuterRef, Prefetch
 from django import forms
 from django.template import Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
 
 from archivebox.config.common import get_config
 from archivebox.misc.util import htmldecode, urldecode
-from archivebox.misc.paginators import AcceleratedPaginator, CountlessPaginator
+from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.logging_util import printable_filesize
-from archivebox.search.admin import SEARCH_RESULT_CACHE_TTL, SearchResultsAdminMixin, SearchResultsChangeList, get_admin_search_cache_key
-from archivebox.core.host_util import build_snapshot_url, build_web_url
+from archivebox.search.admin import SearchResultsAdminMixin, SearchResultsChangeList
+from archivebox.search.views import admin_snapshot_search_stream_view
+from archivebox.core.routes_util import build_snapshot_url, build_web_url
 from archivebox.core.tag_util import get_or_create_tag
-from archivebox.hooks import discover_hooks, get_plugin_icon, get_plugin_name, get_plugins
+from archivebox.plugins.hooks import discover_hooks
+from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_plugins
 
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
 from archivebox.core.admin_archiveresults import render_archiveresults_list
+from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
     PERMISSIONS_META,
@@ -133,56 +127,6 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
         return queryset
 
 
-class SnapshotDepthListFilter(admin.SimpleListFilter):
-    title = "depth"
-    parameter_name = "depth_bucket"
-
-    def lookups(self, request, model_admin):
-        return (
-            ("0", "0 root"),
-            ("1", "1"),
-            ("2", "2"),
-            ("3plus", "3+"),
-        )
-
-    def queryset(self, request, queryset):
-        value = self.value()
-        if value == "0":
-            return queryset.filter(depth=0)
-        if value == "1":
-            return queryset.filter(depth=1)
-        if value == "2":
-            return queryset.filter(depth=2)
-        if value == "3plus":
-            return queryset.filter(depth__gte=3)
-        return queryset
-
-
-class SnapshotRelationListFilter(admin.SimpleListFilter):
-    title = "crawl position"
-    parameter_name = "position"
-
-    def lookups(self, request, model_admin):
-        return (
-            ("root", "Root URL"),
-            ("discovered", "Discovered URL"),
-            ("has_children", "Has discovered URLs"),
-            ("no_children", "No discovered URLs"),
-        )
-
-    def queryset(self, request, queryset):
-        value = self.value()
-        if value == "root":
-            return queryset.filter(parent_snapshot__isnull=True)
-        if value == "discovered":
-            return queryset.filter(parent_snapshot__isnull=False)
-        if value in {"has_children", "no_children"}:
-            child_snapshots = Snapshot.objects.filter(parent_snapshot_id=OuterRef("pk"))
-            queryset = queryset.annotate(has_child_snapshots=Exists(child_snapshots))
-            return queryset.filter(has_child_snapshots=value == "has_children")
-        return queryset
-
-
 class SnapshotArchiveStateListFilter(admin.SimpleListFilter):
     title = "archive state"
     parameter_name = "archive_state"
@@ -243,32 +187,9 @@ class SnapshotSizeListFilter(admin.SimpleListFilter):
         return queryset
 
 
-class SnapshotRetryListFilter(admin.SimpleListFilter):
-    title = "retry"
-    parameter_name = "retry"
-
-    def lookups(self, request, model_admin):
-        return (
-            ("due", "Due now"),
-            ("future", "Scheduled later"),
-            ("none", "No retry time"),
-        )
-
-    def queryset(self, request, queryset):
-        value = self.value()
-        if value == "due":
-            return queryset.filter(retry_at__isnull=False, retry_at__lte=timezone.now())
-        if value == "future":
-            return queryset.filter(retry_at__gt=timezone.now())
-        if value == "none":
-            return queryset.filter(retry_at__isnull=True)
-        return queryset
-
-
 class SnapshotResultHealthListFilter(admin.SimpleListFilter):
     title = "ArchiveResult status"
     parameter_name = "archiveresult_status"
-    SNAPSHOT_FIRST_VALUES = {"succeeded"}
 
     def lookups(self, request, model_admin):
         return (
@@ -282,47 +203,6 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
             ("noresults", ">50% noresults"),
         )
 
-    @staticmethod
-    def _snapshot_total_count_subquery(outer_ref: str = "pk"):
-        return (
-            ArchiveResult.objects.filter(snapshot_id=OuterRef(outer_ref))
-            .order_by()
-            .values("snapshot_id")
-            .annotate(count=Count("pk"))
-            .values("count")
-        )
-
-    @staticmethod
-    def _snapshot_status_count_subquery(status: str, outer_ref: str = "pk"):
-        return (
-            ArchiveResult.objects.filter(snapshot_id=OuterRef(outer_ref), status=status)
-            .order_by()
-            .values("snapshot_id")
-            .annotate(count=Count("pk"))
-            .values("count")
-        )
-
-    def _filter_snapshot_first(self, queryset, status: str):
-        return queryset.annotate(
-            total_results=Subquery(self._snapshot_total_count_subquery(), output_field=IntegerField()),
-            matching_results=Subquery(self._snapshot_status_count_subquery(status), output_field=IntegerField()),
-        ).filter(matching_results__gt=F("total_results") / 2)
-
-    def _filter_status_first(self, queryset, status: str):
-        total_results = self._snapshot_total_count_subquery("snapshot_id")
-        matching_snapshot_ids = (
-            ArchiveResult.objects.filter(status=status)
-            .order_by()
-            .values("snapshot_id")
-            .annotate(
-                matching_results=Count("pk"),
-                total_results=Subquery(total_results, output_field=IntegerField()),
-            )
-            .filter(matching_results__gt=F("total_results") / 2)
-            .values("snapshot_id")
-        )
-        return queryset.filter(pk__in=matching_snapshot_ids)
-
     def queryset(self, request, queryset):
         value = self.value()
         if value:
@@ -340,10 +220,10 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
                 "noresults": ArchiveResult.StatusChoices.NORESULTS,
             }
             if value in status_by_value:
-                status = status_by_value[value]
-                if value in self.SNAPSHOT_FIRST_VALUES:
-                    return self._filter_snapshot_first(queryset, status)
-                return self._filter_status_first(queryset, status)
+                # Start from ArchiveResult.status for every majority-status filter.
+                # The (status, snapshot_id) index keeps this plan stable regardless
+                # of which status is most common in a user's collection.
+                return queryset.filter(pk__in=ArchiveResult.snapshot_ids_with_majority_status(status_by_value[value]))
         return queryset
 
 
@@ -356,28 +236,7 @@ class SnapshotChangeList(SearchResultsChangeList):
             resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
         )
 
-    def _uses_expensive_archiveresult_filter(self, request) -> bool:
-        return bool(request.GET.get(SnapshotResultHealthListFilter.parameter_name))
-
     def get_results(self, request):
-        if self._uses_expensive_archiveresult_filter(request):
-            paginator = CountlessPaginator(self.queryset, self.list_per_page)
-            try:
-                page = paginator.page(self.page_num)
-            except InvalidPage:
-                raise IncorrectLookupParameters
-
-            self.result_count = paginator.count
-            self.show_full_result_count = False
-            self.show_admin_actions = True
-            self.full_result_count = None
-            self.result_list = page.object_list
-            self.can_show_all = False
-            self.multi_page = page.has_next() or self.page_num > 1
-            self.paginator = paginator
-            self.show_search_index_hint = False
-            return
-
         super().get_results(request)
         if request.GET.get("_embedded") == "crawl":
             self.full_result_count = self.result_count
@@ -506,11 +365,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         SnapshotPermissionsListFilter,
         SnapshotStatusListFilter,
         SnapshotResultHealthListFilter,
-        SnapshotDepthListFilter,
-        SnapshotRelationListFilter,
         SnapshotArchiveStateListFilter,
         SnapshotSizeListFilter,
-        SnapshotRetryListFilter,
         "created_at",
         "downloaded_at",
         "crawl__created_by",
@@ -592,7 +448,15 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     )
 
     ordering = ["-timestamp"]
-    actions = ["add_tags", "remove_tags", "resnapshot_snapshot", "update_snapshots", "overwrite_snapshots", "delete_snapshots"]
+    actions = [
+        "add_tags",
+        "remove_tags",
+        "resnapshot_snapshot",
+        "update_snapshots",
+        "overwrite_snapshots",
+        "set_snapshot_permissions",
+        "delete_snapshots",
+    ]
     inlines = []  # Removed TagInline, using TagEditorWidget instead
     list_per_page = 50
 
@@ -614,6 +478,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         extra_context = extra_context or {}
         extra_context["CONFIG"] = request.archivebox_config
+        snapshot = self.get_object(request, object_id)
+        if snapshot and snapshot.status in {
+            Snapshot.StatusChoices.QUEUED,
+            Snapshot.StatusChoices.STARTED,
+            Snapshot.StatusChoices.PAUSED,
+        }:
+            extra_context["progress_auto_expand"] = True
+            extra_context["progress_endpoint"] = progress_endpoint("snapshot", snapshot.id)
         return super().change_view(request, object_id, form_url, extra_context | GLOBAL_CONTEXT)
 
     def changelist_view(self, request, extra_context=None):
@@ -679,155 +551,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return custom_urls + urls
 
     def search_stream_view(self, request):
-        from archivebox.search import iter_query_search_ids
-
-        query = (request.GET.get("q") or "").strip()
-        from archivebox.search import get_search_mode, get_search_mode_base
-
-        search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
-        if not query:
-            return StreamingHttpResponse((), content_type="text/plain")
-
-        search_url = request.GET.get("search_url") or request.get_full_path()
-        target_url = urlsplit(search_url)
-        target_get = QueryDict(target_url.query, mutable=True)
-        for key in ("q", "search_mode", "p", "search_url"):
-            target_get.pop(key, None)
-
-        filter_request = copy(request)
-        filter_request.path = target_url.path or request.path
-        filter_request.path_info = target_url.path or request.path_info
-        filter_request.GET = target_get
-        filter_request.archivebox_config = getattr(request, "archivebox_config", None)
-
-        # Build the same filtered base queryset the changelist uses, but with
-        # the search params stripped. The stream then intersects each wave with
-        # this queryset before writing IDs into the short-lived cache.
-        current_request = getattr(self, "request", None)
-        try:
-            base_queryset = self.get_changelist_instance(filter_request).queryset
-        finally:
-            self.request = current_request
-
-        async def snapshot_ids():
-            seen = set()
-            ids = []
-            last_sent = 0
-            stream_batch_size = 100
-            stream_padding = " " * 4096
-            cache_key = get_admin_search_cache_key(request, search_url)
-            cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-            yield f"0{stream_padding}\n"
-            queue = Queue(maxsize=8)
-            stop_event = threading.Event()
-
-            def emit(item):
-                while not stop_event.is_set():
-                    try:
-                        queue.put(item, timeout=0.1)
-                        return
-                    except Full:
-                        continue
-
-            def run_search():
-                nonlocal last_sent
-                iterator = None
-                try:
-                    search_mode_base = get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None))
-                    iterator = (
-                        self.iter_meta_search_ids(query, base_queryset)
-                        if search_mode_base == "meta"
-                        else self.iter_backend_search_ids(
-                            iter_query_search_ids(query, search_mode=search_mode, config=getattr(request, "archivebox_config", None)),
-                            base_queryset,
-                        )
-                    )
-                    for snapshot_id in iterator:
-                        if stop_event.is_set():
-                            break
-                        snapshot_id = str(snapshot_id).strip().lower()
-                        if len(snapshot_id.replace("-", "")) != 32 or snapshot_id in seen:
-                            continue
-                        seen.add(snapshot_id)
-                        ids.append(snapshot_id)
-                        if len(ids) - last_sent >= stream_batch_size:
-                            cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-                            last_sent = len(ids)
-                            emit(f"{last_sent}{stream_padding}\n")
-                    if not stop_event.is_set() and len(ids) != last_sent:
-                        cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
-                        emit(f"{len(ids)}{stream_padding}\n")
-                except BaseException as err:
-                    emit(err)
-                finally:
-                    if iterator is not None:
-                        try:
-                            iterator.close()
-                        except AttributeError:
-                            pass
-                    cache.set(cache_key, {"ids": ids, "done": True}, SEARCH_RESULT_CACHE_TTL)
-                    emit(None)
-
-            threading.Thread(target=run_search, name="admin-snapshot-search-stream", daemon=True).start()
-            try:
-                while True:
-                    item = await asyncio.to_thread(queue.get)
-                    if item is None:
-                        break
-                    if isinstance(item, BaseException):
-                        raise item
-                    yield item
-            finally:
-                stop_event.set()
-
-        response = StreamingHttpResponse(snapshot_ids(), content_type="text/plain")
-        response["X-Accel-Buffering"] = "no"
-        return response
-
-    def iter_meta_search_ids(self, query, queryset):
-        seen = set()
-        try:
-            snapshot_id = UUID(query)
-        except ValueError:
-            snapshot_id = None
-        if snapshot_id:
-            for pk in queryset.filter(pk=snapshot_id).values_list("pk", flat=True):
-                seen.add(pk)
-                yield pk
-
-        for wave in (
-            Q(timestamp__startswith=query) | Q(url__istartswith=query) | Q(title__istartswith=query),
-            Q(url__icontains=query),
-            Q(title__icontains=query),
-            Q(tags__name__icontains=query),
-        ):
-            for pk in queryset.filter(wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
-                if pk in seen:
-                    continue
-                seen.add(pk)
-                yield pk
-
-    def iter_backend_search_ids(self, iterator, queryset):
-        batch = []
-        seen = set()
-
-        def flush_batch():
-            valid = {str(pk) for pk in queryset.filter(pk__in=batch).values_list("pk", flat=True)}
-            for snapshot_id in batch:
-                if snapshot_id in valid and snapshot_id not in seen:
-                    seen.add(snapshot_id)
-                    yield snapshot_id
-
-        for snapshot_id in iterator:
-            snapshot_id = str(snapshot_id).strip().lower()
-            if len(snapshot_id.replace("-", "")) != 32:
-                continue
-            batch.append(snapshot_id)
-            if len(batch) >= 200:
-                yield from flush_batch()
-                batch = []
-        if batch:
-            yield from flush_batch()
+        return admin_snapshot_search_stream_view(self, request)
 
     def set_permissions_view(self, request, object_id):
         if request.method != "POST":
@@ -852,6 +576,43 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
         return JsonResponse({"permissions": permissions, "icon": icon, "label": label, "fg": fg, "bg": bg})
 
+    @admin.action(description="Permissions ▾")
+    def set_snapshot_permissions(self, request, queryset):
+        permissions = (request.POST.get("permissions") or "").strip().lower()
+        if permissions not in dict(PERMISSIONS_CHOICES):
+            messages.error(request, "Choose a valid permissions value.")
+            return
+        updated = self.update_snapshot_permissions(queryset, permissions)
+        messages.success(request, f"Set permissions to {permissions} on {updated} snapshot(s).")
+
+    def update_snapshot_permissions(self, queryset, permissions):
+        now = timezone.now()
+        updated = 0
+        batch = []
+        snapshots = (
+            queryset.select_related(None)
+            .select_related("crawl")
+            .only("id", "config", "crawl__id", "crawl__permissions")
+            .prefetch_related(None)
+        )
+        for snapshot in snapshots.iterator(chunk_size=500):
+            config = dict(snapshot.config or {})
+            if permissions == snapshot.crawl.permissions:
+                config.pop("PERMISSIONS", None)
+            else:
+                config["PERMISSIONS"] = permissions
+            snapshot.config = config
+            snapshot.modified_at = now
+            batch.append(snapshot)
+            if len(batch) >= 500:
+                Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+                updated += len(batch)
+                batch.clear()
+        if batch:
+            Snapshot.objects.bulk_update(batch, ["config", "modified_at"], batch_size=500)
+            updated += len(batch)
+        return updated
+
     def redo_failed_view(self, request, object_id):
         snapshot = get_object_or_404(Snapshot, pk=object_id)
 
@@ -870,12 +631,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
         return redirect(snapshot.admin_change_url)
 
-    # def get_queryset(self, request):
-    #     # tags_qs = SnapshotTag.objects.all().select_related('tag')
-    #     # prefetch = Prefetch('snapshottag_set', queryset=tags_qs)
-
-    #     self.request = request
-    #     return super().get_queryset(request).prefetch_related('archiveresult_set').distinct()  # .annotate(archiveresult_count=Count('archiveresult'))
     def get_queryset(self, request):
         self.request = request
         ordering_fields = self._get_ordering_fields(request)
@@ -924,10 +679,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         qs = qs.prefetch_related(*prefetches)
         if needs_files_sort:
             qs = qs.annotate(
-                ar_succeeded_count=Count(
-                    "archiveresult",
-                    filter=Q(archiveresult__status="succeeded"),
-                ),
+                ar_succeeded_count=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
             )
         if needs_tags_sort:
             qs = qs.annotate(tag_count=Count("tags", distinct=True))
@@ -1322,33 +1074,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         ordering="ar_succeeded_count",
     )
     def files(self, obj):
-        stats = self._get_progress_stats(obj)
-        if obj.status == Snapshot.StatusChoices.STARTED and stats["total"] > 0:
-            succeeded = stats["succeeded"]
-            failed = stats["failed"]
-            skipped = stats["skipped"]
-            completed = succeeded + failed + skipped + stats["noresults"]
-            return format_html(
-                """<div style="min-width: 96px;">
-                    <div style="display: flex; align-items: center; gap: 6px; margin-bottom: 4px;">
-                        <span class="snapshot-progress-spinner"></span>
-                        <span style="font-size: 11px; color: #64748b;">{}/{} hooks</span>
-                    </div>
-                    <div class="snapshot-progress-bar">
-                        <div class="snapshot-progress-bar-fill" style="background: #3b82f6; width: {}%;"></div>
-                    </div>
-                    <div style="font-size: 10px; color: #94a3b8; margin-top: 2px;">
-                        ✓{} ✗{} ⏳{}
-                    </div>
-                </div>""",
-                completed,
-                stats["total"],
-                stats["percent"],
-                succeeded,
-                failed,
-                stats["running"],
-            )
-
         results = self._get_prefetched_results(obj)
         if results is None:
             results = obj.archiveresult_set.only("plugin", "status", "output_size")
@@ -1576,8 +1301,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             crawl = getattr(obj, "crawl", None)
             snapshot_config = getattr(obj, "config", None) or {}
             crawl_config = getattr(crawl, "config", None) or {}
-            crawl_persona_id = getattr(crawl, "persona_id", None)
-            has_scoped_config = bool(snapshot_config or crawl_config or crawl_persona_id)
+            has_scoped_config = bool(snapshot_config or crawl_config)
 
             if request is not None and not has_scoped_config:
                 cached_total = getattr(request, "archivebox_expected_snapshot_hook_total", None)
@@ -1595,7 +1319,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 if snapshot_config:
                     cache_key = ("snapshot", json.dumps(snapshot_config, sort_keys=True, default=str))
                 else:
-                    cache_key = ("crawl", json.dumps(crawl_config, sort_keys=True, default=str), crawl_persona_id)
+                    cache_key = ("crawl", json.dumps(crawl_config, sort_keys=True, default=str))
                 cached_total = scoped_cache.get(cache_key)
                 if cached_total is None:
                     config = get_config(crawl=crawl, snapshot=obj if snapshot_config else None)
@@ -1702,7 +1426,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         # work-in-progress crawl, not the old snapshot they re-archived from.
         # A snapshot-view redirect would race the runner — the new snapshot
         # may sit queued for a while before the runner creates the DB row.
-        return redirect(f"/admin/crawls/crawl/{crawl.id.hex}/change/#snapshots")
+        return redirect(f"/admin/crawls/crawl/{crawl.id}/change/#snapshots")
 
     @admin.action(
         description="🔄 Redo",

archivebox/core/admin_tags.py

@@ -24,7 +24,7 @@ from archivebox.core.tag_util import (
     normalize_has_snapshots_filter,
     normalize_tag_sort,
 )
-from archivebox.core.host_util import build_snapshot_url
+from archivebox.core.routes_util import build_snapshot_url
 
 
 class TagInline(admin.TabularInline):

archivebox/core/forms.py

@@ -1,14 +1,9 @@
 __package__ = "archivebox.core"
 
-import json
 import re
-from collections.abc import Iterable, Mapping
 from decimal import Decimal, InvalidOperation, ROUND_CEILING
-from pathlib import Path
-from typing import Any
 
 from django import forms
-from django.utils.html import format_html
 
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
 from taggit.utils import edit_string_for_tags, parse_tags
@@ -17,7 +12,13 @@ from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.config.common import get_config, parse_delete_after
 from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
-from archivebox.hooks import get_plugins, discover_plugin_configs, get_plugin_icon
+from archivebox.plugins.discovery import get_plugins
+from archivebox.plugins.forms import (
+    PLUGIN_GROUP_DEFINITIONS,
+    TIMEOUT_INPUT_PATTERN,
+    PluginConfigFormMixin,
+    get_choice_field,
+)
 from archivebox.personas.models import Persona
 
 DEPTH_CHOICES = (
@@ -28,594 +29,6 @@ DEPTH_CHOICES = (
     ("4", "depth = 4 (+ URLs four hops away)"),
 )
 
-PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__"
-PLUGIN_GROUP_DEFINITIONS = (
-    (
-        "main_plugins",
-        "Main",
-        "",
-        "",
-        "",
-        (
-            "dom",
-            "screenshot",
-            "pdf",
-            "singlefile",
-            "wget",
-            "archivedotorg",
-            "chrome_mhtml",
-            "archivewebpage",
-        ),
-    ),
-    (
-        "page_setup_plugins",
-        "Page Setup",
-        "",
-        "",
-        "",
-        (
-            "chrome",
-            "infiniscroll",
-            "modalcloser",
-            "ublock",
-            "istilldontcareaboutcookies",
-            "twocaptcha",
-            "claudechrome",
-        ),
-    ),
-    (
-        "media_plugins",
-        "Media",
-        "",
-        "",
-        "",
-        (
-            "staticfile",
-            "responses",
-            "chrome_screencast",
-            "ytdlp",
-            "gallerydl",
-            "git",
-        ),
-    ),
-    (
-        "text_plugins",
-        "Text",
-        "",
-        "",
-        "",
-        (
-            "readability",
-            "htmltotext",
-            "defuddle",
-            "forumdl",
-            "mercury",
-            "trafilatura",
-            "liteparse",
-            "opendataloader",
-            "papersdl",
-        ),
-    ),
-    (
-        "metadata_plugins",
-        "Metadata",
-        "",
-        "",
-        "",
-        (
-            "title",
-            "favicon",
-            "headers",
-            "redirects",
-            "accessibility",
-            "consolelog",
-            "sslcerts",
-            "dns",
-            "seo",
-            "hashes",
-        ),
-    ),
-    (
-        "postprocessing_plugins",
-        "Postprocessing",
-        "",
-        "",
-        "",
-        (
-            "parse_dom_outlinks",
-            "parse_html_urls",
-            "parse_jsonl_urls",
-            "parse_netscape_urls",
-            "parse_rss_urls",
-            "parse_txt_urls",
-            "claudecode",
-            "claudecodecleanup",
-            "claudecodeextract",
-        ),
-    ),
-)
-HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
-    "apt",
-    "base",
-    "bash",
-    "brew",
-    "cargo",
-    "chromewebstore",
-    "env",
-    "media",
-    "npm",
-    "pip",
-    "puppeteer",
-    "search_backend_ripgrep",
-    "search_backend_sonic",
-    "search_backend_sqlite",
-    "ssl",
-}
-TIMEOUT_INPUT_PATTERN = r"(0|[1-9][0-9]*|[0-9]+(?:\.[0-9]+)?\s*(?:s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours))"
-
-
-def get_plugin_choices():
-    """Get available extractor plugins from discovered hooks."""
-    return [(name, name) for name in get_plugins()]
-
-
-def get_plugin_choice_label(plugin_name: str, plugin_configs: dict[str, dict]) -> str:
-    schema = plugin_configs.get(plugin_name, {})
-    description = str(schema.get("description") or "").strip()
-    if not description:
-        return plugin_name
-    icon_html = get_plugin_icon(plugin_name)
-
-    return format_html(
-        '<span class="plugin-choice-icon">{}</span><span class="plugin-choice-name">{}</span>',
-        icon_html,
-        plugin_name,
-    )
-
-
-def get_choice_field(form: forms.Form, name: str) -> forms.ChoiceField:
-    field = form.fields[name]
-    if not isinstance(field, forms.ChoiceField):
-        raise TypeError(f"{name} must be a ChoiceField")
-    return field
-
-
-def _plugin_config_input_name(plugin_name: str, config_key: str) -> str:
-    return f"{PLUGIN_CONFIG_FIELD_PREFIX}{plugin_name}__{config_key}"
-
-
-def _schema_types(schema: Mapping[str, Any]) -> list[str]:
-    raw_type = schema.get("type") or "string"
-    if isinstance(raw_type, list):
-        return [str(item) for item in raw_type]
-    return [str(raw_type)]
-
-
-def _jsonish(value: Any) -> str:
-    if isinstance(value, str):
-        return value
-    return json.dumps(value, sort_keys=True, default=str)
-
-
-def _same_config_value(left: Any, right: Any) -> bool:
-    return json.dumps(left, sort_keys=True, default=str) == json.dumps(right, sort_keys=True, default=str)
-
-
-def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> Any:
-    schema_types = _schema_types(schema)
-
-    if "boolean" in schema_types:
-        if isinstance(raw_value, bool):
-            return raw_value
-        value = str(raw_value).strip().lower()
-        if value in {"true", "1", "yes", "on"}:
-            return True
-        if value in {"false", "0", "no", "off", ""}:
-            return False
-        raise forms.ValidationError("Must be true or false.")
-
-    if "integer" in schema_types:
-        value = int(str(raw_value).strip())
-        minimum = schema.get("minimum")
-        maximum = schema.get("maximum")
-        if minimum is not None and value < int(minimum):
-            raise forms.ValidationError(f"Must be at least {minimum}.")
-        if maximum is not None and value > int(maximum):
-            raise forms.ValidationError(f"Must be at most {maximum}.")
-        return value
-
-    if "number" in schema_types:
-        value = float(str(raw_value).strip())
-        minimum = schema.get("minimum")
-        maximum = schema.get("maximum")
-        if minimum is not None and value < float(minimum):
-            raise forms.ValidationError(f"Must be at least {minimum}.")
-        if maximum is not None and value > float(maximum):
-            raise forms.ValidationError(f"Must be at most {maximum}.")
-        return value
-
-    if "array" in schema_types:
-        if isinstance(raw_value, list):
-            return raw_value
-        value = str(raw_value).strip()
-        if not value:
-            return []
-        if value.startswith("["):
-            parsed = json.loads(value)
-            if not isinstance(parsed, list):
-                raise forms.ValidationError("Must be a JSON array.")
-            return parsed
-        return [item.strip() for item in value.replace(",", "\n").splitlines() if item.strip()]
-
-    if "object" in schema_types:
-        value = str(raw_value).strip()
-        if not value:
-            return {}
-        parsed = json.loads(value)
-        if not isinstance(parsed, dict):
-            raise forms.ValidationError("Must be a JSON object.")
-        return parsed
-
-    value = str(raw_value)
-    enum = schema.get("enum")
-    if isinstance(enum, list) and enum and value not in {str(item) for item in enum}:
-        raise forms.ValidationError(f"Must be one of: {', '.join(str(item) for item in enum)}.")
-    return value
-
-
-class PluginConfigFormMixin:
-    plugin_groups: list[dict[str, Any]]
-
-    def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
-        all_plugins = get_plugins()
-        plugin_configs = discover_plugin_configs()
-        runtime_config = runtime_config or get_config()
-        self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
-        grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS))
-        other_plugins = tuple(sorted(set(all_plugins) - grouped_plugins - HIDDEN_PLUGIN_CONFIG_UI_PLUGINS))
-
-        for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
-            if field_name in self.fields:
-                get_choice_field(self, field_name).choices = [
-                    (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
-                ]
-
-        if "other_plugins" in self.fields:
-            get_choice_field(self, "other_plugins").choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in other_plugins]
-
-        group_specs = (
-            *PLUGIN_GROUP_DEFINITIONS,
-            ("other_plugins", "Other", "", "", "", other_plugins),
-        )
-        binary_url_lookup = _build_required_binary_url_lookup(plugin_configs, runtime_config)
-        self.plugin_groups = [
-            {
-                "field_name": field_name,
-                "title": title,
-                "note": note,
-                "dom_id": dom_id,
-                "select_all_group": select_all_group,
-                "show_selectors": field_name in self.fields,
-                "plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config, binary_url_lookup),
-            }
-            for field_name, title, note, dom_id, select_all_group, plugin_names in group_specs
-            if any(plugin in all_plugins for plugin in plugin_names)
-        ]
-
-    def _build_plugin_cards(
-        self,
-        field_name: str,
-        plugin_names: Iterable[str],
-        plugin_configs: dict[str, dict[str, Any]],
-        runtime_config: Mapping[str, Any],
-        binary_url_lookup: Mapping[str, str] | None = None,
-    ) -> list[dict[str, Any]]:
-        if field_name in self.fields:
-            choices = list(get_choice_field(self, field_name).choices)
-            selected_values = set(self.data.getlist(field_name)) if self.is_bound else set(get_choice_field(self, field_name).initial or [])
-        else:
-            all_plugins = get_plugins()
-            choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins]
-            selected_values = set()
-
-        cards = []
-        for index, (plugin_name, label) in enumerate(choices):
-            schema = plugin_configs.get(str(plugin_name), {})
-            properties = schema.get("properties") or {}
-            enabled_config_key = f"{str(plugin_name).upper()}_ENABLED"
-            enabled_prop_schema = properties.get(enabled_config_key)
-            if not isinstance(enabled_prop_schema, dict) or "boolean" not in _schema_types(enabled_prop_schema):
-                enabled_config_key = ""
-            config_fields = [
-                self._build_plugin_config_field(str(plugin_name), str(config_key), prop_schema, runtime_config)
-                for config_key, prop_schema in properties.items()
-                if isinstance(prop_schema, dict)
-            ]
-            cards.append(
-                {
-                    "name": str(plugin_name),
-                    "label": label,
-                    "checked": str(plugin_name) in selected_values,
-                    "checkbox_id": f"id_{field_name}_{index}",
-                    "enabled_config_key": enabled_config_key,
-                    "description": str(schema.get("description") or "").strip(),
-                    "source_url": f"https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/{plugin_name}",
-                    "docs_url": f"https://archivebox.github.io/abx-plugins/#{plugin_name}",
-                    "required_plugins": [str(item) for item in schema.get("required_plugins") or []],
-                    "required_binary_links": _build_required_binary_links(
-                        schema.get("required_binaries") or [],
-                        runtime_config,
-                        binary_url_lookup,
-                    ),
-                    "config_fields": config_fields,
-                    "config_count": len(config_fields),
-                },
-            )
-        return cards
-
-    def _build_plugin_config_field(
-        self,
-        plugin_name: str,
-        config_key: str,
-        prop_schema: Mapping[str, Any],
-        runtime_config: Mapping[str, Any],
-    ) -> dict[str, Any]:
-        schema_types = _schema_types(prop_schema)
-        enum = prop_schema.get("enum")
-        input_name = _plugin_config_input_name(plugin_name, config_key)
-        current_value = runtime_config.get(config_key, prop_schema.get("default", ""))
-        if self.is_bound and input_name in self.data:
-            try:
-                current_value = _coerce_plugin_config_value(self.data.get(input_name), prop_schema)
-            except (TypeError, ValueError, json.JSONDecodeError, forms.ValidationError):
-                current_value = self.data.get(input_name)
-
-        default_value = prop_schema.get("default", "")
-        fallback_key = prop_schema.get("x-fallback")
-        default_display = f"{{{fallback_key}}}" if fallback_key else default_value
-        # A field is sensitive if either the schema explicitly marks it
-        # (``x-sensitive``) or the key name matches our credential heuristic
-        # (``*TOKEN*`` / ``*SECRET*`` / ``*API_KEY*`` / ``*APIKEY*``). The
-        # plugin grid lives on user-facing pages, so we redact the value,
-        # render a password input, and on empty submit we preserve the
-        # previously-saved value in ``clean_plugin_config_overrides`` below.
-        from archivebox.config.common import is_sensitive_config_key
-
-        is_sensitive = bool(prop_schema.get("x-sensitive")) or is_sensitive_config_key(config_key)
-        input_value = "" if is_sensitive else _jsonish(current_value)
-        field_kind = "text"
-        input_type = "text"
-        options = []
-
-        if "boolean" in schema_types:
-            field_kind = "boolean"
-            input_value = "true" if bool(current_value) else "false"
-        elif isinstance(enum, list) and enum:
-            field_kind = "select"
-            options = [
-                {
-                    "value": str(option),
-                    "label": str(option),
-                    "selected": str(option) == str(current_value),
-                }
-                for option in enum
-            ]
-        elif "integer" in schema_types or "number" in schema_types:
-            field_kind = "number"
-            input_type = "number"
-        elif "array" in schema_types or "object" in schema_types:
-            field_kind = "json"
-            input_value = "" if is_sensitive else json.dumps(current_value, indent=2, sort_keys=True, default=str)
-        elif is_sensitive:
-            input_type = "password"
-        else:
-            input_value = "" if is_sensitive else str(current_value)
-
-        return {
-            "key": config_key,
-            "input_name": input_name,
-            "kind": field_kind,
-            "input_type": input_type,
-            "value": input_value,
-            "checked": bool(current_value),
-            "options": options,
-            "description": str(prop_schema.get("description") or "").strip(),
-            "default": _jsonish(default_display),
-            "current": "configured"
-            if is_sensitive and current_value
-            else (str(current_value) if "string" in schema_types else _jsonish(current_value)),
-            "current_url": self.plugin_config_binary_urls.get(config_key, "") if str(config_key).endswith("_BINARY") else "",
-            "is_sensitive": is_sensitive,
-            "minimum": prop_schema.get("minimum"),
-            "maximum": prop_schema.get("maximum"),
-            "pattern": prop_schema.get("pattern"),
-            "type_label": " / ".join(schema_types),
-        }
-
-    def clean_plugin_config_overrides(self, effective_config: Mapping[str, Any] | None = None) -> dict[str, Any]:
-        if not self.is_bound:
-            return {}
-
-        effective_config = effective_config or get_config()
-        overrides: dict[str, Any] = {}
-        sources: dict[str, str] = {}
-
-        for plugin_name, schema in discover_plugin_configs().items():
-            for config_key, prop_schema in (schema.get("properties") or {}).items():
-                if not isinstance(prop_schema, dict):
-                    continue
-
-                input_name = _plugin_config_input_name(plugin_name, config_key)
-                if input_name not in self.data:
-                    continue
-
-                raw_value: Any = self.data.get(input_name)
-                if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list):
-                    raw_value = self.data.getlist(input_name)
-
-                from archivebox.config.common import is_sensitive_config_key
-
-                if (prop_schema.get("x-sensitive") or is_sensitive_config_key(config_key)) and raw_value == "":
-                    continue
-
-                try:
-                    coerced_value = _coerce_plugin_config_value(raw_value, prop_schema)
-                except (TypeError, ValueError, json.JSONDecodeError) as err:
-                    self.add_error("config", forms.ValidationError(f"{config_key}: {err}"))
-                    continue
-                except forms.ValidationError as err:
-                    self.add_error("config", forms.ValidationError(f"{config_key}: {err.messages[0]}"))
-                    continue
-
-                base_value = effective_config.get(config_key, prop_schema.get("default", ""))
-                if _same_config_value(coerced_value, base_value):
-                    continue
-
-                existing_value = overrides.get(config_key)
-                if config_key in overrides and not _same_config_value(existing_value, coerced_value):
-                    self.add_error(
-                        "config",
-                        forms.ValidationError(
-                            f"{config_key} was set differently under {sources[config_key]} and {plugin_name}. Set it once in Custom config overrides.",
-                        ),
-                    )
-                    continue
-
-                overrides[config_key] = coerced_value
-                sources[config_key] = plugin_name
-
-        return overrides
-
-    def plugin_config_keys(self) -> set[str]:
-        return {
-            str(config_key)
-            for schema in discover_plugin_configs().values()
-            for config_key, prop_schema in (schema.get("properties") or {}).items()
-            if isinstance(prop_schema, dict)
-        }
-
-
-_BINARY_TEMPLATE_PATTERN = re.compile(r"\{([A-Z_][A-Z0-9_]*)\}")
-
-
-def _resolve_required_binary_name(template_name: str, runtime_config: Mapping[str, Any]) -> str:
-    if "{" not in template_name:
-        return template_name
-
-    def _replace(match: re.Match[str]) -> str:
-        key = match.group(1)
-        try:
-            value = runtime_config.get(key)
-        except Exception:
-            value = None
-        if value is None or value == "":
-            return match.group(0)
-        return str(value)
-
-    resolved = _BINARY_TEMPLATE_PATTERN.sub(_replace, template_name).strip()
-    if not resolved:
-        return template_name
-    return Path(resolved).name if "/" in resolved else resolved
-
-
-def _iter_required_binary_names(
-    required_binaries: Iterable[Any],
-    runtime_config: Mapping[str, Any],
-) -> Iterable[str]:
-    for item in required_binaries or []:
-        if not isinstance(item, dict):
-            continue
-        raw_name = str(item.get("name") or "").strip()
-        if not raw_name:
-            continue
-        resolved = _resolve_required_binary_name(raw_name, runtime_config)
-        if resolved:
-            yield resolved
-
-
-def _build_required_binary_url_lookup(
-    plugin_configs: Mapping[str, dict[str, Any]],
-    runtime_config: Mapping[str, Any],
-) -> dict[str, str]:
-    """Resolve admin URLs for every required binary across all plugin schemas in a single DB query."""
-    from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url
-    from archivebox.machine.models import Binary, Machine
-
-    resolved_names: set[str] = set()
-    for schema in plugin_configs.values():
-        for name in _iter_required_binary_names(schema.get("required_binaries") or [], runtime_config):
-            resolved_names.add(name)
-
-    if not resolved_names:
-        return {}
-
-    machine = Machine.current()
-    name_to_binary: dict[str, Binary] = {}
-    for binary in (
-        Binary.objects.filter(machine=machine, name__in=resolved_names)
-        .exclude(abspath="")
-        .exclude(abspath__isnull=True)
-        .order_by("-modified_at")
-    ):
-        key = binary.name.lower()
-        if key not in name_to_binary:
-            name_to_binary[key] = binary
-
-    return {
-        name: (get_installed_binary_change_url(name, name_to_binary.get(name.lower())) or get_environment_binary_url(name))
-        for name in resolved_names
-    }
-
-
-def _build_required_binary_links(
-    required_binaries: list[dict[str, Any]],
-    runtime_config: Mapping[str, Any],
-    binary_url_lookup: Mapping[str, str] | None = None,
-) -> list[dict[str, str]]:
-    from archivebox.config.views import get_environment_binary_url
-
-    links: list[dict[str, str]] = []
-    seen: set[str] = set()
-    for resolved in _iter_required_binary_names(required_binaries, runtime_config):
-        if resolved in seen:
-            continue
-        seen.add(resolved)
-        url = (binary_url_lookup or {}).get(resolved) or get_environment_binary_url(resolved)
-        links.append({"name": resolved, "url": url})
-    return links
-
-
-def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str, str]:
-    from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url
-    from archivebox.machine.models import Binary, Machine
-
-    binary_keys = {
-        str(config_key)
-        for schema in discover_plugin_configs().values()
-        for config_key, prop_schema in (schema.get("properties") or {}).items()
-        if isinstance(prop_schema, dict) and str(config_key).endswith("_BINARY")
-    }
-    urls: dict[str, str] = {}
-    machine = Machine.current()
-    for key in binary_keys:
-        value = str(runtime_config.get(key) or "").strip()
-        if not value:
-            continue
-        name = Path(value).name if "/" in value else value
-        binary = Binary.objects.get_valid_binary(value, machine=machine)
-        if binary is None and "/" in value:
-            binary = (
-                Binary.objects.exclude(abspath="")
-                .exclude(abspath__isnull=True)
-                .filter(machine=machine, abspath=value)
-                .order_by("-modified_at")
-                .first()
-            )
-        if binary is None and name != value:
-            binary = Binary.objects.get_valid_binary(name, machine=machine)
-        urls[key] = get_installed_binary_change_url(getattr(binary, "name", name), binary) or get_environment_binary_url(name)
-    return urls
-
 
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
     # Basic fields

archivebox/core/middleware.py

@@ -15,7 +15,7 @@ from django.http import HttpResponseForbidden, HttpResponseNotModified
 from archivebox.config.common import get_config
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
-from archivebox.core.host_util import (
+from archivebox.core.routes_util import (
     build_snapshot_url,
     build_admin_url,
     build_web_url,

archivebox/core/models.py

@@ -3,7 +3,7 @@ __package__ = "archivebox.core"
 from typing import TYPE_CHECKING, Optional, Any
 from collections.abc import Iterable, Sequence
 import uuid
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 from datetime import datetime, timedelta
 
 import os
@@ -15,7 +15,7 @@ from statemachine import State, registry
 
 from django.db import models, transaction
 from django.db.models import Case, Q, QuerySet, Sum, Value, When
-from django.db.models.functions import Concat
+from django.db.models.functions import Coalesce, Concat
 from django.db.models.fields.json import KT
 from django.utils.functional import cached_property
 from django.utils.text import slugify
@@ -41,7 +41,7 @@ from archivebox.misc.util import (
     urldecode,
     validate_url_length,
 )
-from archivebox.hooks import (
+from archivebox.plugins.discovery import (
     get_plugins,
     get_plugin_name,
     get_plugin_icon,
@@ -276,18 +276,6 @@ class SnapshotQuerySet(models.QuerySet):
                 raise SystemExit(2)
         return self.filter(q_filter)
 
-    def search(self, patterns: list[str]) -> "SnapshotQuerySet":
-        """Search snapshots using the configured search backend"""
-        from archivebox.search import query_search_index
-
-        qsearch = self.none()
-        for pattern in patterns:
-            try:
-                qsearch |= query_search_index(pattern)
-            except BaseException:
-                raise SystemExit(2)
-        return self.all() & qsearch
-
     # =========================================================================
     # Export Methods
     # =========================================================================
@@ -397,7 +385,7 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig
 
 
 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithStateMachine):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
 
@@ -459,6 +447,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     StatusChoices = ModelWithStateMachine.StatusChoices
     active_state = StatusChoices.STARTED
     delete_after_final_statuses = (StatusChoices.SEALED,)
+    RUNNABLE_STATES = (StatusChoices.QUEUED, StatusChoices.STARTED)
+    OPEN_STATES = (*RUNNABLE_STATES, StatusChoices.PAUSED)
 
     crawl_id: uuid.UUID
     parent_snapshot_id: uuid.UUID | None
@@ -492,6 +482,43 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def __str__(self):
         return f"[{self.id}] {self.url[:64]}"
 
+    @classmethod
+    def crawl_count_subquery(cls, *, status: str | None = None, outer_ref: str = "pk") -> QuerySet:
+        """Return a scalar subquery counting Snapshots for one outer Crawl."""
+        qs = cls.objects.filter(crawl_id=models.OuterRef(outer_ref))
+        if status is not None:
+            qs = qs.filter(status=status)
+        return qs.order_by().values("crawl_id").annotate(count=models.Count("pk")).values("count")
+
+    @classmethod
+    def crawl_count_expr(cls, *, status: str | None = None, outer_ref: str = "pk"):
+        # Use scalar subqueries for sortable Crawl admin counters: SQLite can
+        # probe the (crawl_id, status, modified_at) index per Crawl row instead
+        # of joining/grouping all visible Snapshot rows.
+        return Coalesce(
+            models.Subquery(cls.crawl_count_subquery(status=status, outer_ref=outer_ref), output_field=models.IntegerField()),
+            models.Value(0),
+        )
+
+    @classmethod
+    def crawl_total_and_status_counts(cls, crawl_ids: Iterable[Any], *, status: str) -> dict[str, dict[str, int]]:
+        """Return total and status-filtered Snapshot counts keyed by Crawl ID."""
+        crawl_ids = list(crawl_ids)
+        if not crawl_ids:
+            return {}
+        return {
+            str(row["crawl_id"]): {
+                "total": row["total"],
+                "status": row["status_count"],
+            }
+            for row in cls.objects.filter(crawl_id__in=crawl_ids)
+            .values("crawl_id")
+            .annotate(
+                total=models.Count("pk"),
+                status_count=models.Count("pk", filter=Q(status=status)),
+            )
+        }
+
     def update_and_requeue(self, **kwargs) -> bool:
         """
         Update this Snapshot through the shared retry_at ownership path.
@@ -636,11 +663,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        from archivebox.personas.models import Persona
-
-        persona_ids = Persona.objects.filter(config__has_key="DELETE_AFTER").values_list("id", flat=True)
         return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER") | Q(crawl__persona_id__in=persona_ids),
+            Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER"),
         )
 
     @classmethod
@@ -649,7 +673,14 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         if parsed.scheme not in ("http", "https") or not parsed.hostname:
             return False
 
-        from archivebox.core.host_util import get_admin_host, get_api_host, get_listen_host, get_public_host, get_web_host, split_host_port
+        from archivebox.core.routes_util import (
+            get_admin_host,
+            get_api_host,
+            get_listen_host,
+            get_public_host,
+            get_web_host,
+            split_host_port,
+        )
 
         config = get_config()
         host = parsed.hostname.lower().strip(".")
@@ -742,10 +773,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             if update_fields is not None:
                 kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "fs_version", "modified_at"]))
         elif self.pk:
-            legacy_dir = get_config().ARCHIVE_DIR / self.timestamp
             current_dir = self.get_storage_path_for_version(self._fs_current_version())
-            if legacy_dir.exists() and not legacy_dir.is_symlink() and current_dir.exists() and legacy_dir != current_dir:
-                self.migrate_filesystem_to_current_version(source_dir=legacy_dir)
+            source_dir = Path(self.output_dir)
+            if source_dir.exists() and source_dir != current_dir and not source_dir.is_symlink():
+                self.migrate_filesystem_to_current_version(source_dir=source_dir)
 
         super().save(*args, **kwargs)
 
@@ -814,17 +845,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @staticmethod
     def _fs_current_version() -> str:
-        """Get current ArchiveBox filesystem version (normalized to x.x.0 format)"""
-        from archivebox.config import VERSION
-
-        # Normalize version to x.x.0 format (e.g., "0.9.0rc1" -> "0.9.0")
-        parts = VERSION.split(".")
-        if len(parts) >= 2:
-            major, minor = parts[0], parts[1]
-            # Strip any non-numeric suffix from minor version
-            minor = "".join(c for c in minor if c.isdigit())
-            return f"{major}.{minor}.0"
-        return "0.9.0"  # Fallback if version parsing fails
+        """Get current ArchiveBox filesystem layout version."""
+        return "0.9.4"
 
     @property
     def fs_migration_needed(self) -> bool:
@@ -836,6 +858,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         # Treat 0.7.0 and 0.8.0 as equivalent (both used archive/{timestamp})
         if version in ("0.7.0", "0.8.0"):
             return "0.9.0"
+        if version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3"):
+            return "0.9.4"
         return self._fs_current_version()
 
     @staticmethod
@@ -868,6 +892,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         if source_dir and current == target:
             current_dir = self.get_storage_path_for_version(target, config=runtime_config)
             cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, target_dir=current_dir)
+            crawl_dir = self.crawl.output_dir_for_config(runtime_config)
+            old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex)))
+            if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink():
+                crawl_dir.parent.mkdir(parents=True, exist_ok=True)
+                old_crawl_dir.rename(crawl_dir)
             if cleanup:
                 self._pending_fs_migration_cleanup = cleanup
             return
@@ -877,6 +906,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             migrations = {
                 ("0.7.0", "0.9.0"): self._fs_migrate_from_0_7_0_to_0_9_0,
                 ("0.8.0", "0.9.0"): self._fs_migrate_from_0_8_0_to_0_9_0,
+                ("0.9.0", "0.9.4"): self._fs_migrate_from_0_9_0_to_0_9_4,
+                ("0.9.1", "0.9.4"): self._fs_migrate_from_0_9_0_to_0_9_4,
+                ("0.9.2", "0.9.4"): self._fs_migrate_from_0_9_0_to_0_9_4,
+                ("0.9.3", "0.9.4"): self._fs_migrate_from_0_9_0_to_0_9_4,
             }
 
             migration = migrations.get((current, next_ver))
@@ -897,6 +930,17 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def _fs_migrate_from_0_8_0_to_0_9_0(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None):
         return self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, config=config)
 
+    def _fs_migrate_from_0_9_0_to_0_9_4(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None):
+        runtime_config = config or get_config()
+        target_dir = self.get_storage_path_for_version("0.9.4", config=runtime_config)
+        cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir or self.output_dir, target_dir=target_dir, config=runtime_config)
+        crawl_dir = self.crawl.output_dir_for_config(runtime_config)
+        old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex)))
+        if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink():
+            crawl_dir.parent.mkdir(parents=True, exist_ok=True)
+            old_crawl_dir.rename(crawl_dir)
+        return cleanup
+
     def _fs_migrate_legacy_to_0_9_0(
         self,
         source_dir: Path | None = None,
@@ -1056,7 +1100,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         if version in ("0.7.0", "0.8.0"):
             return runtime_config.ARCHIVE_DIR / self.timestamp
 
-        elif version in ("0.9.0", "1.0.0"):
+        elif version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
             username = self.created_by.username
 
             date_base = self.bookmarked_at or self.created_at
@@ -2055,7 +2099,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @property
     def api_url(self) -> str:
-        return str(reverse_lazy("api-1:get_snapshot", args=[self.id.hex]))
+        return str(reverse_lazy("api-1:get_snapshot", args=[self.id]))
 
     def get_absolute_url(self):
         return f"/{self.archive_path}"
@@ -2162,6 +2206,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         if current_path.exists():
             return current_path
 
+        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
+            hyphen_path = current_path.with_name(str(uuid.UUID(hex=self.id.hex)))
+            if hyphen_path.exists():
+                return hyphen_path
+
         # Check for backwards-compat symlink
         old_path = runtime_config.ARCHIVE_DIR / self.timestamp
         if old_path.is_symlink():
@@ -2250,7 +2299,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         if self.fs_version in ("0.7.0", "0.8.0"):
             return self.legacy_archive_path
 
-        if self.fs_version in ("0.9.0", "1.0.0"):
+        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
             username = "web"
             crawl = getattr(self, "crawl", None)
             if crawl and getattr(crawl, "created_by_id", None):
@@ -2272,7 +2321,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     @cached_property
     def url_path(self) -> str:
         """URL path matching the current snapshot output_dir layout."""
-        if self.fs_version in ("0.9.0", "1.0.0"):
+        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
             return self.archive_path_from_db
 
         output_dir = Path(self.output_dir).resolve()
@@ -2290,7 +2339,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                     username = "web"
                 date_str = parts[2]
                 domain = parts[3]
-                snapshot_id = parts[4]
+                snapshot_id = parts[4].replace("-", "")
                 return f"{username}/{date_str}/{domain}/{snapshot_id}"
 
         try:
@@ -2311,7 +2360,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 username = "web"
             date_str = parts[4]
             domain = parts[5]
-            snapshot_id = parts[6]
+            snapshot_id = parts[6].replace("-", "")
             return f"{username}/{date_str}/{domain}/{snapshot_id}"
 
         # Previous dev layout: users/<username>/snapshots/<YYYYMMDD>/<domain>/<uuid>/
@@ -2321,7 +2370,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 username = "web"
             date_str = parts[3]
             domain = parts[4]
-            snapshot_id = parts[5]
+            snapshot_id = parts[5].replace("-", "")
             return f"{username}/{date_str}/{domain}/{snapshot_id}"
 
         # Legacy layout: archive/<timestamp>/
@@ -2614,7 +2663,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         Creates one ArchiveResult per hook (not per plugin), with hook_name set.
         This enables step-based execution where all hooks in a step can run in parallel.
         """
-        from archivebox.hooks import discover_hooks
+        from archivebox.plugins.hooks import discover_hooks
         from archivebox.config.common import get_config
 
         # Get merged config with crawl-specific PLUGINS filter
@@ -2673,12 +2722,21 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         results = self.archiveresult_set.all()
 
-        # Count by status
-        succeeded = results.filter(status="succeeded").count()
-        failed = results.filter(status="failed").count()
-        running = results.filter(status="started").count()
-        skipped = results.filter(status="skipped").count()
-        noresults = results.filter(status="noresults").count()
+        counts = ArchiveResult.status_counts(
+            results,
+            (
+                ArchiveResult.StatusChoices.SUCCEEDED,
+                ArchiveResult.StatusChoices.FAILED,
+                ArchiveResult.StatusChoices.STARTED,
+                ArchiveResult.StatusChoices.SKIPPED,
+                ArchiveResult.StatusChoices.NORESULTS,
+            ),
+        )
+        succeeded = counts.get(ArchiveResult.StatusChoices.SUCCEEDED, 0)
+        failed = counts.get(ArchiveResult.StatusChoices.FAILED, 0)
+        running = counts.get(ArchiveResult.StatusChoices.STARTED, 0)
+        skipped = counts.get(ArchiveResult.StatusChoices.SKIPPED, 0)
+        noresults = counts.get(ArchiveResult.StatusChoices.NORESULTS, 0)
         total = results.count()
         pending = total - succeeded - failed - running - skipped - noresults
 
@@ -2858,7 +2916,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     def latest_outputs(self, status: str | None = None) -> dict[str, Any]:
         """Get the latest output that each plugin produced"""
-        from archivebox.hooks import get_plugins
+        from archivebox.plugins.discovery import get_plugins
         from django.db.models import Q
 
         latest: dict[str, Any] = {}
@@ -3054,7 +3112,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     def to_dict(self, extended: bool = False) -> dict[str, Any]:
         """Convert Snapshot to a dictionary (replacement for Link._asdict())"""
-        from archivebox.core.host_util import build_snapshot_url
+        from archivebox.core.routes_util import build_snapshot_url
 
         archive_size = self.archive_size
 
@@ -3453,8 +3511,58 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
         plugins = [get_plugin_name(e) for e in get_plugins()]
         return tuple((e, e) for e in plugins)
 
+    @classmethod
+    def snapshot_count_subquery(cls, *, status: str | None = None, outer_ref: str = "pk") -> QuerySet:
+        """Return a scalar subquery counting ArchiveResults for one outer Snapshot.
+
+        Use this instead of filtered join aggregates for per-row Snapshot counts:
+        the scalar form lets SQLite probe the covering ``(snapshot_id, status)``
+        or ``(status, snapshot_id)`` indexes once per visible Snapshot row,
+        instead of joining and grouping the whole candidate Snapshot queryset.
+        """
+        qs = cls.objects.filter(snapshot_id=models.OuterRef(outer_ref))
+        if status is not None:
+            qs = qs.filter(status=status)
+        return qs.order_by().values("snapshot_id").annotate(count=models.Count("pk")).values("count")
+
+    @classmethod
+    def snapshot_count_expr(cls, *, status: str | None = None, outer_ref: str = "pk"):
+        return Coalesce(
+            models.Subquery(cls.snapshot_count_subquery(status=status, outer_ref=outer_ref), output_field=models.IntegerField()),
+            models.Value(0),
+        )
+
+    @classmethod
+    def status_counts(cls, queryset: QuerySet | None = None, statuses: Iterable[str] | None = None) -> dict[str, int]:
+        """Count requested statuses with separate indexed COUNT probes."""
+        qs = queryset if queryset is not None else cls.objects.all()
+        return {status: qs.filter(status=status).count() for status in (statuses or cls.StatusChoices.values)}
+
+    @classmethod
+    def snapshot_ids_with_majority_status(cls, status: str) -> QuerySet:
+        """Return Snapshot IDs where more than half of ArchiveResults have ``status``.
+
+        Start from ArchiveResult.status for every majority-status filter. The
+        ``(status, snapshot_id)`` index keeps the plan predictable even when a
+        user's collection has an unusual status distribution.
+        """
+        return (
+            cls.objects.filter(status=status)
+            .order_by()
+            .values("snapshot_id")
+            .annotate(
+                matching_results=models.Count("pk"),
+                total_results=models.Subquery(
+                    cls.snapshot_count_subquery(outer_ref="snapshot_id"),
+                    output_field=models.IntegerField(),
+                ),
+            )
+            .filter(matching_results__gt=models.F("total_results") / 2)
+            .values("snapshot_id")
+        )
+
     # UUID primary key (migrated from integer in 0029)
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
 
@@ -3540,14 +3648,10 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        from archivebox.personas.models import Persona
-
-        persona_ids = Persona.objects.filter(config__has_key="DELETE_AFTER").values_list("id", flat=True)
         return cls.objects.filter(delete_at__isnull=True).filter(
             Q(config__has_key="DELETE_AFTER")
             | Q(snapshot__config__has_key="DELETE_AFTER")
-            | Q(snapshot__crawl__config__has_key="DELETE_AFTER")
-            | Q(snapshot__crawl__persona_id__in=persona_ids),
+            | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
         )
 
     @property
@@ -4209,7 +4313,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
         from pathlib import Path
         from django.utils import timezone
         from abx_dl.output_files import guess_mimetype
-        from archivebox.hooks import process_hook_records, extract_records_from_process
+        from archivebox.plugins.hooks import process_hook_records, extract_records_from_process
         from archivebox.machine.models import Process
 
         plugin_dir = Path(self.pwd) if self.pwd else None
@@ -4259,7 +4363,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
             # No ArchiveResult record: treat background hooks or clean exits as skipped
             is_background = False
             try:
-                from archivebox.hooks import is_background_hook
+                from archivebox.plugins.hooks import is_background_hook
 
                 is_background = bool(self.hook_name and is_background_hook(self.hook_name))
             except Exception:

archivebox/core/recovery_util.py

@@ -34,7 +34,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     )
     active_child_snapshots = Snapshot.objects.filter(
         crawl_id=OuterRef("pk"),
-        status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED],
+        status__in=Snapshot.OPEN_STATES,
     )
     due_child_snapshots = active_child_snapshots.exclude(status=Snapshot.StatusChoices.PAUSED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__lte=now),
@@ -52,7 +52,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     cleaned["snapshots_queued_without_retry_at"] = Snapshot.objects.filter(
         status=Snapshot.StatusChoices.QUEUED,
         retry_at__isnull=True,
-        crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED],
+        crawl__status__in=Crawl.RUNNABLE_STATES,
     ).update(retry_at=now, modified_at=now)
     backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF)
     orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED).exclude(

archivebox/core/routes_util.py

@@ -257,6 +257,8 @@ def get_snapshot_lookup_key(snapshot_ref: str) -> str:
     match = _SNAPSHOT_SUBDOMAIN_RE.match(value)
     if match:
         return match.group("suffix")
+    if _SNAPSHOT_ID_RE.match(value):
+        return re.sub(r"[^0-9a-fA-F]", "", value).lower()
     return value
 
 

archivebox/core/settings.py

@@ -13,7 +13,7 @@ import archivebox
 
 from archivebox.config.constants import CONSTANTS
 from archivebox.config.common import get_config
-from archivebox.core.host_util import normalize_base_url, get_admin_base_url, get_api_base_url
+from archivebox.core.routes_util import normalize_base_url, get_admin_base_url, get_api_base_url
 from .settings_logging import SETTINGS_LOGGING
 
 
@@ -63,14 +63,15 @@ INSTALLED_APPS = [
     # Our ArchiveBox-provided apps (use fully qualified names)
     # NOTE: Order matters! Apps with migrations that depend on other apps must come AFTER their dependencies
     # "archivebox.config",  # ArchiveBox config settings (no models, not a real Django app)
+    "archivebox.plugins",  # plugin discovery, hook helpers, config UI, and plugin metadata views
+    "archivebox.search",  # search backend query helpers, admin search UI, and daemon integrations
     "archivebox.machine",  # handles collecting and storing information about the host machine, network interfaces, binaries, etc.
     "archivebox.workers",  # handles starting and managing background workers and processes (orchestrators and actors)
     "archivebox.personas",  # handles Persona and session management
     "archivebox.core",  # core django model with Snapshot, ArchiveResult, etc. (crawls depends on this)
     "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
+    "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
     "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
-    # ArchiveBox plugins (hook-based plugins no longer add Django apps)
-    # Use hooks.py discover_hooks() for plugin functionality
     # 3rd-party apps from PyPI that need to be loaded last
     "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
     "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)
@@ -275,7 +276,7 @@ MIGRATION_MODULES = {"signal_webhooks": None}
 
 # Django requires DEFAULT_AUTO_FIELD to subclass AutoField (BigAutoField, SmallAutoField, etc.)
 # Cannot use UUIDField here until Django 6.0 introduces DEFAULT_PK_FIELD setting
-# For now: manually add `id = models.UUIDField(primary_key=True, default=uuid7, ...)` to all models
+# For now: manually add `id = CompactUUIDField(primary_key=True, default=uuid7, ...)` to all models
 # OR inherit from ModelWithUUID base class which provides UUID primary key
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 
@@ -418,6 +419,8 @@ SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
 CSRF_COOKIE_SECURE = False
 SESSION_COOKIE_SECURE = False
 SESSION_COOKIE_HTTPONLY = True
+SESSION_COOKIE_NAME = f"archivebox_sessionid_{CONSTANTS.COLLECTION_ID}"
+CSRF_COOKIE_NAME = f"archivebox_csrftoken_{CONSTANTS.COLLECTION_ID}"
 # Auth cookies are intentionally scoped to the exact host that set them so
 # the admin session is NOT readable from public.* / web.* / api.* — that
 # split is a security boundary, not a UX choice. Subdomains that need to
@@ -530,11 +533,11 @@ ADMIN_DATA_VIEWS = {
         },
         {
             "route": "plugins/",
-            "view": "archivebox.config.views.plugins_list_view",
+            "view": "archivebox.plugins.views.plugins_list_view",
             "name": "Plugins",
             "items": {
                 "route": "<str:key>/",
-                "view": "archivebox.config.views.plugin_detail_view",
+                "view": "archivebox.plugins.views.plugin_detail_view",
                 "name": "plugin",
             },
         },

archivebox/core/settings_logging.py

@@ -7,6 +7,7 @@ import logging
 
 
 from archivebox.config import CONSTANTS
+from archivebox.misc.logging import STDERR
 
 
 IGNORABLE_URL_PATTERNS = [
@@ -182,6 +183,7 @@ SETTINGS_LOGGING = {
             "level": "DEBUG",
             "markup": False,
             "rich_tracebacks": False,  # Use standard Python tracebacks (no frame/box)
+            "console": STDERR,
             "filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"],
         },
         "logfile": {

archivebox/core/sqlite_backend/base.py

@@ -45,26 +45,14 @@ def _format_sql(query: str, params=None) -> str:
 def _log_locked_database(query: str, params=None, *, attempt: int, elapsed: float, retry_interval: float) -> None:
     from rich.console import Console
 
-    from archivebox.misc.db import sqlite_lock_holders
+    from archivebox.misc.db import log_sqlite_lock_holders
 
     console = Console(stderr=True)
     console.print(
         f"[yellow][*] SQLite database is locked for {elapsed:.0f}s; retrying in {retry_interval:g}s... attempt={attempt}[/yellow]",
     )
     console.print(f"[yellow]    Query: {_format_sql(query, params)}[/yellow]")
-    holders = sqlite_lock_holders()
-    if holders:
-        console.print("[yellow]    DB holders:[/yellow]")
-        for holder in holders[:8]:
-            console.print(f"[yellow]    - {holder}[/yellow]")
-        if len(holders) > 8:
-            console.print(f"[yellow]    ... {len(holders) - 8} more[/yellow]")
-    else:
-        console.print("[yellow]    No local process with index.sqlite3 open was visible to this user.[/yellow]")
-    if attempt == 1:
-        console.print(
-            "[dim]    SQLite does not expose the active SQL statement from another process; only local PIDs with the DB open can be shown.[/dim]",
-        )
+    log_sqlite_lock_holders(console)
 
 
 def _connection_in_transaction(connection) -> bool:

archivebox/core/tag_util.py

@@ -12,7 +12,7 @@ from django.http import HttpRequest
 from django.urls import reverse
 
 from archivebox.config.common import get_config
-from archivebox.core.host_util import build_snapshot_url, build_web_url
+from archivebox.core.routes_util import build_snapshot_url, build_web_url
 from archivebox.core.models import Snapshot, SnapshotTag, Tag
 
 
@@ -61,7 +61,6 @@ def get_matching_tags(
     created_by: str = "",
     year: str = "",
     has_snapshots: str = "all",
-    with_snapshot_counts: bool = True,
 ) -> QuerySet[Tag]:
     sort = normalize_tag_sort(sort)
     has_snapshots = normalize_has_snapshots_filter(has_snapshots)
@@ -284,7 +283,6 @@ def build_tag_cards(
         created_by=created_by,
         year=year,
         has_snapshots=has_snapshots,
-        with_snapshot_counts=needs_snapshot_count_annotation,
     )
     if limit is not None:
         queryset = queryset[:limit]

archivebox/core/templatetags/core_tags.py

@@ -7,12 +7,14 @@ from django.utils.html import escape
 
 from pathlib import Path
 
-from archivebox.hooks import (
+from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
+
+from archivebox.plugins.discovery import (
     get_plugin_icon,
     get_plugin_template,
     get_plugin_name,
 )
-from archivebox.core.host_util import (
+from archivebox.core.routes_util import (
     canonical_base_host_for_request,
     get_admin_base_url,
     get_public_base_url,
@@ -27,7 +29,6 @@ register = template.Library()
 _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log")
 _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
 _MHTML_PREVIEW_EXTS = (".mhtml", ".mht")
-_WACZ_PREVIEW_EXTS = (".wacz", ".warc", ".warc.gz")
 
 _MEDIA_FILE_EXTS = {
     ".mp4",
@@ -210,7 +211,7 @@ def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None,
         _is_text_preview_path(path)
         or _is_image_preview_path(path)
         or (path or "").lower().endswith(_MHTML_PREVIEW_EXTS)
-        or (path or "").lower().endswith(_WACZ_PREVIEW_EXTS)
+        or is_archivewebpage_replay_target(path or "")
     ):
         return url
     separator = "&" if "?" in url else "?"
@@ -450,7 +451,7 @@ def _unconfigured_banner_context(request) -> dict:
             from archivebox.machine.models import Machine
 
             machine = Machine.current()
-            machine_admin_url = f"/admin/machine/machine/{machine.id.hex}/change/"
+            machine_admin_url = f"/admin/machine/machine/{machine.id}/change/"
         except Exception:
             machine_admin_url = ""
     return {

archivebox/core/urls.py

@@ -24,8 +24,8 @@ from archivebox.core.views import (
     AddView,
     WebAddView,
     HealthCheckView,
-    live_progress_view,
 )
+from archivebox.progressmonitor.views import live_progress_view
 
 
 # GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306

archivebox/crawls/admin.py

@@ -1,7 +1,8 @@
 __package__ = "archivebox.crawls"
 
 from copy import copy
-from urllib.parse import urlencode
+import json
+from urllib.parse import urlencode, urlparse
 
 from django import forms
 from django.core.paginator import Paginator
@@ -13,15 +14,14 @@ from django.utils.html import escape, format_html, format_html_join
 from django.utils import timezone
 from django.utils.safestring import mark_safe
 from django.contrib import admin, messages
-from django.db.models import Case, CharField, Count, IntegerField, OuterRef, Q, Subquery, Value, When
-from django.db.models.functions import Coalesce
+from django.db.models import Case, CharField, Count, Q, Value, When
 
 
 from django_object_actions import action
 
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
-from archivebox.core.models import Snapshot
+from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
     PERMISSIONS_META,
@@ -34,6 +34,7 @@ from archivebox.core.permissions import (
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.crawls.models import Crawl, CrawlSchedule
 from archivebox.misc.paginators import AcceleratedPaginator
+from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.workers.models import RETRY_AT_MAX
 
 
@@ -70,12 +71,14 @@ def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50,
     if status_filter in valid_statuses:
         filtered_qs = filtered_qs.filter(status=status_filter)
 
+    # Keep ArchiveResult counters as scalar subqueries so the paginated
+    # Snapshot queryset does not become a join+GROUP BY over every result row.
     snapshots_qs = filtered_qs.order_by("-created_at").annotate(
-        total_results=Count("archiveresult"),
-        succeeded_results=Count("archiveresult", filter=Q(archiveresult__status="succeeded")),
-        failed_results=Count("archiveresult", filter=Q(archiveresult__status="failed")),
-        started_results=Count("archiveresult", filter=Q(archiveresult__status="started")),
-        skipped_results=Count("archiveresult", filter=Q(archiveresult__status="skipped")),
+        total_results=ArchiveResult.snapshot_count_expr(),
+        succeeded_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SUCCEEDED),
+        failed_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.FAILED),
+        started_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.STARTED),
+        skipped_results=ArchiveResult.snapshot_count_expr(status=ArchiveResult.StatusChoices.SKIPPED),
         snapshot_permissions=Case(
             When(permissions=PERMISSIONS_PUBLIC, then=Value(PERMISSIONS_PUBLIC)),
             When(permissions=PERMISSIONS_UNLISTED, then=Value(PERMISSIONS_UNLISTED)),
@@ -260,7 +263,7 @@ def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50,
                                         background: {progress_color};
                                         transition: width 0.3s;"></div>
                         </div>
-                        <a href="/admin/core/archiveresult/?snapshot__id__exact={snapshot.id.hex}"
+                        <a href="/admin/core/archiveresult/?snapshot__id__exact={snapshot.id}"
                            style="font-size: 11px; color: #417690; min-width: 35px; text-decoration: none;"
                            title="View archive results">{progress_text}</a>
                     </div>
@@ -381,7 +384,7 @@ class URLFiltersField(forms.Field):
     def to_python(self, value):
         if isinstance(value, dict):
             return value
-        return {"allowlist": "", "denylist": "", "same_domain_only": False, "subpaths_only": False}
+        return {"allowlist": "", "denylist": "", "same_domain_only": False, "subpaths_only": False, "only_new": False}
 
 
 class CrawlAdminForm(forms.ModelForm):
@@ -423,13 +426,140 @@ class CrawlAdminForm(forms.ModelForm):
         config = dict(self.instance.config or {}) if self.instance and self.instance.pk else {}
         if self.instance and self.instance.pk:
             self.initial["tags_editor"] = self.instance.tags_str
+        effective_only_new = self.effective_only_new(self.instance if self.instance and self.instance.pk else None)
+        derived_filter_toggles = self.derive_filter_toggles(
+            self.instance.urls if self.instance and self.instance.pk else "",
+            config.get("URL_ALLOWLIST", ""),
+        )
         self.initial["url_filters"] = {
             "allowlist": config.get("URL_ALLOWLIST", ""),
             "denylist": config.get("URL_DENYLIST", ""),
-            "same_domain_only": False,
-            "subpaths_only": False,
+            "same_domain_only": derived_filter_toggles["same_domain_only"],
+            "subpaths_only": derived_filter_toggles["subpaths_only"],
+            "only_new": effective_only_new,
         }
 
+    @staticmethod
+    def extract_url_line(line):
+        line = str(line or "").strip()
+        if not line or line.startswith("#"):
+            return ""
+        if line.startswith("{"):
+            try:
+                return str(json.loads(line).get("url", "")).strip()
+            except (TypeError, ValueError, json.JSONDecodeError):
+                return ""
+        return line
+
+    @staticmethod
+    def regex_escape(text):
+        escaped = ""
+        for char in str(text or ""):
+            escaped += f"\\{char}" if char in r".*+?^${}()|[]\\" else char
+        return escaped
+
+    @classmethod
+    def generated_host_allowlist(cls, urls):
+        seen = set()
+        domains = []
+        for raw_line in str(urls or "").splitlines():
+            url = cls.extract_url_line(raw_line)
+            if not url:
+                continue
+            parsed = urlparse(url)
+            domain = (parsed.hostname or "").lower()
+            if not domain or domain in seen:
+                continue
+            seen.add(domain)
+            domains.append(domain)
+        if not domains:
+            return ""
+        return "^https?://(" + "|".join(cls.regex_escape(domain) for domain in domains) + ")([:/]|$)"
+
+    @staticmethod
+    def subpath_prefix(pathname):
+        path = str(pathname or "/")
+        while "//" in path:
+            path = path.replace("//", "/")
+        if not path or path == "/":
+            return "/"
+        if path.endswith("/"):
+            return path
+        last_slash = path.rfind("/")
+        last_part = path[last_slash + 1 :]
+        if "." in last_part:
+            return path[: last_slash + 1] or "/"
+        return path
+
+    @staticmethod
+    def parsed_host_and_port(parsed):
+        host = (parsed.hostname or "").lower()
+        if not host:
+            return ""
+        try:
+            port = parsed.port
+        except ValueError:
+            port = None
+        return f"{host}:{port}" if port is not None else host
+
+    @classmethod
+    def generated_subpath_allowlist(cls, urls):
+        seen = set()
+        paths = []
+        for raw_line in str(urls or "").splitlines():
+            url = cls.extract_url_line(raw_line)
+            if not url:
+                continue
+            parsed = urlparse(url)
+            domain = (parsed.hostname or "").lower()
+            if domain:
+                seen.add(domain)
+            host = cls.parsed_host_and_port(parsed)
+            path = cls.subpath_prefix(parsed.path)
+            path_key = f"{host}{path}"
+            if not host or path_key in seen:
+                continue
+            seen.add(path_key)
+            paths.append((host, path))
+        if not paths:
+            return ""
+        patterns = []
+        for host, path in paths:
+            if path == "/":
+                patterns.append(f"^https?://{cls.regex_escape(host)}([/?#]|$)")
+            elif path.endswith("/"):
+                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}")
+            else:
+                patterns.append(f"^https?://{cls.regex_escape(host)}{cls.regex_escape(path)}([/?#]|$)")
+        return "\n".join(patterns)
+
+    @classmethod
+    def derive_filter_toggles(cls, urls, allowlist):
+        normalized_allowlist = "\n".join(Crawl.split_filter_patterns(allowlist))
+        if not normalized_allowlist:
+            return {"same_domain_only": False, "subpaths_only": False}
+        if normalized_allowlist == cls.generated_subpath_allowlist(urls):
+            return {"same_domain_only": True, "subpaths_only": True}
+        if normalized_allowlist == cls.generated_host_allowlist(urls):
+            return {"same_domain_only": True, "subpaths_only": False}
+        return {"same_domain_only": False, "subpaths_only": False}
+
+    @staticmethod
+    def effective_only_new(crawl=None):
+        from archivebox.config.common import get_config
+
+        if crawl is not None:
+            return bool(get_config(crawl=crawl, resolve_plugins=False).ONLY_NEW)
+        return bool(get_config(resolve_plugins=False).ONLY_NEW)
+
+    @staticmethod
+    def inherited_only_new(crawl):
+        crawl_without_only_new = copy(crawl)
+        config = dict(crawl.config or {})
+        config.pop("ONLY_NEW", None)
+        crawl_without_only_new.config = config
+        return CrawlAdminForm.effective_only_new(crawl_without_only_new)
+
     def clean_tags_editor(self):
         tags_str = self.cleaned_data.get("tags_editor", "")
         tag_names = []
@@ -452,6 +582,7 @@ class CrawlAdminForm(forms.ModelForm):
             "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
             "same_domain_only": bool(value.get("same_domain_only")),
             "subpaths_only": bool(value.get("subpaths_only")),
+            "only_new": bool(value.get("only_new")),
         }
 
     def save(self, commit=True):
@@ -463,6 +594,14 @@ class CrawlAdminForm(forms.ModelForm):
                 url_filters.get("allowlist", ""),
                 url_filters.get("denylist", ""),
             )
+            config = dict(instance.config or {})
+            only_new = bool(url_filters.get("only_new"))
+            inherited_only_new = self.inherited_only_new(instance)
+            if only_new != inherited_only_new:
+                config["ONLY_NEW"] = only_new
+            else:
+                config.pop("ONLY_NEW", None)
+            instance.config = config
         if commit:
             instance.save()
             instance.apply_crawl_config_filters()
@@ -623,19 +762,11 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         crawl_ids = [crawl.pk for crawl in crawl_list]
         if not crawl_ids:
             return
-        counts = {
-            str(row["crawl_id"]): row
-            for row in Snapshot.objects.filter(crawl_id__in=crawl_ids)
-            .values("crawl_id")
-            .annotate(
-                num_snapshots_cached=Count("pk"),
-                num_archived_snapshots_cached=Count("pk", filter=Q(status=Snapshot.StatusChoices.SEALED)),
-            )
-        }
+        counts = Snapshot.crawl_total_and_status_counts(crawl_ids, status=Snapshot.StatusChoices.SEALED)
         for crawl in crawl_list:
             row = counts.get(str(crawl.pk), {})
-            crawl.num_snapshots_cached = row.get("num_snapshots_cached", 0)
-            crawl.num_archived_snapshots_cached = row.get("num_archived_snapshots_cached", 0)
+            crawl.num_snapshots_cached = row.get("total", 0)
+            crawl.num_archived_snapshots_cached = row.get("status", 0)
 
     def get_queryset(self, request):
         """Keep joins page-local while computing per-row snapshot counts in the page query."""
@@ -649,25 +780,9 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
             )
         )
         if self.should_annotate_snapshot_counts(request):
-            snapshot_count = (
-                Snapshot.objects.filter(crawl_id=OuterRef("pk")).order_by().values("crawl_id").annotate(count=Count("pk")).values("count")
-            )
-            archived_snapshot_count = (
-                Snapshot.objects.filter(crawl_id=OuterRef("pk"), status=Snapshot.StatusChoices.SEALED)
-                .order_by()
-                .values("crawl_id")
-                .annotate(count=Count("pk"))
-                .values("count")
-            )
             queryset = queryset.annotate(
-                num_snapshots_cached=Coalesce(
-                    Subquery(snapshot_count, output_field=IntegerField()),
-                    Value(0),
-                ),
-                num_archived_snapshots_cached=Coalesce(
-                    Subquery(archived_snapshot_count, output_field=IntegerField()),
-                    Value(0),
-                ),
+                num_snapshots_cached=Snapshot.crawl_count_expr(),
+                num_archived_snapshots_cached=Snapshot.crawl_count_expr(status=Snapshot.StatusChoices.SEALED),
             )
         return queryset
 
@@ -684,6 +799,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
             "crawl_stop_reason": self.stop_reason_for_crawl(crawl) if crawl else "",
             "crawl_snapshots_changelist": self.snapshots_changelist(crawl) if crawl else "",
         }
+        if crawl and crawl.status in {
+            Crawl.StatusChoices.QUEUED,
+            Crawl.StatusChoices.STARTED,
+            Crawl.StatusChoices.PAUSED,
+        }:
+            extra_context["progress_auto_expand"] = True
+            extra_context["progress_endpoint"] = progress_endpoint("crawl", crawl.id)
         return super().change_view(request, object_id, form_url, extra_context)
 
     def add_view(self, request, form_url="", extra_context=None):
@@ -742,7 +864,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         # hold SQLite behind the request for minutes on large archives. The
         # Crawl row is the scheduler signal; the runner observes PAUSED and
         # owns child-row lifecycle work.
-        paused = queryset.exclude(status__in=[Crawl.StatusChoices.SEALED, Crawl.StatusChoices.PAUSED]).update(
+        paused = queryset.exclude(status__in=Crawl.INACTIVE_STATES).update(
             status=Crawl.StatusChoices.PAUSED,
             retry_at=RETRY_AT_MAX,
             modified_at=timezone.now(),
@@ -757,7 +879,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         # Keep resume symmetrical with pause: one tight scheduler UPDATE, no
         # save() hooks and no child fanout in the request path. Paused child
         # rows become runnable through their own resume/maintenance paths.
-        resumed = queryset.filter(status__in=[Crawl.StatusChoices.PAUSED, Crawl.StatusChoices.SEALED]).update(
+        resumed = queryset.filter(status__in=Crawl.INACTIVE_STATES).update(
             status=Crawl.StatusChoices.QUEUED,
             retry_at=timezone.now(),
             modified_at=timezone.now(),
@@ -777,11 +899,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
         Snapshot.objects.filter(
             crawl_id__in=crawl_ids,
-            status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-                Snapshot.StatusChoices.PAUSED,
-            ],
+            status__in=Snapshot.OPEN_STATES,
         ).filter(
             Q(retry_at__isnull=True) | Q(retry_at__gt=now),
         ).update(
@@ -799,7 +917,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         )
         messages.success(request, f"Sealed {sealed} crawl(s). The runner will finish cleanup on the next sweep.")
 
-    @admin.action(description="Set Permissions ▾")
+    @admin.action(description="Permissions ▾")
     def set_crawl_permissions(self, request, queryset):
         permissions = (request.POST.get("permissions") or "").strip().lower()
         if permissions not in PERMISSIONS_VALUES:
@@ -836,7 +954,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
             messages.error(request, "Cannot recrawl: original crawl has no URLs.")
             return redirect("admin:crawls_crawl_change", obj.id)
 
-        new_crawl = Crawl.objects.create(
+        new_crawl = Crawl.create_scheduler_row(
             urls=obj.urls,
             max_depth=obj.max_depth,
             tags_str=obj.tags_str,
@@ -861,22 +979,17 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         return format_html('<span class="crawl-stop-reason">{}</span>', reason)
 
     def stop_reason_for_crawl(self, obj):
-        from abx_dl.limits import CrawlLimitState
-
         if obj.pk in self.stop_reason_cache:
             return self.stop_reason_cache[obj.pk]
 
         output_dir = obj.output_dir_for_config(self.crawl_admin_base_config)
         config = self.limit_config_for_crawl(obj, output_dir)
-        reason = ""
-        if (output_dir / ".abx-dl" / "limits.json").exists():
-            config["CRAWL_DIR"] = str(output_dir)
-            reason = CrawlLimitState.from_config(config).get_stop_reason() or ""
-
-        max_urls = int(config["CRAWL_MAX_URLS"] or 0)
-        if not reason and max_urls > 0 and obj.num_snapshots_cached >= max_urls and obj.count_urls_for_limit() >= max_urls:
-            reason = "crawl_max_urls"
-
+        reason = obj.stop_reason(
+            config=config,
+            output_dir=output_dir,
+            num_snapshots=obj.num_snapshots_cached,
+            num_sealed_snapshots=obj.num_archived_snapshots_cached,
+        )
         self.stop_reason_cache[obj.pk] = reason
         return reason
 
@@ -901,7 +1014,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
     @admin.display(description="Status", ordering="status")
     def status_with_stop_reason(self, obj):
         status = "PAUSED" if obj.is_paused else str(obj.status or "").upper()
-        reason = self.stop_reason_for_crawl(obj) if obj.status == Crawl.StatusChoices.SEALED else ""
+        reason = self.stop_reason_for_crawl(obj) if obj.is_paused or obj.status == Crawl.StatusChoices.SEALED else ""
         if reason:
             reason_label = reason.removeprefix("crawl_").replace("_", " ")
             return format_html(

archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py

@@ -0,0 +1,33 @@
+from django.db import migrations
+
+
+def freeze_existing_crawl_configs(apps, schema_editor):
+    from archivebox.config.common import build_crawl_config_snapshot
+    from archivebox.personas.models import Persona
+    from django.contrib.auth import get_user_model
+
+    Crawl = apps.get_model("crawls", "Crawl")
+    User = get_user_model()
+    db_alias = schema_editor.connection.alias
+
+    for crawl in Crawl.objects.using(db_alias).select_related("persona", "created_by").iterator(chunk_size=200):
+        current_config = dict(crawl.config or {})
+        persona = Persona.objects.using(db_alias).filter(pk=crawl.persona_id).first()
+        user = User.objects.using(db_alias).filter(pk=crawl.created_by_id).first()
+        frozen_config = build_crawl_config_snapshot(
+            user=user,
+            persona=persona,
+            overrides=current_config,
+        )
+        if frozen_config != current_config:
+            Crawl.objects.using(db_alias).filter(pk=crawl.pk).update(config=frozen_config)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("crawls", "0017_drop_stale_crawl_limit_columns"),
+    ]
+
+    operations = [
+        migrations.RunPython(freeze_existing_crawl_configs, migrations.RunPython.noop),
+    ]

archivebox/crawls/migrations/0019_crawlschedule_config.py

@@ -0,0 +1,25 @@
+from django.db import migrations, models
+
+
+def copy_template_config_to_schedule(apps, schema_editor):
+    CrawlSchedule = apps.get_model("crawls", "CrawlSchedule")
+    db_alias = schema_editor.connection.alias
+
+    for schedule in CrawlSchedule.objects.using(db_alias).select_related("template").iterator(chunk_size=200):
+        template_config = dict(schedule.template.config or {}) if schedule.template_id else {}
+        CrawlSchedule.objects.using(db_alias).filter(pk=schedule.pk).update(config=template_config)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("crawls", "0018_freeze_crawl_config_snapshots"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="crawlschedule",
+            name="config",
+            field=models.JSONField(blank=True, default=dict, null=True),
+        ),
+        migrations.RunPython(copy_template_config_to_schedule, migrations.RunPython.noop),
+    ]

archivebox/crawls/models.py

@@ -8,7 +8,7 @@ import json
 import re
 from itertools import islice
 from datetime import timedelta
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 from urllib.parse import urlparse
 
@@ -42,7 +42,7 @@ if TYPE_CHECKING:
 
 
 class CrawlSchedule(ModelWithUUID, ModelWithNotes):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     modified_at = models.DateTimeField(auto_now=True)
@@ -50,6 +50,7 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
     template: "Crawl" = models.ForeignKey("Crawl", on_delete=models.CASCADE, null=False, blank=False)  # type: ignore
     schedule = models.CharField(max_length=64, blank=False, null=False)
     is_enabled = models.BooleanField(default=True)
+    config = models.JSONField(default=dict, null=True, blank=True)
     label = models.CharField(max_length=64, blank=True, null=False, default="")
     notes = models.TextField(blank=True, null=False, default="")
 
@@ -102,13 +103,17 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         return self.is_enabled and self.next_run_at <= now
 
     def enqueue(self, queued_at=None) -> "Crawl":
+        from archivebox.config.common import build_crawl_config_snapshot
+
         queued_at = queued_at or timezone.now()
         template = self.template
         label = template.label or self.label
+        persona = template.persona if template.persona_id else None
+        user = template.created_by if template.created_by_id else None
 
         return Crawl.objects.create(
             urls=template.urls,
-            config=template.config or {},
+            config=build_crawl_config_snapshot(user=user, persona=persona, overrides=self.config or {}),
             max_depth=template.max_depth,
             tags_str=template.tags_str,
             persona_id=template.persona_id,
@@ -122,7 +127,7 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
 
 
 class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithStateMachine):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     modified_at = models.DateTimeField(auto_now=True)
@@ -162,6 +167,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     StatusChoices = ModelWithStateMachine.StatusChoices
     active_state = StatusChoices.STARTED
     delete_after_final_statuses = (StatusChoices.SEALED,)
+    RUNNABLE_STATES = (StatusChoices.QUEUED, StatusChoices.STARTED)
+    INACTIVE_STATES = (StatusChoices.PAUSED, StatusChoices.SEALED)
 
     schedule_id: uuid.UUID | None
     sm: "CrawlMachine"
@@ -238,11 +245,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         # child Snapshot through its own state machine. Active children that
         # are already due need no write; the runner will claim them as-is.
         active_children = self.snapshot_set.filter(
-            status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-                Snapshot.StatusChoices.PAUSED,
-            ],
+            status__in=Snapshot.OPEN_STATES,
         )
         return active_children.filter(
             Q(retry_at__isnull=True) | Q(retry_at__gt=now),
@@ -259,10 +262,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         # Snapshot runner claim performs the real pause transition and cascades
         # its own ArchiveResults, keeping request/admin transactions tiny.
         active_children = self.snapshot_set.filter(
-            status__in=[
-                Snapshot.StatusChoices.QUEUED,
-                Snapshot.StatusChoices.STARTED,
-            ],
+            status__in=Snapshot.RUNNABLE_STATES,
         )
         return active_children.filter(
             Q(retry_at__isnull=True) | Q(retry_at__gt=now),
@@ -273,10 +273,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     @classmethod
     def missing_delete_at_candidates(cls):
-        from archivebox.personas.models import Persona
-
-        persona_ids = Persona.objects.filter(config__has_key="DELETE_AFTER").values_list("id", flat=True)
-        return cls.objects.filter(delete_at__isnull=True).filter(Q(config__has_key="DELETE_AFTER") | Q(persona_id__in=persona_ids))
+        return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER")
 
     def save(self, *args, **kwargs):
         update_fields = kwargs.get("update_fields")
@@ -287,11 +284,16 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             previous_tag_names = set(self.parse_tag_names(old_crawl.tags_str or ""))
 
         config = dict(self.config or {})
+        is_new = self._state.adding or old_crawl is None
+        persona = self.persona if self.persona_id else None
+        user = self.created_by if self.created_by_id else None
+        if is_new:
+            from archivebox.config.common import build_crawl_config_snapshot
+
+            config = build_crawl_config_snapshot(user=user, persona=persona, overrides=config)
         if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
             from archivebox.config.common import get_config
 
-            persona = self.persona if self.persona_id else None
-            user = self.created_by if self.created_by_id else None
             config["PERMISSIONS"] = normalize_permissions(get_config(persona=persona, user=user, include_machine=True).PERMISSIONS)
         if "CRAWL_MAX_CONCURRENT_SNAPSHOTS" in config:
             raw_concurrency = config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]
@@ -338,7 +340,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     @property
     def api_url(self) -> str:
-        return str(reverse_lazy("api-1:get_crawl", args=[self.id.hex]))
+        return str(reverse_lazy("api-1:get_crawl", args=[self.id]))
 
     @staticmethod
     def parse_tag_names(tags: Iterable[str] | str, *, pattern: str = r",") -> list[str]:
@@ -491,7 +493,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     def output_dir(self) -> Path:
         from archivebox.config.common import get_config
 
-        return self.output_dir_for_config(get_config(resolve_plugins=False))
+        output_dir = self.output_dir_for_config(get_config(resolve_plugins=False))
+        hyphen_dir = output_dir.with_name(str(uuid.UUID(hex=self.id.hex)))
+        return output_dir if output_dir.exists() or not hyphen_dir.exists() else hyphen_dir
 
     def get_urls_list(self) -> list[str]:
         """Get list of URLs from urls field, filtering out comments and empty lines."""
@@ -771,35 +775,108 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         from archivebox.personas.models import Persona
 
         if self.persona_id:
-            persona = Persona.objects.filter(id=self.persona_id).first()
-            if persona is not None:
-                return persona
-
-        default_persona_name = str((self.config or {}).get("DEFAULT_PERSONA") or "").strip()
-        if default_persona_name:
-            persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default")
-            persona.ensure_dirs()
-            return persona
+            return Persona.objects.filter(id=self.persona_id).first()
 
         return None
 
-    def limit_stop_reason(self) -> str:
-        from abx_dl.limits import CrawlLimitState
-        from archivebox.config.common import get_config
+    @staticmethod
+    def _config_value(config: Mapping[str, Any] | Any, key: str, default: Any = None) -> Any:
+        if isinstance(config, Mapping):
+            return config.get(key, default)
+        return getattr(config, key, default)
 
-        config = get_config(crawl=self, include_machine=False)
-        if (self.output_dir / ".abx-dl" / "limits.json").exists():
-            config["CRAWL_DIR"] = str(self.output_dir)
-            stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
+    @classmethod
+    def create_scheduler_row(cls, **kwargs) -> "Crawl":
+        from archivebox.base_models.models import normalize_config_json_values
+        from archivebox.config.common import build_crawl_config_snapshot
+
+        now = timezone.now()
+        kwargs.setdefault("created_at", now)
+        kwargs.setdefault("modified_at", now)
+        config = normalize_config_json_values(kwargs.get("config") or {})
+        user = kwargs.get("created_by")
+        persona = kwargs.get("persona")
+        if user is None and kwargs.get("created_by_id"):
+            from django.contrib.auth import get_user_model
+
+            user = get_user_model().objects.filter(pk=kwargs["created_by_id"]).first()
+        if persona is None and kwargs.get("persona_id"):
+            from archivebox.personas.models import Persona
+
+            persona = Persona.objects.filter(pk=kwargs["persona_id"]).first()
+        kwargs["config"] = build_crawl_config_snapshot(user=user, persona=persona, overrides=config)
+        crawl = cls(**kwargs)
+        if crawl.delete_at is None:
+            crawl.set_delete_at_from_config()
+        cls.objects.bulk_create([crawl])
+        return crawl
+
+    def limit_stop_reason(
+        self,
+        *,
+        config: Mapping[str, Any] | Any | None = None,
+        output_dir: Path | None = None,
+        num_snapshots: int | None = None,
+    ) -> str:
+        from abx_dl.limits import CrawlLimitState
+
+        if config is None:
+            from archivebox.config.common import get_config
+
+            config = get_config(crawl=self, include_machine=False)
+        if output_dir is None:
+            output_dir = self.output_dir
+
+        limits_path = output_dir / ".abx-dl" / "limits.json"
+        if limits_path.exists():
+            config_with_crawl_dir = {**dict(config.items())} if isinstance(config, Mapping) else config
+            config_with_crawl_dir["CRAWL_DIR"] = str(output_dir)
+            stop_reason = CrawlLimitState.from_config(config_with_crawl_dir).get_stop_reason()
             if stop_reason:
                 return stop_reason
 
-        max_urls = int(config.CRAWL_MAX_URLS or 0)
-        if max_urls > 0 and self.snapshot_set.count() >= max_urls and self.count_urls_for_limit() >= max_urls:
+        max_urls = int(self._config_value(config, "CRAWL_MAX_URLS", 0) or 0)
+        if num_snapshots is None:
+            num_snapshots = self.snapshot_set.count()
+        if max_urls > 0 and num_snapshots >= max_urls and self.count_urls_for_limit() >= max_urls:
             return "crawl_max_urls"
 
         return ""
 
+    def lifecycle_stop_reason(self, *, num_snapshots: int | None = None, num_sealed_snapshots: int | None = None) -> str:
+        if self.is_paused:
+            return "paused"
+
+        if self.status != self.StatusChoices.SEALED:
+            return ""
+
+        if num_snapshots is None:
+            num_snapshots = self.snapshot_set.count()
+        if num_snapshots == 0:
+            return "no_viable_urls"
+
+        if num_sealed_snapshots is None:
+            from archivebox.core.models import Snapshot
+
+            num_sealed_snapshots = self.snapshot_set.filter(status=Snapshot.StatusChoices.SEALED).count()
+        if num_sealed_snapshots >= num_snapshots:
+            return "done"
+
+        return ""
+
+    def stop_reason(
+        self,
+        *,
+        config: Mapping[str, Any] | Any | None = None,
+        output_dir: Path | None = None,
+        num_snapshots: int | None = None,
+        num_sealed_snapshots: int | None = None,
+    ) -> str:
+        return self.limit_stop_reason(config=config, output_dir=output_dir, num_snapshots=num_snapshots) or self.lifecycle_stop_reason(
+            num_snapshots=num_snapshots,
+            num_sealed_snapshots=num_sealed_snapshots,
+        )
+
     def add_url(self, entry: dict) -> bool:
         """
         Add a URL to the crawl queue if not already present.
@@ -1220,7 +1297,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             The root Snapshot for this crawl, or None for system crawls that don't create snapshots
         """
         import time
-        from archivebox.hooks import run_hook, discover_hooks, process_hook_records, is_finite_background_hook
+        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records, is_finite_background_hook
         from archivebox.config.common import get_config
         from archivebox.machine.models import Binary, Machine
 
@@ -1284,7 +1361,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 except Exception:
                     return set()
 
-            from archivebox.hooks import extract_records_from_process
+            from archivebox.plugins.hooks import extract_records_from_process
 
             records = []
             # Finite background hooks can exit before their completed Process
@@ -1409,7 +1486,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     def cleanup(self):
         """Clean up background hooks and run on_CrawlEnd hooks."""
-        from archivebox.hooks import run_hook, discover_hooks
+        from archivebox.plugins.hooks import run_hook, discover_hooks
 
         # Clean up .pid files from output directory
         if self.output_dir.exists():

archivebox/machine/models.py

@@ -7,7 +7,7 @@ import sys
 import uuid
 import socket
 from pathlib import Path
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 from datetime import timedelta, datetime
 from typing import TYPE_CHECKING, Any, cast
 
@@ -98,7 +98,7 @@ def _get_process_binary_env_keys(plugin_name: str, hook_path: str, env: dict[str
         add(f"{plugin_key}_BINARY")
 
     try:
-        from archivebox.hooks import discover_plugin_configs
+        from archivebox.plugins.discovery import discover_plugin_configs
 
         plugin_schema = discover_plugin_configs().get(plugin_name, {})
         schema_keys = [key for key in (plugin_schema.get("properties") or {}) if key.endswith("_BINARY")]
@@ -173,7 +173,7 @@ class MachineManager(models.Manager):
 
 
 class Machine(ModelWithHealthStats):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
     guid = models.CharField(max_length=64, default=None, null=False, unique=True, editable=False)
@@ -377,7 +377,7 @@ class NetworkInterfaceManager(models.Manager):
 
 
 class NetworkInterface(ModelWithHealthStats):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
     machine = models.ForeignKey(Machine, on_delete=models.CASCADE, default=None, null=False)
@@ -492,15 +492,15 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
     Installation is synchronous during queued→installed transition.
     If installation fails, Binary stays in queued with retry_at set for later retry.
 
-    State machine calls run() which executes on_BinaryRequest__* hooks
-    to install the binary using the specified providers.
+    State machine calls run(), which emits an abxpkg BinaryRequestEvent through
+    the ArchiveBox runner and installs the binary using the specified providers.
     """
 
     class StatusChoices(models.TextChoices):
         QUEUED = "queued", "Queued"
         INSTALLED = "installed", "Installed"
 
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
     machine = models.ForeignKey(Machine, on_delete=models.CASCADE, null=False)
@@ -583,7 +583,8 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
         """
         from archivebox.config.common import get_config
 
-        return get_config().DATA_DIR / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
+        data_dir = get_config().DATA_DIR
+        return data_dir / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
 
     def to_json(self) -> dict:
         """
@@ -720,101 +721,11 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
 
     def run(self):
         """
-        Execute binary installation by running on_BinaryRequest__* hooks.
-
-        Called by BinaryMachine when entering 'started' state.
-        Runs ALL on_BinaryRequest__* hooks - each hook checks binproviders
-        and decides if it can handle this binary. First hook to succeed wins.
-        Updates status to SUCCEEDED or FAILED based on hook output.
+        Execute binary installation through the ArchiveBox binary runner.
         """
-        import json
-        from archivebox.hooks import discover_hooks, run_hook
-        from archivebox.config.common import get_config
+        from archivebox.services.runner import run_binary
 
-        # Get merged config (Binary doesn't have crawl/snapshot context).
-        config = get_config()
-
-        # ArchiveBox installs the puppeteer package and Chromium in separate
-        # hook phases. Suppress puppeteer's bundled browser download during the
-        # package install step so the dedicated chromium hook owns that work.
-        if self.name == "puppeteer":
-            config.setdefault("PUPPETEER_SKIP_DOWNLOAD", "true")
-            config.setdefault("PUPPETEER_SKIP_CHROMIUM_DOWNLOAD", "true")
-
-        # Create output directory
-        output_dir = self.output_dir
-        output_dir.mkdir(parents=True, exist_ok=True)
-
-        # Discover ALL on_BinaryRequest__* hooks
-        hooks = discover_hooks("BinaryRequest", config=config)
-        if not hooks:
-            # No hooks available - stay queued, will retry later
-            return
-
-        allowed_binproviders = self._allowed_binproviders()
-
-        # Run each hook - they decide if they can handle this binary
-        for hook in hooks:
-            plugin_name = hook.parent.name
-            if allowed_binproviders is not None and plugin_name not in allowed_binproviders:
-                continue
-
-            plugin_output_dir = output_dir / plugin_name
-            plugin_output_dir.mkdir(parents=True, exist_ok=True)
-
-            overrides_json = None
-            if self.overrides:
-                overrides_json = json.dumps(self.overrides)
-
-            # Run the hook
-            process = run_hook(
-                hook,
-                output_dir=plugin_output_dir,
-                config=config,
-                timeout=600,  # 10 min timeout for binary installation
-                binary_id=str(self.id),
-                machine_id=str(self.machine_id),
-                name=self.name,
-                binproviders=self.binproviders,
-                overrides=overrides_json,
-            )
-
-            # Background hook (unlikely for binary installation, but handle it)
-            if process is None:
-                continue
-
-            # Failed or skipped hook - try next one
-            if process.exit_code != 0:
-                continue
-
-            # Parse JSONL output to check for successful installation
-            from archivebox.hooks import extract_records_from_process, process_hook_records
-
-            records = extract_records_from_process(process)
-            if records:
-                process_hook_records(records, overrides={})
-            binary_records = [record for record in records if record.get("type") == "Binary" and record.get("abspath")]
-            if binary_records:
-                record = binary_records[0]
-                # Update self from successful installation
-                self.abspath = record["abspath"]
-                self.version = record.get("version", "")
-                self.sha256 = record.get("sha256", "")
-                self.binprovider = record.get("binprovider", "env")
-                self.status = self.StatusChoices.INSTALLED
-                self.save()
-
-                # Maintain the optional human-facing LIB_BIN_DIR convenience symlink.
-                from archivebox.config.common import get_config
-
-                lib_bin_dir = get_config().LIB_BIN_DIR
-                if lib_bin_dir:
-                    self.symlink_to_lib_bin_after_commit(lib_bin_dir)
-
-                return
-
-        # No hook succeeded - leave status as QUEUED (will retry later)
-        # Don't set to FAILED since we don't have that status anymore
+        run_binary(str(self.id))
 
     def cleanup(self):
         """
@@ -1044,7 +955,7 @@ class Process(ModelWithDeleteAfter, models.Model):
         BINARY = "binary", "Binary"
 
     # Primary fields
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
 

archivebox/misc/checks.py

@@ -1,8 +1,11 @@
 __package__ = "archivebox.misc"
 
 import os
+import signal
 import sys
 import time
+import threading
+from contextlib import contextmanager
 from pathlib import Path
 
 from rich import print
@@ -20,6 +23,42 @@ from rich.panel import Panel
 # that the check is called after django.setup() has been called
 
 
+def _migration_interrupt_message(*, before_apply: bool = False) -> str:
+    status = "Migration cancelled before any changes were applied." if before_apply else "Migration interrupted."
+    return (
+        f"\n[X] {status}\n"
+        "    Database migrations are atomic; interrupted migration work is rolled back or left unapplied,\n"
+        "    so no partially-applied migration is recorded and no data loss has occurred.\n\n"
+        "    To continue the upgrade, run:\n"
+        "        archivebox init\n"
+    )
+
+
+@contextmanager
+def _exit_on_migration_interrupt():
+    if threading.current_thread() is not threading.main_thread():
+        yield
+        return
+
+    handled_signals = (signal.SIGINT, signal.SIGTERM)
+    previous_handlers = {sig: signal.getsignal(sig) for sig in handled_signals}
+
+    def handle_shutdown(_signum, _frame):
+        try:
+            os.write(sys.stderr.fileno(), _migration_interrupt_message().encode())
+        except Exception:
+            pass
+        os._exit(130)
+
+    try:
+        for sig in handled_signals:
+            signal.signal(sig, handle_shutdown)
+        yield
+    finally:
+        for sig, previous_handler in previous_handlers.items():
+            signal.signal(sig, previous_handler)
+
+
 def check_data_folder(config=None, **config_kwargs) -> None:
     from archivebox import DATA_DIR
     from archivebox.config import CONSTANTS
@@ -109,14 +148,19 @@ def check_migrations(*, blocking: bool = True, auto_apply: bool = False, cancel_
             try:
                 time.sleep(cancel_delay)
             except KeyboardInterrupt:
-                print("[red][X] Migration cancelled before any changes were applied.[/red]", file=sys.stderr)
+                print(_migration_interrupt_message(before_apply=True), file=sys.stderr)
                 raise SystemExit(130) from None
 
             # Always delegate to Django's migration executor. It records each
             # migration only after it succeeds, so power loss or SIGKILL leaves
             # unapplied work visible here and the next startup resumes normally.
             print("[yellow][*] Applying database migrations...[/yellow]", file=sys.stderr)
-            apply_migrations(stdout=sys.stderr, stderr=sys.stderr, verbosity=1)
+            try:
+                with _exit_on_migration_interrupt():
+                    apply_migrations(stdout=sys.stderr, stderr=sys.stderr, verbosity=1)
+            except KeyboardInterrupt:
+                print(_migration_interrupt_message(), file=sys.stderr)
+                raise SystemExit(130) from None
             return pending_migrations()
         if blocking:
             raise SystemExit(3)
@@ -159,10 +203,7 @@ def check_not_root():
 
     if IS_ROOT and not (is_getting_help or is_getting_version):
         print("[yellow][!] Running ArchiveBox as root is not recommended.[/yellow]", file=sys.stderr)
-        print(
-            "    Chrome and other plugins run as non-root for security, if DATA_DIR is owned by root it can prevent archiving from succeeding.",
-            file=sys.stderr,
-        )
+        print("    Root-owned DATA_DIR files may be inaccessible to non-root users later.", file=sys.stderr)
         print("        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root", file=sys.stderr)
 
 
@@ -208,6 +249,7 @@ def check_data_dir_permissions(config=None, **config_kwargs):
             f"[violet]Hint:[/violet] Change the current ownership [red]{data_dir_uid}[/red]:{data_dir_gid} (PUID:PGID) to the user & group that will run ArchiveBox, e.g.:",
         )
         STDERR.print(f"    [grey53]sudo[/grey53] chown -R [blue]{DEFAULT_PUID}:{DEFAULT_PGID}[/blue] {DATA_DIR.resolve()}")
+        STDERR.print("    Avoid recursive chown on very large archives unless you know the full tree needs repair.")
         STDERR.print()
         STDERR.print("[blue]More info:[/blue]")
         STDERR.print(

archivebox/misc/db.py

@@ -146,6 +146,18 @@ def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]
     return holders
 
 
+def log_sqlite_lock_holders(console: Any, *, db_path: Path = DATA_DIR / "index.sqlite3", limit: int = 8) -> None:
+    holders = sqlite_lock_holders(db_path)
+    if holders:
+        console.print("[yellow]    DB holders:[/yellow]")
+        for holder in holders[:limit]:
+            console.print(f"[yellow]    - {holder}[/yellow]")
+        if len(holders) > limit:
+            console.print(f"[yellow]    ... {len(holders) - limit} more[/yellow]")
+    else:
+        console.print("[yellow]    No local process with index.sqlite3 open was visible to this user.[/yellow]")
+
+
 def sqlite_lock_error(error: BaseException) -> bool:
     from django.db import OperationalError as DjangoOperationalError
 
@@ -157,7 +169,6 @@ def retry_sqlite_locks(action: Callable[[], Any], *, label: str, stderr: TextIO
     from rich.console import Console
 
     console = Console(file=stderr or None, stderr=stderr is None)
-    attempts = 0
     while True:
         try:
             return action()
@@ -168,22 +179,9 @@ def retry_sqlite_locks(action: Callable[[], Any], *, label: str, stderr: TextIO
             if not sqlite_lock_error(err):
                 raise
 
-        attempts += 1
         connections.close_all()
-        holders = sqlite_lock_holders()
         console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in 5s...[/yellow]")
-        if holders:
-            console.print("[yellow]    DB holders:[/yellow]")
-            for holder in holders[:8]:
-                console.print(f"[yellow]    - {holder}[/yellow]")
-            if len(holders) > 8:
-                console.print(f"[yellow]    ... {len(holders) - 8} more[/yellow]")
-        else:
-            console.print("[yellow]    No local process with index.sqlite3 open was visible to this user.[/yellow]")
-        if attempts == 1:
-            console.print(
-                "[dim]    SQLite does not expose the active SQL statement from another process; only the owning local PIDs can be shown.[/dim]",
-            )
+        log_sqlite_lock_holders(console)
         with console.status("[yellow]Waiting for SQLite database lock to clear...[/yellow]", spinner="dots"):
             time.sleep(5.0)
 

archivebox/misc/serve_static.py

@@ -24,6 +24,7 @@ from django.http import StreamingHttpResponse, Http404, HttpResponse, HttpRespon
 from django.utils._os import safe_join
 from django.utils.http import http_date
 from django.utils.translation import gettext as _
+from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
 from archivebox.config.common import get_config
 from archivebox.misc.logging_util import printable_filesize
 
@@ -97,7 +98,7 @@ def _cache_policy(config=None, **config_kwargs) -> str:
 
 
 def _render_mhtml_preview_document(filename: str, output_path: str) -> str:
-    from archivebox.hooks import get_plugin_template
+    from archivebox.plugins.discovery import get_plugin_template
 
     template_str = get_plugin_template("chrome_mhtml", "full", fallback=False)
     if not template_str:
@@ -792,9 +793,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
         bool(request.GET.get("preview")) and content_type.startswith("image/") and not content_type.startswith("image/svg+xml")
     )
     preview_as_mhtml_html = bool(request.GET.get("preview")) and fullpath.suffix.lower() in {".mhtml", ".mht"}
-    preview_as_archivewebpage_html = bool(request.GET.get("preview")) and (
-        fullpath.suffix.lower() in {".wacz", ".warc"} or fullpath.name.lower().endswith(".warc.gz")
-    )
+    preview_as_archivewebpage_html = bool(request.GET.get("preview")) and archivewebpage_replay.is_replay_target(fullpath.name)
 
     # Respect the If-Modified-Since header for non-markdown responses.
     if not (content_type.startswith("text/plain") or content_type.startswith("text/html")):
@@ -863,48 +862,32 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
             pass
 
     if preview_as_archivewebpage_html:
-        # POLICY EXCEPTION: archivebox normally does not depend on any specific
-        # plugin. The WACZ/WARC embedded-replay viewer (ui.js + sw.js + the
-        # rendered preview HTML) is plugin-owned, but it has to be served at
-        # plugin-defined paths on the snapshot host so the same-origin service
-        # worker registration works. There is no clean generic "plugin
-        # contributes a preview handler" extension hook in archivebox yet, so
-        # we conditionally import the archivewebpage plugin's
-        # ``replay_preview`` module here. If the plugin is not installed, the
-        # import fails and we fall through to default static-file serving.
         try:
-            from abx_plugins.plugins.archivewebpage import replay_preview as _awp_preview
-        except ImportError:
-            _awp_preview = None
-        if _awp_preview is not None:
-            try:
-                raw_query = request.GET.copy()
-                raw_query.pop("preview", None)
-                raw_output_path = request.path
-                if raw_query:
-                    raw_output_path = f"{raw_output_path}?{raw_query.urlencode()}"
-                snapshot_url_fallback = getattr(request, "archivebox_snapshot_url", "") or ""
-                rendered = _awp_preview.render_preview_html(
-                    fullpath.name,
-                    raw_output_path,
-                    wacz_path=fullpath,
-                    fallback_url=snapshot_url_fallback,
-                )
-                response = HttpResponse(rendered, content_type="text/html; charset=utf-8")
-                response.headers["Last-Modified"] = http_date(statobj.st_mtime)
-                if etag:
-                    response.headers["ETag"] = etag
-                    response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
-                else:
-                    response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
-                response.headers["Content-Disposition"] = f'inline; filename="{fullpath.stem}.html"'
-                for key, value in _awp_preview.preview_response_headers().items():
-                    response.headers[key] = value
-                if encoding:
-                    response.headers["Content-Encoding"] = encoding
-                return response
-            except Exception:
-                pass
+            raw_query = request.GET.copy()
+            raw_query.pop("preview", None)
+            raw_output_path = request.path
+            if raw_query:
+                raw_output_path = f"{raw_output_path}?{raw_query.urlencode()}"
+            body, preview_content_type, headers = archivewebpage_replay.render_preview_response(
+                fullpath.name,
+                raw_output_path,
+                wacz_path=fullpath,
+                fallback_url=getattr(request, "archivebox_snapshot_url", "") or "",
+                last_modified=http_date(statobj.st_mtime),
+                etag=etag or "",
+                cache_control=(
+                    f"{_cache_policy(config=config)}, max-age=31536000, immutable"
+                    if etag
+                    else f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
+                ),
+                content_encoding=encoding or "",
+            )
+            response = HttpResponse(body, content_type=preview_content_type)
+            for key, value in headers.items():
+                response.headers[key] = value
+            return response
+        except Exception:
+            pass
 
     if preview_as_mhtml_html:
         try:

archivebox/personas/forms.py

@@ -6,8 +6,8 @@ from django import forms
 from django.utils.safestring import mark_safe
 
 from archivebox.config.common import get_config
-from archivebox.core.forms import PluginConfigFormMixin
 from archivebox.core.permissions import PERMISSIONS_CHOICES
+from archivebox.plugins.forms import PluginConfigFormMixin
 from archivebox.personas.importers import (
     PersonaImportResult,
     PersonaImportSource,

archivebox/personas/models.py

@@ -25,7 +25,7 @@ from django.utils import timezone
 
 from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
 from archivebox.base_models.models import ModelWithConfig, get_or_create_system_user_pk
-from archivebox.uuid_compat import uuid7
+from archivebox.uuid_compat import CompactUUIDField, uuid7
 
 _fcntl: Any | None = None
 try:
@@ -81,7 +81,7 @@ class Persona(ModelWithConfig):
         persona.CHROME_USER_DATA_DIR  # -> Path to chrome_profile
     """
 
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     name = models.CharField(max_length=64, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)

archivebox/plugins/__init__.py

@@ -0,0 +1 @@
+__package__ = "archivebox.plugins"

archivebox/plugins/apps.py

@@ -0,0 +1,9 @@
+__package__ = "archivebox.plugins"
+
+from django.apps import AppConfig
+
+
+class PluginsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "archivebox.plugins"
+    verbose_name = "Plugins"

archivebox/plugins/discovery.py

@@ -0,0 +1,377 @@
+__package__ = "archivebox.plugins"
+
+import json
+import os
+from collections.abc import Iterable
+from functools import lru_cache
+from pathlib import Path
+from typing import Any, Protocol, TypedDict
+
+from abx_plugins import get_plugins_dir
+from django.utils.safestring import mark_safe
+
+from archivebox.config.constants import CONSTANTS
+
+
+class ConfigLookup(Protocol):
+    def get(self, key: str, default: Any = None) -> Any: ...
+
+    def items(self) -> Iterable[tuple[str, Any]]: ...
+
+
+class PluginSpecialConfig(TypedDict):
+    enabled: bool
+    timeout: int
+    binary: str
+
+
+BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
+USER_PLUGINS_DIR = Path(
+    os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or str(CONSTANTS.USER_PLUGINS_DIR),
+).expanduser()
+
+
+def iter_plugin_dirs() -> list[Path]:
+    """Iterate over all built-in and user plugin directories."""
+    plugin_dirs: list[Path] = []
+
+    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
+        if not base_dir.exists():
+            continue
+
+        for plugin_dir in base_dir.iterdir():
+            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
+                plugin_dirs.append(plugin_dir)
+
+    return plugin_dirs
+
+
+@lru_cache(maxsize=1)
+def get_plugins() -> list[str]:
+    """
+    Get list of available plugins by discovering plugin directories.
+
+    Returns plugin directory names for any plugin that exposes hooks, config.json,
+    or a standardized templates/icon.html asset. This includes non-extractor
+    plugins such as binary providers and shared base plugins.
+    """
+    plugins = []
+
+    for plugin_dir in iter_plugin_dirs():
+        has_hooks = any(plugin_dir.glob("on_*__*.*"))
+        has_config = (plugin_dir / "config.json").exists()
+        has_icon = (plugin_dir / "templates" / "icon.html").exists()
+        if has_hooks or has_config or has_icon:
+            plugins.append(plugin_dir.name)
+
+    return sorted(set(plugins))
+
+
+def get_plugin_name(plugin: str) -> str:
+    """
+    Get the base plugin name without numeric prefix.
+
+    Examples:
+        '10_title' -> 'title'
+        '26_readability' -> 'readability'
+        '50_parse_html_urls' -> 'parse_html_urls'
+    """
+    parts = plugin.split("_", 1)
+    if len(parts) == 2 and parts[0].isdigit():
+        return parts[1]
+    return plugin
+
+
+def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any) -> list[str]:
+    """
+    Get the list of enabled plugins based on config and available hooks.
+
+    Filters plugins by USE_/SAVE_ flags. Only returns plugins that are enabled.
+    """
+    if config is None:
+        from archivebox.config.common import get_config
+
+        config = get_config(**config_kwargs)
+
+    def normalize_enabled_plugins(value: Any) -> list[str]:
+        if value is None:
+            return []
+        if isinstance(value, str):
+            raw = value.strip()
+            if not raw:
+                return []
+            if raw.startswith("["):
+                try:
+                    parsed = json.loads(raw)
+                except json.JSONDecodeError:
+                    parsed = None
+                if isinstance(parsed, list):
+                    return [str(plugin).strip() for plugin in parsed if str(plugin).strip()]
+            return [plugin.strip() for plugin in raw.split(",") if plugin.strip()]
+        if isinstance(value, (list, tuple, set)):
+            return [str(plugin).strip() for plugin in value if str(plugin).strip()]
+        return [str(value).strip()] if str(value).strip() else []
+
+    plugins_override = config.get("PLUGINS")
+    if plugins_override:
+        return normalize_enabled_plugins(plugins_override)
+
+    enabled = []
+    for plugin in get_plugins():
+        plugin_config = get_plugin_special_config(plugin, config)
+        if plugin_config["enabled"]:
+            enabled.append(plugin)
+
+    return enabled
+
+
+def discover_plugins_that_provide_interface(
+    module_name: str,
+    required_attrs: list[str],
+    plugin_prefix: str | None = None,
+) -> dict[str, Any]:
+    """
+    Discover plugins that provide a specific Python module with required interface.
+
+    This enables dynamic plugin discovery for features like search backends,
+    storage backends, etc. without hardcoding imports.
+    """
+    import importlib.util
+
+    backends = {}
+
+    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
+        if not base_dir.exists():
+            continue
+
+        for plugin_dir in base_dir.iterdir():
+            if not plugin_dir.is_dir():
+                continue
+
+            plugin_name = plugin_dir.name
+            if plugin_prefix and not plugin_name.startswith(plugin_prefix):
+                continue
+
+            module_path = plugin_dir / f"{module_name}.py"
+            if not module_path.exists():
+                continue
+
+            try:
+                spec = importlib.util.spec_from_file_location(
+                    f"archivebox.dynamic_plugins.{plugin_name}.{module_name}",
+                    module_path,
+                )
+                if spec is None or spec.loader is None:
+                    continue
+
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)
+
+                if not all(hasattr(module, attr) for attr in required_attrs):
+                    continue
+
+                if plugin_prefix:
+                    backend_name = plugin_name[len(plugin_prefix) :]
+                else:
+                    backend_name = plugin_name
+
+                backends[backend_name] = module
+
+            except Exception:
+                continue
+
+    return backends
+
+
+def get_search_backends() -> dict[str, Any]:
+    """
+    Discover all available search backend plugins.
+
+    Search backends must provide a search.py module with:
+        - search(query: str) -> List[str]  (returns snapshot IDs)
+        - flush(snapshot_ids: Iterable[str]) -> None
+    """
+    return discover_plugins_that_provide_interface(
+        module_name="search",
+        required_attrs=["search", "flush"],
+        plugin_prefix="search_backend_",
+    )
+
+
+def discover_plugin_configs() -> dict[str, dict[str, Any]]:
+    """
+    Discover all plugin config.json schemas.
+
+    Each plugin can define a config.json file with JSONSchema defining
+    its configuration options.
+    """
+    configs = {}
+
+    for plugin_dir in iter_plugin_dirs():
+        config_path = plugin_dir / "config.json"
+        if not config_path.exists():
+            continue
+
+        try:
+            with open(config_path) as f:
+                schema = json.load(f)
+
+            if not isinstance(schema, dict):
+                continue
+            if schema.get("type") != "object":
+                continue
+            if "properties" not in schema:
+                continue
+
+            configs[plugin_dir.name] = schema
+
+        except (json.JSONDecodeError, OSError) as e:
+            import sys
+
+            print(f"Warning: Failed to load config.json from {plugin_dir.name}: {e}", file=sys.stderr)
+            continue
+
+    return configs
+
+
+def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited: set[str] | None = None) -> PluginSpecialConfig:
+    """
+    Extract special config keys for a plugin following naming conventions.
+
+    ArchiveBox recognizes 3 special config key patterns per plugin:
+        - {PLUGIN}_ENABLED: Enable/disable toggle (default True)
+        - {PLUGIN}_TIMEOUT: Plugin-specific timeout (fallback to TIMEOUT, default 300)
+        - {PLUGIN}_BINARY: Primary binary path (default to plugin_name)
+    """
+    plugin_upper = plugin_name.upper()
+
+    plugins_whitelist = config.get("PLUGINS", "")
+    if plugins_whitelist:
+        plugin_configs = discover_plugin_configs()
+        plugin_names = {p.strip().lower() for p in plugins_whitelist.split(",") if p.strip()}
+        pending = list(plugin_names)
+
+        while pending:
+            current = pending.pop()
+            schema = plugin_configs.get(current, {})
+            required_plugins = schema.get("required_plugins", [])
+            if not isinstance(required_plugins, list):
+                continue
+
+            for required_plugin in required_plugins:
+                required_plugin_name = str(required_plugin).strip().lower()
+                if not required_plugin_name or required_plugin_name in plugin_names:
+                    continue
+                plugin_names.add(required_plugin_name)
+                pending.append(required_plugin_name)
+
+        if plugin_name.lower() not in plugin_names:
+            enabled = False
+        else:
+            enabled_key = f"{plugin_upper}_ENABLED"
+            enabled = config.get(enabled_key)
+            if enabled is None:
+                enabled = True
+            elif isinstance(enabled, str):
+                enabled = enabled.lower() not in ("false", "0", "no", "")
+    else:
+        enabled_key = f"{plugin_upper}_ENABLED"
+        enabled = config.get(enabled_key)
+        if enabled is None:
+            enabled = True
+        elif isinstance(enabled, str):
+            enabled = enabled.lower() not in ("false", "0", "no", "")
+
+    plugin_configs = discover_plugin_configs()
+    plugin_name_lower = plugin_name.lower()
+
+    if enabled:
+        visited = _visited or set()
+        if plugin_name_lower not in visited:
+            next_visited = visited | {plugin_name_lower}
+            schema = plugin_configs.get(plugin_name_lower, {})
+            required_plugins = schema.get("required_plugins", [])
+            if isinstance(required_plugins, list):
+                for required_plugin in required_plugins:
+                    required_plugin_name = str(required_plugin).strip()
+                    if not required_plugin_name:
+                        continue
+                    required_config = get_plugin_special_config(required_plugin_name, config, _visited=next_visited)
+                    if not required_config["enabled"]:
+                        enabled = False
+                        break
+
+    timeout_key = f"{plugin_upper}_TIMEOUT"
+    timeout = config.get(timeout_key) or config.get("TIMEOUT", 300)
+
+    binary_key = f"{plugin_upper}_BINARY"
+    binary = config.get(binary_key, plugin_name)
+
+    return {
+        "enabled": bool(enabled),
+        "timeout": int(timeout),
+        "binary": str(binary),
+    }
+
+
+DEFAULT_TEMPLATES = {
+    "icon": """
+        <span title="{{ plugin }}" style="display:inline-flex; width:20px; height:20px; align-items:center; justify-content:center;">
+            {{ icon }}
+        </span>
+    """,
+    "card": """
+        <iframe src="{{ output_path }}"
+                class="card-img-top"
+                style="width: 100%; height: 100%; border: none;"
+                sandbox="allow-same-origin allow-scripts allow-forms"
+                loading="lazy"
+                fetchpriority="low">
+        </iframe>
+    """,
+    "full": """
+        <iframe src="{{ output_path }}"
+                class="full-page-iframe"
+                style="width: 100%; height: 100vh; border: none;"
+                sandbox="allow-same-origin allow-scripts allow-forms">
+        </iframe>
+    """,
+}
+
+
+@lru_cache(maxsize=None)
+def get_plugin_template(plugin: str, template_name: str, fallback: bool = True) -> str | None:
+    """
+    Get a plugin template by plugin name and template type.
+
+    Args:
+        plugin: Plugin name (e.g., 'screenshot', '15_singlefile')
+        template_name: One of 'icon', 'card', 'full'
+        fallback: If True, return default template if plugin template not found
+    """
+    base_name = get_plugin_name(plugin)
+    if base_name in ("yt-dlp", "youtube-dl"):
+        base_name = "ytdlp"
+
+    for plugin_dir in iter_plugin_dirs():
+        if plugin_dir.name == base_name or plugin_dir.name.endswith(f"_{base_name}"):
+            template_path = plugin_dir / "templates" / f"{template_name}.html"
+            if template_path.exists():
+                return template_path.read_text()
+
+    if fallback:
+        return DEFAULT_TEMPLATES.get(template_name, "")
+
+    return None
+
+
+@lru_cache(maxsize=None)
+def get_plugin_icon(plugin: str) -> str:
+    """
+    Get the icon for a plugin from its icon.html template.
+    """
+    icon_template = get_plugin_template(plugin, "icon", fallback=False)
+    if icon_template:
+        return mark_safe(icon_template.strip())
+
+    return mark_safe("📁")

archivebox/plugins/forms.py

@@ -0,0 +1,599 @@
+__package__ = "archivebox.plugins"
+
+import json
+import re
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+from typing import Any
+
+from django import forms
+from django.utils.html import format_html
+
+from archivebox.config.common import get_config
+from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_icon, get_plugins
+
+
+PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__"
+PLUGIN_GROUP_DEFINITIONS = (
+    (
+        "main_plugins",
+        "Main",
+        "",
+        "",
+        "",
+        (
+            "dom",
+            "screenshot",
+            "pdf",
+            "singlefile",
+            "wget",
+            "archivedotorg",
+            "chrome_mhtml",
+            "archivewebpage",
+        ),
+    ),
+    (
+        "page_setup_plugins",
+        "Page Setup",
+        "",
+        "",
+        "",
+        (
+            "chrome",
+            "infiniscroll",
+            "modalcloser",
+            "ublock",
+            "istilldontcareaboutcookies",
+            "twocaptcha",
+            "claudechrome",
+        ),
+    ),
+    (
+        "media_plugins",
+        "Media",
+        "",
+        "",
+        "",
+        (
+            "staticfile",
+            "responses",
+            "chrome_screencast",
+            "ytdlp",
+            "gallerydl",
+            "git",
+        ),
+    ),
+    (
+        "text_plugins",
+        "Text",
+        "",
+        "",
+        "",
+        (
+            "readability",
+            "htmltotext",
+            "defuddle",
+            "forumdl",
+            "mercury",
+            "trafilatura",
+            "liteparse",
+            "opendataloader",
+            "papersdl",
+        ),
+    ),
+    (
+        "metadata_plugins",
+        "Metadata",
+        "",
+        "",
+        "",
+        (
+            "title",
+            "favicon",
+            "headers",
+            "redirects",
+            "accessibility",
+            "consolelog",
+            "sslcerts",
+            "dns",
+            "seo",
+            "hashes",
+        ),
+    ),
+    (
+        "postprocessing_plugins",
+        "Postprocessing",
+        "",
+        "",
+        "",
+        (
+            "parse_dom_outlinks",
+            "parse_html_urls",
+            "parse_jsonl_urls",
+            "parse_netscape_urls",
+            "parse_rss_urls",
+            "parse_txt_urls",
+            "claudecode",
+            "claudecodecleanup",
+            "claudecodeextract",
+        ),
+    ),
+)
+HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
+    "apt",
+    "base",
+    "bash",
+    "brew",
+    "cargo",
+    "chromewebstore",
+    "env",
+    "media",
+    "npm",
+    "pip",
+    "puppeteer",
+    "search_backend_ripgrep",
+    "search_backend_sonic",
+    "search_backend_sqlite",
+    "ssl",
+}
+TIMEOUT_INPUT_PATTERN = r"(0|[1-9][0-9]*|[0-9]+(?:\.[0-9]+)?\s*(?:s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours))"
+
+
+def get_plugin_choices():
+    """Get available extractor plugins from discovered hooks."""
+    return [(name, name) for name in get_plugins()]
+
+
+def get_plugin_choice_label(plugin_name: str, plugin_configs: dict[str, dict]) -> str:
+    schema = plugin_configs.get(plugin_name, {})
+    description = str(schema.get("description") or "").strip()
+    if not description:
+        return plugin_name
+    icon_html = get_plugin_icon(plugin_name)
+
+    return format_html(
+        '<span class="plugin-choice-icon">{}</span><span class="plugin-choice-name">{}</span>',
+        icon_html,
+        plugin_name,
+    )
+
+
+def get_choice_field(form: forms.Form, name: str) -> forms.ChoiceField:
+    field = form.fields[name]
+    if not isinstance(field, forms.ChoiceField):
+        raise TypeError(f"{name} must be a ChoiceField")
+    return field
+
+
+def _plugin_config_input_name(plugin_name: str, config_key: str) -> str:
+    return f"{PLUGIN_CONFIG_FIELD_PREFIX}{plugin_name}__{config_key}"
+
+
+def _schema_types(schema: Mapping[str, Any]) -> list[str]:
+    raw_type = schema.get("type") or "string"
+    if isinstance(raw_type, list):
+        return [str(item) for item in raw_type]
+    return [str(raw_type)]
+
+
+def _jsonish(value: Any) -> str:
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, sort_keys=True, default=str)
+
+
+def _same_config_value(left: Any, right: Any) -> bool:
+    return json.dumps(left, sort_keys=True, default=str) == json.dumps(right, sort_keys=True, default=str)
+
+
+def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> Any:
+    schema_types = _schema_types(schema)
+
+    if "boolean" in schema_types:
+        if isinstance(raw_value, bool):
+            return raw_value
+        value = str(raw_value).strip().lower()
+        if value in {"true", "1", "yes", "on"}:
+            return True
+        if value in {"false", "0", "no", "off", ""}:
+            return False
+        raise forms.ValidationError("Must be true or false.")
+
+    if "integer" in schema_types:
+        value = int(str(raw_value).strip())
+        minimum = schema.get("minimum")
+        maximum = schema.get("maximum")
+        if minimum is not None and value < int(minimum):
+            raise forms.ValidationError(f"Must be at least {minimum}.")
+        if maximum is not None and value > int(maximum):
+            raise forms.ValidationError(f"Must be at most {maximum}.")
+        return value
+
+    if "number" in schema_types:
+        value = float(str(raw_value).strip())
+        minimum = schema.get("minimum")
+        maximum = schema.get("maximum")
+        if minimum is not None and value < float(minimum):
+            raise forms.ValidationError(f"Must be at least {minimum}.")
+        if maximum is not None and value > float(maximum):
+            raise forms.ValidationError(f"Must be at most {maximum}.")
+        return value
+
+    if "array" in schema_types:
+        if isinstance(raw_value, list):
+            return raw_value
+        value = str(raw_value).strip()
+        if not value:
+            return []
+        if value.startswith("["):
+            parsed = json.loads(value)
+            if not isinstance(parsed, list):
+                raise forms.ValidationError("Must be a JSON array.")
+            return parsed
+        return [item.strip() for item in value.replace(",", "\n").splitlines() if item.strip()]
+
+    if "object" in schema_types:
+        value = str(raw_value).strip()
+        if not value:
+            return {}
+        parsed = json.loads(value)
+        if not isinstance(parsed, dict):
+            raise forms.ValidationError("Must be a JSON object.")
+        return parsed
+
+    value = str(raw_value)
+    enum = schema.get("enum")
+    if isinstance(enum, list) and enum and value not in {str(item) for item in enum}:
+        raise forms.ValidationError(f"Must be one of: {', '.join(str(item) for item in enum)}.")
+    return value
+
+
+class PluginConfigFormMixin:
+    plugin_groups: list[dict[str, Any]]
+
+    def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
+        all_plugins = get_plugins()
+        plugin_configs = discover_plugin_configs()
+        runtime_config = runtime_config or get_config()
+        self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
+        grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS))
+        other_plugins = tuple(sorted(set(all_plugins) - grouped_plugins - HIDDEN_PLUGIN_CONFIG_UI_PLUGINS))
+
+        for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
+            if field_name in self.fields:
+                get_choice_field(self, field_name).choices = [
+                    (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
+                ]
+
+        if "other_plugins" in self.fields:
+            get_choice_field(self, "other_plugins").choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in other_plugins]
+
+        group_specs = (
+            *PLUGIN_GROUP_DEFINITIONS,
+            ("other_plugins", "Other", "", "", "", other_plugins),
+        )
+        binary_url_lookup = _build_required_binary_url_lookup(plugin_configs, runtime_config)
+        self.plugin_groups = [
+            {
+                "field_name": field_name,
+                "title": title,
+                "note": note,
+                "dom_id": dom_id,
+                "select_all_group": select_all_group,
+                "show_selectors": field_name in self.fields,
+                "plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config, binary_url_lookup),
+            }
+            for field_name, title, note, dom_id, select_all_group, plugin_names in group_specs
+            if any(plugin in all_plugins for plugin in plugin_names)
+        ]
+
+    def _build_plugin_cards(
+        self,
+        field_name: str,
+        plugin_names: Iterable[str],
+        plugin_configs: dict[str, dict[str, Any]],
+        runtime_config: Mapping[str, Any],
+        binary_url_lookup: Mapping[str, str] | None = None,
+    ) -> list[dict[str, Any]]:
+        if field_name in self.fields:
+            choices = list(get_choice_field(self, field_name).choices)
+            selected_values = set(self.data.getlist(field_name)) if self.is_bound else set(get_choice_field(self, field_name).initial or [])
+        else:
+            all_plugins = get_plugins()
+            choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins]
+            selected_values = set()
+
+        cards = []
+        for index, (plugin_name, label) in enumerate(choices):
+            schema = plugin_configs.get(str(plugin_name), {})
+            properties = schema.get("properties") or {}
+            enabled_config_key = f"{str(plugin_name).upper()}_ENABLED"
+            enabled_prop_schema = properties.get(enabled_config_key)
+            if not isinstance(enabled_prop_schema, dict) or "boolean" not in _schema_types(enabled_prop_schema):
+                enabled_config_key = ""
+            config_fields = [
+                self._build_plugin_config_field(str(plugin_name), str(config_key), prop_schema, runtime_config)
+                for config_key, prop_schema in properties.items()
+                if isinstance(prop_schema, dict)
+            ]
+            cards.append(
+                {
+                    "name": str(plugin_name),
+                    "label": label,
+                    "checked": str(plugin_name) in selected_values,
+                    "checkbox_id": f"id_{field_name}_{index}",
+                    "enabled_config_key": enabled_config_key,
+                    "description": str(schema.get("description") or "").strip(),
+                    "source_url": f"https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/{plugin_name}",
+                    "docs_url": f"https://archivebox.github.io/abx-plugins/#{plugin_name}",
+                    "required_plugins": [str(item) for item in schema.get("required_plugins") or []],
+                    "required_binary_links": _build_required_binary_links(
+                        schema.get("required_binaries") or [],
+                        runtime_config,
+                        binary_url_lookup,
+                    ),
+                    "config_fields": config_fields,
+                    "config_count": len(config_fields),
+                },
+            )
+        return cards
+
+    def _build_plugin_config_field(
+        self,
+        plugin_name: str,
+        config_key: str,
+        prop_schema: Mapping[str, Any],
+        runtime_config: Mapping[str, Any],
+    ) -> dict[str, Any]:
+        schema_types = _schema_types(prop_schema)
+        enum = prop_schema.get("enum")
+        input_name = _plugin_config_input_name(plugin_name, config_key)
+        current_value = runtime_config.get(config_key, prop_schema.get("default", ""))
+        if self.is_bound and input_name in self.data:
+            try:
+                current_value = _coerce_plugin_config_value(self.data.get(input_name), prop_schema)
+            except (TypeError, ValueError, json.JSONDecodeError, forms.ValidationError):
+                current_value = self.data.get(input_name)
+
+        default_value = prop_schema.get("default", "")
+        fallback_key = prop_schema.get("x-fallback")
+        default_display = f"{{{fallback_key}}}" if fallback_key else default_value
+        from archivebox.config.common import is_sensitive_config_key
+
+        is_sensitive = bool(prop_schema.get("x-sensitive")) or is_sensitive_config_key(config_key)
+        input_value = "" if is_sensitive else _jsonish(current_value)
+        field_kind = "text"
+        input_type = "text"
+        options = []
+
+        if "boolean" in schema_types:
+            field_kind = "boolean"
+            input_value = "true" if bool(current_value) else "false"
+        elif isinstance(enum, list) and enum:
+            field_kind = "select"
+            options = [
+                {
+                    "value": str(option),
+                    "label": str(option),
+                    "selected": str(option) == str(current_value),
+                }
+                for option in enum
+            ]
+        elif "integer" in schema_types or "number" in schema_types:
+            field_kind = "number"
+            input_type = "number"
+        elif "array" in schema_types or "object" in schema_types:
+            field_kind = "json"
+            input_value = "" if is_sensitive else json.dumps(current_value, indent=2, sort_keys=True, default=str)
+        elif is_sensitive:
+            input_type = "password"
+        else:
+            input_value = "" if is_sensitive else str(current_value)
+
+        return {
+            "key": config_key,
+            "input_name": input_name,
+            "kind": field_kind,
+            "input_type": input_type,
+            "value": input_value,
+            "checked": bool(current_value),
+            "options": options,
+            "description": str(prop_schema.get("description") or "").strip(),
+            "default": _jsonish(default_display),
+            "current": "configured"
+            if is_sensitive and current_value
+            else (str(current_value) if "string" in schema_types else _jsonish(current_value)),
+            "current_url": self.plugin_config_binary_urls.get(config_key, "") if str(config_key).endswith("_BINARY") else "",
+            "is_sensitive": is_sensitive,
+            "minimum": prop_schema.get("minimum"),
+            "maximum": prop_schema.get("maximum"),
+            "pattern": prop_schema.get("pattern"),
+            "type_label": " / ".join(schema_types),
+        }
+
+    def clean_plugin_config_overrides(self, effective_config: Mapping[str, Any] | None = None) -> dict[str, Any]:
+        if not self.is_bound:
+            return {}
+
+        effective_config = effective_config or get_config()
+        overrides: dict[str, Any] = {}
+        sources: dict[str, str] = {}
+
+        for plugin_name, schema in discover_plugin_configs().items():
+            for config_key, prop_schema in (schema.get("properties") or {}).items():
+                if not isinstance(prop_schema, dict):
+                    continue
+
+                input_name = _plugin_config_input_name(plugin_name, config_key)
+                if input_name not in self.data:
+                    continue
+
+                raw_value: Any = self.data.get(input_name)
+                if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list):
+                    raw_value = self.data.getlist(input_name)
+
+                from archivebox.config.common import SENSITIVE_CONFIG_VALUE_REDACTED, is_sensitive_config_key
+
+                if (prop_schema.get("x-sensitive") or is_sensitive_config_key(config_key)) and raw_value in (
+                    "",
+                    SENSITIVE_CONFIG_VALUE_REDACTED,
+                ):
+                    continue
+
+                try:
+                    coerced_value = _coerce_plugin_config_value(raw_value, prop_schema)
+                except (TypeError, ValueError, json.JSONDecodeError) as err:
+                    self.add_error("config", forms.ValidationError(f"{config_key}: {err}"))
+                    continue
+                except forms.ValidationError as err:
+                    self.add_error("config", forms.ValidationError(f"{config_key}: {err.messages[0]}"))
+                    continue
+
+                base_value = effective_config.get(config_key, prop_schema.get("default", ""))
+                if _same_config_value(coerced_value, base_value):
+                    continue
+
+                existing_value = overrides.get(config_key)
+                if config_key in overrides and not _same_config_value(existing_value, coerced_value):
+                    self.add_error(
+                        "config",
+                        forms.ValidationError(
+                            f"{config_key} was set differently under {sources[config_key]} and {plugin_name}. Set it once in Custom config overrides.",
+                        ),
+                    )
+                    continue
+
+                overrides[config_key] = coerced_value
+                sources[config_key] = plugin_name
+
+        return overrides
+
+    def plugin_config_keys(self) -> set[str]:
+        return {
+            str(config_key)
+            for schema in discover_plugin_configs().values()
+            for config_key, prop_schema in (schema.get("properties") or {}).items()
+            if isinstance(prop_schema, dict)
+        }
+
+
+_BINARY_TEMPLATE_PATTERN = re.compile(r"\{([A-Z_][A-Z0-9_]*)\}")
+
+
+def _resolve_required_binary_name(template_name: str, runtime_config: Mapping[str, Any]) -> str:
+    if "{" not in template_name:
+        return template_name
+
+    def _replace(match: re.Match[str]) -> str:
+        key = match.group(1)
+        try:
+            value = runtime_config.get(key)
+        except Exception:
+            value = None
+        if value is None or value == "":
+            return match.group(0)
+        return str(value)
+
+    resolved = _BINARY_TEMPLATE_PATTERN.sub(_replace, template_name).strip()
+    if not resolved:
+        return template_name
+    return Path(resolved).name if "/" in resolved else resolved
+
+
+def _iter_required_binary_names(
+    required_binaries: Iterable[Any],
+    runtime_config: Mapping[str, Any],
+) -> Iterable[str]:
+    for item in required_binaries or []:
+        if not isinstance(item, dict):
+            continue
+        raw_name = str(item.get("name") or "").strip()
+        if not raw_name:
+            continue
+        resolved = _resolve_required_binary_name(raw_name, runtime_config)
+        if resolved:
+            yield resolved
+
+
+def _build_required_binary_url_lookup(
+    plugin_configs: Mapping[str, dict[str, Any]],
+    runtime_config: Mapping[str, Any],
+) -> dict[str, str]:
+    """Resolve admin URLs for every required binary across all plugin schemas in a single DB query."""
+    from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url
+    from archivebox.machine.models import Binary, Machine
+
+    resolved_names: set[str] = set()
+    for schema in plugin_configs.values():
+        for name in _iter_required_binary_names(schema.get("required_binaries") or [], runtime_config):
+            resolved_names.add(name)
+
+    if not resolved_names:
+        return {}
+
+    machine = Machine.current()
+    name_to_binary: dict[str, Binary] = {}
+    for binary in (
+        Binary.objects.filter(machine=machine, name__in=resolved_names)
+        .exclude(abspath="")
+        .exclude(abspath__isnull=True)
+        .order_by("-modified_at")
+    ):
+        key = binary.name.lower()
+        if key not in name_to_binary:
+            name_to_binary[key] = binary
+
+    return {
+        name: (get_installed_binary_change_url(name, name_to_binary.get(name.lower())) or get_environment_binary_url(name))
+        for name in resolved_names
+    }
+
+
+def _build_required_binary_links(
+    required_binaries: list[dict[str, Any]],
+    runtime_config: Mapping[str, Any],
+    binary_url_lookup: Mapping[str, str] | None = None,
+) -> list[dict[str, str]]:
+    from archivebox.config.views import get_environment_binary_url
+
+    links: list[dict[str, str]] = []
+    seen: set[str] = set()
+    for resolved in _iter_required_binary_names(required_binaries, runtime_config):
+        if resolved in seen:
+            continue
+        seen.add(resolved)
+        url = (binary_url_lookup or {}).get(resolved) or get_environment_binary_url(resolved)
+        links.append({"name": resolved, "url": url})
+    return links
+
+
+def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str, str]:
+    from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url
+    from archivebox.machine.models import Binary, Machine
+
+    binary_keys = {
+        str(config_key)
+        for schema in discover_plugin_configs().values()
+        for config_key, prop_schema in (schema.get("properties") or {}).items()
+        if isinstance(prop_schema, dict) and str(config_key).endswith("_BINARY")
+    }
+    urls: dict[str, str] = {}
+    machine = Machine.current()
+    for key in binary_keys:
+        value = str(runtime_config.get(key) or "").strip()
+        if not value:
+            continue
+        name = Path(value).name if "/" in value else value
+        binary = Binary.objects.get_valid_binary(value, machine=machine)
+        if binary is None and "/" in value:
+            binary = (
+                Binary.objects.exclude(abspath="")
+                .exclude(abspath__isnull=True)
+                .filter(machine=machine, abspath=value)
+                .order_by("-modified_at")
+                .first()
+            )
+        if binary is None and name != value:
+            binary = Binary.objects.get_valid_binary(name, machine=machine)
+        urls[key] = get_installed_binary_change_url(getattr(binary, "name", name), binary) or get_environment_binary_url(name)
+    return urls

archivebox/plugins/hooks.py

@@ -41,37 +41,28 @@ API:
     is_background_hook(name)  -> bool           Check if hook is background (.bg suffix)
 """
 
-__package__ = "archivebox"
+__package__ = "archivebox.plugins"
 
-import os
 import json
-from collections.abc import Iterable, Mapping
-from functools import lru_cache
+import os
+from collections.abc import Mapping
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, TypedDict
+from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard
 
-from abx_plugins import get_plugins_dir
-from django.utils.safestring import mark_safe
 from archivebox.config.constants import CONSTANTS
 from archivebox.config.version import VERSION
 from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
+from archivebox.plugins.discovery import (
+    BUILTIN_PLUGINS_DIR,
+    USER_PLUGINS_DIR,
+    ConfigLookup,
+    get_plugin_special_config,
+)
 
 if TYPE_CHECKING:
     from archivebox.machine.models import Process
 
 
-class ConfigLookup(Protocol):
-    def get(self, key: str, default: Any = None) -> Any: ...
-
-    def items(self) -> Iterable[tuple[str, Any]]: ...
-
-
-class PluginSpecialConfig(TypedDict):
-    enabled: bool
-    timeout: int
-    binary: str
-
-
 class ConfigDump(Protocol):
     def as_dict(self) -> dict[str, Any]: ...
 
@@ -88,13 +79,6 @@ def _config_to_overrides(config: ConfigLookup | Mapping[str, Any] | None) -> dic
     return dict(config.items())
 
 
-# Plugin directories
-BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
-USER_PLUGINS_DIR = Path(
-    os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or str(CONSTANTS.USER_PLUGINS_DIR),
-).expanduser()
-
-
 # =============================================================================
 # Hook Step Extraction
 # =============================================================================
@@ -125,21 +109,6 @@ def is_finite_background_hook(hook_name: str) -> bool:
     return ".finite.bg." in hook_name
 
 
-def iter_plugin_dirs() -> list[Path]:
-    """Iterate over all built-in and user plugin directories."""
-    plugin_dirs: list[Path] = []
-
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        if not base_dir.exists():
-            continue
-
-        for plugin_dir in base_dir.iterdir():
-            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
-                plugin_dirs.append(plugin_dir)
-
-    return plugin_dirs
-
-
 def normalize_hook_event_name(event_name: str) -> str | None:
     """
     Normalize a hook event family or event class name to its on_* prefix.
@@ -319,11 +288,11 @@ def run_hook(
     """
     from archivebox.machine.models import Process, Machine, NetworkInterface
     from archivebox.config.common import get_config
-    from archivebox.config.constants import CONSTANTS
     import sys
 
     config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
     resolved_config = get_config(overrides=_config_to_overrides(config), **config_scope)
+    hook_config = resolved_config.for_crawl_execution()
 
     # Auto-detect timeout from plugin config if not explicitly provided
     if timeout is None:
@@ -463,7 +432,7 @@ def run_hook(
         "SNAP_DIR",
         "CRAWL_DIR",
     }
-    for key, value in resolved_config.items():
+    for key, value in hook_config.items():
         if key in SKIP_KEYS:
             continue  # Already handled specially above, don't overwrite
         if value is None:
@@ -598,474 +567,6 @@ def collect_urls_from_plugins(snapshot_dir: Path) -> list[dict[str, Any]]:
     return urls
 
 
-@lru_cache(maxsize=1)
-def get_plugins() -> list[str]:
-    """
-    Get list of available plugins by discovering plugin directories.
-
-    Returns plugin directory names for any plugin that exposes hooks, config.json,
-    or a standardized templates/icon.html asset. This includes non-extractor
-    plugins such as binary providers and shared base plugins.
-    """
-    plugins = []
-
-    for plugin_dir in iter_plugin_dirs():
-        has_hooks = any(plugin_dir.glob("on_*__*.*"))
-        has_config = (plugin_dir / "config.json").exists()
-        has_icon = (plugin_dir / "templates" / "icon.html").exists()
-        if has_hooks or has_config or has_icon:
-            plugins.append(plugin_dir.name)
-
-    return sorted(set(plugins))
-
-
-def get_plugin_name(plugin: str) -> str:
-    """
-    Get the base plugin name without numeric prefix.
-
-    Examples:
-        '10_title' -> 'title'
-        '26_readability' -> 'readability'
-        '50_parse_html_urls' -> 'parse_html_urls'
-    """
-    # Split on first underscore after any leading digits
-    parts = plugin.split("_", 1)
-    if len(parts) == 2 and parts[0].isdigit():
-        return parts[1]
-    return plugin
-
-
-def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any) -> list[str]:
-    """
-    Get the list of enabled plugins based on config and available hooks.
-
-    Filters plugins by USE_/SAVE_ flags. Only returns plugins that are enabled.
-
-    Args:
-        config: Optional pre-merged config dict from get_config().
-        **config_kwargs: Scope/override args forwarded to get_config() when config is not supplied.
-
-    Returns:
-        Plugin names sorted alphabetically (numeric prefix controls order).
-
-    Example:
-        from archivebox.config.common import get_config
-        config = get_config(crawl=my_crawl, snapshot=my_snapshot)
-        enabled = get_enabled_plugins(config)  # ['wget', 'media', 'chrome', ...]
-    """
-    # Get merged config if not provided
-    if config is None:
-        from archivebox.config.common import get_config
-
-        config = get_config(**config_kwargs)
-
-    def normalize_enabled_plugins(value: Any) -> list[str]:
-        if value is None:
-            return []
-        if isinstance(value, str):
-            raw = value.strip()
-            if not raw:
-                return []
-            if raw.startswith("["):
-                try:
-                    parsed = json.loads(raw)
-                except json.JSONDecodeError:
-                    parsed = None
-                if isinstance(parsed, list):
-                    return [str(plugin).strip() for plugin in parsed if str(plugin).strip()]
-            return [plugin.strip() for plugin in raw.split(",") if plugin.strip()]
-        if isinstance(value, (list, tuple, set)):
-            return [str(plugin).strip() for plugin in value if str(plugin).strip()]
-        return [str(value).strip()] if str(value).strip() else []
-
-    # Support explicit PLUGINS override
-    plugins_override = config.get("PLUGINS")
-    if plugins_override:
-        return normalize_enabled_plugins(plugins_override)
-
-    # Filter all plugins by enabled status
-    all_plugins = get_plugins()
-    enabled = []
-
-    for plugin in all_plugins:
-        plugin_config = get_plugin_special_config(plugin, config)
-        if plugin_config["enabled"]:
-            enabled.append(plugin)
-
-    return enabled
-
-
-def discover_plugins_that_provide_interface(
-    module_name: str,
-    required_attrs: list[str],
-    plugin_prefix: str | None = None,
-) -> dict[str, Any]:
-    """
-    Discover plugins that provide a specific Python module with required interface.
-
-    This enables dynamic plugin discovery for features like search backends,
-    storage backends, etc. without hardcoding imports.
-
-    Args:
-        module_name: Name of the module to look for (e.g., 'search')
-        required_attrs: List of attributes the module must have (e.g., ['search', 'flush'])
-        plugin_prefix: Optional prefix to filter plugins (e.g., 'search_backend_')
-
-    Returns:
-        Dict mapping backend names to imported modules.
-        Backend name is derived from plugin directory name minus the prefix.
-        e.g., search_backend_sqlite -> 'sqlite'
-
-    Example:
-        backends = discover_plugins_that_provide_interface(
-            module_name='search',
-            required_attrs=['search', 'flush'],
-            plugin_prefix='search_backend_',
-        )
-        # Returns: {'sqlite': <module>, 'sonic': <module>, 'ripgrep': <module>}
-    """
-    import importlib.util
-
-    backends = {}
-
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        if not base_dir.exists():
-            continue
-
-        for plugin_dir in base_dir.iterdir():
-            if not plugin_dir.is_dir():
-                continue
-
-            plugin_name = plugin_dir.name
-
-            # Filter by prefix if specified
-            if plugin_prefix and not plugin_name.startswith(plugin_prefix):
-                continue
-
-            # Look for the module file
-            module_path = plugin_dir / f"{module_name}.py"
-            if not module_path.exists():
-                continue
-
-            try:
-                # Import the module dynamically
-                spec = importlib.util.spec_from_file_location(
-                    f"archivebox.dynamic_plugins.{plugin_name}.{module_name}",
-                    module_path,
-                )
-                if spec is None or spec.loader is None:
-                    continue
-
-                module = importlib.util.module_from_spec(spec)
-                spec.loader.exec_module(module)
-
-                # Check for required attributes
-                if not all(hasattr(module, attr) for attr in required_attrs):
-                    continue
-
-                # Derive backend name from plugin directory name
-                if plugin_prefix:
-                    backend_name = plugin_name[len(plugin_prefix) :]
-                else:
-                    backend_name = plugin_name
-
-                backends[backend_name] = module
-
-            except Exception:
-                # Skip plugins that fail to import
-                continue
-
-    return backends
-
-
-def get_search_backends() -> dict[str, Any]:
-    """
-    Discover all available search backend plugins.
-
-    Search backends must provide a search.py module with:
-        - search(query: str) -> List[str]  (returns snapshot IDs)
-        - flush(snapshot_ids: Iterable[str]) -> None
-
-    Returns:
-        Dict mapping backend names to their modules.
-        e.g., {'sqlite': <module>, 'sonic': <module>, 'ripgrep': <module>}
-    """
-    return discover_plugins_that_provide_interface(
-        module_name="search",
-        required_attrs=["search", "flush"],
-        plugin_prefix="search_backend_",
-    )
-
-
-def discover_plugin_configs() -> dict[str, dict[str, Any]]:
-    """
-    Discover all plugin config.json schemas.
-
-    Each plugin can define a config.json file with JSONSchema defining
-    its configuration options. This function discovers and loads all such schemas.
-
-    The config.json files use JSONSchema draft-07 with custom extensions:
-        - x-fallback: Global config key to use as fallback
-        - x-aliases: List of old/alternative config key names
-
-    Returns:
-        Dict mapping plugin names to their parsed JSONSchema configs.
-        e.g., {'wget': {...schema...}, 'chrome': {...schema...}}
-
-    Example config.json:
-        {
-            "$schema": "http://json-schema.org/draft-07/schema#",
-            "type": "object",
-            "properties": {
-                "SAVE_WGET": {"type": "boolean", "default": true},
-                "WGET_TIMEOUT": {"type": "integer", "default": 60, "x-fallback": "TIMEOUT"}
-            }
-        }
-    """
-    configs = {}
-
-    for plugin_dir in iter_plugin_dirs():
-        config_path = plugin_dir / "config.json"
-        if not config_path.exists():
-            continue
-
-        try:
-            with open(config_path) as f:
-                schema = json.load(f)
-
-            # Basic validation: must be an object with properties
-            if not isinstance(schema, dict):
-                continue
-            if schema.get("type") != "object":
-                continue
-            if "properties" not in schema:
-                continue
-
-            configs[plugin_dir.name] = schema
-
-        except (json.JSONDecodeError, OSError) as e:
-            # Log warning but continue - malformed config shouldn't break discovery
-            import sys
-
-            print(f"Warning: Failed to load config.json from {plugin_dir.name}: {e}", file=sys.stderr)
-            continue
-
-    return configs
-
-
-def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited: set[str] | None = None) -> PluginSpecialConfig:
-    """
-    Extract special config keys for a plugin following naming conventions.
-
-    ArchiveBox recognizes 3 special config key patterns per plugin:
-        - {PLUGIN}_ENABLED: Enable/disable toggle (default True)
-        - {PLUGIN}_TIMEOUT: Plugin-specific timeout (fallback to TIMEOUT, default 300)
-        - {PLUGIN}_BINARY: Primary binary path (default to plugin_name)
-
-    These allow ArchiveBox to:
-        - Skip disabled plugins (optimization)
-        - Enforce plugin-specific timeouts automatically
-        - Discover plugin binaries for validation
-
-    Args:
-        plugin_name: Plugin name (e.g., 'wget', 'media', 'chrome')
-        config: Merged config dict from get_config() (properly merges file, env, machine, crawl, snapshot)
-
-    Returns:
-        Dict with standardized keys:
-            {
-                'enabled': True,         # bool
-                'timeout': 60,           # int, seconds
-                'binary': 'wget',        # str, path or name
-            }
-
-    Examples:
-        >>> from archivebox.config.common import get_config
-        >>> config = get_config(crawl=my_crawl, snapshot=my_snapshot)
-        >>> get_plugin_special_config('wget', config)
-        {'enabled': True, 'timeout': 120, 'binary': '/usr/bin/wget'}
-    """
-    plugin_upper = plugin_name.upper()
-
-    # 1. Enabled: Check PLUGINS whitelist first, then PLUGINNAME_ENABLED (default True)
-    # Old names (USE_*, SAVE_*) are aliased in config.json via x-aliases
-
-    # Check if PLUGINS whitelist is specified (e.g., --plugins=wget,favicon)
-    plugins_whitelist = config.get("PLUGINS", "")
-    if plugins_whitelist:
-        # PLUGINS whitelist is specified - include transitive required_plugins from
-        # config.json so selecting a plugin also enables its declared plugin-level
-        # dependencies (e.g. singlefile -> chrome).
-        plugin_configs = discover_plugin_configs()
-        plugin_names = {p.strip().lower() for p in plugins_whitelist.split(",") if p.strip()}
-        pending = list(plugin_names)
-
-        while pending:
-            current = pending.pop()
-            schema = plugin_configs.get(current, {})
-            required_plugins = schema.get("required_plugins", [])
-            if not isinstance(required_plugins, list):
-                continue
-
-            for required_plugin in required_plugins:
-                required_plugin_name = str(required_plugin).strip().lower()
-                if not required_plugin_name or required_plugin_name in plugin_names:
-                    continue
-                plugin_names.add(required_plugin_name)
-                pending.append(required_plugin_name)
-
-        if plugin_name.lower() not in plugin_names:
-            # Plugin not in whitelist - explicitly disabled
-            enabled = False
-        else:
-            # Plugin is in whitelist - check if explicitly disabled by PLUGINNAME_ENABLED
-            enabled_key = f"{plugin_upper}_ENABLED"
-            enabled = config.get(enabled_key)
-            if enabled is None:
-                enabled = True  # Default to enabled if in whitelist
-            elif isinstance(enabled, str):
-                enabled = enabled.lower() not in ("false", "0", "no", "")
-    else:
-        # No PLUGINS whitelist - use PLUGINNAME_ENABLED (default True)
-        enabled_key = f"{plugin_upper}_ENABLED"
-        enabled = config.get(enabled_key)
-        if enabled is None:
-            enabled = True
-        elif isinstance(enabled, str):
-            # Handle string values from config file ("true"/"false")
-            enabled = enabled.lower() not in ("false", "0", "no", "")
-
-    plugin_configs = discover_plugin_configs()
-    plugin_name_lower = plugin_name.lower()
-
-    if enabled:
-        visited = _visited or set()
-        if plugin_name_lower not in visited:
-            next_visited = visited | {plugin_name_lower}
-            schema = plugin_configs.get(plugin_name_lower, {})
-            required_plugins = schema.get("required_plugins", [])
-            if isinstance(required_plugins, list):
-                for required_plugin in required_plugins:
-                    required_plugin_name = str(required_plugin).strip()
-                    if not required_plugin_name:
-                        continue
-                    required_config = get_plugin_special_config(required_plugin_name, config, _visited=next_visited)
-                    if not required_config["enabled"]:
-                        enabled = False
-                        break
-
-    # 2. Timeout: PLUGINNAME_TIMEOUT (fallback to TIMEOUT, default 300)
-    timeout_key = f"{plugin_upper}_TIMEOUT"
-    timeout = config.get(timeout_key) or config.get("TIMEOUT", 300)
-
-    # 3. Binary: PLUGINNAME_BINARY (default to plugin_name)
-    binary_key = f"{plugin_upper}_BINARY"
-    binary = config.get(binary_key, plugin_name)
-
-    return {
-        "enabled": bool(enabled),
-        "timeout": int(timeout),
-        "binary": str(binary),
-    }
-
-
-# =============================================================================
-# Plugin Template Discovery
-# =============================================================================
-#
-# Plugins can provide custom templates for rendering their output in the UI.
-# Templates are discovered by filename convention inside each plugin's templates/ dir:
-#
-#     abx_plugins/plugins/<plugin_name>/
-#         templates/
-#             icon.html          # Icon for admin table view (small inline HTML)
-#             card.html          # Preview card for snapshot header
-#             full.html          # Fullscreen view template
-#
-# Template context variables available:
-#     {{ result }}         - ArchiveResult object
-#     {{ snapshot }}       - Parent Snapshot object
-#     {{ output_path }}    - Path to output file/dir relative to snapshot dir
-#     {{ plugin }}         - Plugin name (e.g., 'screenshot', 'singlefile')
-#
-
-# Default templates used when plugin doesn't provide one
-DEFAULT_TEMPLATES = {
-    "icon": """
-        <span title="{{ plugin }}" style="display:inline-flex; width:20px; height:20px; align-items:center; justify-content:center;">
-            {{ icon }}
-        </span>
-    """,
-    "card": """
-        <iframe src="{{ output_path }}"
-                class="card-img-top"
-                style="width: 100%; height: 100%; border: none;"
-                sandbox="allow-same-origin allow-scripts allow-forms"
-                loading="lazy"
-                fetchpriority="low">
-        </iframe>
-    """,
-    "full": """
-        <iframe src="{{ output_path }}"
-                class="full-page-iframe"
-                style="width: 100%; height: 100vh; border: none;"
-                sandbox="allow-same-origin allow-scripts allow-forms">
-        </iframe>
-    """,
-}
-
-
-@lru_cache(maxsize=None)
-def get_plugin_template(plugin: str, template_name: str, fallback: bool = True) -> str | None:
-    """
-    Get a plugin template by plugin name and template type.
-
-    Args:
-        plugin: Plugin name (e.g., 'screenshot', '15_singlefile')
-        template_name: One of 'icon', 'card', 'full'
-        fallback: If True, return default template if plugin template not found
-
-    Returns:
-        Template content as string, or None if not found and fallback=False.
-    """
-    base_name = get_plugin_name(plugin)
-    if base_name in ("yt-dlp", "youtube-dl"):
-        base_name = "ytdlp"
-
-    for plugin_dir in iter_plugin_dirs():
-        # Match by directory name (exact or partial)
-        if plugin_dir.name == base_name or plugin_dir.name.endswith(f"_{base_name}"):
-            template_path = plugin_dir / "templates" / f"{template_name}.html"
-            if template_path.exists():
-                return template_path.read_text()
-
-    # Fall back to default template if requested
-    if fallback:
-        return DEFAULT_TEMPLATES.get(template_name, "")
-
-    return None
-
-
-@lru_cache(maxsize=None)
-def get_plugin_icon(plugin: str) -> str:
-    """
-    Get the icon for a plugin from its icon.html template.
-
-    Args:
-        plugin: Plugin name (e.g., 'screenshot', '15_singlefile')
-
-    Returns:
-        Icon HTML/emoji string.
-    """
-    # Try plugin-provided icon template
-    icon_template = get_plugin_template(plugin, "icon", fallback=False)
-    if icon_template:
-        return mark_safe(icon_template.strip())
-
-    # Fall back to generic folder icon
-    return mark_safe("📁")
-
-
 # =============================================================================
 # Hook Result Processing Helpers
 # =============================================================================

archivebox/plugins/views.py

@@ -0,0 +1,462 @@
+__package__ = "archivebox.plugins"
+
+import html
+import json
+import re
+from typing import Any
+from collections.abc import Callable
+from urllib.parse import quote
+
+from django.http import HttpRequest
+from django.utils.html import format_html
+from django.utils.safestring import mark_safe
+
+from admin_data_views.typing import ItemContext, SectionData, TableContext
+from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
+
+from archivebox.config.views import get_environment_binary_url, is_superuser
+from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, discover_plugin_configs, iter_plugin_dirs
+
+
+ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
+ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
+LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
+LIVE_PLUGIN_BASE_URL = "/admin/environment/plugins/"
+
+
+JSON_TOKEN_RE = re.compile(
+    r'(?P<key>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")(?=\s*:)'
+    r'|(?P<string>"(?:\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*")'
+    r"|(?P<boolean>\btrue\b|\bfalse\b)"
+    r"|(?P<null>\bnull\b)"
+    r"|(?P<number>-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)",
+)
+
+
+def render_code_block(text: str, *, highlighted: bool = False) -> str:
+    code = html.escape(text, quote=False)
+
+    if highlighted:
+
+        def _wrap_token(match: re.Match[str]) -> str:
+            styles = {
+                "key": "color: #0550ae;",
+                "string": "color: #0a7f45;",
+                "boolean": "color: #8250df; font-weight: 600;",
+                "null": "color: #6e7781; font-style: italic;",
+                "number": "color: #b35900;",
+            }
+            token_type = next(name for name, value in match.groupdict().items() if value is not None)
+            return f'<span style="{styles[token_type]}">{match.group(0)}</span>'
+
+        code = JSON_TOKEN_RE.sub(_wrap_token, code)
+
+    return (
+        '<pre style="max-height: 600px; overflow: auto; background: #f6f8fa; '
+        'border: 1px solid #d0d7de; border-radius: 6px; padding: 12px; margin: 0;">'
+        '<code style="font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, '
+        "'Liberation Mono', monospace; white-space: pre; line-height: 1.5;\">"
+        f"{code}"
+        "</code></pre>"
+    )
+
+
+def render_highlighted_json_block(value: Any) -> str:
+    return render_code_block(json.dumps(value, indent=2, ensure_ascii=False), highlighted=True)
+
+
+def get_plugin_docs_url(plugin_name: str) -> str:
+    return f"{ABX_PLUGINS_DOCS_BASE_URL}#{plugin_name}"
+
+
+def get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str:
+    return f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/{quote(hook_name)}"
+
+
+def get_live_config_url(key: str) -> str:
+    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
+
+
+def get_machine_admin_url() -> str | None:
+    try:
+        from archivebox.machine.models import Machine
+
+        machine = Machine.current()
+        return getattr(machine, "admin_change_url", None) or f"/admin/machine/machine/{machine.id}/change/"
+    except Exception:
+        return None
+
+
+def render_code_tag_list(values: list[str]) -> str:
+    if not values:
+        return '<span style="color: #6e7781;">(none)</span>'
+
+    tags = "".join(
+        str(
+            format_html(
+                '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
+                'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
+                value,
+            ),
+        )
+        for value in values
+    )
+    return f'<div style="display: flex; flex-wrap: wrap;">{tags}</div>'
+
+
+def render_link_tag_list(values: list[str], url_resolver: Callable[[str], str] | None = None) -> str:
+    if not values:
+        return '<span style="color: #6e7781;">(none)</span>'
+
+    tags = []
+    for value in values:
+        if url_resolver is None:
+            tags.append(
+                str(
+                    format_html(
+                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
+                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>',
+                        value,
+                    ),
+                ),
+            )
+        else:
+            tags.append(
+                str(
+                    format_html(
+                        '<a href="{}" style="text-decoration: none;">'
+                        '<code style="display: inline-block; margin: 0 6px 6px 0; padding: 2px 6px; '
+                        'background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 999px;">{}</code>'
+                        "</a>",
+                        url_resolver(value),
+                        value,
+                    ),
+                ),
+            )
+    return f'<div style="display: flex; flex-wrap: wrap;">{"".join(tags)}</div>'
+
+
+def render_plugin_metadata_html(config: dict[str, Any]) -> str:
+    required_binaries = [
+        str(item.get("name")) for item in (config.get("required_binaries") or []) if isinstance(item, dict) and item.get("name")
+    ]
+    rows = (
+        ("Title", config.get("title") or "(none)"),
+        ("Description", config.get("description") or "(none)"),
+        ("Required Plugins", mark_safe(render_link_tag_list(config.get("required_plugins") or [], get_plugin_docs_url))),
+        ("Required Binaries", mark_safe(render_link_tag_list(required_binaries, get_environment_binary_url))),
+        ("Output MIME Types", mark_safe(render_code_tag_list(config.get("output_mimetypes") or []))),
+    )
+
+    rendered_rows = "".join(
+        str(
+            format_html(
+                '<div style="margin: 0 0 14px 0;"><div style="font-weight: 600; margin-bottom: 4px;">{}</div><div>{}</div></div>',
+                label,
+                value,
+            ),
+        )
+        for label, value in rows
+    )
+    return f'<div style="margin: 4px 0 0 0;">{rendered_rows}</div>'
+
+
+def render_property_links(prop_name: str, prop_info: dict[str, Any], machine_admin_url: str | None) -> str:
+    links = [
+        str(format_html('<a href="{}">Computed value</a>', get_live_config_url(prop_name))),
+    ]
+    if machine_admin_url:
+        links.append(str(format_html('<a href="{}">Edit override</a>', machine_admin_url)))
+
+    fallback = prop_info.get("x-fallback")
+    if isinstance(fallback, str) and fallback:
+        links.append(str(format_html('<a href="{}">Fallback: <code>{}</code></a>', get_live_config_url(fallback), fallback)))
+
+    aliases = prop_info.get("x-aliases") or []
+    if isinstance(aliases, list):
+        for alias in aliases:
+            if isinstance(alias, str) and alias:
+                links.append(str(format_html('<a href="{}">Alias: <code>{}</code></a>', get_live_config_url(alias), alias)))
+
+    default = prop_info.get("default")
+    if prop_name.endswith("_BINARY") and isinstance(default, str) and default:
+        links.append(str(format_html('<a href="{}">Binary: <code>{}</code></a>', get_environment_binary_url(default), default)))
+
+    return " &nbsp; ".join(links)
+
+
+def render_config_properties_html(properties: dict[str, Any], machine_admin_url: str | None) -> str:
+    header_links = [
+        str(format_html('<a href="{}">Dependencies</a>', "/admin/environment/binaries/")),
+        str(format_html('<a href="{}">Installed Binaries</a>', "/admin/machine/binary/")),
+    ]
+    if machine_admin_url:
+        header_links.insert(0, str(format_html('<a href="{}">Machine Config Editor</a>', machine_admin_url)))
+
+    cards = [
+        f'<div style="margin: 0 0 16px 0;">{" &nbsp; | &nbsp; ".join(header_links)}</div>',
+    ]
+
+    for prop_name, prop_info in properties.items():
+        prop_type = prop_info.get("type", "unknown")
+        if isinstance(prop_type, list):
+            prop_type = " | ".join(str(type_name) for type_name in prop_type)
+        prop_desc = prop_info.get("description", "")
+
+        default_html = ""
+        if "default" in prop_info:
+            default_html = str(
+                format_html(
+                    '<div style="margin-top: 6px;"><b>Default:</b> <code>{}</code></div>',
+                    prop_info["default"],
+                ),
+            )
+
+        description_html = prop_desc or mark_safe('<span style="color: #6e7781;">(no description)</span>')
+        cards.append(
+            str(
+                format_html(
+                    '<div style="margin: 0 0 14px 0; padding: 12px; background: #f6f8fa; border: 1px solid #d0d7de; border-radius: 6px;">'
+                    '<div style="margin-bottom: 6px;">'
+                    '<a href="{}" style="font-weight: 600;"><code>{}</code></a>'
+                    ' <span style="color: #6e7781;">({})</span>'
+                    "</div>"
+                    '<div style="margin-bottom: 6px;">{}</div>'
+                    '<div style="font-size: 0.95em;">{}</div>'
+                    "{}"
+                    "</div>",
+                    get_live_config_url(prop_name),
+                    prop_name,
+                    prop_type,
+                    description_html,
+                    mark_safe(render_property_links(prop_name, prop_info, machine_admin_url)),
+                    mark_safe(default_html),
+                ),
+            ),
+        )
+
+    return "".join(cards)
+
+
+def render_hook_links_html(plugin_name: str, hooks: list[str], source: str) -> str:
+    if not hooks:
+        return '<span style="color: #6e7781;">(none)</span>'
+
+    items = []
+    for hook_name in hooks:
+        if source == "builtin":
+            items.append(
+                str(
+                    format_html(
+                        '<div style="margin: 0 0 8px 0;"><a href="{}" target="_blank" rel="noopener noreferrer"><code>{}</code></a></div>',
+                        get_plugin_hook_source_url(plugin_name, hook_name),
+                        hook_name,
+                    ),
+                ),
+            )
+        else:
+            items.append(
+                str(
+                    format_html(
+                        '<div style="margin: 0 0 8px 0;"><code>{}</code></div>',
+                        hook_name,
+                    ),
+                ),
+            )
+    return "".join(items)
+
+
+def get_filesystem_plugins() -> dict[str, dict[str, Any]]:
+    """Discover plugins from filesystem directories."""
+    plugins = {}
+
+    for base_dir, source in [(BUILTIN_PLUGINS_DIR, "builtin"), (USER_PLUGINS_DIR, "user")]:
+        if not base_dir.exists():
+            continue
+
+        for plugin_dir in base_dir.iterdir():
+            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
+                plugin_id = f"{source}.{plugin_dir.name}"
+
+                hooks = []
+                for ext in ("sh", "py", "js"):
+                    hooks.extend(plugin_dir.glob(f"on_*__*.{ext}"))
+
+                config_file = plugin_dir / "config.json"
+                config_data = None
+                if config_file.exists():
+                    try:
+                        with open(config_file) as f:
+                            config_data = json.load(f)
+                    except (json.JSONDecodeError, OSError):
+                        config_data = None
+
+                plugins[plugin_id] = {
+                    "id": plugin_id,
+                    "name": plugin_dir.name,
+                    "path": str(plugin_dir),
+                    "source": source,
+                    "hooks": [str(h.name) for h in hooks],
+                    "config": config_data,
+                }
+
+    return plugins
+
+
+def find_plugin_for_config_key(key: str) -> str | None:
+    for plugin_name, schema in discover_plugin_configs().items():
+        if key in (schema.get("properties") or {}):
+            return plugin_name
+    return None
+
+
+def get_config_definition_link(key: str) -> tuple[str, str]:
+    plugin_name = find_plugin_for_config_key(key)
+    if not plugin_name:
+        return (
+            f"https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{quote(key)}&type=code",
+            "archivebox/config",
+        )
+
+    plugin_dir = next((path.resolve() for path in iter_plugin_dirs() if path.name == plugin_name), None)
+    if plugin_dir:
+        builtin_root = BUILTIN_PLUGINS_DIR.resolve()
+        if plugin_dir.is_relative_to(builtin_root):
+            return (
+                f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/config.json",
+                f"abx_plugins/plugins/{plugin_name}/config.json",
+            )
+
+        user_root = USER_PLUGINS_DIR.resolve()
+        if plugin_dir.is_relative_to(user_root):
+            return (
+                f"{LIVE_PLUGIN_BASE_URL}user.{quote(plugin_name)}/",
+                f"data/custom_plugins/{plugin_name}/config.json",
+            )
+
+    return (
+        f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/",
+        f"abx_plugins/plugins/{plugin_name}/config.json",
+    )
+
+
+@render_with_table_view
+def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
+    assert is_superuser(request), "Must be a superuser to view configuration settings."
+
+    rows = {
+        "Name": [],
+        "Source": [],
+        "Path": [],
+        "Hooks": [],
+        "Config": [],
+    }
+
+    plugins = get_filesystem_plugins()
+
+    for plugin_id, plugin in plugins.items():
+        rows["Name"].append(ItemLink(plugin["name"], key=plugin_id))
+        rows["Source"].append(plugin["source"])
+        rows["Path"].append(format_html("<code>{}</code>", plugin["path"]))
+        rows["Hooks"].append(", ".join(plugin["hooks"]) or "(none)")
+
+        if plugin.get("config"):
+            config_properties = plugin["config"].get("properties", {})
+            config_count = len(config_properties)
+            rows["Config"].append(f"✅ {config_count} properties" if config_count > 0 else "✅ present")
+        else:
+            rows["Config"].append("❌ none")
+
+    if not plugins:
+        rows["Name"].append("(no plugins found)")
+        rows["Source"].append("-")
+        rows["Path"].append(mark_safe("<code>abx_plugins/plugins/</code> or <code>data/custom_plugins/</code>"))
+        rows["Hooks"].append("-")
+        rows["Config"].append("-")
+
+    return TableContext(
+        title="Installed plugins",
+        table=rows,
+    )
+
+
+@render_with_item_view
+def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
+    assert is_superuser(request), "Must be a superuser to view configuration settings."
+
+    plugins = get_filesystem_plugins()
+
+    plugin = plugins.get(key)
+    if not plugin:
+        return ItemContext(
+            slug=key,
+            title=f"Plugin not found: {key}",
+            data=[],
+        )
+
+    docs_url = get_plugin_docs_url(plugin["name"])
+    machine_admin_url = get_machine_admin_url()
+    fields = {
+        "id": plugin["id"],
+        "name": plugin["name"],
+        "source": plugin["source"],
+    }
+
+    sections: list[SectionData] = [
+        {
+            "name": plugin["name"],
+            "description": format_html(
+                '<code>{}</code><br/><a href="{}" target="_blank" rel="noopener noreferrer">ABX Plugin Docs</a>',
+                plugin["path"],
+                docs_url,
+            ),
+            "fields": fields,
+            "help_texts": {},
+        },
+    ]
+
+    if plugin["hooks"]:
+        sections.append(
+            {
+                "name": "Hooks",
+                "description": mark_safe(render_hook_links_html(plugin["name"], plugin["hooks"], plugin["source"])),
+                "fields": {},
+                "help_texts": {},
+            },
+        )
+
+    if plugin.get("config"):
+        sections.append(
+            {
+                "name": "Plugin Metadata",
+                "description": mark_safe(render_plugin_metadata_html(plugin["config"])),
+                "fields": {},
+                "help_texts": {},
+            },
+        )
+
+        sections.append(
+            {
+                "name": "config.json",
+                "description": mark_safe(render_highlighted_json_block(plugin["config"])),
+                "fields": {},
+                "help_texts": {},
+            },
+        )
+
+        config_properties = plugin["config"].get("properties", {})
+        if config_properties:
+            sections.append(
+                {
+                    "name": "Config Properties",
+                    "description": mark_safe(render_config_properties_html(config_properties, machine_admin_url)),
+                    "fields": {},
+                    "help_texts": {},
+                },
+            )
+
+    return ItemContext(
+        slug=key,
+        title=plugin["name"],
+        data=sections,
+    )

archivebox/progressmonitor/__init__.py

@@ -0,0 +1 @@
+__package__ = "archivebox.progressmonitor"

archivebox/progressmonitor/apps.py

@@ -0,0 +1,8 @@
+__package__ = "archivebox.progressmonitor"
+
+from django.apps import AppConfig
+
+
+class ProgressMonitorConfig(AppConfig):
+    name = "archivebox.progressmonitor"
+    label = "progressmonitor"

archivebox/progressmonitor/templates/progressmonitor/progress_monitor.html

@@ -755,101 +755,6 @@
         white-space: nowrap;
     }
 
-    /* Thumbnail Strip */
-    #progress-monitor .thumbnail-strip {
-        display: flex;
-        gap: 8px;
-        padding: 10px 16px;
-        background: rgba(0,0,0,0.15);
-        border-top: 1px solid #21262d;
-        overflow-x: auto;
-        scrollbar-width: thin;
-        scrollbar-color: #30363d #0d1117;
-    }
-    #progress-monitor .thumbnail-strip::-webkit-scrollbar {
-        height: 6px;
-    }
-    #progress-monitor .thumbnail-strip::-webkit-scrollbar-track {
-        background: #0d1117;
-    }
-    #progress-monitor .thumbnail-strip::-webkit-scrollbar-thumb {
-        background: #30363d;
-        border-radius: 3px;
-    }
-    #progress-monitor .thumbnail-strip::-webkit-scrollbar-thumb:hover {
-        background: #484f58;
-    }
-    #progress-monitor .thumbnail-strip.empty {
-        display: none;
-    }
-    #progress-monitor .thumbnail-item {
-        flex-shrink: 0;
-        position: relative;
-        width: 64px;
-        height: 48px;
-        border-radius: 4px;
-        overflow: hidden;
-        border: 1px solid #30363d;
-        background: #161b22;
-        cursor: pointer;
-        transition: transform 0.2s, border-color 0.2s, box-shadow 0.2s;
-    }
-    #progress-monitor .thumbnail-item:hover {
-        transform: scale(1.1);
-        border-color: #58a6ff;
-        box-shadow: 0 0 12px rgba(88, 166, 255, 0.3);
-        z-index: 10;
-    }
-    #progress-monitor .thumbnail-item.new {
-        animation: thumbnail-pop 0.4s ease-out;
-    }
-    @keyframes thumbnail-pop {
-        0% { transform: scale(0.5); opacity: 0; }
-        50% { transform: scale(1.15); }
-        100% { transform: scale(1); opacity: 1; }
-    }
-    #progress-monitor .thumbnail-item img {
-        width: 100%;
-        height: 100%;
-        object-fit: cover;
-    }
-    #progress-monitor .thumbnail-item .thumbnail-fallback {
-        width: 100%;
-        height: 100%;
-        display: flex;
-        align-items: center;
-        justify-content: center;
-        font-size: 20px;
-        color: #8b949e;
-        background: linear-gradient(135deg, #21262d 0%, #161b22 100%);
-    }
-    #progress-monitor .thumbnail-item .thumbnail-plugin {
-        position: absolute;
-        bottom: 0;
-        left: 0;
-        right: 0;
-        padding: 2px 4px;
-        font-size: 8px;
-        font-weight: 600;
-        text-transform: uppercase;
-        color: #fff;
-        background: rgba(0,0,0,0.7);
-        text-align: center;
-        white-space: nowrap;
-        overflow: hidden;
-        text-overflow: ellipsis;
-    }
-    #progress-monitor .thumbnail-label {
-        display: flex;
-        align-items: center;
-        gap: 6px;
-        padding: 0 4px;
-        color: #8b949e;
-        font-size: 10px;
-        text-transform: uppercase;
-        letter-spacing: 0.5px;
-        flex-shrink: 0;
-    }
     #progress-monitor .pid-label {
         display: inline-flex;
         align-items: center;
@@ -1048,9 +953,10 @@
 
 </style>
 
-<div id="progress-monitor" class="collapsed"
+<div id="progress-monitor" class="{% if not progress_auto_expand %}collapsed{% endif %}"
      data-progress-endpoint="{{ progress_endpoint|default:'/progress.json' }}"
-     data-progress-scope="{{ progress_scope|default:'global' }}">
+     data-progress-scope="{{ progress_scope|default:'global' }}"
+     data-auto-expand="{% if progress_auto_expand %}1{% else %}0{% endif %}">
     <div class="header-bar">
         <div class="header-left">
             <div class="orchestrator-status">
@@ -1078,7 +984,9 @@
             </div>
         </div>
         <div class="header-right">
-            <button class="toggle-btn" id="progress-collapse" title="Toggle details" aria-expanded="false">Details</button>
+            <button class="toggle-btn" id="progress-collapse" title="Toggle details" aria-expanded="{% if progress_auto_expand %}true{% else %}false{% endif %}">
+                {% if progress_auto_expand %}Hide{% else %}Details{% endif %}
+            </button>
         </div>
     </div>
 
@@ -1099,13 +1007,12 @@
     const crawlTree = document.getElementById('crawl-tree');
     const idleMessage = document.getElementById('idle-message');
     const screencastPanel = document.getElementById('screencast-panel');
-    const thumbnailStrip = null;
 
     let pollInterval = null;
     let pollDelayMs = 1000;
     let idleTicks = 0;
-    let isCollapsed = true;
-    let knownThumbnailIds = new Set();
+    let isCollapsed = monitor.dataset.autoExpand !== '1';
+    const snapshotMedia = new Map();
 
     function getApiKey() {
         return (window.ARCHIVEBOX_API_KEY || '').trim();
@@ -1144,14 +1051,55 @@
     function escapeAttr(value) {
         return escapeHtml(value).replace(/`/g, '&#96;');
     }
+    function stableSnapshotMedia(snapshot) {
+        // The progress endpoint can briefly omit media URLs while ArchiveResult
+        // rows settle. Once this page has seen a snapshot image URL, keep using
+        // it so the preview/favicon branch does not flicker back to placeholder.
+        const snapshotId = String(snapshot.id || '');
+        const media = snapshotMedia.get(snapshotId) || {};
+        if (snapshot.favicon_url) media.favicon_url ||= snapshot.favicon_url;
+        if (snapshot.preview_url) {
+            media.preview_url ||= snapshot.preview_url;
+            media.preview_link ||= snapshot.preview_link;
+            media.preview_fallbacks ||= snapshot.preview_fallbacks || [];
+        }
+        if (snapshotId) snapshotMedia.set(snapshotId, media);
+        return media;
+    }
+    function preserveStableMediaNodes(nextRoot) {
+        // Snapshot preview/favicon files are immutable once visible. Keep the
+        // loaded DOM nodes across poll renders so the browser never reloads
+        // them while progress text/badges continue updating around them.
+        const currentCards = new Map(
+            Array.from(crawlTree.querySelectorAll('.snapshot-item[data-snapshot-id]')).map(card => [card.dataset.snapshotId, card])
+        );
+        nextRoot.querySelectorAll('.snapshot-item[data-snapshot-id]').forEach(nextCard => {
+            const currentCard = currentCards.get(nextCard.dataset.snapshotId);
+            if (!currentCard) return;
+            currentCard.querySelectorAll('[data-stable-media]').forEach(currentNode => {
+                const nextNode = nextCard.querySelector(`[data-stable-media="${currentNode.dataset.stableMedia}"]`);
+                if (nextNode) nextNode.replaceWith(currentNode);
+            });
+        });
+    }
+    function replaceCrawlTree(html) {
+        const template = document.createElement('template');
+        template.innerHTML = html;
+        preserveStableMediaNodes(template.content);
+        crawlTree.replaceChildren(...template.content.childNodes);
+    }
     window.nextPreviewFallback = function(img) {
         const fallbacks = (img.dataset.fallbacks || '').split(',').filter(Boolean);
         if (fallbacks.length > 0) {
             img.src = fallbacks.shift();
             img.dataset.fallbacks = fallbacks.join(',');
         } else {
-            img.closest('.snapshot-preview')?.classList.add('placeholder');
-            img.closest('.snapshot-preview').innerHTML = '<span>▤</span>';
+            const preview = img.closest('.snapshot-preview');
+            if (preview) {
+                preview.removeAttribute('data-stable-media');
+                preview.classList.add('placeholder');
+                preview.innerHTML = '<span>▤</span>';
+            }
         }
     };
     function formatUrl(url) {
@@ -1182,10 +1130,6 @@
         return icons[plugin] || '•';
     }
 
-
-    function renderThumbnail(thumb, isNew) { return null; }
-    function updateThumbnails(thumbnails) {}
-
     function formatDuration(seconds) {
         seconds = Math.max(0, Math.floor(seconds || 0));
         const hours = Math.floor(seconds / 3600);
@@ -1273,10 +1217,10 @@
 
     function updateScreencastPanel(data) {
         const hasWork = (data.active_crawls || []).length > 0 ||
-            (data.crawls_pending || 0) > 0 ||
-            (data.crawls_started || 0) > 0 ||
-            (data.snapshots_pending || 0) > 0 ||
-            (data.snapshots_started || 0) > 0;
+            (data.crawls_queued || 0) > 0 ||
+            (data.crawls_active || 0) > 0 ||
+            (data.snapshots_queued || 0) > 0 ||
+            (data.snapshots_active || 0) > 0;
         let target = activeScreencastTarget(data);
         if (!target) {
             const currentFrame = screencastPanel.querySelector('.screencast-frame img');
@@ -1505,11 +1449,12 @@
         const snapshotDurationHtml = renderDurationBadge(snapshot.started);
         const titleText = snapshot.title || formatUrl(snapshot.full_url || snapshot.url);
         const urlText = snapshot.full_url || snapshot.url || '';
-        const faviconHtml = snapshot.favicon_url
-            ? `<img class="snapshot-favicon" src="${escapeAttr(snapshot.favicon_url)}" alt="" decoding="async" loading="lazy" onerror="this.remove()">`
+        const media = stableSnapshotMedia(snapshot);
+        const faviconHtml = media.favicon_url
+            ? `<img class="snapshot-favicon" data-stable-media="favicon" src="${escapeAttr(media.favicon_url)}" alt="" decoding="async" loading="lazy" onerror="this.remove()">`
             : '';
-        const previewHtml = snapshot.preview_url
-            ? `<a class="snapshot-preview" href="${escapeAttr(snapshot.preview_link || snapshot.view_url || adminUrl)}" title="Open snapshot output"><img src="${escapeAttr(snapshot.preview_url)}" alt="" decoding="async" loading="lazy" data-fallbacks="${escapeAttr((snapshot.preview_fallbacks || []).join(','))}" onerror="nextPreviewFallback(this)"></a>`
+        const previewHtml = media.preview_url
+            ? `<a class="snapshot-preview" data-stable-media="preview" href="${escapeAttr(media.preview_link || snapshot.view_url || adminUrl)}" title="Open snapshot output"><img src="${escapeAttr(media.preview_url)}" alt="" decoding="async" loading="lazy" data-fallbacks="${escapeAttr((media.preview_fallbacks || []).join(','))}" onerror="nextPreviewFallback(this)"></a>`
             : `<a class="snapshot-preview placeholder" href="${escapeAttr(snapshot.view_url || adminUrl)}" title="Open snapshot"><span>${statusIcon}</span></a>`;
 
         let extractorHtml = '';
@@ -1549,7 +1494,7 @@
                         : 'Waiting for extractors...';
 
         return `
-            <div class="snapshot-item">
+            <div class="snapshot-item" data-snapshot-id="${escapeAttr(snapshot.id || '')}">
                 <div class="snapshot-header">
                     ${previewHtml}
                     <a class="snapshot-header-link" href="${adminUrl}">
@@ -1742,12 +1687,19 @@
             const el = document.getElementById(id);
             if (el) el.textContent = Number(value || 0).toLocaleString();
         }
+        function setOrchestratorState(state, label) {
+            const dot = document.getElementById('orchestrator-dot');
+            dot.classList.remove('stopped', 'idle', 'running');
+            dot.classList.add(state);
+            document.getElementById('orchestrator-text').textContent = label;
+            return dot;
+        }
 
         // Calculate if there's activity
         const hasActivity = data.active_crawls.length > 0 ||
-                           data.crawls_pending > 0 || data.crawls_started > 0 ||
-                           data.snapshots_pending > 0 || data.snapshots_started > 0 ||
-                           data.archiveresults_pending > 0 || data.archiveresults_started > 0;
+                           data.crawls_queued > 0 || data.crawls_active > 0 ||
+                           data.snapshots_queued > 0 || data.snapshots_active > 0 ||
+                           data.archiveresults_queued > 0 || data.archiveresults_active > 0;
         if (hasActivity) {
             idleTicks = 0;
             if (pollDelayMs !== 1000) {
@@ -1761,29 +1713,20 @@
         }
 
         // Update orchestrator status - show "Running" only when there are active workers.
-        const dot = document.getElementById('orchestrator-dot');
-        const text = document.getElementById('orchestrator-text');
         const pidEl = document.getElementById('orchestrator-pid');
         const hasWorkers = data.total_workers > 0;
         const hasBlockedCrawl = (data.active_crawls || []).some(c => c.worker_state === 'crashed');
+        let dot = null;
 
         if (hasWorkers) {
-            dot.classList.remove('stopped', 'idle');
-            dot.classList.add('running');
-            text.textContent = 'Running';
+            dot = setOrchestratorState('running', 'Running');
         } else if (hasActivity && hasBlockedCrawl && !data.orchestrator_running) {
-            dot.classList.remove('idle', 'running');
-            dot.classList.add('stopped');
-            text.textContent = 'Runner stopped';
+            dot = setOrchestratorState('stopped', 'Runner stopped');
         } else if (hasActivity) {
-            dot.classList.remove('stopped', 'running');
-            dot.classList.add('idle');
-            text.textContent = data.orchestrator_running ? 'Idle' : 'Waiting';
+            dot = setOrchestratorState('idle', data.orchestrator_running ? 'Idle' : 'Waiting');
         } else {
             // No activity - show as idle (whether orchestrator process exists or not)
-            dot.classList.remove('stopped', 'running');
-            dot.classList.add('idle');
-            text.textContent = 'Idle';
+            dot = setOrchestratorState('idle', 'Idle');
         }
 
         if (data.orchestrator_pid) {
@@ -1798,15 +1741,16 @@
         dot.classList.add('flash');
         setTimeout(() => dot.classList.remove('flash'), 300);
 
-        // Update stats
-        setCount('crawls-active', data.crawls_active ?? data.crawls_started);
-        setCount('crawls-queued', data.crawls_queued ?? data.crawls_pending);
-        setCount('snapshots-active', data.snapshots_active ?? data.snapshots_started);
-        setCount('snapshots-queued', data.snapshots_queued ?? data.snapshots_pending);
-        setCount('downloads-active', data.downloads_active ?? data.downloads_started);
-        setCount('downloads-queued', data.downloads_queued ?? data.downloads_pending);
-        setCount('indexing-active', data.indexing_active ?? data.indexing_started);
-        setCount('indexing-queued', data.indexing_queued ?? data.indexing_pending);
+        [
+            ['crawls-active', data.crawls_active],
+            ['crawls-queued', data.crawls_queued],
+            ['snapshots-active', data.snapshots_active],
+            ['snapshots-queued', data.snapshots_queued],
+            ['downloads-active', data.downloads_active],
+            ['downloads-queued', data.downloads_queued],
+            ['indexing-active', data.indexing_active],
+            ['indexing-queued', data.indexing_queued],
+        ].forEach(([id, value]) => setCount(id, value));
         updateScreencastPanel(data);
 
         // Render crawl tree
@@ -1816,24 +1760,23 @@
             const queuedCrawlsNote = queuedCrawlsHidden > 0
                 ? `<div class="progress-overflow-note">${queuedCrawlsHidden} more queued crawl${queuedCrawlsHidden === 1 ? '' : 's'} not shown</div>`
                 : '';
-            crawlTree.innerHTML = data.active_crawls.map(c => renderCrawl(c)).join('') + queuedCrawlsNote;
+            replaceCrawlTree(data.active_crawls.map(c => renderCrawl(c)).join('') + queuedCrawlsNote);
         } else if (hasActivity) {
             idleMessage.style.display = 'none';
-            crawlTree.innerHTML = `
+            replaceCrawlTree(`
                 <div class="idle-message">
-                    ${data.snapshots_started || 0} snapshots processing, ${data.archiveresults_started || 0} extractors running
+                    ${data.snapshots_active || 0} snapshots processing, ${data.archiveresults_active || 0} extractors running
                 </div>
-            `;
+            `);
         } else {
             idleMessage.style.display = '';
             // Build the URL for recent crawls (last 24 hours)
             var yesterday = new Date(Date.now() - 24*60*60*1000).toISOString().split('T')[0];
             var recentUrl = '/admin/crawls/crawl/?created_at__gte=' + yesterday + '&o=-1';
-            idleMessage.innerHTML = `No active crawls (${data.crawls_pending || 0} pending, ${data.crawls_started || 0} started, <a href="${recentUrl}" style="color: #58a6ff;">${data.crawls_recent || 0} recent</a>)`;
-            crawlTree.innerHTML = '';
+            idleMessage.innerHTML = `No active crawls (${data.crawls_queued || 0} pending, ${data.crawls_active || 0} started, <a href="${recentUrl}" style="color: #58a6ff;">${data.crawls_recent || 0} recent</a>)`;
+            replaceCrawlTree('');
         }
 
-        // Recent thumbnails removed
         updateDurationBadges();
     }
 
@@ -1900,69 +1843,24 @@
         btn.textContent = busy ? '…' : label;
     }
 
-    function cancelCrawl(crawlId, btn) {
-        if (!crawlId) return;
+    function patchProgressItem(url, action, btn, errorLabel) {
+        if (!url || !action) return;
         setActionButtonState(btn, true);
 
-        fetch(buildApiUrl(`/api/v1/crawls/crawl/${crawlId}`), {
-            method: 'PATCH',
-            headers: buildApiHeaders(),
-            credentials: 'same-origin',
-            body: JSON.stringify({ action: 'cancel' }),
-        })
-        .then(response => response.json().then(data => ({response, data})))
-        .then(({response, data}) => {
-            if (!response.ok) throw new Error(data.detail || data.error || `HTTP ${response.status}`);
-            fetchProgress();
-        })
-        .catch(error => {
-            console.error('Cancel crawl failed:', error);
-            setActionButtonState(btn, false);
-        });
-    }
-
-    function setCrawlPaused(crawlId, action, btn) {
-        if (!crawlId) return;
-        setActionButtonState(btn, true);
-
-        fetch(buildApiUrl(`/api/v1/crawls/crawl/${crawlId}`), {
+        fetch(buildApiUrl(url), {
             method: 'PATCH',
             headers: buildApiHeaders(),
             credentials: 'same-origin',
             body: JSON.stringify({ action: action }),
         })
-        .then(response => response.json())
-        .then(data => {
-            if (data.error) {
-                console.error('Crawl action error:', data.error);
-            }
+        .then(response => response.json().then(data => ({response, data})))
+        .then(({response, data}) => {
+            if (!response.ok) throw new Error(data.detail || data.error || `HTTP ${response.status}`);
+            if (data.error) console.error(`${errorLabel} error:`, data.error);
             fetchProgress();
         })
         .catch(error => {
-            console.error('Crawl action failed:', error);
-            setActionButtonState(btn, false);
-        });
-    }
-
-    function cancelSnapshot(snapshotId, btn) {
-        if (!snapshotId) return;
-        setActionButtonState(btn, true);
-
-        fetch(buildApiUrl(`/api/v1/core/snapshot/${snapshotId}`), {
-            method: 'PATCH',
-            headers: buildApiHeaders(),
-            credentials: 'same-origin',
-            body: JSON.stringify({ action: 'cancel' }),
-        })
-        .then(response => response.json())
-        .then(data => {
-            if (data.error) {
-                console.error('Cancel snapshot error:', data.error);
-            }
-            fetchProgress();
-        })
-        .catch(error => {
-            console.error('Cancel snapshot failed:', error);
+            console.error(`${errorLabel} failed:`, error);
             setActionButtonState(btn, false);
         });
     }
@@ -1977,7 +1875,12 @@
         if (actionBtn) {
             event.preventDefault();
             event.stopPropagation();
-            setCrawlPaused(actionBtn.dataset.crawlId, actionBtn.dataset.crawlAction, actionBtn);
+            patchProgressItem(
+                actionBtn.dataset.crawlId ? `/api/v1/crawls/crawl/${actionBtn.dataset.crawlId}` : '',
+                actionBtn.dataset.crawlAction,
+                actionBtn,
+                'Crawl action',
+            );
             return;
         }
         const btn = event.target.closest('.cancel-item-btn');
@@ -1987,9 +1890,9 @@
 
         const cancelType = btn.dataset.cancelType;
         if (cancelType === 'crawl') {
-            cancelCrawl(btn.dataset.crawlId, btn);
+            patchProgressItem(btn.dataset.crawlId ? `/api/v1/crawls/crawl/${btn.dataset.crawlId}` : '', 'cancel', btn, 'Cancel crawl');
         } else if (cancelType === 'snapshot') {
-            cancelSnapshot(btn.dataset.snapshotId, btn);
+            patchProgressItem(btn.dataset.snapshotId ? `/api/v1/core/snapshot/${btn.dataset.snapshotId}` : '', 'cancel', btn, 'Cancel snapshot');
         }
     });
 

archivebox/progressmonitor/views.py

@@ -0,0 +1,953 @@
+__package__ = "archivebox.progressmonitor"
+
+from functools import lru_cache
+from pathlib import Path
+from typing import Literal
+
+from django.db.models import CharField, Count, Q, Sum
+from django.db.models.functions import Cast
+from django.http import HttpResponse, JsonResponse
+from django.utils import timezone
+
+from abx_dl.events import PROCESS_EXIT_SKIPPED
+
+from archivebox.config import CONSTANTS
+from archivebox.config.common import get_config
+from archivebox.core.routes_util import build_snapshot_url, build_web_url, get_api_base_url
+from archivebox.core.permissions import can_view_snapshot, is_admin_user
+from archivebox.plugins.discovery import discover_plugin_configs
+from archivebox.misc.logging_util import printable_filesize
+
+
+def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_id: object | None = None) -> str:
+    """Return the canonical same-origin progress endpoint for monitor embeds."""
+    if not scope or object_id is None:
+        return "/progress.json"
+    return f"/progress.json?{scope}_id={str(object_id).replace('-', '')}"
+
+
+@lru_cache(maxsize=1)
+def _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]]:
+    plugin_configs = discover_plugin_configs()
+    download_plugin_names = frozenset(
+        plugin_name
+        for plugin_name, plugin_config in plugin_configs.items()
+        if plugin_config.get("output_mimetypes") and not plugin_name.startswith("search_backend_")
+    )
+    indexing_plugin_names = frozenset(plugin_name for plugin_name in plugin_configs if plugin_name.startswith("search_backend_"))
+    return download_plugin_names, indexing_plugin_names
+
+
+def live_progress_view(request):
+    """Simple JSON endpoint for live progress status - used by admin progress monitor."""
+    try:
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import Snapshot, ArchiveResult
+        from archivebox.machine.models import Process, Machine
+
+        snapshot_id_filter = (request.GET.get("snapshot_id") or "").strip().replace("-", "")
+        crawl_id_filter = (request.GET.get("crawl_id") or "").strip().replace("-", "")
+        is_admin = is_admin_user(request)
+
+        scoped_snapshot = None
+        if snapshot_id_filter:
+            import uuid as _uuid
+
+            try:
+                _uuid.UUID(snapshot_id_filter)
+            except (TypeError, ValueError):
+                return JsonResponse({"error": "Invalid snapshot_id"}, status=400)
+            scoped_snapshot = Snapshot.objects.filter(id=snapshot_id_filter).select_related("crawl").first()
+            if scoped_snapshot is None or not can_view_snapshot(request, scoped_snapshot):
+                return JsonResponse({"error": "Permission denied"}, status=403)
+        elif crawl_id_filter:
+            # Crawl-only scope still requires staff: there's no per-crawl ACL helper,
+            # and a crawl can mix snapshot permissions levels.
+            if not is_admin:
+                return JsonResponse({"error": "Permission denied"}, status=403)
+        else:
+            if not is_admin:
+                return JsonResponse({"error": "Permission denied"}, status=403)
+
+        request_config = request.archivebox_config
+        now = timezone.now()
+        crawl_scope = Crawl.objects.all()
+        snapshot_scope = Snapshot.objects.all()
+        archiveresult_scope = ArchiveResult.objects.all()
+        if is_admin and not request.user.is_superuser:
+            crawl_scope = crawl_scope.filter(created_by=request.user)
+            snapshot_scope = snapshot_scope.filter(crawl__created_by=request.user)
+            archiveresult_scope = archiveresult_scope.filter(snapshot__crawl__created_by=request.user)
+        if scoped_snapshot is not None:
+            snapshot_scope = Snapshot.objects.filter(id=scoped_snapshot.id)
+            crawl_scope = Crawl.objects.filter(id=scoped_snapshot.crawl_id)
+            archiveresult_scope = ArchiveResult.objects.filter(snapshot_id=scoped_snapshot.id)
+        elif crawl_id_filter:
+            snapshot_scope = snapshot_scope.filter(crawl_id=crawl_id_filter)
+            crawl_scope = crawl_scope.filter(id=crawl_id_filter)
+            archiveresult_scope = archiveresult_scope.filter(snapshot__crawl_id=crawl_id_filter)
+
+        def is_current_run_timestamp(event_ts, run_started_at) -> bool:
+            if run_started_at is None:
+                return True
+            if event_ts is None:
+                return False
+            return event_ts >= run_started_at
+
+        def archiveresult_matches_current_run(ar, run_started_at) -> bool:
+            if run_started_at is None:
+                return True
+            if ar.status in (
+                ArchiveResult.StatusChoices.QUEUED,
+                ArchiveResult.StatusChoices.STARTED,
+                ArchiveResult.StatusChoices.BACKOFF,
+            ):
+                return True
+            event_ts = ar.end_ts or ar.start_ts or ar.modified_at or ar.created_at
+            return is_current_run_timestamp(event_ts, run_started_at)
+
+        def hook_details(hook_name: str, plugin: str = "setup") -> tuple[str, str, str, str]:
+            normalized_hook_name = Path(hook_name).name if hook_name else ""
+            if not normalized_hook_name:
+                return (plugin, plugin, "unknown", "")
+
+            phase = "unknown"
+            if normalized_hook_name == "InstallEvent":
+                phase = "install"
+            elif normalized_hook_name.startswith("on_CrawlSetup__"):
+                phase = "crawl"
+            elif normalized_hook_name.startswith("on_Snapshot__"):
+                phase = "snapshot"
+            elif normalized_hook_name.startswith("on_BinaryRequest__"):
+                phase = "binary"
+
+            label = normalized_hook_name
+            if "__" in normalized_hook_name:
+                label = normalized_hook_name.split("__", 1)[1]
+            label = label.rsplit(".", 1)[0]
+            if len(label) > 3 and label[:2].isdigit() and label[2] == "_":
+                label = label[3:]
+            label = label.replace("_", " ").strip() or plugin
+
+            return (plugin, label, phase, normalized_hook_name)
+
+        def process_label(cmd: list[str] | None) -> tuple[str, str, str, str]:
+            hook_path = ""
+            if isinstance(cmd, list) and cmd:
+                first = cmd[0]
+                if isinstance(first, str):
+                    hook_path = first
+
+            if not hook_path:
+                return ("", "setup", "unknown", "")
+
+            return hook_details(Path(hook_path).name, plugin=Path(hook_path).parent.name or "setup")
+
+        def archiveresult_output_path(ar) -> str | None:
+            output_file_map = ar.output_files if isinstance(ar.output_files, dict) else {}
+
+            def is_root_relative(path: str) -> bool:
+                metadata = output_file_map.get(path) or {}
+                return bool(isinstance(metadata, dict) and metadata.get("root_relative"))
+
+            if ar.output_str:
+                raw_output = str(ar.output_str).strip()
+                if ar._looks_like_output_path(raw_output, ar.plugin):
+                    output_path = Path(raw_output)
+                    if output_path.is_absolute():
+                        return None
+
+                    if raw_output.startswith(f"{ar.plugin}/"):
+                        candidates = [raw_output]
+                    elif len(output_path.parts) == 1:
+                        candidates = [f"{ar.plugin}/{raw_output}", raw_output]
+                    else:
+                        candidates = [raw_output]
+
+                    if raw_output in output_file_map and is_root_relative(raw_output):
+                        return raw_output
+
+                    for relative_path in candidates:
+                        plugin_relative = relative_path.removeprefix(f"{ar.plugin}/")
+                        if relative_path in output_file_map:
+                            return f"{ar.plugin}/{relative_path}" if not relative_path.startswith(f"{ar.plugin}/") else relative_path
+                        if plugin_relative in output_file_map:
+                            return f"{ar.plugin}/{plugin_relative}"
+
+            output_file_paths = list(output_file_map.keys())
+            if output_file_paths:
+                fallback_path = ArchiveResult._fallback_output_file_path(output_file_paths, ar.plugin, output_file_map)
+                if fallback_path:
+                    if is_root_relative(fallback_path):
+                        return fallback_path
+                    return f"{ar.plugin}/{fallback_path}"
+
+            return None
+
+        def snapshot_output_url(snapshot, output_path: str) -> str:
+            return build_snapshot_url(str(snapshot["id"]), output_path, request=request, config=request_config)
+
+        def snapshot_archive_path(snapshot) -> str:
+            if snapshot["fs_version"] in ("0.7.0", "0.8.0"):
+                return f"{CONSTANTS.ARCHIVE_DIR_NAME}/{snapshot['timestamp']}"
+            crawl = crawls_by_id.get(str(snapshot["crawl_id"]))
+            username = "web"
+            if crawl is not None and crawl["created_by_id"]:
+                username = crawl["created_by__username"]
+            if username == "system":
+                username = "web"
+            date_base = snapshot["bookmarked_at"] or snapshot["created_at"]
+            date_str = date_base.strftime("%Y%m%d") if date_base else "unknown"
+            domain = Snapshot.extract_domain_from_url(snapshot["url"])
+            return f"{username}/{date_str}/{domain}/{snapshot['id']}"
+
+        def snapshot_view_url(snapshot, output_path: str = "") -> str:
+            anchor = f"#{output_path}" if output_path else ""
+            return build_web_url(
+                f"/{snapshot_archive_path(snapshot)}/index.html{anchor}",
+                request=request,
+                config=request_config,
+            )
+
+        def snapshot_display_url(url: str) -> str:
+            url = str(url or "")
+            return url if len(url) <= 96 else f"{url[:93]}..."
+
+        api_base = get_api_base_url(request=request, config=request_config) if scoped_snapshot is not None else ""
+
+        def screencast_frame_url(crawl_id: str, crawl_dir: Path) -> str:
+            frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
+            try:
+                frame_stat = frame_path.stat()
+            except OSError:
+                return ""
+            if frame_stat.st_size <= 0:
+                return ""
+            if now.timestamp() - frame_stat.st_mtime > 15:
+                return ""
+            rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
+            return f"{api_base}{rel}" if api_base else rel
+
+        machine_id = Machine.current().id
+        orchestrator_proc = (
+            Process.objects.filter(
+                machine_id=machine_id,
+                process_type=Process.TypeChoices.ORCHESTRATOR,
+                status=Process.StatusChoices.RUNNING,
+            )
+            .only("id", "pid", "started_at", "machine_id", "process_type", "status")
+            .order_by("-started_at")
+            .first()
+            if machine_id is not None
+            else None
+        )
+        runner_worker = None
+        orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running)
+        if not orchestrator_proc_running:
+            try:
+                from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+
+                supervisor = get_existing_supervisord_process(quiet=True)
+                runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
+            except Exception:
+                runner_worker = None
+
+        runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"))
+        runner_worker_pid = runner_worker.get("pid") if runner_worker else None
+        orchestrator_running = orchestrator_proc_running or runner_worker_running
+        orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid
+
+        # Get model counts by status
+        crawl_status_counts = Crawl.status_counts(
+            crawl_scope,
+            (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.PAUSED),
+        )
+        crawls_queued = crawl_status_counts.get(Crawl.StatusChoices.QUEUED, 0)
+        crawls_active = crawl_status_counts.get(Crawl.StatusChoices.STARTED, 0)
+
+        # Get recent crawls (last 24 hours)
+        from datetime import timedelta
+
+        one_day_ago = now - timedelta(days=1)
+        paused_crawl_cutoff = now - timedelta(hours=12)
+        crawls_recent = crawl_scope.filter(created_at__gte=one_day_ago).count()
+
+        snapshot_status_counts = Snapshot.status_counts(
+            snapshot_scope,
+            Snapshot.OPEN_STATES,
+        )
+        snapshots_queued = snapshot_status_counts.get(Snapshot.StatusChoices.QUEUED, 0)
+        snapshots_active = snapshot_status_counts.get(Snapshot.StatusChoices.STARTED, 0)
+
+        download_plugin_names, indexing_plugin_names = _live_progress_plugin_names()
+        result_statuses = (
+            ArchiveResult.StatusChoices.QUEUED,
+            ArchiveResult.StatusChoices.STARTED,
+        )
+        archiveresult_status_counts = ArchiveResult.status_counts(archiveresult_scope, result_statuses)
+        download_scope = archiveresult_scope.filter(
+            plugin__in=download_plugin_names,
+            snapshot__status__in=Snapshot.RUNNABLE_STATES,
+            snapshot__crawl__status__in=Crawl.RUNNABLE_STATES,
+        )
+        indexing_scope = archiveresult_scope.filter(plugin__in=indexing_plugin_names)
+        download_status_counts = ArchiveResult.status_counts(download_scope, result_statuses)
+        indexing_status_counts = ArchiveResult.status_counts(indexing_scope, result_statuses)
+        archiveresults_queued = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
+        archiveresults_active = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
+
+        downloads_queued = download_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
+        downloads_active = download_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
+        indexing_queued = indexing_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
+        indexing_active = indexing_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
+
+        # Build hierarchical active crawls with nested snapshots and archive results
+        max_active_crawls = 10
+        max_queued_crawls = 10
+        max_started_snapshots_per_crawl = 50
+        max_queued_snapshots_per_crawl = 50
+
+        active_crawl_fields = (
+            "id",
+            "created_at",
+            "created_by_id",
+            "modified_at",
+            "urls",
+            "config",
+            "max_depth",
+            "tags_str",
+            "persona_id",
+            "status",
+            "retry_at",
+            "label",
+            "created_by__id",
+            "created_by__username",
+        )
+        started_crawls = list(
+            crawl_scope.filter(status=Crawl.StatusChoices.STARTED)
+            .values(*active_crawl_fields)
+            .order_by("-modified_at")[:max_active_crawls],
+        )
+        paused_crawls = list(
+            crawl_scope.filter(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
+            .values(*active_crawl_fields)
+            .order_by("-modified_at")[:max_active_crawls],
+        )
+        queued_crawls = list(
+            crawl_scope.filter(status=Crawl.StatusChoices.QUEUED).values(*active_crawl_fields).order_by("-modified_at")[:max_queued_crawls],
+        )
+        queued_crawls_hidden = max(crawls_queued - len(queued_crawls), 0)
+        active_crawls_list = started_crawls + paused_crawls + queued_crawls
+        for crawl in active_crawls_list:
+            crawl["id"] = str(crawl["id"])
+            if crawl["persona_id"]:
+                crawl["persona_id"] = str(crawl["persona_id"])
+        persona_details_by_id: dict[str, dict[str, str]] = {}
+        persona_details_by_name: dict[str, dict[str, str]] = {}
+        persona_objects_by_id = {}
+        persona_objects_by_name = {}
+        persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
+        persona_names = {
+            str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default") for crawl in active_crawls_list if not crawl["persona_id"]
+        }
+        if persona_ids or persona_names:
+            from archivebox.personas.models import Persona
+
+            for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name", "config"):
+                persona_details = {
+                    "name": persona.name,
+                    "admin_url": f"/admin/personas/persona/{persona.pk}/change/",
+                }
+                persona_details_by_id[str(persona.id)] = persona_details
+                persona_details_by_name[persona.name] = persona_details
+                persona_objects_by_id[str(persona.id)] = persona
+                persona_objects_by_name[persona.name] = persona
+        active_crawl_ids = [crawl["id"] for crawl in active_crawls_list]
+        active_crawl_objects = {}
+        if active_crawl_ids:
+            for crawl_obj in Crawl.objects.filter(id__in=active_crawl_ids).select_related("created_by", "persona"):
+                crawl_obj._runtime_config = request_config
+                active_crawl_objects[str(crawl_obj.id)] = crawl_obj
+        snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids}
+        cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
+        crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
+        queued_snapshot_overflow_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
+        active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids)
+        if active_crawl_ids:
+            for row in active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")):
+                snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"]
+
+            for row in (
+                active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True)
+                .values("crawl_id")
+                .annotate(count=Count("id"))
+            ):
+                cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"]
+
+            for row in (
+                active_snapshot_scope.filter(
+                    status=Snapshot.StatusChoices.SEALED,
+                )
+                .values("crawl_id")
+                .annotate(size=Sum("output_size"))
+            ):
+                crawl_output_sizes_by_crawl[str(row["crawl_id"])] = int(row["size"] or 0)
+
+        crawl_process_pids: dict[str, int] = {}
+        snapshot_process_pids: dict[str, int] = {}
+        process_records_by_crawl: dict[str, list[tuple[dict[str, object], object | None]]] = {}
+        process_records_by_snapshot: dict[str, list[tuple[dict[str, object], object | None]]] = {}
+        seen_process_records: set[str] = set()
+        crawls_by_id = {str(crawl["id"]): crawl for crawl in active_crawls_list}
+        started_snapshot_fields = (
+            "id_str",
+            "created_at",
+            "modified_at",
+            "url",
+            "timestamp",
+            "bookmarked_at",
+            "crawl_id_str",
+            "title",
+            "downloaded_at",
+            "fs_version",
+            "status",
+        )
+        queued_snapshot_fields = (
+            "id_str",
+            "url",
+            "crawl_id_str",
+            "title",
+            "status",
+        )
+        snapshots = []
+        for crawl_id in active_crawl_ids:
+            crawl_snapshot_scope = active_snapshot_scope.filter(crawl_id=crawl_id)
+            snapshots.extend(
+                crawl_snapshot_scope.filter(status=Snapshot.StatusChoices.STARTED)
+                .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
+                .values(*started_snapshot_fields)
+                .order_by("-modified_at")[:max_started_snapshots_per_crawl],
+            )
+            queued_snapshots = list(
+                crawl_snapshot_scope.filter(status=Snapshot.StatusChoices.QUEUED)
+                .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
+                .values(
+                    *queued_snapshot_fields,
+                )
+                .order_by("modified_at")[:max_queued_snapshots_per_crawl],
+            )
+            queued_snapshot_overflow_by_crawl[str(crawl_id)] = max(
+                snapshot_counts_by_crawl.get(str(crawl_id), {}).get(Snapshot.StatusChoices.QUEUED, 0) - len(queued_snapshots),
+                0,
+            )
+            snapshots.extend(queued_snapshots)
+
+        for snapshot in snapshots:
+            # Process.pwd points at Snapshot.output_dir, which uses CompactUUID
+            # hex path components. Keep progress IDs compact too so process rows
+            # can be matched without carrying dashed/undashed variants.
+            snapshot["id"] = str(snapshot.pop("id_str")).replace("-", "")
+            snapshot["crawl_id"] = str(snapshot.pop("crawl_id_str")).replace("-", "")
+        snapshots_by_id = {str(snapshot["id"]): snapshot for snapshot in snapshots}
+        displayed_snapshots_by_crawl: dict[str, list[Snapshot]] = {str(crawl_id): [] for crawl_id in active_crawl_ids}
+        for snapshot in snapshots:
+            crawl_snapshots = displayed_snapshots_by_crawl.setdefault(str(snapshot["crawl_id"]), [])
+            crawl_snapshots.append(snapshot)
+        displayed_snapshot_ids = [
+            snapshot["id"] for crawl_snapshots in displayed_snapshots_by_crawl.values() for snapshot in crawl_snapshots
+        ]
+        detailed_snapshot_ids = [snapshot["id"] for snapshot in snapshots if snapshot["status"] != Snapshot.StatusChoices.QUEUED]
+        process_value_fields = ("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "modified_at")
+        if active_crawl_ids or displayed_snapshot_ids:
+            process_scope = Process.objects.filter(
+                machine_id=machine_id,
+                process_type__in=[
+                    Process.TypeChoices.HOOK,
+                    Process.TypeChoices.BINARY,
+                ],
+            )
+            running_processes = process_scope.filter(status=Process.StatusChoices.RUNNING).values(*process_value_fields)
+            recent_processes = (
+                process_scope.filter(modified_at__gte=now - timedelta(minutes=10)).values(*process_value_fields).order_by("-modified_at")
+            )
+        else:
+            running_processes = Process.objects.none()
+            recent_processes = Process.objects.none()
+
+        archiveresults_by_snapshot: dict[str, list[ArchiveResult]] = {str(snapshot_id): [] for snapshot_id in detailed_snapshot_ids}
+        if detailed_snapshot_ids:
+            displayed_archiveresults = (
+                archiveresult_scope.filter(snapshot_id__in=detailed_snapshot_ids)
+                .select_related("process")
+                .only(
+                    "id",
+                    "snapshot_id",
+                    "plugin",
+                    "hook_name",
+                    "status",
+                    "output_str",
+                    "output_files",
+                    "output_size",
+                    "start_ts",
+                    "end_ts",
+                    "created_at",
+                    "modified_at",
+                    "process_id",
+                    "process__id",
+                    "process__pid",
+                    "process__started_at",
+                    "process__timeout",
+                )
+                .order_by("snapshot_id", "start_ts", "created_at")
+            )
+            for archiveresult in displayed_archiveresults:
+                archiveresults_by_snapshot.setdefault(str(archiveresult.snapshot_id), []).append(archiveresult)
+
+        def find_snapshot_for_process(proc_pwd: Path) -> Snapshot | None:
+            for path_part in reversed(proc_pwd.parts):
+                snapshot = snapshots_by_id.get(path_part)
+                if snapshot:
+                    return snapshot
+            return None
+
+        def find_crawl_for_process(proc_pwd: Path) -> Crawl | None:
+            for path_part in reversed(proc_pwd.parts):
+                crawl = crawls_by_id.get(path_part)
+                if crawl:
+                    return crawl
+            return None
+
+        running_worker_ids: set[str] = set()
+        for proc in running_processes:
+            if not proc["pwd"]:
+                continue
+            proc_pwd = Path(proc["pwd"])
+            matched_snapshot = find_snapshot_for_process(proc_pwd)
+            matched_crawl = (
+                crawls_by_id.get(str(matched_snapshot["crawl_id"])) if matched_snapshot is not None else find_crawl_for_process(proc_pwd)
+            )
+            if matched_snapshot is None:
+                if matched_crawl is None:
+                    continue
+                crawl_id = str(matched_crawl["id"])
+                snapshot_id = ""
+            else:
+                crawl_id = str(matched_snapshot["crawl_id"])
+                snapshot_id = str(matched_snapshot["id"])
+            running_worker_ids.add(str(proc["id"]))
+            _plugin, _label, phase, _hook_name = process_label(proc["cmd"])
+            if crawl_id and proc["pid"]:
+                crawl_process_pids.setdefault(crawl_id, proc["pid"])
+            if phase == "snapshot" and snapshot_id and proc["pid"]:
+                snapshot_process_pids.setdefault(snapshot_id, proc["pid"])
+
+        for proc in recent_processes:
+            if not proc["pwd"]:
+                continue
+            proc_pwd = Path(proc["pwd"])
+            matched_snapshot = find_snapshot_for_process(proc_pwd)
+            matched_crawl = (
+                crawls_by_id.get(str(matched_snapshot["crawl_id"])) if matched_snapshot is not None else find_crawl_for_process(proc_pwd)
+            )
+            if matched_snapshot is None and matched_crawl is None:
+                continue
+            crawl_id = str(matched_snapshot["crawl_id"] if matched_snapshot is not None else matched_crawl["id"])
+            snapshot_id = str(matched_snapshot["id"]) if matched_snapshot is not None else ""
+
+            plugin, label, phase, hook_name = process_label(proc["cmd"])
+
+            record_scope = str(snapshot_id) if phase == "snapshot" and snapshot_id else str(crawl_id)
+            proc_key = f"{record_scope}:{plugin}:{label}:{proc['status']}:{proc['exit_code']}"
+            if proc_key in seen_process_records:
+                continue
+            seen_process_records.add(proc_key)
+
+            status = (
+                "started"
+                if proc["status"] == Process.StatusChoices.RUNNING
+                else (
+                    "skipped"
+                    if proc["exit_code"] == PROCESS_EXIT_SKIPPED or (phase == "binary" and proc["exit_code"] not in (None, 0))
+                    else ("failed" if proc["exit_code"] not in (None, 0) else "succeeded")
+                )
+            )
+            payload: dict[str, object] = {
+                "id": str(proc["id"]),
+                "plugin": plugin,
+                "label": label,
+                "hook_name": hook_name,
+                "status": status,
+                "phase": phase,
+                "source": "process",
+                "process_id": str(proc["id"]),
+            }
+            if status == "started" and proc["pid"]:
+                payload["pid"] = proc["pid"]
+            proc_started_at = proc["started_at"] or proc["modified_at"]
+            if phase == "snapshot" and snapshot_id:
+                process_records_by_snapshot.setdefault(snapshot_id, []).append((payload, proc_started_at))
+            elif crawl_id:
+                process_records_by_crawl.setdefault(crawl_id, []).append((payload, proc_started_at))
+
+        active_crawls = []
+        total_workers = len(running_worker_ids)
+        for crawl in active_crawls_list:
+            crawl_id = str(crawl["id"])
+            crawl_snapshot_counts = snapshot_counts_by_crawl.get(crawl_id, {})
+            total_snapshots = sum(crawl_snapshot_counts.values())
+            completed_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.SEALED, 0)
+            started_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.STARTED, 0)
+            pending_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.QUEUED, 0)
+            cancelled_snapshots = cancelled_snapshot_counts_by_crawl.get(crawl_id, 0)
+
+            # Count URLs in the crawl (for when snapshots haven't been created yet)
+            urls_count = 0
+            if crawl["urls"]:
+                urls_count = len([u for u in crawl["urls"].split("\n") if u.strip() and not u.startswith("#")])
+
+            # Calculate crawl progress
+            crawl_progress = int((completed_snapshots / total_snapshots) * 100) if total_snapshots > 0 else 0
+            crawl_run_started_at = crawl["created_at"]
+            crawl_setup_plugins = [
+                payload
+                for payload, proc_started_at in process_records_by_crawl.get(crawl_id, [])
+                if is_current_run_timestamp(proc_started_at, crawl_run_started_at)
+            ]
+            crawl_setup_total = len(crawl_setup_plugins)
+            crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded")
+            crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed")
+            crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued")
+            crawl_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
+            crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id.replace('-', '')}/change/" if crawl_screencast_url else ""
+
+            # Get active snapshots for this crawl (already prefetched)
+            active_snapshots_for_crawl = []
+            for snapshot in displayed_snapshots_by_crawl.get(crawl_id, []):
+                snapshot_run_started_at = snapshot.get("downloaded_at") or snapshot.get("created_at")
+                # Get archive results only for displayed active snapshots. Large crawls can
+                # contain thousands of sealed snapshots, and prefetching all their results
+                # makes the progress endpoint compete with the runner.
+                snapshot_results = [
+                    ar
+                    for ar in archiveresults_by_snapshot.get(str(snapshot["id"]), [])
+                    if archiveresult_matches_current_run(ar, snapshot_run_started_at)
+                ]
+                if snapshot["status"] == Snapshot.StatusChoices.QUEUED:
+                    snapshot_results = []
+
+                plugin_progress_values: list[int] = []
+                all_plugins: list[dict[str, object]] = []
+                seen_plugin_keys: set[str] = set()
+                snapshot_title = (
+                    str(snapshot["title"] or "")
+                    if snapshot["status"] == Snapshot.StatusChoices.QUEUED
+                    else Snapshot._normalize_title_candidate(snapshot["title"], snapshot_url=snapshot["url"])
+                )
+                snapshot_favicon_url = ""
+                snapshot_preview_url = ""
+                snapshot_preview_link = ""
+                snapshot_screencast_url = ""
+                snapshot_screencast_link = ""
+                snapshot_fallback_urls: list[str] = []
+                result_by_plugin = {result.plugin: result for result in snapshot_results}
+                title_result = result_by_plugin.get("title")
+                if not snapshot_title and title_result is not None and title_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
+                    snapshot_title = Snapshot._normalize_title_candidate(title_result.output_str, snapshot_url=snapshot["url"])
+                favicon_result = result_by_plugin.get("favicon")
+                if favicon_result is not None and favicon_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
+                    favicon_path = archiveresult_output_path(favicon_result) or "favicon/favicon.ico"
+                    snapshot_favicon_url = snapshot_output_url(snapshot, favicon_path)
+                screenshot_result = result_by_plugin.get("screenshot")
+                if screenshot_result is not None and screenshot_result.status == ArchiveResult.StatusChoices.SUCCEEDED:
+                    snapshot_preview_link = snapshot_view_url(snapshot)
+                    screenshot_path = archiveresult_output_path(screenshot_result) or "screenshot/screenshot.png"
+                    snapshot_preview_url = snapshot_output_url(snapshot, screenshot_path)
+                    snapshot_preview_link = snapshot_view_url(snapshot, screenshot_path)
+                    if snapshot_favicon_url:
+                        snapshot_fallback_urls.append(snapshot_favicon_url)
+                elif snapshot_favicon_url:
+                    snapshot_preview_url = snapshot_favicon_url
+
+                if snapshot["status"] == Snapshot.StatusChoices.STARTED:
+                    snapshot_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
+                    snapshot_screencast_link = snapshot_view_url(snapshot) if snapshot_screencast_url else ""
+
+                def plugin_sort_key(ar):
+                    status_order = {
+                        ArchiveResult.StatusChoices.STARTED: 0,
+                        ArchiveResult.StatusChoices.QUEUED: 1,
+                        ArchiveResult.StatusChoices.SUCCEEDED: 2,
+                        ArchiveResult.StatusChoices.NORESULTS: 3,
+                        ArchiveResult.StatusChoices.FAILED: 4,
+                    }
+                    return (status_order.get(ar.status, 5), ar.plugin, ar.hook_name or "")
+
+                for ar in sorted(snapshot_results, key=plugin_sort_key):
+                    status = ar.status
+                    process = ar.process_record
+                    progress_value = 0
+                    if status in (
+                        ArchiveResult.StatusChoices.SUCCEEDED,
+                        ArchiveResult.StatusChoices.FAILED,
+                        ArchiveResult.StatusChoices.SKIPPED,
+                        ArchiveResult.StatusChoices.NORESULTS,
+                    ):
+                        progress_value = 100
+                    elif status == ArchiveResult.StatusChoices.STARTED:
+                        started_at = ar.start_ts or (process.started_at if process else None)
+                        timeout = process.timeout if process else 120
+                        if started_at and timeout:
+                            elapsed = max(0.0, (now - started_at).total_seconds())
+                            progress_value = int(min(99, max(1, (elapsed / float(timeout)) * 100)))
+                        else:
+                            progress_value = 1
+                    else:
+                        progress_value = 0
+
+                    plugin_progress_values.append(progress_value)
+                    plugin, label, phase, hook_name = hook_details(ar.hook_name or ar.plugin, plugin=ar.plugin)
+
+                    plugin_payload = {
+                        "id": str(ar.id),
+                        "plugin": ar.plugin,
+                        "label": label,
+                        "hook_name": hook_name,
+                        "phase": phase,
+                        "status": status,
+                        "process_id": str(process.id) if process else None,
+                        "admin_url": f"/admin/core/archiveresult/{ar.id}/change/",
+                    }
+                    output_path = archiveresult_output_path(ar)
+                    if output_path:
+                        plugin_payload["output_path"] = output_path
+                        plugin_payload["output_url"] = snapshot_view_url(snapshot, output_path)
+                    if status == ArchiveResult.StatusChoices.STARTED and process:
+                        plugin_payload["pid"] = process.pid
+                    if status == ArchiveResult.StatusChoices.STARTED:
+                        plugin_payload["progress"] = progress_value
+                        plugin_payload["timeout"] = process.timeout if process else 120
+                    plugin_payload["source"] = "archiveresult"
+                    all_plugins.append(plugin_payload)
+                    seen_plugin_keys.add(str(process.id) if process else f"{ar.plugin}:{hook_name}")
+
+                for proc_payload, proc_started_at in process_records_by_snapshot.get(str(snapshot["id"]), []):
+                    if not is_current_run_timestamp(proc_started_at, snapshot_run_started_at):
+                        continue
+                    proc_key = str(proc_payload.get("process_id") or f"{proc_payload.get('plugin')}:{proc_payload.get('hook_name')}")
+                    if proc_key in seen_plugin_keys:
+                        continue
+                    seen_plugin_keys.add(proc_key)
+                    all_plugins.append(proc_payload)
+
+                    proc_status = proc_payload.get("status")
+                    if proc_status in ("succeeded", "failed", "skipped"):
+                        plugin_progress_values.append(100)
+                    elif proc_status == "started":
+                        plugin_progress_values.append(1)
+                    else:
+                        plugin_progress_values.append(0)
+
+                total_plugins = len(all_plugins)
+                completed_plugins = sum(1 for item in all_plugins if item.get("status") == "succeeded")
+                failed_plugins = sum(1 for item in all_plugins if item.get("status") == "failed")
+                pending_plugins = sum(1 for item in all_plugins if item.get("status") == "queued")
+
+                snapshot_progress = int(sum(plugin_progress_values) / len(plugin_progress_values)) if plugin_progress_values else 0
+                worker_state = "running" if snapshot_process_pids.get(str(snapshot["id"])) else "waiting"
+                if (
+                    snapshot["status"] == Snapshot.StatusChoices.STARTED
+                    and worker_state == "waiting"
+                    and not all_plugins
+                    and snapshot["modified_at"]
+                    and (now - snapshot["modified_at"]).total_seconds() > 30
+                ):
+                    worker_state = "waiting" if orchestrator_running else "crashed"
+
+                if snapshot["status"] == Snapshot.StatusChoices.QUEUED and not snapshot_process_pids.get(str(snapshot["id"])):
+                    compact_snapshot = [
+                        str(snapshot["id"]),
+                        snapshot_display_url(snapshot["url"]),
+                    ]
+                    if snapshot_title:
+                        compact_snapshot.append(snapshot_title)
+                    active_snapshots_for_crawl.append(compact_snapshot)
+                    continue
+
+                snapshot_payload = {
+                    "id": str(snapshot["id"]),
+                    "url": snapshot_display_url(snapshot["url"]),
+                    "title": snapshot_title,
+                    "status": snapshot["status"],
+                    "worker_state": worker_state,
+                }
+                if snapshot["status"] != Snapshot.StatusChoices.QUEUED or all_plugins or snapshot_process_pids.get(str(snapshot["id"])):
+                    snapshot_payload.update(
+                        {
+                            "view_url": snapshot_view_url(snapshot),
+                            "started": (snapshot["downloaded_at"] or snapshot["created_at"]).isoformat()
+                            if (snapshot["downloaded_at"] or snapshot["created_at"])
+                            else None,
+                            "progress": snapshot_progress,
+                            "total_plugins": total_plugins,
+                            "completed_plugins": completed_plugins,
+                            "failed_plugins": failed_plugins,
+                            "pending_plugins": pending_plugins,
+                            "all_plugins": all_plugins,
+                        },
+                    )
+                    if snapshot_favicon_url:
+                        snapshot_payload["favicon_url"] = snapshot_favicon_url
+                    if snapshot_preview_url:
+                        snapshot_payload["preview_url"] = snapshot_preview_url
+                        snapshot_payload["preview_link"] = snapshot_preview_link
+                    if snapshot_screencast_url:
+                        snapshot_payload["screencast_url"] = snapshot_screencast_url
+                        snapshot_payload["screencast_link"] = snapshot_screencast_link
+                    if snapshot_fallback_urls:
+                        snapshot_payload["preview_fallbacks"] = snapshot_fallback_urls
+                    if snapshot_process_pids.get(str(snapshot["id"])):
+                        snapshot_payload["worker_pid"] = snapshot_process_pids[str(snapshot["id"])]
+
+                active_snapshots_for_crawl.append(snapshot_payload)
+
+            # Check if crawl can start (for debugging stuck crawls)
+            can_start = bool(crawl["urls"])
+            urls_preview = crawl["urls"][:60] if crawl["urls"] else None
+            crawl_tags = [tag.strip() for tag in (crawl["tags_str"] or "").replace("\n", ",").split(",") if tag.strip()]
+            persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
+            persona_name = persona_details["name"] if persona_details else str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default")
+            persona_details = persona_details or persona_details_by_name.get(persona_name)
+            crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
+            avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
+            crawl_obj = active_crawl_objects[crawl_id]
+            effective_crawl_config = get_config(crawl=crawl_obj, resolve_plugins=False)
+            max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0)
+            crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0)
+            crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0)
+            snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0)
+
+            # Check if retry_at is in the future (would prevent worker from claiming)
+            retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False
+            is_paused = crawl_obj.is_paused
+            seconds_until_retry = (
+                0 if is_paused else int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
+            )
+            crawl_worker_state = (
+                "running"
+                if crawl_process_pids.get(crawl_id)
+                or any(isinstance(snapshot, dict) and snapshot.get("worker_pid") for snapshot in active_snapshots_for_crawl)
+                else "waiting"
+            )
+            if is_paused:
+                crawl_worker_state = "paused"
+            elif (
+                crawl["status"] == Crawl.StatusChoices.STARTED
+                and crawl_worker_state == "waiting"
+                and (started_snapshots or pending_snapshots)
+            ):
+                crawl_worker_state = "waiting" if orchestrator_running else "crashed"
+
+            active_crawls.append(
+                {
+                    "id": crawl_id,
+                    "label": (next((line.strip() for line in (crawl["urls"] or "").splitlines() if line.strip()), "") or crawl_id)[:60],
+                    "status": crawl["status"],
+                    "is_paused": is_paused,
+                    "started": crawl["created_at"].isoformat() if crawl["created_at"] else None,
+                    "progress": crawl_progress,
+                    "created_by": crawl["created_by__username"],
+                    "persona": persona_name,
+                    "persona_admin_url": persona_details["admin_url"] if persona_details else None,
+                    "max_depth": crawl["max_depth"],
+                    "max_urls": max_urls,
+                    "max_crawl_size": crawl_max_size,
+                    "crawl_timeout": crawl_timeout,
+                    "max_snapshot_size": snapshot_max_size,
+                    "max_crawl_size_display": printable_filesize(crawl_max_size) if crawl_max_size else "unlimited",
+                    "crawl_timeout_display": f"{crawl_timeout}s" if crawl_timeout else "unlimited",
+                    "max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited",
+                    "crawl_output_size": crawl_output_size,
+                    "avg_snapshot_size": avg_snapshot_size,
+                    "crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B",
+                    "avg_snapshot_size_display": printable_filesize(avg_snapshot_size) if avg_snapshot_size else "0 B",
+                    "tags": crawl_tags,
+                    "urls_count": urls_count,
+                    "total_snapshots": total_snapshots,
+                    "completed_snapshots": completed_snapshots,
+                    "started_snapshots": started_snapshots,
+                    "failed_snapshots": 0,
+                    "pending_snapshots": pending_snapshots,
+                    "cancelled_snapshots": cancelled_snapshots,
+                    "setup_plugins": crawl_setup_plugins,
+                    "setup_total_plugins": crawl_setup_total,
+                    "setup_completed_plugins": crawl_setup_completed,
+                    "setup_failed_plugins": crawl_setup_failed,
+                    "setup_pending_plugins": crawl_setup_pending,
+                    "screencast_url": crawl_screencast_url,
+                    "screencast_link": crawl_screencast_link,
+                    "active_snapshots": active_snapshots_for_crawl,
+                    "queued_snapshots_hidden": queued_snapshot_overflow_by_crawl.get(crawl_id, 0),
+                    "can_start": can_start,
+                    "urls_preview": urls_preview,
+                    "retry_at_future": retry_at_future,
+                    "seconds_until_retry": seconds_until_retry,
+                    "worker_pid": crawl_process_pids.get(crawl_id),
+                    "worker_state": crawl_worker_state,
+                },
+            )
+
+        payload = {
+            "is_admin": is_admin,
+            "scope": {
+                "snapshot_id": str(scoped_snapshot.id) if scoped_snapshot is not None else "",
+                "crawl_id": crawl_id_filter,
+            },
+            "orchestrator_running": orchestrator_running,
+            "orchestrator_pid": orchestrator_pid,
+            "total_workers": total_workers,
+            "crawls_active": crawls_active,
+            "crawls_queued": crawls_queued,
+            "crawls_recent": crawls_recent,
+            "snapshots_active": snapshots_active,
+            "snapshots_queued": snapshots_queued,
+            "archiveresults_active": archiveresults_active,
+            "archiveresults_queued": archiveresults_queued,
+            "downloads_active": downloads_active,
+            "downloads_queued": downloads_queued,
+            "indexing_active": indexing_active,
+            "indexing_queued": indexing_queued,
+            "active_crawls": active_crawls,
+            "queued_crawls_hidden": queued_crawls_hidden,
+            "server_time": timezone.now().isoformat(),
+        }
+        try:
+            import ujson
+
+            return HttpResponse(ujson.dumps(payload), content_type="application/json")
+        except ImportError:
+            return JsonResponse(payload)
+    except Exception as e:
+        import traceback
+
+        return JsonResponse(
+            {
+                "error": str(e),
+                "traceback": traceback.format_exc(),
+                "orchestrator_running": False,
+                "total_workers": 0,
+                "crawls_active": 0,
+                "crawls_queued": 0,
+                "crawls_recent": 0,
+                "snapshots_active": 0,
+                "snapshots_queued": 0,
+                "archiveresults_active": 0,
+                "archiveresults_queued": 0,
+                "downloads_active": 0,
+                "downloads_queued": 0,
+                "indexing_active": 0,
+                "indexing_queued": 0,
+                "active_crawls": [],
+                "server_time": timezone.now().isoformat(),
+            },
+            status=500,
+        )

archivebox/search/__init__.py

@@ -1,353 +1 @@
-"""
-Search module for ArchiveBox.
-
-Search indexing is handled by search backend hooks in plugins:
-    abx_plugins/plugins/search_backend_*/on_Snapshot__*_index_*.py
-
-This module provides the query interface that dynamically discovers
-search backend plugins using the hooks system.
-
-Search backends must provide a search.py module with:
-    - search(query: str) -> List[str]  (returns snapshot IDs)
-    - flush(snapshot_ids: Iterable[str]) -> None
-"""
-
 __package__ = "archivebox.search"
-
-import os
-from contextlib import contextmanager
-from typing import Any
-
-from django.db.models import Case, IntegerField, Q, QuerySet, Value, When
-
-from archivebox.misc.util import enforce_types
-from archivebox.misc.logging import stderr
-from archivebox.config.common import get_config
-
-
-# Cache discovered backends to avoid repeated filesystem scans
-_search_backends_cache: dict | None = None
-SEARCH_MODES = ("meta", "contents", "deep")
-SEARCH_BACKEND_UI_NAMES = {
-    "rg": "ripgrep",
-    "sonic": "sonic",
-    "fts": "sqlite",
-}
-MAX_SEARCH_RANK_IDS = 500
-
-
-@contextmanager
-def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
-    """Expose ArchiveBox collection roots to in-process search backends."""
-    config = config or get_config(**config_kwargs)
-    updates = {}
-    for key, value in config.items():
-        if value is None:
-            continue
-        if isinstance(value, (str, int, float, bool, os.PathLike)):
-            updates[str(key)] = str(value)
-    updates["DATA_DIR"] = str(config.DATA_DIR)
-    updates["SNAP_DIR"] = str(config.USERS_DIR)
-    previous = {key: os.environ.get(key) for key in updates}
-    os.environ.update(updates)
-    try:
-        yield
-    finally:
-        for key, value in previous.items():
-            if value is None:
-                os.environ.pop(key, None)
-            else:
-                os.environ[key] = value
-
-
-def normalize_search_backend_name(backend_name: str | None) -> str:
-    return (backend_name or "").strip().lower().replace("-", "_")
-
-
-def get_search_backend_display_name(backend_name: str) -> str:
-    backend_name = normalize_search_backend_name(backend_name)
-    return next((ui_name for ui_name, canonical_name in SEARCH_BACKEND_UI_NAMES.items() if canonical_name == backend_name), backend_name)
-
-
-def get_default_search_mode(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
-    backends = get_available_backends()
-    if backend_name in backends:
-        return f"deep:{backend_name}"
-    if "ripgrep" in backends:
-        return "deep:ripgrep"
-    return "contents"
-
-
-def get_search_mode(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    normalized = (search_mode or "").strip().lower().replace(" ", "")
-    if normalized in SEARCH_MODES:
-        return normalized
-    if ":" in normalized:
-        mode, backend_name = normalized.split(":", 1)
-        backend_name = normalize_search_backend_name(backend_name)
-        if mode == "deep" and backend_name in get_available_backends():
-            return f"{mode}:{backend_name}"
-    return get_default_search_mode(config=config, **config_kwargs)
-
-
-def get_search_mode_base(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return get_search_mode(search_mode, config=config, **config_kwargs).split(":", 1)[0]
-
-
-def get_search_mode_backend(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str | None:
-    normalized = get_search_mode(search_mode, config=config, **config_kwargs)
-    if ":" not in normalized:
-        return None
-    return normalized.split(":", 1)[1]
-
-
-def get_search_mode_options(config: dict[str, Any] | None = None, **config_kwargs: Any) -> list[dict[str, str]]:
-    config = config or get_config(**config_kwargs)
-    backends = get_available_backends()
-    configured_backend = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
-    backend_names = [
-        *([configured_backend] if configured_backend in backends else []),
-        *(name for name in sorted(backends) if name != configured_backend),
-    ]
-    options = [
-        {"value": "meta", "label": "meta"},
-        {"value": "contents", "label": "contents"},
-    ]
-    if backend_names:
-        options.extend(
-            {
-                "value": f"deep:{backend_name}",
-                "label": f"deep: {get_search_backend_display_name(backend_name)}",
-            }
-            for backend_name in backend_names
-        )
-    else:
-        options.append({"value": "deep", "label": "deep"})
-    return options
-
-
-def prioritize_metadata_matches(
-    base_queryset: QuerySet,
-    metadata_queryset: QuerySet,
-    fulltext_queryset: QuerySet,
-    *,
-    deep_queryset: QuerySet | None = None,
-    ordering: list[str] | tuple[str, ...] | None = None,
-) -> QuerySet:
-    metadata_ids = list(metadata_queryset.values_list("pk", flat=True).distinct()[: MAX_SEARCH_RANK_IDS + 1])
-    metadata_id_set = set(metadata_ids)
-    fulltext_ids = [
-        pk for pk in fulltext_queryset.values_list("pk", flat=True).distinct()[: MAX_SEARCH_RANK_IDS + 1] if pk not in metadata_id_set
-    ]
-    fulltext_id_set = set(fulltext_ids)
-    deep_ids = []
-    if deep_queryset is not None:
-        deep_ids = [
-            pk
-            for pk in deep_queryset.values_list("pk", flat=True).distinct()[: MAX_SEARCH_RANK_IDS + 1]
-            if pk not in metadata_id_set and pk not in fulltext_id_set
-        ]
-
-    if not metadata_ids and not fulltext_ids and not deep_ids:
-        return base_queryset.none()
-
-    if any(len(ids) > MAX_SEARCH_RANK_IDS for ids in (metadata_ids, fulltext_ids, deep_ids)):
-        search_filter = Q()
-        if metadata_ids:
-            search_filter |= Q(pk__in=metadata_queryset.values("pk").distinct())
-        if fulltext_ids:
-            search_filter |= Q(pk__in=fulltext_queryset.values("pk").distinct())
-        if deep_queryset is not None and deep_ids:
-            search_filter |= Q(pk__in=deep_queryset.values("pk").distinct())
-        qs = base_queryset.filter(search_filter)
-        if ordering is not None:
-            qs = qs.order_by(*ordering)
-        return qs.distinct()
-
-    qs = base_queryset.filter(pk__in=[*metadata_ids, *fulltext_ids, *deep_ids]).annotate(
-        search_rank=Case(
-            When(pk__in=metadata_ids, then=Value(0)),
-            When(pk__in=fulltext_ids, then=Value(1)),
-            default=Value(2),
-            output_field=IntegerField(),
-        ),
-    )
-
-    if ordering is not None:
-        qs = qs.order_by("search_rank", *ordering)
-
-    return qs.distinct()
-
-
-def get_available_backends() -> dict:
-    """
-    Discover all available search backend plugins.
-
-    Uses the hooks system to find plugins with search.py modules.
-    Results are cached after first call.
-    """
-    global _search_backends_cache
-
-    if _search_backends_cache is None:
-        from archivebox.hooks import get_search_backends
-
-        _search_backends_cache = get_search_backends()
-
-    return _search_backends_cache
-
-
-def get_backend(config: dict[str, Any] | None = None, **config_kwargs: Any) -> Any:
-    """
-    Get the configured search backend module.
-
-    Discovers available backends via the hooks system and returns
-    the one matching SEARCH_BACKEND_ENGINE configuration.
-
-    Falls back to 'ripgrep' if configured backend is not found.
-    """
-    config = config or get_config(**config_kwargs)
-    backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
-    backends = get_available_backends()
-
-    if backend_name in backends:
-        return backends[backend_name]
-
-    # Fallback to ripgrep if available (no index needed)
-    if "ripgrep" in backends:
-        return backends["ripgrep"]
-
-    # No backends found
-    available = list(backends.keys())
-    raise RuntimeError(
-        f'Search backend "{backend_name}" not found. Available backends: {available or "none"}',
-    )
-
-
-@enforce_types
-def query_search_index(
-    query: str,
-    search_mode: str | None = None,
-    config: dict[str, Any] | None = None,
-    max_results: int | None = None,
-    **config_kwargs: Any,
-) -> QuerySet:
-    """
-    Search for snapshots matching the query.
-
-    Returns a QuerySet of Snapshot objects matching the search.
-    """
-    from archivebox.core.models import Snapshot
-
-    config = config or get_config(**config_kwargs)
-    search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config)
-    search_mode_base = get_search_mode_base(search_mode, config=config)
-    if search_mode_base == "meta":
-        return Snapshot.objects.none()
-
-    snapshot_pks = list(iter_query_search_ids(query, search_mode=search_mode, config=config, max_results=max_results))
-    return Snapshot.objects.filter(pk__in=list(dict.fromkeys(snapshot_pks)))
-
-
-def iter_query_search_ids(
-    query: str,
-    search_mode: str | None = None,
-    config: dict[str, Any] | None = None,
-    max_results: int | None = None,
-    **config_kwargs: Any,
-):
-    """Yield snapshot IDs from configured search backends as soon as each backend produces them."""
-    config = config or get_config(**config_kwargs)
-    search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config)
-    search_mode_base = get_search_mode_base(search_mode, config=config)
-    forced_backend = get_search_mode_backend(search_mode, config=config)
-    if search_mode_base == "meta":
-        return
-
-    backends = get_available_backends()
-    configured_backend = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
-    if forced_backend:
-        if forced_backend not in backends:
-            raise RuntimeError(
-                f'Search backend "{forced_backend}" not found. Available backends: {list(backends) or "none"}',
-            )
-        backend_names = [forced_backend]
-    elif search_mode_base == "deep":
-        backend_names = [
-            *([configured_backend] if configured_backend in backends and configured_backend != "ripgrep" else []),
-            *(name for name in backends if name not in {configured_backend, "ripgrep"}),
-            *(["ripgrep"] if "ripgrep" in backends else []),
-        ]
-    elif configured_backend in backends:
-        backend_names = [configured_backend]
-    elif "ripgrep" in backends:
-        backend_names = ["ripgrep"]
-    else:
-        get_backend()
-        return
-
-    if "sonic" in backend_names:
-        from archivebox.core.takeover_util import ensure_daemon_stack
-
-        ensure_daemon_stack(reason="search query")
-
-    errors: list[Exception] = []
-    successful_backends = 0
-    seen: set[str] = set()
-    try:
-        for backend_name in backend_names:
-            backend = backends[backend_name]
-            try:
-                with search_backend_env(config=config):
-                    if hasattr(backend, "iter_search"):
-                        ids = backend.iter_search(query, search_mode=search_mode_base)
-                    elif backend_name == "ripgrep":
-                        ids = backend.search(query, search_mode=search_mode_base)
-                    else:
-                        ids = backend.search(query)
-                    for snapshot_id in ids:
-                        if snapshot_id in seen:
-                            continue
-                        seen.add(snapshot_id)
-                        yield snapshot_id
-                        if max_results and len(seen) >= max_results:
-                            return
-                successful_backends += 1
-            except Exception as err:
-                errors.append(err)
-                if search_mode_base != "deep" or forced_backend:
-                    raise
-    except Exception as err:
-        stderr()
-        stderr(
-            f"[X] The search backend threw an exception={err}:",
-            color="red",
-        )
-        raise
-    else:
-        if not successful_backends and errors and search_mode_base == "deep":
-            raise errors[0]
-
-
-@enforce_types
-def flush_search_index(snapshots: QuerySet, config: dict[str, Any] | None = None, **config_kwargs: Any) -> None:
-    """
-    Remove snapshots from the search index.
-    """
-    config = config or get_config(**config_kwargs)
-    if not snapshots:
-        return
-
-    backend = get_backend(config=config)
-    snapshot_pks = [str(pk) for pk in snapshots.values_list("pk", flat=True)]
-
-    try:
-        with search_backend_env(config=config):
-            backend.flush(snapshot_pks)
-    except Exception as err:
-        stderr()
-        stderr(
-            f"[X] The search backend threw an exception={err}:",
-            color="red",
-        )

archivebox/search/admin.py

@@ -1,55 +1,31 @@
 __package__ = "archivebox.search"
 
-import hashlib
-import json
-
 from django.contrib import admin
 from django.contrib.admin.views.main import ChangeList
-from django.core.cache import cache
 
-from archivebox.search import (
-    get_search_backend_display_name,
+from archivebox.search.config import (
     get_default_search_mode,
     get_search_mode,
     get_search_mode_backend,
     get_search_mode_base,
     get_search_mode_options,
-    query_search_index,
 )
-
-
-SEARCH_RESULT_CACHE_TTL = 60
-
-
-def get_admin_search_cache_key(request, url: str | None = None) -> str:
-    # Search streams publish IDs for one exact changelist URL. Keeping the URL
-    # whole makes sidebar filters, ordering, and user scope part of the key.
-    payload = json.dumps(
-        {
-            "user": str(request.user.pk or "anon"),
-            "url": url or request.get_full_path(),
-        },
-        sort_keys=True,
-    )
-    return f"abx:admin-search:{hashlib.sha256(payload.encode()).hexdigest()}"
-
-
-def get_cached_admin_search_ids(request) -> list[str] | None:
-    cached = cache.get(get_admin_search_cache_key(request))
-    if isinstance(cached, dict):
-        return cached.get("ids") or []
-    return None
+from archivebox.search.query import query_search_index
+from archivebox.search.views import get_cached_admin_search_ids
 
 
 class SearchResultsChangeList(ChangeList):
+    """Django admin ChangeList with ArchiveBox search mode state."""
+
     def __init__(self, request, *args, **kwargs):
+        """Capture normalized search mode before Django builds results."""
         self.search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
         self.search_mode_backend = get_search_mode_backend(self.search_mode, config=getattr(request, "archivebox_config", None))
-        self.search_backend_label = get_search_backend_display_name(self.search_mode_backend) if self.search_mode_backend else ""
         super().__init__(request, *args, **kwargs)
         self.embedded_changelist = request.GET.get("_embedded") == "crawl"
 
     def get_results(self, request):
+        """Populate normal admin results plus search-index hint state."""
         super().get_results(request)
         self.show_search_index_hint = bool(
             self.opts.model_name == "snapshot"
@@ -60,6 +36,7 @@ class SearchResultsChangeList(ChangeList):
         )
 
     def get_filters_params(self, params=None):
+        """Remove UI-only search params before admin filter processing."""
         lookup_params = super().get_filters_params(params)
         lookup_params.pop("search_mode", None)
         lookup_params.pop("_embedded", None)
@@ -68,21 +45,26 @@ class SearchResultsChangeList(ChangeList):
 
 
 class SearchResultsAdminMixin(admin.ModelAdmin):
+    """Mixin that routes admin searches through ArchiveBox search modes."""
+
     show_search_mode_selector = True
 
     def get_changelist(self, request, **kwargs):
+        """Return the ArchiveBox search-aware ChangeList class."""
         return SearchResultsChangeList
 
     def get_default_search_mode(self):
+        """Return the default search mode for the current request config."""
         request = getattr(self, "request", None)
         return get_default_search_mode(config=getattr(request, "archivebox_config", None))
 
     def get_search_mode_options(self):
+        """Return selector options for the current request config."""
         request = getattr(self, "request", None)
         return get_search_mode_options(config=getattr(request, "archivebox_config", None))
 
     def get_search_results(self, request, queryset, search_term: str):
-        """Enhances the search queryset with results from the search backend"""
+        """Apply admin search semantics to a changelist queryset."""
 
         search_term = search_term.strip()
         if not search_term:

archivebox/search/apps.py

@@ -0,0 +1,11 @@
+__package__ = "archivebox.search"
+
+from django.apps import AppConfig
+
+
+class SearchConfig(AppConfig):
+    """Register search templates and admin integration with Django."""
+
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "archivebox.search"
+    verbose_name = "Search"

archivebox/search/backends.py

@@ -0,0 +1,69 @@
+__package__ = "archivebox.search"
+
+import os
+from contextlib import contextmanager
+from typing import Any
+
+from archivebox.config.common import get_config
+
+
+_search_backends_cache: dict | None = None
+
+
+@contextmanager
+def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
+    """Temporarily expose resolved config through os.environ for backend code."""
+    config = config or get_config(**config_kwargs)
+    updates = {}
+    for key, value in config.items():
+        if value is None:
+            continue
+        if isinstance(value, (str, int, float, bool, os.PathLike)):
+            updates[str(key)] = str(value)
+    updates["DATA_DIR"] = str(config.DATA_DIR)
+    updates["SNAP_DIR"] = str(config.USERS_DIR)
+    previous = {key: os.environ.get(key) for key in updates}
+    os.environ.update(updates)
+    try:
+        yield
+    finally:
+        for key, value in previous.items():
+            if value is None:
+                os.environ.pop(key, None)
+            else:
+                os.environ[key] = value
+
+
+def normalize_search_backend_name(backend_name: str | None) -> str:
+    """Normalize a backend name for config and plugin lookup."""
+    return (backend_name or "").strip().lower().replace("-", "_")
+
+
+def get_available_backends() -> dict:
+    """Discover search backend plugin modules and cache them in memory."""
+    global _search_backends_cache
+
+    if _search_backends_cache is None:
+        from archivebox.plugins.discovery import get_search_backends
+
+        _search_backends_cache = get_search_backends()
+
+    return _search_backends_cache
+
+
+def get_backend(config: dict[str, Any] | None = None, **config_kwargs: Any) -> Any:
+    """Resolve the configured search backend module."""
+    config = config or get_config(**config_kwargs)
+    backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
+    backends = get_available_backends()
+
+    if backend_name in backends:
+        return backends[backend_name]
+
+    if "ripgrep" in backends:
+        return backends["ripgrep"]
+
+    available = list(backends.keys())
+    raise RuntimeError(
+        f'Search backend "{backend_name}" not found. Available backends: {available or "none"}',
+    )

archivebox/search/config.py

@@ -0,0 +1,77 @@
+__package__ = "archivebox.search"
+
+from typing import Any
+
+from archivebox.config.common import get_config
+from archivebox.search.backends import get_available_backends, normalize_search_backend_name
+
+
+SEARCH_MODES = ("meta", "contents", "deep")
+
+
+def get_default_search_mode(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    """Choose the default search mode from config and discovered backends."""
+    config = config or get_config(**config_kwargs)
+    backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
+    backends = get_available_backends()
+    if backend_name in backends:
+        return f"deep:{backend_name}"
+    if "ripgrep" in backends:
+        return "deep:ripgrep"
+    return "contents"
+
+
+def get_search_mode(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    """Normalize a user-supplied search mode or fall back to the default."""
+    normalized = (search_mode or "").strip().lower().replace(" ", "")
+    if normalized == "content":
+        normalized = "contents"
+    if normalized in SEARCH_MODES:
+        return normalized
+    if ":" in normalized:
+        mode, backend_name = normalized.split(":", 1)
+        backend_name = normalize_search_backend_name(backend_name)
+        if mode == "content":
+            mode = "contents"
+        if mode in {"contents", "deep"} and backend_name in get_available_backends():
+            return f"{mode}:{backend_name}"
+    return get_default_search_mode(config=config, **config_kwargs)
+
+
+def get_search_mode_base(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    """Return the mode portion of a normalized search mode."""
+    return get_search_mode(search_mode, config=config, **config_kwargs).split(":", 1)[0]
+
+
+def get_search_mode_backend(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str | None:
+    """Return the backend portion of a backend-qualified search mode."""
+    normalized = get_search_mode(search_mode, config=config, **config_kwargs)
+    if ":" not in normalized:
+        return None
+    return normalized.split(":", 1)[1]
+
+
+def get_search_mode_options(config: dict[str, Any] | None = None, **config_kwargs: Any) -> list[dict[str, str]]:
+    """Build search mode choices for admin and public selectors."""
+    config = config or get_config(**config_kwargs)
+    backends = get_available_backends()
+    configured_backend = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
+    backend_names = [
+        *([configured_backend] if configured_backend in backends else []),
+        *(name for name in sorted(backends) if name != configured_backend),
+    ]
+    options = [
+        {"value": "meta", "label": "meta"},
+        {"value": "contents", "label": "contents"},
+    ]
+    if backend_names:
+        options.extend(
+            {
+                "value": f"deep:{backend_name}",
+                "label": f"deep:{backend_name}",
+            }
+            for backend_name in backend_names
+        )
+    else:
+        options.append({"value": "deep", "label": "deep"})
+    return options

archivebox/search/query.py

@@ -0,0 +1,299 @@
+__package__ = "archivebox.search"
+
+from typing import Any
+
+from django.db import connection
+from django.db.models import Case, IntegerField, Q, QuerySet, Value, When
+
+from archivebox.config.common import get_config
+from archivebox.misc.logging import stderr
+from archivebox.misc.util import enforce_types
+from archivebox.search.backends import get_available_backends, get_backend, normalize_search_backend_name, search_backend_env
+from archivebox.search.config import get_search_mode, get_search_mode_backend, get_search_mode_base
+
+
+MAX_SEARCH_RANK_IDS = 500
+
+
+def escape_like_query(query: str) -> str:
+    """Escape a string for SQLite LIKE matching."""
+    return query.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+
+
+def crawl_config_values_search_wave(query: str) -> Q | None:
+    """Build a Snapshot Q predicate matching values inside Crawl.config."""
+    if connection.vendor != "sqlite":
+        return None
+
+    from archivebox.crawls.models import Crawl
+
+    pattern = f"%{escape_like_query(query).lower()}%"
+    matching_crawls = Crawl.objects.extra(
+        where=[
+            """
+            EXISTS (
+                SELECT 1
+                FROM json_tree(config)
+                WHERE json_tree.atom IS NOT NULL
+                  AND LOWER(CAST(json_tree.atom AS TEXT)) LIKE %s ESCAPE '\\'
+            )
+            """,
+        ],
+        params=[pattern],
+    )
+    return Q(crawl_id__in=matching_crawls.values("pk"))
+
+
+def snapshot_metadata_search_waves(query: str, *, include_id_matches: bool = False) -> list[Q]:
+    """Build ordered metadata predicates for Snapshot search."""
+    waves = []
+    if include_id_matches:
+        waves.append(Q(id__istartswith=query) | Q(id__iendswith=query))
+
+    waves.extend(
+        [
+            Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query),
+            Q(tags__name__icontains=query),
+            Q(notes__icontains=query) | Q(crawl__notes__icontains=query) | Q(crawl__label__icontains=query),
+            Q(crawl__created_by__username=query),
+        ],
+    )
+
+    config_wave = crawl_config_values_search_wave(query)
+    if config_wave is not None:
+        waves.append(config_wave)
+    return waves
+
+
+def prioritize_metadata_matches(
+    base_queryset: QuerySet,
+    metadata_queryset: QuerySet,
+    fulltext_queryset: QuerySet,
+    *,
+    deep_queryset: QuerySet | None = None,
+    ordering: list[str] | tuple[str, ...] | None = None,
+) -> QuerySet:
+    """Rank metadata hits before backend full-text hits."""
+    metadata_ids = list(metadata_queryset.values_list("pk", flat=True).distinct()[: MAX_SEARCH_RANK_IDS + 1])
+    metadata_id_set = set(metadata_ids)
+    fulltext_ids = [
+        pk for pk in fulltext_queryset.values_list("pk", flat=True).distinct()[: MAX_SEARCH_RANK_IDS + 1] if pk not in metadata_id_set
+    ]
+    fulltext_id_set = set(fulltext_ids)
+    deep_ids = []
+    if deep_queryset is not None:
+        deep_ids = [
+            pk
+            for pk in deep_queryset.values_list("pk", flat=True).distinct()[: MAX_SEARCH_RANK_IDS + 1]
+            if pk not in metadata_id_set and pk not in fulltext_id_set
+        ]
+
+    if not metadata_ids and not fulltext_ids and not deep_ids:
+        return base_queryset.none()
+
+    if any(len(ids) > MAX_SEARCH_RANK_IDS for ids in (metadata_ids, fulltext_ids, deep_ids)):
+        search_filter = Q()
+        if metadata_ids:
+            search_filter |= Q(pk__in=metadata_queryset.values("pk").distinct())
+        if fulltext_ids:
+            search_filter |= Q(pk__in=fulltext_queryset.values("pk").distinct())
+        if deep_queryset is not None and deep_ids:
+            search_filter |= Q(pk__in=deep_queryset.values("pk").distinct())
+        qs = base_queryset.filter(search_filter)
+        if ordering is not None:
+            qs = qs.order_by(*ordering)
+        return qs.distinct()
+
+    qs = base_queryset.filter(pk__in=[*metadata_ids, *fulltext_ids, *deep_ids]).annotate(
+        search_rank=Case(
+            When(pk__in=metadata_ids, then=Value(0)),
+            When(pk__in=fulltext_ids, then=Value(1)),
+            default=Value(2),
+            output_field=IntegerField(),
+        ),
+    )
+
+    if ordering is not None:
+        qs = qs.order_by("search_rank", *ordering)
+
+    return qs.distinct()
+
+
+def apply_snapshot_search(
+    base_queryset: QuerySet,
+    query: str,
+    *,
+    search_mode: str | None = None,
+    config: dict[str, Any] | None = None,
+    ordering: list[str] | tuple[str, ...] | None = None,
+    max_results: int | None = None,
+    skip_backend_when_metadata_satisfies_limit: bool = False,
+    include_metadata_for_forced_backend: bool = False,
+    include_id_matches: bool = False,
+) -> QuerySet:
+    """Apply shared CLI/API/public/admin Snapshot search semantics."""
+    query = (query or "").strip()
+    if not query:
+        return base_queryset
+
+    config = config or get_config()
+    search_mode = get_search_mode(search_mode, config=config)
+    search_mode_base = get_search_mode_base(search_mode, config=config)
+    search_mode_backend = get_search_mode_backend(search_mode, config=config)
+    metadata_filter = Q()
+    for wave in snapshot_metadata_search_waves(query, include_id_matches=include_id_matches):
+        metadata_filter |= wave
+    metadata_queryset = base_queryset.filter(metadata_filter)
+
+    if search_mode_base == "meta":
+        return metadata_queryset.distinct()
+
+    if skip_backend_when_metadata_satisfies_limit and max_results:
+        metadata_ids = list(metadata_queryset.values_list("pk", flat=True).distinct()[:max_results])
+        if len(metadata_ids) >= max_results:
+            return metadata_queryset.distinct()
+
+    if search_mode_base == "deep":
+        fulltext_search_mode = f"contents:{search_mode_backend}" if search_mode_backend else "contents"
+        fulltext_queryset = query_search_index(query, search_mode=fulltext_search_mode, config=config, max_results=max_results)
+        deep_queryset = query_search_index(query, search_mode=search_mode, config=config, max_results=max_results)
+        return prioritize_metadata_matches(
+            base_queryset,
+            metadata_queryset,
+            fulltext_queryset,
+            deep_queryset=deep_queryset,
+            ordering=ordering,
+        )
+
+    backend_queryset = query_search_index(query, search_mode=search_mode, config=config, max_results=max_results)
+    if search_mode_backend and not include_metadata_for_forced_backend:
+        return base_queryset.filter(pk__in=backend_queryset.values("pk")).distinct()
+
+    return prioritize_metadata_matches(
+        base_queryset,
+        metadata_queryset,
+        backend_queryset,
+        ordering=ordering,
+    )
+
+
+@enforce_types
+def query_search_index(
+    query: str,
+    search_mode: str | None = None,
+    config: dict[str, Any] | None = None,
+    max_results: int | None = None,
+    **config_kwargs: Any,
+) -> QuerySet:
+    """Return a Snapshot queryset from backend search IDs."""
+    from archivebox.core.models import Snapshot
+
+    config = config or get_config(**config_kwargs)
+    search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config)
+    search_mode_base = get_search_mode_base(search_mode, config=config)
+    if search_mode_base == "meta":
+        return Snapshot.objects.none()
+
+    snapshot_pks = list(iter_query_search_ids(query, search_mode=search_mode, config=config, max_results=max_results))
+    return Snapshot.objects.filter(pk__in=list(dict.fromkeys(snapshot_pks)))
+
+
+def iter_query_search_ids(
+    query: str,
+    search_mode: str | None = None,
+    config: dict[str, Any] | None = None,
+    max_results: int | None = None,
+    **config_kwargs: Any,
+):
+    """Yield snapshot IDs from configured search backend modules."""
+    config = config or get_config(**config_kwargs)
+    search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config)
+    search_mode_base = get_search_mode_base(search_mode, config=config)
+    forced_backend = get_search_mode_backend(search_mode, config=config)
+    if search_mode_base == "meta":
+        return
+
+    backends = get_available_backends()
+    configured_backend = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
+    if forced_backend:
+        if forced_backend not in backends:
+            raise RuntimeError(
+                f'Search backend "{forced_backend}" not found. Available backends: {list(backends) or "none"}',
+            )
+        backend_names = [forced_backend]
+    elif search_mode_base == "deep":
+        backend_names = [
+            *([configured_backend] if configured_backend in backends and configured_backend != "ripgrep" else []),
+            *(name for name in backends if name not in {configured_backend, "ripgrep"}),
+            *(["ripgrep"] if "ripgrep" in backends else []),
+        ]
+    elif configured_backend in backends:
+        backend_names = [configured_backend]
+    elif "ripgrep" in backends:
+        backend_names = ["ripgrep"]
+    else:
+        get_backend()
+        return
+
+    if "sonic" in backend_names:
+        from archivebox.core.takeover_util import ensure_daemon_stack
+
+        ensure_daemon_stack(reason="search query")
+
+    errors: list[Exception] = []
+    successful_backends = 0
+    seen: set[str] = set()
+    try:
+        for backend_name in backend_names:
+            backend = backends[backend_name]
+            try:
+                with search_backend_env(config=config):
+                    if hasattr(backend, "iter_search"):
+                        ids = backend.iter_search(query, search_mode=search_mode_base)
+                    elif backend_name == "ripgrep":
+                        ids = backend.search(query, search_mode=search_mode_base)
+                    else:
+                        ids = backend.search(query)
+                    for snapshot_id in ids:
+                        if snapshot_id in seen:
+                            continue
+                        seen.add(snapshot_id)
+                        yield snapshot_id
+                        if max_results and len(seen) >= max_results:
+                            return
+                successful_backends += 1
+            except Exception as err:
+                errors.append(err)
+                if search_mode_base != "deep" or forced_backend:
+                    raise
+    except Exception as err:
+        stderr()
+        stderr(
+            f"[X] The search backend threw an exception={err}:",
+            color="red",
+        )
+        raise
+    else:
+        if not successful_backends and errors and search_mode_base == "deep":
+            raise errors[0]
+
+
+@enforce_types
+def flush_search_index(snapshots: QuerySet, config: dict[str, Any] | None = None, **config_kwargs: Any) -> None:
+    """Remove Snapshot IDs from the configured search backend index."""
+    config = config or get_config(**config_kwargs)
+    if not snapshots:
+        return
+
+    backend = get_backend(config=config)
+    snapshot_pks = [str(pk) for pk in snapshots.values_list("pk", flat=True)]
+
+    try:
+        with search_backend_env(config=config):
+            backend.flush(snapshot_pks)
+    except Exception as err:
+        stderr()
+        stderr(
+            f"[X] The search backend threw an exception={err}:",
+            color="red",
+        )

archivebox/search/views.py

@@ -0,0 +1,202 @@
+__package__ = "archivebox.search"
+
+import asyncio
+import hashlib
+import json
+import threading
+from copy import copy
+from queue import Full, Queue
+from urllib.parse import urlsplit
+from uuid import UUID
+
+from django.core.cache import cache
+from django.db.models import Q
+from django.http import QueryDict, StreamingHttpResponse
+
+from archivebox.search.config import get_search_mode, get_search_mode_base
+from archivebox.search.query import crawl_config_values_search_wave, iter_query_search_ids
+
+
+SEARCH_RESULT_CACHE_TTL = 60
+
+
+def get_admin_search_cache_key(request, url: str | None = None) -> str:
+    """Build the cache key for one user and changelist URL."""
+    # Search streams publish IDs for one exact changelist URL. Keeping the URL
+    # whole makes sidebar filters, ordering, and user scope part of the key.
+    payload = json.dumps(
+        {
+            "user": str(request.user.pk or "anon"),
+            "url": url or request.get_full_path(),
+        },
+        sort_keys=True,
+    )
+    return f"abx:admin-search:{hashlib.sha256(payload.encode()).hexdigest()}"
+
+
+def get_cached_admin_search_ids(request) -> list[str] | None:
+    """Return streamed admin search IDs from Django cache."""
+    cached = cache.get(get_admin_search_cache_key(request))
+    if isinstance(cached, dict):
+        return cached.get("ids") or []
+    return None
+
+
+def iter_admin_meta_search_ids(query, queryset):
+    """Yield metadata search matches from a filtered Snapshot queryset."""
+    seen = set()
+    try:
+        snapshot_id = UUID(query)
+    except ValueError:
+        snapshot_id = None
+    if snapshot_id:
+        for pk in queryset.filter(pk=snapshot_id).values_list("pk", flat=True):
+            seen.add(pk)
+            yield pk
+
+    waves = [
+        Q(timestamp__startswith=query) | Q(url__istartswith=query) | Q(title__istartswith=query),
+        Q(url__icontains=query),
+        Q(title__icontains=query),
+        Q(tags__name__icontains=query),
+        Q(notes__icontains=query) | Q(crawl__notes__icontains=query) | Q(crawl__label__icontains=query),
+        Q(crawl__created_by__username=query),
+    ]
+    config_wave = crawl_config_values_search_wave(query)
+    if config_wave is not None:
+        waves.append(config_wave)
+
+    for wave in waves:
+        for pk in queryset.filter(wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
+            if pk in seen:
+                continue
+            seen.add(pk)
+            yield pk
+
+
+def iter_admin_backend_search_ids(iterator, queryset):
+    """Yield backend search IDs that still match the filtered queryset."""
+    batch = []
+    seen = set()
+
+    def flush_batch():
+        valid = {str(pk) for pk in queryset.filter(pk__in=batch).values_list("pk", flat=True)}
+        for snapshot_id in batch:
+            if snapshot_id in valid and snapshot_id not in seen:
+                seen.add(snapshot_id)
+                yield snapshot_id
+
+    for snapshot_id in iterator:
+        snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
+        if len(snapshot_id) != 32:
+            continue
+        batch.append(snapshot_id)
+        if len(batch) >= 200:
+            yield from flush_batch()
+            batch = []
+    if batch:
+        yield from flush_batch()
+
+
+def admin_snapshot_search_stream_view(model_admin, request):
+    """Stream admin Snapshot search progress and cache matching IDs."""
+    query = (request.GET.get("q") or "").strip()
+    search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
+    if not query:
+        return StreamingHttpResponse((), content_type="text/plain")
+
+    search_url = request.GET.get("search_url") or request.get_full_path()
+    target_url = urlsplit(search_url)
+    target_get = QueryDict(target_url.query, mutable=True)
+    for key in ("q", "search_mode", "p", "search_url"):
+        target_get.pop(key, None)
+
+    filter_request = copy(request)
+    filter_request.path = target_url.path or request.path
+    filter_request.path_info = target_url.path or request.path_info
+    filter_request.GET = target_get
+    filter_request.archivebox_config = getattr(request, "archivebox_config", None)
+
+    # Build the same filtered base queryset the changelist uses, but with the
+    # search params stripped. The stream intersects each wave with this queryset
+    # before writing IDs into the short-lived cache consumed by the changelist.
+    current_request = getattr(model_admin, "request", None)
+    try:
+        base_queryset = model_admin.get_changelist_instance(filter_request).queryset
+    finally:
+        model_admin.request = current_request
+
+    async def snapshot_ids():
+        seen = set()
+        ids = []
+        last_sent = 0
+        stream_batch_size = 100
+        stream_padding = " " * 4096
+        cache_key = get_admin_search_cache_key(request, search_url)
+        cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
+        yield f"0{stream_padding}\n"
+        queue = Queue(maxsize=8)
+        stop_event = threading.Event()
+
+        def emit(item):
+            while not stop_event.is_set():
+                try:
+                    queue.put(item, timeout=0.1)
+                    return
+                except Full:
+                    continue
+
+        def run_search():
+            nonlocal last_sent
+            iterator = None
+            try:
+                search_mode_base = get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None))
+                iterator = (
+                    iter_admin_meta_search_ids(query, base_queryset)
+                    if search_mode_base == "meta"
+                    else iter_admin_backend_search_ids(
+                        iter_query_search_ids(query, search_mode=search_mode, config=getattr(request, "archivebox_config", None)),
+                        base_queryset,
+                    )
+                )
+                for snapshot_id in iterator:
+                    if stop_event.is_set():
+                        break
+                    snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
+                    if len(snapshot_id) != 32 or snapshot_id in seen:
+                        continue
+                    seen.add(snapshot_id)
+                    ids.append(snapshot_id)
+                    if len(ids) - last_sent >= stream_batch_size:
+                        cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
+                        last_sent = len(ids)
+                        emit(f"{last_sent}{stream_padding}\n")
+                if not stop_event.is_set() and len(ids) != last_sent:
+                    cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
+                    emit(f"{len(ids)}{stream_padding}\n")
+            except BaseException as err:
+                emit(err)
+            finally:
+                if iterator is not None:
+                    try:
+                        iterator.close()
+                    except AttributeError:
+                        pass
+                cache.set(cache_key, {"ids": ids, "done": True}, SEARCH_RESULT_CACHE_TTL)
+                emit(None)
+
+        threading.Thread(target=run_search, name="admin-snapshot-search-stream", daemon=True).start()
+        try:
+            while True:
+                item = await asyncio.to_thread(queue.get)
+                if item is None:
+                    break
+                if isinstance(item, BaseException):
+                    raise item
+                yield item
+        finally:
+            stop_event.set()
+
+    response = StreamingHttpResponse(snapshot_ids(), content_type="text/plain")
+    response["X-Accel-Buffering"] = "no"
+    return response

archivebox/services/__init__.py

@@ -1,5 +1,5 @@
 from .archive_result_service import ArchiveResultService
-from .binary_service import BinaryService
+from .binary_service import ArchiveBoxBinaryCacheBackend
 from .crawl_service import CrawlService
 from .machine_service import MachineService
 from .process_service import ProcessService
@@ -9,7 +9,7 @@ from .tag_service import TagService
 
 __all__ = [
     "ArchiveResultService",
-    "BinaryService",
+    "ArchiveBoxBinaryCacheBackend",
     "CrawlService",
     "MachineService",
     "ProcessService",

archivebox/services/binary_service.py

@@ -1,38 +1,51 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import Any
 
 from asgiref.sync import sync_to_async
 
-from abx_dl.events import BinaryRequestEvent, BinaryEvent
-from abx_dl.services.base import BaseService
+from abxpkg import Binary as AbxBinary
+from abxpkg import BinProvider, PROVIDER_CLASS_BY_NAME
+from abxpkg.binary_service import BinaryRequestEvent
 
 
-class BinaryService(BaseService):
-    LISTENS_TO = [BinaryRequestEvent, BinaryEvent]
-    EMITS = []
+_LIB_DIR_MANAGED_PROVIDERS = {
+    "bash",
+    "cargo",
+    "deno",
+    "gem",
+    "goget",
+    "nix",
+    "npm",
+    "pip",
+    "puppeteer",
+    "uv",
+}
 
-    def __init__(self, bus):
-        super().__init__(bus)
-        self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent)
-        self.bus.on(BinaryEvent, self.on_BinaryEvent)
 
-    async def on_BinaryRequestEvent(self, event: BinaryRequestEvent) -> str | None:
+class ArchiveBoxBinaryCacheBackend:
+    """ArchiveBox machine.Binary projection backend for abxpkg BinaryCacheService."""
+
+    async def get(self, request: BinaryRequestEvent) -> AbxBinary | None:
+        from archivebox.config.common import get_config
         from archivebox.machine.models import Binary, Machine, _canonical_binary_name
 
         machine = await sync_to_async(Machine.current, thread_sensitive=True)()
-        binary_name = _canonical_binary_name(event.name)
+        binary_name = _canonical_binary_name(request.name)
         if not binary_name:
             return None
+
         existing = await Binary.objects.filter(machine=machine, name=binary_name).afirst()
         cache_invalidated = False
         if existing and existing.status == Binary.StatusChoices.INSTALLED:
             changed = False
-            if event.binproviders and existing.binproviders != event.binproviders:
-                existing.binproviders = event.binproviders
+            requested_binproviders = _binproviders_to_str(request.binproviders)
+            if requested_binproviders and existing.binproviders != requested_binproviders:
+                existing.binproviders = requested_binproviders
                 changed = True
-            if event.overrides and existing.overrides != event.overrides:
-                existing.overrides = event.overrides
+            if request.overrides and existing.overrides != request.overrides:
+                existing.overrides = request.overrides
                 changed = True
             if changed:
                 existing.status = Binary.StatusChoices.QUEUED
@@ -43,8 +56,8 @@ class BinaryService(BaseService):
             await Binary.objects.acreate(
                 machine=machine,
                 name=binary_name,
-                binproviders=event.binproviders,
-                overrides=event.overrides or {},
+                binproviders=_binproviders_to_str(request.binproviders),
+                overrides=request.overrides or {},
                 status=Binary.StatusChoices.QUEUED,
             )
 
@@ -62,108 +75,141 @@ class BinaryService(BaseService):
                 installed.retry_at = None
                 await installed.asave(update_fields=["status", "retry_at", "modified_at"])
                 installed = None
-            if installed is not None and event.overrides and installed.overrides != event.overrides:
+            if installed is not None and request.overrides and installed.overrides != request.overrides:
                 installed.status = Binary.StatusChoices.QUEUED
                 installed.retry_at = None
                 await installed.asave(update_fields=["status", "retry_at", "modified_at"])
                 installed = None
-        cached = None
-        if installed is not None:
-            from archivebox.config.common import get_config
-            from abxpkg import BinProvider, PROVIDER_CLASS_BY_NAME
+        if installed is None:
+            return None
 
-            binary_env: dict[str, str] = {}
-            installed_path = Path(installed.abspath).expanduser().resolve(strict=False)
-            active_lib_dir = (
-                Path(str((await sync_to_async(get_config, thread_sensitive=True)()).get("LIB_DIR", "")))
-                .expanduser()
-                .resolve(
-                    strict=False,
-                )
-            )
-            provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip()
-            if active_lib_dir and provider_name in {"npm", "pip", "puppeteer", "uv", "deno", "gem", "cargo", "goget", "nix", "bash"}:
-                try:
-                    installed_path.relative_to(active_lib_dir)
-                except ValueError:
-                    installed.status = Binary.StatusChoices.QUEUED
-                    installed.retry_at = None
-                    await installed.asave(update_fields=["status", "retry_at", "modified_at"])
-                    installed = None
-            if installed is None:
+        installed_path = Path(installed.abspath).expanduser().resolve(strict=False)
+        active_lib_dir = (
+            Path(str((await sync_to_async(get_config, thread_sensitive=True)()).get("LIB_DIR", ""))).expanduser().resolve(strict=False)
+        )
+        provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip()
+        if active_lib_dir and provider_name in _LIB_DIR_MANAGED_PROVIDERS:
+            try:
+                installed_path.relative_to(active_lib_dir)
+            except ValueError:
+                installed.status = Binary.StatusChoices.QUEUED
+                installed.retry_at = None
+                await installed.asave(update_fields=["status", "retry_at", "modified_at"])
                 return None
-            provider_class = PROVIDER_CLASS_BY_NAME.get(provider_name)
-            if provider_class is not None:
-                provider = provider_class()
-                overrides = installed.overrides if isinstance(installed.overrides, dict) else {}
-                provider_overrides = overrides.get(provider_name)
-                if isinstance(provider_overrides, dict):
-                    provider = provider.get_provider_with_overrides(
-                        overrides={installed.name: provider_overrides},
-                    )
-                binary_env = BinProvider.build_exec_env(
-                    providers=[provider],
-                    base_env={},
-                )
-            cached = {
-                "abspath": installed.abspath,
-                "version": installed.version or "",
-                "sha256": installed.sha256 or "",
-                "binproviders": installed.binproviders or "",
-                "binprovider": installed.binprovider or "",
-                "machine_id": str(installed.machine_id),
-                "overrides": installed.overrides or {},
+
+        provider = _provider_for_name(provider_name, installed.name, installed.overrides)
+        binary_env = BinProvider.build_exec_env(providers=[provider], base_env={}) if provider is not None else {}
+        provider_names = _provider_names(installed.binproviders or request.binproviders or "env")
+        return AbxBinary.model_validate(
+            {
+                "name": request.name,
+                "description": request.description,
+                "binproviders": _providers_for_names(provider_names),
+                "overrides": installed.overrides or request.overrides or {},
+                "loaded_binprovider": provider,
+                "loaded_abspath": installed.abspath,
+                "loaded_version": installed.version or None,
+                "loaded_sha256": installed.sha256 or None,
                 "env": binary_env,
-            }
-        if cached is not None:
-            binary_event = BinaryEvent(
-                name=event.name,
-                plugin_name=event.plugin_name,
-                hook_name=event.hook_name,
-                abspath=cached["abspath"],
-                version=cached["version"],
-                sha256=cached["sha256"],
-                binproviders=event.binproviders or cached["binproviders"],
-                binprovider=cached["binprovider"],
-                overrides=event.overrides or cached["overrides"],
-                env=cached["env"],
-                binary_id=event.binary_id,
-                machine_id=cached["machine_id"],
-            )
-            await event.emit(binary_event).now()
-            return binary_event.abspath
-        return None
-
-    async def on_BinaryEvent(self, event: BinaryEvent) -> None:
-        from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-        from archivebox.config.common import get_config
-
-        machine = await sync_to_async(Machine.current, thread_sensitive=True)()
-        binary_name = _canonical_binary_name(event.name)
-        if not binary_name:
-            return
-        binary, _ = await Binary.objects.aget_or_create(
-            machine=machine,
-            name=binary_name,
-            defaults={
-                "status": Binary.StatusChoices.QUEUED,
             },
         )
-        binary.abspath = event.abspath
-        if event.version:
-            binary.version = event.version
-        if event.sha256:
-            binary.sha256 = event.sha256
-        if event.binproviders:
-            binary.binproviders = event.binproviders
-        if event.binprovider:
-            binary.binprovider = event.binprovider
-        if event.overrides and binary.overrides != event.overrides:
-            binary.overrides = event.overrides
-        binary.status = Binary.StatusChoices.INSTALLED
-        binary.retry_at = None
-        await binary.asave(
+
+    async def set(self, request: BinaryRequestEvent | None, binary: AbxBinary) -> None:
+        from archivebox.config.common import get_config
+        from archivebox.machine.models import Binary, Machine, _canonical_binary_name
+
+        machine = await sync_to_async(Machine.current, thread_sensitive=True)()
+        binary_name = _canonical_binary_name(binary.name)
+        if not binary_name:
+            return
+        request_context = request.extra_context if request is not None else {}
+        binary_id = str(request_context.get("binary_id") or "")
+        if binary_id:
+            existing = await Binary.objects.filter(id=binary_id).afirst()
+        else:
+            existing = None
+        if existing is None:
+            existing, _created = await Binary.objects.aget_or_create(
+                machine=machine,
+                name=binary_name,
+                defaults={"status": Binary.StatusChoices.QUEUED},
+            )
+
+        existing.abspath = str(binary.loaded_abspath or "")
+        if binary.loaded_version:
+            existing.version = str(binary.loaded_version)
+        if binary.loaded_sha256:
+            existing.sha256 = str(binary.loaded_sha256)
+        existing.binproviders = _binproviders_to_str(
+            request.binproviders if request is not None else [provider.name for provider in binary.binproviders],
+        )
+        if binary.loaded_binprovider is not None:
+            existing.binprovider = binary.loaded_binprovider.name
+        existing.overrides = request.overrides if request is not None and request.overrides is not None else binary.overrides
+        existing.status = Binary.StatusChoices.INSTALLED
+        existing.retry_at = None
+        await existing.asave(
             update_fields=["abspath", "version", "sha256", "binproviders", "binprovider", "overrides", "status", "retry_at", "modified_at"],
         )
         lib_bin_dir = await sync_to_async(lambda: get_config().LIB_BIN_DIR, thread_sensitive=True)()
-        await sync_to_async(binary.symlink_to_lib_bin_after_commit, thread_sensitive=True)(lib_bin_dir)
+        await sync_to_async(existing.symlink_to_lib_bin_after_commit, thread_sensitive=True)(lib_bin_dir)
+
+    async def invalidate(self, request: BinaryRequestEvent, binary: AbxBinary, reason: str) -> None:
+        from archivebox.machine.models import Binary, Machine, _canonical_binary_name
+
+        machine = await sync_to_async(Machine.current, thread_sensitive=True)()
+        binary_name = _canonical_binary_name(request.name)
+        if not binary_name:
+            return
+        installed = (
+            await Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED)
+            .exclude(abspath="")
+            .exclude(abspath__isnull=True)
+            .order_by("-modified_at")
+            .afirst()
+        )
+        if installed is None:
+            return
+        installed.status = Binary.StatusChoices.QUEUED
+        installed.retry_at = None
+        await installed.asave(update_fields=["status", "retry_at", "modified_at"])
+
+
+def _provider_names(binproviders: str | list[str] | None) -> list[str]:
+    if isinstance(binproviders, str):
+        raw_names = [part.strip() for part in binproviders.split(",")]
+    elif binproviders:
+        raw_names = [str(part).strip() for part in binproviders]
+    else:
+        raw_names = ["env"]
+    names: list[str] = []
+    for name in raw_names:
+        if name and name not in names:
+            names.append(name)
+    return names or ["env"]
+
+
+def _binproviders_to_str(binproviders: str | list[str] | None) -> str:
+    return ",".join(_provider_names(binproviders))
+
+
+def _providers_for_names(names: list[str]) -> list[BinProvider]:
+    providers: list[BinProvider] = []
+    for name in names:
+        provider_class = PROVIDER_CLASS_BY_NAME.get(name)
+        if provider_class is not None:
+            providers.append(provider_class())
+    return providers
+
+
+def _provider_for_name(provider_name: str, binary_name: str, overrides: dict[str, Any] | None) -> BinProvider | None:
+    provider_class = PROVIDER_CLASS_BY_NAME.get(provider_name)
+    if provider_class is None:
+        return None
+    provider = provider_class()
+    provider_overrides = overrides.get(provider_name) if isinstance(overrides, dict) else None
+    if isinstance(provider_overrides, dict):
+        provider = provider.get_provider_with_overrides(
+            overrides={binary_name: provider_overrides},
+        )
+    return provider

archivebox/services/crawl_service.py

@@ -27,7 +27,7 @@ class CrawlService(BaseService):
         await (
             Crawl.objects.filter(id=self.crawl_id)
             .exclude(
-                status__in=[Crawl.StatusChoices.PAUSED, Crawl.StatusChoices.SEALED],
+                status__in=Crawl.INACTIVE_STATES,
             )
             .aupdate(
                 status=Crawl.StatusChoices.STARTED,
@@ -42,7 +42,7 @@ class CrawlService(BaseService):
         await (
             Crawl.objects.filter(id=self.crawl_id)
             .exclude(
-                status__in=[Crawl.StatusChoices.PAUSED, Crawl.StatusChoices.SEALED],
+                status__in=Crawl.INACTIVE_STATES,
             )
             .aupdate(
                 status=Crawl.StatusChoices.STARTED,
@@ -61,7 +61,7 @@ class CrawlService(BaseService):
         await (
             Crawl.objects.filter(id=self.crawl_id)
             .exclude(
-                status__in=[Crawl.StatusChoices.PAUSED, Crawl.StatusChoices.SEALED],
+                status__in=Crawl.INACTIVE_STATES,
             )
             .aupdate(
                 status=Crawl.StatusChoices.STARTED,
@@ -77,14 +77,12 @@ class CrawlService(BaseService):
         crawl = await Crawl.objects.aget(id=self.crawl_id)
         if crawl.is_paused or crawl.status == Crawl.StatusChoices.SEALED:
             return
-        is_finished = not await crawl.snapshot_set.filter(
-            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED],
-        ).aexists()
+        is_finished = not await crawl.snapshot_set.filter(status__in=Snapshot.OPEN_STATES).aexists()
         if not is_finished:
             await (
                 Crawl.objects.filter(id=self.crawl_id)
                 .exclude(
-                    status__in=[Crawl.StatusChoices.PAUSED, Crawl.StatusChoices.SEALED],
+                    status__in=Crawl.INACTIVE_STATES,
                 )
                 .aupdate(
                     status=Crawl.StatusChoices.STARTED,
@@ -97,7 +95,7 @@ class CrawlService(BaseService):
         await (
             Crawl.objects.filter(id=self.crawl_id)
             .exclude(
-                status__in=[Crawl.StatusChoices.PAUSED, Crawl.StatusChoices.SEALED],
+                status__in=Crawl.INACTIVE_STATES,
             )
             .aupdate(
                 status=Crawl.StatusChoices.SEALED,

archivebox/services/process_service.py

@@ -146,11 +146,14 @@ class ProcessService(BaseService):
         self._ensure_completed_worker()
         await self._completed_queue.put(event)
 
-    async def on_CrawlCleanupEvent__flush_completed(self, event: CrawlCleanupEvent) -> None:
+    async def flush_completed(self) -> None:
         await self._completed_queue.join()
 
+    async def on_CrawlCleanupEvent__flush_completed(self, event: CrawlCleanupEvent) -> None:
+        await self.flush_completed()
+
     async def on_CrawlCompletedEvent__flush_completed(self, event: CrawlCompletedEvent) -> None:
-        await self._completed_queue.join()
+        await self.flush_completed()
 
     async def _save_completed_process_to_db(self, event: ProcessCompletedEvent) -> None:
         from archivebox.machine.models import Process

archivebox/services/runner.py

@@ -23,8 +23,8 @@ from django.utils import timezone
 from rich.console import Console
 from rich.text import Text
 
+from abxpkg.binary_service import BinaryCacheService, BinaryRequestEvent, BinaryService
 from abx_dl.events import (
-    BinaryRequestEvent,
     CrawlAbortEvent,
     CrawlCleanupEvent,
     CrawlCompletedEvent,
@@ -51,14 +51,14 @@ from abx_dl.orchestrator import (
     setup_services as setup_abx_services,
 )
 from abx_dl.services.process_service import ProcessService as HookProcessService
-from abx_dl.services.binary_service import BinaryService as HookBinaryService
+from abx_dl.services.binary_service import PluginBinariesService as HookPluginBinariesService
 from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService
 from abx_dl.cli import LiveBusUI
 from abxbus import BaseEvent
 from abxbus.event_bus import EventBus, get_current_event, in_handler_context
 from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError
 
-from archivebox.config.configset import BaseConfigSet
+from archivebox.config.common import ArchiveBoxBaseConfig
 from archivebox.core.recovery_util import recover_orchestrator_state
 from archivebox.misc.db import run_db_analyze_batch
 from archivebox.core.shutdown_util import foreground_shutdown_signals
@@ -66,7 +66,7 @@ from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
 from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS
 
 from .archive_result_service import ArchiveResultService
-from .binary_service import BinaryService
+from .binary_service import ArchiveBoxBinaryCacheBackend
 from .crawl_service import CrawlService
 from .machine_service import MachineService
 from .process_service import ProcessService as PersistedProcessService
@@ -122,16 +122,12 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str
     return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
 
 
-def _normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | None) -> dict[str, Any]:
-    if config is None:
-        return {}
-    if isinstance(config, BaseConfigSet):
-        config = config.model_dump(mode="json")
-    elif isinstance(config, str):
-        config = json.loads(config)
-    else:
-        config = dict(config)
-    return {key: value for key, value in json.loads(json.dumps(config, default=str)).items() if value is not None}
+def _normalize_runtime_config(config: ArchiveBoxBaseConfig | Mapping[str, Any] | str | None) -> dict[str, Any]:
+    from archivebox.config.common import normalize_runtime_config
+
+    if isinstance(config, ArchiveBoxBaseConfig):
+        return config.for_crawl_execution()
+    return normalize_runtime_config(config)
 
 
 def _runner_task_context() -> contextvars.Context:
@@ -146,6 +142,11 @@ def _is_external_task_cancelled(error: asyncio.CancelledError) -> bool:
     return not isinstance(error, (EventHandlerAbortedError, EventHandlerCancelledError))
 
 
+def _register_binary_services(bus) -> None:
+    BinaryCacheService(bus, backend=ArchiveBoxBinaryCacheBackend())
+    BinaryService(bus)
+
+
 async def _emit_machine_config(
     bus,
     *,
@@ -160,18 +161,16 @@ async def _emit_machine_config(
         config_type="user",
     )
     if parent_event is not None:
-        await parent_event.emit(user_event).now()
-    else:
-        await bus.emit(user_event).now()
+        user_event.event_parent_id = parent_event.event_id
+    await bus.emit(user_event).now()
     if derived_machine_config:
         derived_event = MachineEvent(
             config=derived_machine_config,
             config_type="derived",
         )
         if parent_event is not None:
-            await parent_event.emit(derived_event).now()
-        else:
-            await bus.emit(derived_event).now()
+            derived_event.event_parent_id = parent_event.event_id
+        await bus.emit(derived_event).now()
 
 
 async def _run_event_now(event, timeout: float | None = None):
@@ -238,7 +237,7 @@ class CrawlRunner:
         HookProcessService(self.bus, emit_jsonl=False, interactive_tty=interactive_interrupts)
         register_sonic_daemon_event_handler(self.bus)
         PersistedProcessService(self.bus)
-        BinaryService(self.bus)
+        _register_binary_services(self.bus)
         TagService(self.bus)
         CrawlService(self.bus, crawl_id=str(crawl.id))
         MachineService(self.bus)
@@ -411,7 +410,7 @@ class CrawlRunner:
             return
         current_event = crawl_start_event or get_current_event()
         if isinstance(current_event, CrawlStartEvent):
-            task = asyncio.create_task(self.run_snapshot(snapshot_id, current_event))
+            task = asyncio.create_task(self.run_snapshot(snapshot_id, current_event), context=_runner_task_context())
         elif in_handler_context():
             return
         else:
@@ -553,7 +552,7 @@ class CrawlRunner:
             return
         pending_snapshot_ids = await sync_to_async(
             lambda: list(
-                self.crawl.snapshot_set.filter(status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED])
+                self.crawl.snapshot_set.filter(status__in=Snapshot.RUNNABLE_STATES)
                 .exclude(id__in=active_snapshot_ids)
                 .filter(retry_at__lte=timezone.now())
                 .order_by("depth", "created_at")
@@ -568,7 +567,7 @@ class CrawlRunner:
     def load_run_state(self) -> list[str]:
         from archivebox.config.common import get_config
         from archivebox.core.models import Snapshot
-        from archivebox.hooks import discover_hooks
+        from archivebox.plugins.hooks import discover_hooks
         from archivebox.machine.models import Machine, NetworkInterface, Process, _sanitize_machine_config
 
         self.primary_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else ""
@@ -608,7 +607,7 @@ class CrawlRunner:
         if self.crawl.is_paused:
             return []
         pending_snapshots = list(
-            self.crawl.snapshot_set.filter(status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED])
+            self.crawl.snapshot_set.filter(status__in=Snapshot.RUNNABLE_STATES)
             .filter(retry_at__lte=timezone.now())
             .order_by("depth", "created_at"),
         )
@@ -742,7 +741,7 @@ class CrawlRunner:
     async def enqueue_discovered_snapshots_from_outputs(self, snapshot_payload: dict[str, Any]) -> None:
         from archivebox.core.models import Snapshot
         from archivebox.config.common import get_config
-        from archivebox.hooks import collect_urls_from_plugins
+        from archivebox.plugins.hooks import collect_urls_from_plugins
 
         await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)()
         if self.crawl.is_paused and not self.allow_maintenance_on_inactive_crawl:
@@ -843,7 +842,9 @@ class CrawlRunner:
             emit_jsonl=False,
             abort_requested=self.crawl_is_cancelled,
             MachineService=None,
-            BinaryService=HookBinaryService,
+            PluginBinariesService=HookPluginBinariesService,
+            BinaryCacheService=None,
+            BinaryService=None,
             ProcessService=None,
             ArchiveResultService=None,
             TagService=None,
@@ -1055,7 +1056,8 @@ class CrawlRunner:
                     event_timeout=snapshot_phase_timeout,
                     event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout),
                 )
-                emitted_snapshot_event = crawl_start_event.emit(snapshot_event)
+                snapshot_event.event_parent_id = crawl_start_event.event_id
+                emitted_snapshot_event = self.bus.emit(snapshot_event)
                 await _run_event_now(emitted_snapshot_event, snapshot_phase_timeout)
                 completed_snapshot = await self.bus.find(
                     SnapshotCompletedEvent,
@@ -1081,11 +1083,7 @@ class CrawlRunner:
                         self.crawl.sm.seal()
                         if self.crawl.status == self.crawl.StatusChoices.STARTED
                         and not self.crawl.snapshot_set.filter(
-                            status__in=[
-                                self.crawl.snapshot_set.model.StatusChoices.QUEUED,
-                                self.crawl.snapshot_set.model.StatusChoices.STARTED,
-                                self.crawl.snapshot_set.model.StatusChoices.PAUSED,
-                            ],
+                            status__in=self.crawl.snapshot_set.model.OPEN_STATES,
                         ).exists()
                         else None
                     ),
@@ -1169,8 +1167,8 @@ async def _run_binary(binary_id: str) -> None:
     config["ABX_RUNTIME"] = "archivebox"
     config = _normalize_runtime_config(config)
     bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0)
-    PersistedProcessService(bus)
-    BinaryService(bus)
+    process_service = PersistedProcessService(bus)
+    _register_binary_services(bus)
     TagService(bus)
     ArchiveResultService(bus)
     MachineService(bus)
@@ -1185,6 +1183,8 @@ async def _run_binary(binary_id: str) -> None:
         persist_derived=False,
         auto_install=True,
         emit_jsonl=False,
+        BinaryCacheService=None,
+        BinaryService=None,
     )
     await _emit_machine_config(bus, config=config, derived_config=derived_config)
 
@@ -1192,17 +1192,20 @@ async def _run_binary(binary_id: str) -> None:
         await bus.emit(
             BinaryRequestEvent(
                 name=binary.name,
-                plugin_name="archivebox",
-                hook_name="on_BinaryRequest__archivebox_run",
-                output_dir=str(binary.output_dir),
-                binary_id=str(binary.id),
-                machine_id=str(binary.machine_id),
                 binproviders=binary.binproviders,
                 overrides=binary.overrides or None,
+                extra_context={
+                    "plugin_name": "archivebox",
+                    "hook_name": "on_BinaryRequest__archivebox_run",
+                    "output_dir": str(binary.output_dir),
+                    "binary_id": str(binary.id),
+                    "machine_id": str(binary.machine_id),
+                },
             ),
         ).now(first_result=True)
     finally:
         await bus.wait_until_idle()
+        await process_service.flush_completed()
 
 
 def run_binary(binary_id: str) -> None:
@@ -1291,7 +1294,7 @@ def run_due_crawl(crawl, *, lock_seconds: int, interactive_interrupts: bool = Fa
         now = timezone.now()
         snapshot_count = crawl.snapshot_set.count()
         due_active_snapshots = crawl.snapshot_set.filter(
-            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
+            status__in=Snapshot.RUNNABLE_STATES,
             retry_at__lte=now,
         ).exists()
         if snapshot_count and due_active_snapshots:
@@ -1334,11 +1337,7 @@ def run_due_crawl(crawl, *, lock_seconds: int, interactive_interrupts: bool = Fa
 
             next_snapshot_retry = (
                 crawl.snapshot_set.filter(
-                    status__in=[
-                        Snapshot.StatusChoices.QUEUED,
-                        Snapshot.StatusChoices.STARTED,
-                        Snapshot.StatusChoices.PAUSED,
-                    ],
+                    status__in=Snapshot.OPEN_STATES,
                     retry_at__gt=now,
                 )
                 .order_by("retry_at", "created_at")
@@ -1433,13 +1432,15 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
             return False
         snapshot.refresh_from_db()
         snapshot.finalize_completed_upload_results()
+        maintenance_ran = False
         if snapshot.fs_migration_needed:
-            # Final snapshots can still need maintenance after an old data-dir
-            # migration. Run the filesystem/json save path before queued search
-            # backfill rows so both maintenance streams stay ordered without
-            # changing Snapshot.status away from SEALED.
-            _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
-            return run_snapshot_maintenance(str(snapshot.id))
+            # Final snapshots can still need filesystem/json maintenance after
+            # a data-dir migration, but queued ArchiveResult rows are the actual
+            # runnable work. Do the metadata rewrite first, then continue into
+            # the targeted plugin path in the same tick so large migrations do
+            # not starve search/index backfills behind a full maintenance pass.
+            maintenance_ran = run_snapshot_maintenance(str(snapshot.id))
+            snapshot.refresh_from_db()
         selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
         if selected_plugins:
             _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
@@ -1451,6 +1452,8 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
                 interactive_interrupts=interactive_interrupts,
             )
             return True
+        if maintenance_ran:
+            return True
         return run_snapshot_maintenance(str(snapshot.id))
 
     if snapshot.status == Snapshot.StatusChoices.STARTED:
@@ -1521,7 +1524,7 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
     config = _normalize_runtime_config(config)
     bus = create_bus(name="ArchiveBox_install", total_timeout=3600.0)
     PersistedProcessService(bus)
-    BinaryService(bus)
+    _register_binary_services(bus)
     TagService(bus)
     ArchiveResultService(bus)
     MachineService(bus)
@@ -1594,6 +1597,8 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
                         emit_jsonl=False,
                         bus=bus,
                         MachineService=None,
+                        BinaryCacheService=None,
+                        BinaryService=None,
                     )
                 finally:
                     try:
@@ -1730,7 +1735,7 @@ def run_pending_crawls(
     from archivebox.config.common import get_config
     from archivebox.crawls.models import Crawl, CrawlSchedule
     from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.hooks import discover_plugin_configs
+    from archivebox.plugins.discovery import discover_plugin_configs
     from archivebox.machine.models import Process
 
     crawl_claim_lock_seconds = 10
@@ -1796,8 +1801,8 @@ def run_pending_crawls(
         if not maintenance_only:
             active_snapshots = Snapshot.objects.filter(
                 retry_at__lte=timezone.now(),
-                crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED],
-                status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
+                crawl__status__in=Crawl.RUNNABLE_STATES,
+                status__in=Snapshot.RUNNABLE_STATES,
             )
             if crawl_id:
                 active_snapshots = active_snapshots.filter(crawl_id=crawl_id)
@@ -1850,10 +1855,7 @@ def run_pending_crawls(
             pausing_snapshots = Snapshot.objects.filter(
                 retry_at__lte=timezone.now(),
                 crawl__status=Crawl.StatusChoices.PAUSED,
-                status__in=[
-                    Snapshot.StatusChoices.QUEUED,
-                    Snapshot.StatusChoices.STARTED,
-                ],
+                status__in=Snapshot.RUNNABLE_STATES,
             )
             if crawl_id:
                 pausing_snapshots = pausing_snapshots.filter(crawl_id=crawl_id)

archivebox/templates/admin/actions.html

@@ -17,22 +17,21 @@
       {% block actions-counter %}
       {% if actions_selection_counter %}
         <span class="action-summary" data-page-count="{{ cl.result_list|length }}" data-result-count="{{ cl.result_count }}" data-full-result-count="{{ cl.full_result_count|default:cl.result_count }}">
-          <span class="action-selected-count">0 / {{ cl.result_list|length|intcomma }} selected</span>
+          <span class="action-selected-count">
+            <span class="action-selected-current">0</span>
+            /
+            <a class="action-page-select" href="#" title="{% translate "Select all rows on this page" %}">{{ cl.result_list|length|intcomma }}</a>
+            selected
+          </span>
           <span class="action-counter hidden" data-actions-icnt="{{ cl.result_list|length }}" style="display: none !important;" aria-hidden="true">{{ selection_note }}</span>
           {% if cl.opts.model_name == 'snapshot' %}
           <span class="action-total-count">
             {% if cl.full_result_count and cl.full_result_count != cl.result_count %}
-              <span class="question hidden">
-                  <a role="button" href="#" title="{% translate "Select all matching rows across all pages" %}">{{ cl.result_count|intcomma }}</a>
-              </span>
               <a class="action-match-count-static action-total-select" href="#" title="{% translate "Select all matching rows across all pages" %}">{{ cl.result_count|intcomma }}</a>
               /
-              <a href="?" title="{% translate "Show all rows" %}">{{ cl.full_result_count|intcomma }}</a>
+              <a class="action-total-reset" href="?" title="{% translate "Show all rows" %}">{{ cl.full_result_count|intcomma }}</a>
               total
             {% else %}
-              <span class="question hidden">
-                  <a role="button" href="#" title="{% translate "Select all matching rows across all pages" %}">{{ cl.result_count|intcomma }}</a>
-              </span>
               <a class="action-total-reset" href="?" title="{% translate "Show all rows" %}">{{ cl.result_count|intcomma }}</a>
               total
             {% endif %}
@@ -40,15 +39,11 @@
           {% endif %}
           {% if cl.result_count != cl.result_list|length %}
           <span class="all hidden">{{ selection_note_all }}</span>
-          {% if cl.opts.model_name != 'snapshot' %}
           <span class="question hidden">
               <a role="button" href="#" title="{% translate "Click here to select the objects across all pages" %}">{% blocktranslate with cl.result_count as total_count %}Select all {{ total_count }} {{ module_name }}{% endblocktranslate %}</a>
           </span>
-          {% endif %}
-          {% if cl.opts.model_name != 'snapshot' %}
           <span class="clear hidden"><a role="button" href="#">{% translate "Clear selection" %}</a></span>
           {% endif %}
-          {% endif %}
         </span>
       {% endif %}
       {% endblock %}

archivebox/templates/admin/base.html

@@ -1180,6 +1180,10 @@
                 margin-left: auto;
                 flex: 0 0 auto;
             }
+            body.change-list #changelist .actions .action-buttons[hidden],
+            body.change-list #changelist .actions .actions-tags-with-buttons[hidden] {
+                display: none !important;
+            }
 
             .actions label {
                 display: inline-flex;
@@ -1247,7 +1251,8 @@
                 display: inline-block;
             }
 
-            .action-rearchive-select {
+            .action-rearchive-select,
+            .action-permissions-select {
                 position: absolute;
                 left: 0;
                 bottom: 0;
@@ -1701,7 +1706,7 @@
                 </div>
 
                 {% if has_permission %}
-                    {% include 'admin/progress_monitor.html' %}
+                    {% include 'progressmonitor/progress_monitor.html' %}
                 {% endif %}
 
                 {% block breadcrumbs %}
@@ -1920,10 +1925,14 @@
                         })
                         return
                     }
-                    if (action_type === 'set_crawl_permissions') {
+                    if (action_type === 'set_crawl_permissions' || action_type === 'set_snapshot_permissions') {
                         const wrapper = $('<span></span>')
                             .addClass('action-permissions-wrapper')
                             .appendTo(buttons)
+                        const select = $('<select></select>')
+                            .addClass('action-permissions-select')
+                            .attr('aria-label', 'Permissions')
+                            .appendTo(wrapper)
                         $('<button>')
                             .attr('type', 'button')
                             .attr('name', action_type)
@@ -1932,41 +1941,40 @@
                             .click(function (e) {
                                 e.preventDefault()
                                 e.stopPropagation()
-                                wrapper.toggleClass('is-open')
-                                wrapper.find('.action-permissions-menu').prop('hidden', !wrapper.hasClass('is-open'))
+                                const el = select[0]
+                                if (typeof el.showPicker === 'function') {
+                                    try { el.showPicker() } catch (err) { el.focus(); el.click() }
+                                } else {
+                                    el.focus()
+                                    el.click()
+                                }
                                 return false
                             })
                             .appendTo(wrapper)
-                        const menu = $('<span></span>')
-                            .addClass('action-permissions-menu')
-                            .attr('hidden', true)
-                            .appendTo(wrapper)
+                        $('<option></option>').attr('value', '').attr('hidden', true).appendTo(select)
                         ;[
                             ['public', '👥', 'Public'],
                             ['unlisted', '🔗', 'Unlisted'],
                             ['private', '🔒', 'Private'],
                         ].forEach(function (choice) {
-                            $('<button>')
-                                .attr('type', 'button')
-                                .addClass('action-permissions-menu-item')
-                                .attr('data-permissions', choice[0])
-                                .html('<span aria-hidden="true">' + choice[1] + '</span><span>' + choice[2] + '</span>')
-                                .click(function (e) {
-                                    e.preventDefault()
-                                    e.stopPropagation()
-                                    let permissionsInput = container.find('input[name=permissions]')
-                                    if (!permissionsInput.length) {
-                                        permissionsInput = $('<input type="hidden" name="permissions">').appendTo(container)
-                                    }
-                                    permissionsInput.val(choice[0])
-                                    container.find('select[name=action]')
-                                        .val(action_type)
-                                        .trigger('change')
-                                    $('#changelist-form button[name="index"]').click()
-                                    document.querySelector('#logo').outerHTML = '<div class="loader"></div>'
-                                    return false
-                                })
-                                .appendTo(menu)
+                            $('<option></option>')
+                                .attr('value', choice[0])
+                                .text(choice[1] + ' ' + choice[2])
+                                .appendTo(select)
+                        })
+                        select.change(function () {
+                            const permissions = this.value
+                            if (!permissions) return
+                            let permissionsInput = container.find('input[name=permissions]')
+                            if (!permissionsInput.length) {
+                                permissionsInput = $('<input type="hidden" name="permissions">').appendTo(container)
+                            }
+                            permissionsInput.val(permissions)
+                            container.find('select[name=action]')
+                                .val(action_type)
+                                .trigger('change')
+                            $('#changelist-form button[name="index"]').click()
+                            document.querySelector('#logo').outerHTML = '<div class="loader"></div>'
                         })
                         return
                     }
@@ -2016,15 +2024,18 @@
                         .appendTo(buttons)
                 })
                 console.log('Converted', buttons.children().length, 'admin actions from dropdown to buttons')
-                const crawlAddLink = document.querySelector('.model-crawl.change-list .object-tools a.addlink')
-                const crawlActionButtons = document.querySelector('.model-crawl.change-list .actions-top .action-buttons')
-                if (crawlAddLink && crawlActionButtons && !crawlActionButtons.querySelector('.crawl-add-action')) {
-                    const addButton = crawlAddLink.cloneNode(true)
-                    addButton.classList.add('button', 'crawl-add-action')
-                    addButton.classList.remove('addlink')
-                    addButton.textContent = 'Add Crawl'
-                    crawlActionButtons.appendChild(addButton)
-                }
+                ;['.model-crawl.change-list', '.model-snapshot.change-list'].forEach(function(selector) {
+                    const scope = document.querySelector(selector)
+                    const addLink = scope?.querySelector('.object-tools a.addlink')
+                    const actionButtons = scope?.querySelector('.actions-top .action-buttons')
+                    if (addLink && actionButtons && !actionButtons.querySelector('.admin-add-action')) {
+                        const addButton = addLink.cloneNode(true)
+                        addButton.classList.add('button', 'admin-add-action')
+                        addButton.classList.remove('addlink')
+                        addButton.textContent = '➕ Add'
+                        actionButtons.appendChild(addButton)
+                    }
+                })
                 const tagContainer = document.querySelector('.actions-tags')
                 if (tagContainer) {
                     const tagButtons = buttons.find('button[name="add_tags"], button[name="remove_tags"]')
@@ -2033,32 +2044,103 @@
                         tagButtons.appendTo(tagContainer)
                     }
                 }
-                document.addEventListener('click', function(event) {
-                    document.querySelectorAll('.action-permissions-wrapper.is-open').forEach(function(wrapper) {
-                        if (!wrapper.contains(event.target)) {
-                            wrapper.classList.remove('is-open')
-                            const menu = wrapper.querySelector('.action-permissions-menu')
-                            if (menu) menu.hidden = true
-                        }
-                    })
-                })
                 if (window.jQuery && window.jQuery.fn.select2) {
                     window.jQuery('select[multiple]').select2();
                 }
+                updateActionControlVisibility()
             }
-            function updateTagWidgetVisibility() {
-                const tagContainer = document.querySelector('.actions-tags');
-                if (!tagContainer) return;
-                const checked = document.querySelectorAll('#changelist-form input.action-select:checked').length;
-                tagContainer.style.display = tagContainer.classList.contains('actions-tags-with-buttons') || checked > 0 ? 'inline-flex' : 'none';
+            function updateActionControlVisibility() {
+                const selectAcross = document.querySelector('div.actions input.select-across')?.value === '1'
+                const checked = selectAcross
+                    ? Number(document.querySelector('.action-summary')?.dataset.resultCount || 0)
+                    : document.querySelectorAll('#changelist-form input.action-select:checked').length
+                document.querySelectorAll('.action-buttons, .actions-tags-with-buttons').forEach(function(el) {
+                    el.hidden = checked === 0
+                })
             }
             function setupActionSummary() {
                 const summary = document.querySelector('.action-summary')
                 if (!summary || summary.dataset.summaryReady) return
                 summary.dataset.summaryReady = '1'
                 const selectedCount = summary.querySelector('.action-selected-count')
+                const selectedCurrent = summary.querySelector('.action-selected-current')
+                const pageSelect = summary.querySelector('.action-page-select')
                 const counter = summary.querySelector('.action-counter')
                 const formatter = new Intl.NumberFormat()
+                const selectAllParam = '_archivebox_select_all'
+                const getSelectedCount = function() {
+                    const selectAcross = document.querySelector('div.actions input.select-across')?.value === '1'
+                    if (selectAcross) {
+                        return Number(summary.dataset.resultCount || 0)
+                    }
+                    const match = counter?.textContent.match(/(\d+)\s+of\s+(\d+)\s+selected/)
+                    return match ? Number(match[1]) : document.querySelectorAll('#changelist-form input.action-select:checked').length
+                }
+                const selectCurrentResultsWithDjango = function() {
+                    const allToggle = document.getElementById('action-toggle')
+                    const questionLink = summary.querySelector('.question a')
+                    const selectAcross = document.querySelector('div.actions input.select-across')
+                    const pageCount = Number(summary.dataset.pageCount || 0)
+                    const resultCount = Number(summary.dataset.resultCount || pageCount)
+
+                    if (allToggle) {
+                        if (!allToggle.checked) {
+                            allToggle.click()
+                        }
+                        if (questionLink && resultCount > pageCount) {
+                            questionLink.click()
+                        }
+                        return true
+                    }
+
+                    document.querySelectorAll('#changelist-form input.action-select').forEach(function(checkbox) {
+                        if (!checkbox.checked) {
+                            checkbox.checked = true
+                            checkbox.dispatchEvent(new Event('change', { bubbles: true }))
+                        }
+                        const card = checkbox.closest('.card')
+                        if (card) {
+                            card.classList.add('selected-card')
+                        }
+                    })
+                    if (selectAcross && resultCount > pageCount) {
+                        selectAcross.value = '1'
+                    }
+                    return false
+                }
+                const selectCurrentPageWithDjango = function() {
+                    const allToggle = document.getElementById('action-toggle')
+
+                    if (allToggle) {
+                        if (!allToggle.checked) {
+                            allToggle.click()
+                        }
+                        return true
+                    }
+
+                    document.querySelectorAll('#changelist-form input.action-select').forEach(function(checkbox) {
+                        if (!checkbox.checked) {
+                            checkbox.checked = true
+                            checkbox.dispatchEvent(new Event('change', { bubbles: true }))
+                        }
+                        const card = checkbox.closest('.card')
+                        if (card) {
+                            card.classList.add('selected-card')
+                        }
+                    })
+                    return false
+                }
+                const selectAllRowsWithDjango = function() {
+                    const resultCount = Number(summary.dataset.resultCount || 0)
+                    const fullResultCount = Number(summary.dataset.fullResultCount || resultCount)
+                    if (fullResultCount > resultCount) {
+                        const url = new URL(window.location.href)
+                        url.search = '?' + selectAllParam + '=1'
+                        window.location.href = url.toString()
+                        return
+                    }
+                    selectCurrentResultsWithDjango()
+                }
                 const update = function() {
                     if (!selectedCount || !counter) return
                     const match = counter.textContent.match(/(\d+)\s+of\s+(\d+)\s+selected/)
@@ -2068,36 +2150,49 @@
                         : (match ? Number(match[1]) : document.querySelectorAll('#changelist-form input.action-select:checked').length)
                     const pageCount = match ? Number(match[2]) : Number(summary.dataset.pageCount || 0)
                     const selectedLimit = selectAcross ? Number(summary.dataset.resultCount || pageCount) : pageCount
-                    selectedCount.textContent = formatter.format(selected) + ' / ' + formatter.format(selectedLimit) + ' selected'
+                    if (selectedCurrent && pageSelect) {
+                        selectedCurrent.textContent = formatter.format(selected)
+                        pageSelect.textContent = formatter.format(selectedLimit)
+                    } else {
+                        selectedCount.textContent = formatter.format(selected) + ' / ' + formatter.format(selectedLimit) + ' selected'
+                    }
                     summary.classList.toggle('action-summary-has-selection', selected > 0)
                     summary.classList.toggle('action-summary-select-across', selectAcross)
                     const totalCount = summary.querySelector('.action-total-count')
                     if (totalCount) {
-                        totalCount.hidden = selectAcross
+                        totalCount.hidden = false
                     }
+                    updateActionControlVisibility()
                 }
                 new MutationObserver(update).observe(counter, { childList: true, characterData: true, subtree: true })
                 document.querySelector('#changelist-form')?.addEventListener('change', function() {
                     window.setTimeout(update, 0)
                 })
                 summary.addEventListener('click', function(event) {
+                    const pageSelect = event.target.closest('.action-page-select')
                     const explicitSelectAll = event.target.closest('.action-total-select, .action-total-count .question a')
-                    const selectedTotalClick = event.target.closest('.action-total-reset') && summary.classList.contains('action-summary-has-selection')
-                    if (explicitSelectAll || selectedTotalClick) {
-                        const questionLink = summary.querySelector('.action-total-count .question a')
-                        const allToggle = document.getElementById('action-toggle')
-                        const pageCount = Number(summary.dataset.pageCount || 0)
-                        const resultCount = Number(summary.dataset.resultCount || pageCount)
+                    const totalReset = event.target.closest('.action-total-reset')
+                    if (pageSelect || explicitSelectAll || (totalReset && getSelectedCount() === 0)) {
                         event.preventDefault()
-                        if (allToggle && !allToggle.checked) {
-                            allToggle.click()
-                        }
-                        if (questionLink && resultCount > pageCount) {
-                            questionLink.click()
+                        if (pageSelect) {
+                            selectCurrentPageWithDjango()
+                        } else if (totalReset) {
+                            selectAllRowsWithDjango()
+                        } else {
+                            selectCurrentResultsWithDjango()
                         }
                     }
                     window.setTimeout(update, 0)
                 })
+                if (new URLSearchParams(window.location.search).get(selectAllParam) === '1') {
+                    const cleanUrl = new URL(window.location.href)
+                    cleanUrl.searchParams.delete(selectAllParam)
+                    window.history.replaceState(null, '', cleanUrl.toString())
+                    window.setTimeout(function() {
+                        selectAllRowsWithDjango()
+                        update()
+                    }, 0)
+                }
                 update()
             }
             function setupSearchModeSelect() {
@@ -2230,7 +2325,7 @@
                 const form = document.querySelector('#changelist-form')
                 if (form && !form.dataset.archiveboxActionsReady) {
                     form.dataset.archiveboxActionsReady = '1'
-                    form.addEventListener('change', updateTagWidgetVisibility)
+                    form.addEventListener('change', updateActionControlVisibility)
                 }
             }
             function fixInlineAddRow() {
@@ -2358,7 +2453,7 @@
                     fixInlineAddRow()
                     setupSnapshotGridListToggle()
                 }
-                updateTagWidgetVisibility()
+                updateActionControlVisibility()
                 setupActionSummary()
                 setupSearchModeSelect()
                 setupEmbeddedChangelistSearch()

archivebox/templates/admin/change_list_panel.html

@@ -2,7 +2,7 @@
 
 {% if cl.model_admin.show_search_mode_selector %}
   {% with current_search_mode=cl.params.search_mode|default:cl.model_admin.get_default_search_mode %}
-    <div class="module{% if cl.has_filters %} filtered{% endif %}{% if current_search_mode == 'contents' %} search-mode-contents{% elif current_search_mode == 'deep' %} search-mode-deep{% endif %}" id="changelist">
+    <div class="module{% if cl.has_filters %} filtered{% endif %}{% if current_search_mode == 'contents' %} search-mode-contents{% elif current_search_mode|slice:':4' == 'deep' %} search-mode-deep{% endif %}" id="changelist">
   {% endwith %}
 {% else %}
     <div class="module{% if cl.has_filters %} filtered{% endif %}" id="changelist">

archivebox/templates/admin/change_list_results.html

@@ -54,7 +54,7 @@
 {% elif cl.show_search_index_hint %}
 <div class="results search-empty-state">
   <p>
-    0 results from deep: {{ cl.search_backend_label }}.
+    0 results from {{ cl.search_mode }}.
     If this looks wrong, the search index may need to be updated:
     <code>archivebox update --index-only</code>
   </p>

archivebox/templates/admin/personas/persona/change_form.html

@@ -267,6 +267,6 @@ document.addEventListener('DOMContentLoaded', function () {
 <section id="plugin-config" class="module aligned persona-plugin-config">
     <h2>Plugin Config</h2>
     <p>These typed controls update the same Persona config JSON shown above. Shared config keys stay synced across plugin sections.</p>
-    {% include "core/plugin_config_grid.html" with plugin_groups=adminform.form.plugin_groups %}
+    {% include "plugins/plugin_config_grid.html" with plugin_groups=adminform.form.plugin_groups %}
 </section>
 {% endblock %}

archivebox/templates/admin/snapshots_grid.html

@@ -208,7 +208,7 @@
     {% for obj in results %}
       <div class="card{% if obj.status == 'started' %} archiving{% endif %}">
           <div class="card-info card-meta">
-            <input type="checkbox" name="_selected_action" value="{{obj.pk}}"/>
+            <input type="checkbox" class="action-select" name="_selected_action" value="{{obj.pk}}"/>
             <a href="{% url 'admin:core_snapshot_change' obj.pk %}" class="card-date">
               <span class="timestamp">{{obj.bookmarked_at}}</span>
             </a>

archivebox/templates/admin/submit_line.html

@@ -0,0 +1,16 @@
+{% load i18n admin_urls %}
+<div class="submit-row">
+{% block submit-row %}
+{% if show_save and not original %}<input type="submit" value="{% translate 'Save' %}" class="default" name="_save">{% endif %}
+{% if show_save_as_new %}<input type="submit" value="{% translate 'Save as new' %}" name="_saveasnew">{% endif %}
+{% if show_save_and_continue %}<input type="submit" value="{% if can_change %}{% translate 'Save' %}{% else %}{% translate 'Save and view' %}{% endif %}" class="{% if original %}default{% endif %}" name="_continue">{% endif %}
+{% if show_close %}
+    {% url opts|admin_urlname:'changelist' as changelist_url %}
+    <a role="button" href="{% add_preserved_filters changelist_url %}" class="closelink">{% translate 'Close' %}</a>
+{% endif %}
+{% if show_delete_link and original %}
+    {% url opts|admin_urlname:'delete' original.pk|admin_urlquote as delete_url %}
+    <a role="button" href="{% add_preserved_filters delete_url %}" class="deletelink">{% translate "Delete" %}</a>
+{% endif %}
+{% endblock %}
+</div>

archivebox/templates/core/add.html

@@ -264,7 +264,7 @@
                         <button type="button" class="preset-btn" data-preset="clear-all">✗ Clear All</button>
                     </div>
 
-                    {% include "core/plugin_config_grid.html" with plugin_groups=form.plugin_groups %}
+                    {% include "plugins/plugin_config_grid.html" with plugin_groups=form.plugin_groups %}
                 </div>
 
                 <!-- Advanced options (collapsible) -->

archivebox/templates/core/navigation.html

@@ -1,7 +1,7 @@
 {% load i18n static %}
 
 <div id="user-tools">
-    <a href="{% url 'add' %}">Add ➕</a> &nbsp; &nbsp;
+    <a href="{% url 'add' %}" class="navbar-add-link">Add ➕</a> &nbsp; &nbsp;
     <a href="/admin/crawls/crawl/">Crawls</a> |
     <a href="{% url 'Home' %}">Snapshots</a> |
     <a href="/admin/core/archiveresult/?o=-1">Log</a> |

archivebox/templates/core/public_index.html

@@ -481,7 +481,7 @@
                         <tr>
                             <td colspan="7" class="public-search-empty">
                                 {% if show_search_index_hint %}
-                                    0 results from deep: {{ search_backend_label }}. If this looks wrong, the search index may need to be updated:
+                                    0 results from {{ search_mode }}. If this looks wrong, the search index may need to be updated:
                                     <code>archivebox update --index-only</code>
                                 {% else %}
                                     No snapshots found.

archivebox/templates/core/snapshot.html

@@ -1048,7 +1048,7 @@
         <div id="main-frame-wrapper" class="full-page-wrapper" data-has-outputs="{{ has_outputs|yesno:'1,0' }}" data-snapshot-state="{{ snapshot_state }}">
             {% if snapshot_state == 'queued' or snapshot_state == 'started' or snapshot_state == 'paused' %}
             <div id="snapshot-progress-wrapper">
-                {% include "admin/progress_monitor.html" with progress_endpoint=progress_endpoint progress_scope="snapshot" %}
+                {% include "progressmonitor/progress_monitor.html" with progress_endpoint=progress_endpoint progress_scope="snapshot" %}
             </div>
             {% endif %}
             {% if has_outputs %}

archivebox/tests/conftest.py

@@ -484,7 +484,7 @@ def build_test_env(port: int, **extra: str) -> dict[str, str]:
             "USE_COLOR": "False",
             "SHOW_PROGRESS": "False",
             "TIMEOUT": "30",
-            "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
+            "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*|example\.com",
             "SAVE_ARCHIVEDOTORG": "False",
             "SAVE_TITLE": "False",
             "SAVE_FAVICON": "False",

archivebox/tests/fixtures.py

@@ -35,6 +35,7 @@ def disable_extractors_dict():
             "SAVE_ARCHIVEDOTORG": "false",
             "SAVE_TITLE": "false",
             "SAVE_FAVICON": "false",
+            "PLUGINS": "__archivebox_test_no_plugins__",
         },
     )
     return env

archivebox/tests/test_archive_result_service.py

@@ -2,7 +2,8 @@ from pathlib import Path
 import pytest
 
 
-from abx_dl.events import ArchiveResultEvent, BinaryRequestEvent, ProcessEvent, ProcessStartedEvent
+from abxpkg.binary_service import BinaryRequestEvent
+from abx_dl.events import ArchiveResultEvent, ProcessEvent, ProcessStartedEvent
 from abx_dl.orchestrator import create_bus
 from abx_dl.output_files import OutputFile
 

archivebox/tests/test_cli_add.py

@@ -269,7 +269,7 @@ def test_add_records_selected_persona_on_crawl(tmp_path, process, disable_extrac
         crawl = Crawl.objects.get()
 
     assert crawl.persona_id
-    assert crawl.config.get("DEFAULT_PERSONA") is None
+    assert crawl.config["ACTIVE_PERSONA"] == "Default"
     assert (tmp_path / "personas" / "Default" / "chrome_profile").is_dir()
 
 

archivebox/tests/test_cli_list.py

@@ -11,7 +11,7 @@ import subprocess
 import pytest
 
 from archivebox.core.models import Snapshot
-from archivebox.tests.conftest import run_queued_crawls
+from archivebox.tests.conftest import create_test_url, parse_jsonl_output, run_archivebox_cmd, run_queued_crawls
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
 pytestmark = pytest.mark.django_db(transaction=True)
@@ -171,25 +171,122 @@ def test_list_allows_sort_with_limit(tmp_path, process, disable_extractors_dict)
     assert len(rows) == 2
 
 
-def test_list_search_meta_matches_metadata(tmp_path, process, disable_extractors_dict):
-    """Test that list --search=meta applies metadata search to the queryset."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
-        check=True,
-    )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+def test_snapshot_list_search_meta(initialized_archive):
+    """snapshot list should support metadata search mode."""
+    url = create_test_url(domain="meta-search-example.com")
+    run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
 
-    result = subprocess.run(
-        ["archivebox", "list", "--search=meta", "example.com"],
-        capture_output=True,
-        text=True,
-        timeout=30,
+    stdout, stderr, code = run_archivebox_cmd(
+        ["snapshot", "list", "--search=meta", "meta-search-example.com"],
+        data_dir=initialized_archive,
     )
 
-    rows = _parse_jsonl(result.stdout)
-    assert result.returncode == 0, result.stderr
-    assert len(rows) == 1
-    assert rows[0]["url"] == "https://example.com"
+    assert code == 0, f"Command failed: {stderr}"
+    records = parse_jsonl_output(stdout)
+    assert len(records) == 1
+    assert "meta-search-example.com" in records[0]["url"]
+
+
+def test_list_search_meta_matches_metadata(initialized_archive):
+    """top-level list --search=meta should apply metadata search to the queryset."""
+    url = create_test_url(domain="top-level-meta-search-example.com")
+    run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+
+    stdout, stderr, code = run_archivebox_cmd(
+        ["list", "--search=meta", "top-level-meta-search-example.com"],
+        data_dir=initialized_archive,
+    )
+
+    assert code == 0, f"Command failed: {stderr}"
+    records = parse_jsonl_output(stdout)
+    assert len(records) == 1
+    assert "top-level-meta-search-example.com" in records[0]["url"]
+
+
+def test_search_command_finds_snapshots(initialized_archive):
+    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+
+    stdout, stderr, code = run_archivebox_cmd(["search", "example"], data_dir=initialized_archive)
+
+    assert code == 0, stderr
+    assert "example" in stdout
+
+
+def test_search_command_returns_no_results_for_missing_term(initialized_archive):
+    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+
+    _stdout, _stderr, code = run_archivebox_cmd(["search", "nonexistentterm12345"], data_dir=initialized_archive)
+
+    assert code in [0, 1]
+
+
+def test_search_command_on_empty_archive(initialized_archive):
+    _stdout, _stderr, code = run_archivebox_cmd(["search", "anything"], data_dir=initialized_archive)
+
+    assert code in [0, 1]
+
+
+def test_search_command_json_outputs_matching_snapshots(initialized_archive):
+    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+
+    stdout, stderr, code = run_archivebox_cmd(["search", "--json"], data_dir=initialized_archive)
+
+    assert code == 0, stderr
+    payload = json.loads(stdout)
+    assert any("example.com" in row.get("url", "") for row in payload)
+
+
+def test_search_command_json_with_headers_wraps_links_payload(initialized_archive):
+    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+
+    stdout, stderr, code = run_archivebox_cmd(["search", "--json", "--with-headers"], data_dir=initialized_archive)
+
+    assert code == 0, stderr
+    payload = json.loads(stdout)
+    links = payload.get("links", payload)
+    assert any("example.com" in row.get("url", "") for row in links)
+
+
+def test_search_command_html_outputs_markup(initialized_archive):
+    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+
+    stdout, stderr, code = run_archivebox_cmd(["search", "--html"], data_dir=initialized_archive)
+
+    assert code == 0, stderr
+    assert "<" in stdout
+
+
+def test_search_command_csv_outputs_requested_column(initialized_archive):
+    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+
+    stdout, stderr, code = run_archivebox_cmd(["search", "--csv", "url", "--with-headers"], data_dir=initialized_archive)
+
+    assert code == 0, stderr
+    assert "url" in stdout
+    assert "example.com" in stdout
+
+
+def test_search_command_with_headers_requires_structured_output_format(initialized_archive):
+    _stdout, stderr, code = run_archivebox_cmd(["search", "--with-headers"], data_dir=initialized_archive)
+
+    assert code != 0
+    assert "requires" in stderr.lower() or "json" in stderr.lower()
+
+
+def test_search_command_sort_option_runs_successfully(initialized_archive):
+    for url in ["https://iana.org", "https://example.com"]:
+        run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+
+    stdout, stderr, code = run_archivebox_cmd(["search", "--csv", "url", "--sort=url"], data_dir=initialized_archive)
+
+    assert code == 0, stderr
+    assert "example.com" in stdout or "iana.org" in stdout
+
+
+def test_search_command_help_lists_supported_filters(initialized_archive):
+    stdout, _stderr, code = run_archivebox_cmd(["search", "--help"], data_dir=initialized_archive)
+
+    assert code == 0
+    assert "--filter-type" in stdout or "-f" in stdout
+    assert "--status" in stdout
+    assert "--sort" in stdout

archivebox/tests/test_cli_piping.py

@@ -138,7 +138,7 @@ def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl():
 
 def test_collect_urls_from_plugins_reads_only_parser_outputs(tmp_path):
     """Parser extractor `urls.jsonl` outputs should be discoverable for recursive piping."""
-    from archivebox.hooks import collect_urls_from_plugins
+    from archivebox.plugins.hooks import collect_urls_from_plugins
 
     (tmp_path / "wget").mkdir()
     (tmp_path / "wget" / "urls.jsonl").write_text(
@@ -163,7 +163,7 @@ def test_collect_urls_from_plugins_reads_only_parser_outputs(tmp_path):
 
 
 def test_collect_urls_from_plugins_trims_markdown_suffixes(tmp_path):
-    from archivebox.hooks import collect_urls_from_plugins
+    from archivebox.plugins.hooks import collect_urls_from_plugins
 
     (tmp_path / "parse_html_urls").mkdir()
     (tmp_path / "parse_html_urls" / "urls.jsonl").write_text(
@@ -177,7 +177,7 @@ def test_collect_urls_from_plugins_trims_markdown_suffixes(tmp_path):
 
 
 def test_collect_urls_from_plugins_trims_trailing_punctuation(tmp_path):
-    from archivebox.hooks import collect_urls_from_plugins
+    from archivebox.plugins.hooks import collect_urls_from_plugins
 
     (tmp_path / "parse_html_urls").mkdir()
     (tmp_path / "parse_html_urls" / "urls.jsonl").write_text(

archivebox/tests/test_cli_run.py

@@ -1218,6 +1218,67 @@ class TestRecoverOrchestratorState:
         assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
         assert snapshot.retry_at is None
 
+    def test_run_due_snapshot_runs_queued_plugin_after_fs_migration(self, monkeypatch):
+        from django.utils import timezone
+
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import ArchiveResult, Snapshot
+        from archivebox.services import runner
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=timezone.now(),
+        )
+        Snapshot.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
+        snapshot.refresh_from_db()
+        result = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="search_backend_sonic",
+            hook_name="on_Snapshot__91_index_sonic",
+            status=ArchiveResult.StatusChoices.QUEUED,
+        )
+        calls = []
+
+        def fake_run_crawl(crawl_id, *, snapshot_ids=None, selected_plugins=None, **kwargs):
+            calls.append((crawl_id, snapshot_ids, selected_plugins, kwargs))
+            ArchiveResult.objects.filter(pk=result.pk).update(
+                status=ArchiveResult.StatusChoices.NORESULTS,
+                start_ts=timezone.now(),
+                end_ts=timezone.now(),
+                output_str="No indexable content",
+            )
+
+        monkeypatch.setattr(
+            runner,
+            "_snapshot_hook_names_by_plugin",
+            lambda: {"search_backend_sonic": frozenset({"on_Snapshot__91_index_sonic"})},
+        )
+        monkeypatch.setattr(runner, "run_crawl", fake_run_crawl)
+
+        assert runner.run_due_snapshot(snapshot, lock_seconds=60) is True
+
+        snapshot.refresh_from_db()
+        result.refresh_from_db()
+        assert snapshot.fs_version == Snapshot._fs_current_version()
+        assert result.status == ArchiveResult.StatusChoices.NORESULTS
+        assert calls == [
+            (
+                str(crawl.id),
+                [str(snapshot.id)],
+                ["search_backend_sonic"],
+                {"process_discovered_snapshots_inline": True, "interactive_interrupts": False},
+            ),
+        ]
+
     def test_run_due_snapshot_fails_obsolete_queued_hook_name(self):
         from django.utils import timezone
 

archivebox/tests/test_cli_search.py

@@ -1,223 +0,0 @@
-#!/usr/bin/env python3
-"""
-Tests for archivebox search command.
-Verify search queries snapshots from DB.
-"""
-
-import json
-import os
-import subprocess
-
-import pytest
-
-from archivebox.tests.conftest import run_queued_crawls
-
-pytestmark = pytest.mark.django_db(transaction=True)
-
-
-def test_search_finds_snapshots(tmp_path, process, disable_extractors_dict):
-    """Test that search command finds matching snapshots."""
-    os.chdir(tmp_path)
-
-    # Add snapshots
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
-    )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
-
-    # Search for it
-    result = subprocess.run(
-        ["archivebox", "search", "example"],
-        capture_output=True,
-        text=True,
-        timeout=30,
-    )
-
-    assert result.returncode == 0
-    assert "example" in result.stdout
-
-
-def test_search_returns_no_results_for_missing_term(tmp_path, process, disable_extractors_dict):
-    """Test search returns empty for non-existent term."""
-    os.chdir(tmp_path)
-
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
-    )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
-
-    result = subprocess.run(
-        ["archivebox", "search", "nonexistentterm12345"],
-        capture_output=True,
-        text=True,
-        timeout=30,
-    )
-
-    # Should complete with no results
-    assert result.returncode in [0, 1]
-
-
-def test_search_on_empty_archive(tmp_path, process):
-    """Test search works on empty archive."""
-    os.chdir(tmp_path)
-
-    result = subprocess.run(
-        ["archivebox", "search", "anything"],
-        capture_output=True,
-        text=True,
-        timeout=30,
-    )
-
-    # Should complete without error
-    assert result.returncode in [0, 1]
-
-
-def test_search_json_outputs_matching_snapshots(tmp_path, process, disable_extractors_dict):
-    """Test that search --json returns parseable matching snapshot rows."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
-        check=True,
-    )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
-
-    result = subprocess.run(
-        ["archivebox", "search", "--json"],
-        capture_output=True,
-        text=True,
-        timeout=30,
-    )
-
-    assert result.returncode == 0, result.stderr
-    payload = json.loads(result.stdout)
-    assert any("example.com" in row.get("url", "") for row in payload)
-
-
-def test_search_json_with_headers_wraps_links_payload(tmp_path, process, disable_extractors_dict):
-    """Test that search --json --with-headers returns a headers envelope."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
-        check=True,
-    )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
-
-    result = subprocess.run(
-        ["archivebox", "search", "--json", "--with-headers"],
-        capture_output=True,
-        text=True,
-        timeout=30,
-    )
-
-    assert result.returncode == 0, result.stderr
-    payload = json.loads(result.stdout)
-    links = payload.get("links", payload)
-    assert any("example.com" in row.get("url", "") for row in links)
-
-
-def test_search_html_outputs_markup(tmp_path, process, disable_extractors_dict):
-    """Test that search --html renders an HTML response."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
-        check=True,
-    )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
-
-    result = subprocess.run(
-        ["archivebox", "search", "--html"],
-        capture_output=True,
-        text=True,
-        timeout=30,
-    )
-
-    assert result.returncode == 0, result.stderr
-    assert "<" in result.stdout
-
-
-def test_search_csv_outputs_requested_column(tmp_path, process, disable_extractors_dict):
-    """Test that search --csv emits the requested fields."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
-        check=True,
-    )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
-
-    result = subprocess.run(
-        ["archivebox", "search", "--csv", "url", "--with-headers"],
-        capture_output=True,
-        text=True,
-        timeout=30,
-    )
-
-    assert result.returncode == 0, result.stderr
-    assert "url" in result.stdout
-    assert "example.com" in result.stdout
-
-
-def test_search_with_headers_requires_structured_output_format(tmp_path, process):
-    """Test that --with-headers is rejected without --json, --html, or --csv."""
-    os.chdir(tmp_path)
-
-    result = subprocess.run(
-        ["archivebox", "search", "--with-headers"],
-        capture_output=True,
-        text=True,
-        timeout=30,
-    )
-
-    assert result.returncode != 0
-    assert "requires" in result.stderr.lower() or "json" in result.stderr.lower()
-
-
-def test_search_sort_option_runs_successfully(tmp_path, process, disable_extractors_dict):
-    """Test that search --sort accepts sortable fields."""
-    os.chdir(tmp_path)
-    for url in ["https://iana.org", "https://example.com"]:
-        subprocess.run(
-            ["archivebox", "add", "--index-only", "--depth=0", url],
-            capture_output=True,
-            env=disable_extractors_dict,
-            check=True,
-        )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
-
-    result = subprocess.run(
-        ["archivebox", "search", "--csv", "url", "--sort=url"],
-        capture_output=True,
-        text=True,
-        timeout=30,
-    )
-
-    assert result.returncode == 0, result.stderr
-    assert "example.com" in result.stdout or "iana.org" in result.stdout
-
-
-def test_search_help_lists_supported_filters(tmp_path, process):
-    """Test that search --help documents the available filters and output modes."""
-    os.chdir(tmp_path)
-
-    result = subprocess.run(
-        ["archivebox", "search", "--help"],
-        capture_output=True,
-        text=True,
-        timeout=30,
-    )
-
-    assert result.returncode == 0
-    assert "--filter-type" in result.stdout or "-f" in result.stdout
-    assert "--status" in result.stdout
-    assert "--sort" in result.stdout

archivebox/tests/test_cli_server.py

@@ -15,6 +15,73 @@ from datetime import datetime
 from types import SimpleNamespace
 
 
+def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path, monkeypatch):
+    """Admin sessions must survive `archivebox server` restarts for a collection."""
+    from archivebox.config.collection import write_config_file
+
+    os.chdir(tmp_path)
+    (tmp_path / ".archivebox_id").write_text("testcoll")
+    monkeypatch.setenv("BASE_URL", "http://archivebox.localhost:9292")
+
+    first = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import os;"
+                "os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'archivebox.core.settings');"
+                "import django;"
+                "django.setup();"
+                "from django.conf import settings;"
+                "print(settings.SECRET_KEY);"
+                "print(settings.SESSION_ENGINE);"
+                "print(settings.SESSION_COOKIE_NAME);"
+                "print(settings.SESSION_COOKIE_DOMAIN);"
+                "print(settings.SESSION_COOKIE_SECURE);"
+                "print(settings.SESSION_EXPIRE_AT_BROWSER_CLOSE)"
+            ),
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    first_lines = first.stdout.strip().splitlines()
+    assert first_lines[0], first.stderr
+
+    # Simulate the next `archivebox server` process, reading only persisted
+    # collection config. If SECRET_KEY falls back to the random default_factory
+    # here, Django will reject existing signed session cookies after restart.
+    monkeypatch.delenv("BASE_URL", raising=False)
+    write_config_file({"BASE_URL": "http://archivebox.localhost:9292"})
+    second = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import os;"
+                "os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'archivebox.core.settings');"
+                "import django;"
+                "django.setup();"
+                "from django.conf import settings;"
+                "print(settings.SECRET_KEY);"
+                "print(settings.SESSION_ENGINE);"
+                "print(settings.SESSION_COOKIE_NAME);"
+                "print(settings.SESSION_COOKIE_DOMAIN);"
+                "print(settings.SESSION_COOKIE_SECURE);"
+                "print(settings.SESSION_EXPIRE_AT_BROWSER_CLOSE)"
+            ),
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+
+    assert second.stdout.strip().splitlines() == first_lines
+    assert first_lines[1] == "django.contrib.sessions.backends.db"
+    assert first_lines[2].startswith("archivebox_sessionid_")
+    assert first_lines[3:] == ["None", "False", "False"]
+
+
 def test_sqlite_connections_use_explicit_busy_timeout():
     from archivebox.core.settings import SQLITE_CONNECTION_OPTIONS
 

archivebox/tests/test_cli_snapshot.py

@@ -199,21 +199,6 @@ class TestSnapshotList:
         records = parse_jsonl_output(stdout)
         assert len(records) == 2
 
-    def test_list_search_meta(self, initialized_archive):
-        """snapshot list should support metadata search mode."""
-        url = create_test_url(domain="meta-search-example.com")
-        run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
-
-        stdout, stderr, code = run_archivebox_cmd(
-            ["snapshot", "list", "--search=meta", "meta-search-example.com"],
-            data_dir=initialized_archive,
-        )
-
-        assert code == 0, f"Command failed: {stderr}"
-        records = parse_jsonl_output(stdout)
-        assert len(records) == 1
-        assert "meta-search-example.com" in records[0]["url"]
-
 
 class TestSnapshotUpdate:
     """Tests for `archivebox snapshot update`."""

archivebox/tests/test_crawl_admin.py

@@ -1,3 +1,4 @@
+import re
 from typing import cast
 
 import pytest
@@ -65,6 +66,75 @@ def test_crawl_admin_add_view_renders_url_filter_alias_fields(client, admin_user
     assert b"Subpaths only" in response.content
 
 
+def test_crawl_admin_change_view_checks_effective_only_new(client, admin_user):
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        config={},
+        created_by=admin_user,
+    )
+    client.login(username="crawladmin", password="testpassword")
+
+    response = client.get(
+        reverse("admin:crawls_crawl_change", args=[crawl.pk]),
+        HTTP_HOST=ADMIN_HOST,
+    )
+
+    assert response.status_code == 200
+    assert b"Effective ONLY_NEW" not in response.content
+    assert b'id="id_url_filters_only_new" name="url_filters_only_new" value="1" checked' in response.content
+
+
+def test_crawl_admin_change_view_derives_url_filter_shortcut_toggles(client, admin_user):
+    crawl = Crawl.objects.create(
+        urls="https://example.com/docs/page.html",
+        config={"URL_ALLOWLIST": r"^https?://example\.com/docs/"},
+        created_by=admin_user,
+    )
+    client.login(username="crawladmin", password="testpassword")
+
+    response = client.get(
+        reverse("admin:crawls_crawl_change", args=[crawl.pk]),
+        HTTP_HOST=ADMIN_HOST,
+    )
+
+    assert response.status_code == 200
+    assert b'id="id_url_filters_same_domain_only" name="url_filters_same_domain_only" value="1" checked' in response.content
+    assert b'id="id_url_filters_subpaths_only" name="url_filters_subpaths_only" value="1" checked' in response.content
+
+
+def test_admin_change_submit_row_uses_single_save_continue_button(client, admin_user, crawl):
+    client.login(username="crawladmin", password="testpassword")
+
+    response = client.get(
+        reverse("admin:crawls_crawl_change", args=[crawl.pk]),
+        HTTP_HOST=ADMIN_HOST,
+    )
+
+    assert response.status_code == 200
+    submit_rows = re.findall(r'<div class="submit-row">.*?</div>', response.content.decode(), flags=re.DOTALL)
+    assert submit_rows
+    for row in submit_rows:
+        assert 'name="_save"' not in row
+        assert 'name="_addanother"' not in row
+        assert 'value="Save and continue editing"' not in row
+        assert 'value="Save"' in row
+        assert 'name="_continue"' in row
+
+
+def test_admin_add_submit_row_hides_save_and_add_another(client, admin_user):
+    client.login(username="crawladmin", password="testpassword")
+
+    response = client.get(
+        reverse("admin:crawls_crawl_add"),
+        HTTP_HOST=ADMIN_HOST,
+    )
+
+    assert response.status_code == 200
+    submit_rows = re.findall(r'<div class="submit-row">.*?</div>', response.content.decode(), flags=re.DOTALL)
+    assert submit_rows
+    assert all('name="_addanother"' not in row for row in submit_rows)
+
+
 def test_crawl_schedule_admin_add_redirects_to_add_page_schedule_field(client, admin_user):
     client.login(username="crawladmin", password="testpassword")
 
@@ -349,6 +419,67 @@ def test_create_snapshots_from_urls_respects_max_urls(admin_user):
     assert crawl.add_url({"url": "https://example.com/extra", "depth": 1}) is False
 
 
+def test_crawl_stop_reason_reports_no_viable_urls_for_sealed_empty_crawl(admin_user):
+    crawl = Crawl.objects.create(
+        urls="https://example.com/already-known",
+        status=Crawl.StatusChoices.SEALED,
+        retry_at=None,
+        created_by=admin_user,
+    )
+
+    assert crawl.stop_reason() == "no_viable_urls"
+
+
+def test_crawl_stop_reason_reports_done_for_sealed_crawl_with_all_snapshots_sealed(admin_user):
+    crawl = Crawl.objects.create(
+        urls="https://example.com/done",
+        status=Crawl.StatusChoices.SEALED,
+        retry_at=None,
+        created_by=admin_user,
+    )
+    Snapshot.objects.create(
+        url="https://example.com/done",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.SEALED,
+        timestamp="1700000000.010",
+    )
+
+    assert crawl.stop_reason() == "done"
+
+
+def test_crawl_stop_reason_reports_paused_for_paused_crawl(admin_user):
+    crawl = Crawl.objects.create(
+        urls="https://example.com/paused",
+        status=Crawl.StatusChoices.PAUSED,
+        created_by=admin_user,
+    )
+
+    assert crawl.stop_reason() == "paused"
+
+
+def test_crawl_stop_reason_keeps_specific_limit_reason_over_lifecycle_fallback(admin_user):
+    crawl = Crawl.objects.create(
+        urls="\n".join(
+            [
+                "https://example.com/root",
+                "https://example.com/about",
+            ],
+        ),
+        config={"CRAWL_MAX_URLS": 1},
+        status=Crawl.StatusChoices.SEALED,
+        retry_at=None,
+        created_by=admin_user,
+    )
+    Snapshot.objects.create(
+        url="https://example.com/root",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.SEALED,
+        timestamp="1700000000.011",
+    )
+
+    assert crawl.stop_reason() == "crawl_max_urls"
+
+
 def test_create_snapshots_from_urls_respects_only_new_exact_url_matches(admin_user):
     existing_crawl = Crawl.objects.create(urls="https://example.com/existing", created_by=admin_user)
     Snapshot.objects.create(

archivebox/tests/test_frozen_crawl_config.py

@@ -0,0 +1,203 @@
+import pytest
+from django.test import RequestFactory
+from django.utils import timezone
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+SENSITIVE_SECRET = "raw-twocaptcha-secret-for-frozen-crawl-test"
+UPDATED_SECRET = "updated-secret-that-must-not-affect-old-crawl"
+
+
+@pytest.fixture
+def archivebox_db(initialized_archive):
+    from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+    with use_archivebox_db(initialized_archive):
+        yield initialized_archive
+
+
+def _user(username="frozen-config-admin"):
+    from django.contrib.auth import get_user_model
+
+    return get_user_model().objects.create_superuser(
+        username=username,
+        email=f"{username}@example.com",
+        password="testpassword",
+    )
+
+
+def _persona(user, *, name="Frozen Persona", secret=SENSITIVE_SECRET, user_agent="Frozen UA"):
+    from archivebox.personas.models import Persona
+
+    persona = Persona.objects.create(
+        name=name,
+        created_by=user,
+        config={
+            "PERMISSIONS": "private",
+            "USER_AGENT": user_agent,
+            "TWOCAPTCHA_API_KEY": secret,
+            "DELETE_AFTER": "2h",
+        },
+    )
+    persona.ensure_dirs()
+    return persona
+
+
+def test_crawl_save_freezes_full_raw_persona_config_and_redacts_public_serialization(archivebox_db):
+    from archivebox.config.common import SENSITIVE_CONFIG_VALUE_REDACTED, get_config
+    from archivebox.crawls.models import Crawl
+
+    user = _user()
+    persona = _persona(user)
+
+    crawl = Crawl.objects.create(
+        urls="https://example.com/frozen",
+        persona=persona,
+        created_by=user,
+        config={"CRAWL_MAX_CONCURRENT_SNAPSHOTS": 3},
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=timezone.now(),
+    )
+
+    assert "TIMEOUT" in crawl.config
+    assert "CHECK_SSL_VALIDITY" in crawl.config
+    assert crawl.config["USER_AGENT"] == "Frozen UA"
+    assert crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET
+    assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 3
+    assert "CRAWL_DIR" not in crawl.config
+    assert "SNAP_DIR" not in crawl.config
+    assert "DEBUG" not in crawl.config
+    assert "SECRET_KEY" not in crawl.config
+    assert "PUBLIC_ADD_VIEW" not in crawl.config
+    assert "DATABASE_NAME" not in crawl.config
+
+    persona.config["USER_AGENT"] = "Mutated UA"
+    persona.config["TWOCAPTCHA_API_KEY"] = UPDATED_SECRET
+    persona.save(update_fields=["config"])
+
+    runtime_config = get_config(crawl=crawl)
+    assert runtime_config.USER_AGENT == "Frozen UA"
+    assert runtime_config.TWOCAPTCHA_API_KEY == SENSITIVE_SECRET
+    redacted_runtime_config = get_config(crawl=crawl, redact_sensitive=True)
+    assert redacted_runtime_config.USER_AGENT == "Frozen UA"
+    assert redacted_runtime_config.TWOCAPTCHA_API_KEY == SENSITIVE_CONFIG_VALUE_REDACTED
+    execution_config = runtime_config.for_crawl_execution()
+    assert execution_config["DEBUG"] is False
+    assert execution_config["CRAWL_DIR"] == str(crawl.output_dir)
+    assert "SECRET_KEY" not in execution_config
+    assert "PUBLIC_ADD_VIEW" not in execution_config
+    assert "DATABASE_NAME" not in execution_config
+
+    public_json = crawl.to_json()
+    assert public_json["config"]["TWOCAPTCHA_API_KEY"] == SENSITIVE_CONFIG_VALUE_REDACTED
+    assert SENSITIVE_SECRET not in str(public_json)
+
+
+def test_snapshot_config_overlays_frozen_crawl_without_re_reading_persona(archivebox_db):
+    from archivebox.config.common import get_config
+    from archivebox.core.models import Snapshot
+    from archivebox.crawls.models import Crawl
+
+    user = _user("frozen-config-snapshot-admin")
+    persona = _persona(user, name="Frozen Snapshot Persona", user_agent="Crawl UA")
+    crawl = Crawl.objects.create(urls="https://example.com/root", persona=persona, created_by=user, config={"TIMEOUT": 11})
+    snapshot = Snapshot.objects.create(
+        url="https://example.com/root",
+        crawl=crawl,
+        config={"TIMEOUT": 22, "ANTHROPIC_API_KEY": "snapshot-secret"},
+    )
+
+    persona.config["TIMEOUT"] = 99
+    persona.save(update_fields=["config"])
+
+    runtime_config = get_config(crawl=crawl, snapshot=snapshot)
+    assert runtime_config.USER_AGENT == "Crawl UA"
+    assert runtime_config.TIMEOUT == 22
+    assert runtime_config.ANTHROPIC_API_KEY == "snapshot-secret"
+    assert snapshot.config == {"TIMEOUT": 22, "ANTHROPIC_API_KEY": "snapshot-secret"}
+
+
+def test_config_scopes_are_derived_from_section_and_field_metadata():
+    from archivebox.config.common import ArchiveBoxConfig
+
+    assert ArchiveBoxConfig.scope_for_key("TIMEOUT") == "crawl_frozen"
+    assert ArchiveBoxConfig.scope_for_key("DEBUG") == "crawl_execution"
+    assert ArchiveBoxConfig.scope_for_key("CRAWL_DIR") == "crawl_execution"
+    assert ArchiveBoxConfig.scope_for_key("SECRET_KEY") == "server"
+    assert ArchiveBoxConfig.scope_for_key("DATABASE_NAME") == "server"
+
+
+def test_api_create_and_cli_add_store_full_frozen_config(archivebox_db):
+    from archivebox.api.v1_crawls import CrawlCreateSchema, CrawlSchema, create_crawl
+    from archivebox.cli.archivebox_add import add
+    from archivebox.config.common import SENSITIVE_CONFIG_VALUE_REDACTED
+
+    user = _user("frozen-config-api-admin")
+    request = RequestFactory().post("/api/v1/crawls")
+    request.user = user
+
+    api_crawl = create_crawl(
+        request,
+        CrawlCreateSchema(
+            urls=["https://example.com/api"],
+            max_depth=0,
+            tags=[],
+            tags_str="",
+            label="API frozen config",
+            notes="",
+            config={"TWOCAPTCHA_API_KEY": SENSITIVE_SECRET, "TIMEOUT": 33, "SECRET_KEY": "must-not-freeze", "PUBLIC_ADD_VIEW": True},
+        ),
+    )
+    assert "CHECK_SSL_VALIDITY" in api_crawl.config
+    assert api_crawl.config["TIMEOUT"] == 33
+    assert api_crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET
+    assert "SECRET_KEY" not in api_crawl.config
+    assert "PUBLIC_ADD_VIEW" not in api_crawl.config
+    assert CrawlSchema.resolve_config(api_crawl)["TWOCAPTCHA_API_KEY"] == SENSITIVE_CONFIG_VALUE_REDACTED
+
+    cli_crawl, _snapshots = add(
+        "https://example.com/cli",
+        bg=True,
+        created_by_id=user.pk,
+        config={"TWOCAPTCHA_API_KEY": SENSITIVE_SECRET, "TIMEOUT": 44},
+    )
+    assert "CHECK_SSL_VALIDITY" in cli_crawl.config
+    assert cli_crawl.config["TIMEOUT"] == 44
+    assert cli_crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET
+
+
+def test_schedule_enqueue_refreezes_using_current_template_persona_defaults(archivebox_db):
+    from archivebox.crawls.models import Crawl, CrawlSchedule
+
+    user = _user("frozen-config-schedule-admin")
+    persona = _persona(user, name="Frozen Schedule Persona", user_agent="Initial schedule UA")
+    template = Crawl.objects.create(
+        urls="https://example.com/scheduled",
+        persona=persona,
+        created_by=user,
+        config={"TIMEOUT": 55, "SECRET_KEY": "template-secret-must-not-freeze", "PUBLIC_ADD_VIEW": True},
+        status=Crawl.StatusChoices.PAUSED,
+    )
+    schedule = CrawlSchedule.objects.create(
+        template=template,
+        schedule="daily",
+        created_by=user,
+        config={"TIMEOUT": 55, "SECRET_KEY": "schedule-secret-must-not-freeze", "PUBLIC_ADD_VIEW": True},
+    )
+
+    assert schedule.config["TIMEOUT"] == 55
+    assert "SECRET_KEY" in schedule.config
+
+    persona.config["USER_AGENT"] = "Current schedule UA"
+    persona.config["TWOCAPTCHA_API_KEY"] = UPDATED_SECRET
+    persona.save(update_fields=["config"])
+
+    child = schedule.enqueue()
+    assert child.config["TIMEOUT"] == 55
+    assert child.config["USER_AGENT"] == "Current schedule UA"
+    assert child.config["TWOCAPTCHA_API_KEY"] == UPDATED_SECRET
+    assert "SECRET_KEY" not in child.config
+    assert "PUBLIC_ADD_VIEW" not in child.config
+    assert template.config["USER_AGENT"] == "Initial schedule UA"
+    assert template.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET

archivebox/tests/test_hooks.py

@@ -47,7 +47,9 @@ def create_test_plugin_structure(plugins_dir: Path) -> None:
 def run_plugin_discovery_subprocess(tmp_path: Path, plugins_dir: Path, script: str):
     env = os.environ.copy()
     env["ARCHIVEBOX_USER_PLUGINS_DIR"] = str(plugins_dir)
-    env["DATA_DIR"] = str(tmp_path / "data")
+    data_dir = tmp_path / "data"
+    data_dir.mkdir()
+    env["DATA_DIR"] = str(data_dir)
     env["PYTHONPATH"] = str(REPO_ROOT) + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "")
     subprocess_script = "\n".join(
         [
@@ -67,7 +69,7 @@ def run_plugin_discovery_subprocess(tmp_path: Path, plugins_dir: Path, script: s
             "-c",
             subprocess_script,
         ],
-        cwd=tmp_path,
+        cwd=data_dir,
         env=env,
         capture_output=True,
         text=True,
@@ -87,37 +89,37 @@ class TestBackgroundHookDetection:
 
     def test_bg_js_suffix_detected(self):
         """Hooks with .bg.js suffix should be detected as background."""
-        from archivebox.hooks import is_background_hook
+        from archivebox.plugins.hooks import is_background_hook
 
         assert is_background_hook("on_Snapshot__21_consolelog.daemon.bg.js")
 
     def test_bg_py_suffix_detected(self):
         """Hooks with .bg.py suffix should be detected as background."""
-        from archivebox.hooks import is_background_hook
+        from archivebox.plugins.hooks import is_background_hook
 
         assert is_background_hook("on_Snapshot__24_responses.finite.bg.py")
 
     def test_bg_sh_suffix_detected(self):
         """Hooks with .bg.sh suffix should be detected as background."""
-        from archivebox.hooks import is_background_hook
+        from archivebox.plugins.hooks import is_background_hook
 
         assert is_background_hook("on_Snapshot__23_ssl.daemon.bg.sh")
 
     def test_legacy_background_suffix_detected(self):
         """Hooks with __background in stem should be detected (backwards compat)."""
-        from archivebox.hooks import is_background_hook
+        from archivebox.plugins.hooks import is_background_hook
 
         assert is_background_hook("on_Snapshot__21_consolelog__background.js")
 
     def test_foreground_hook_not_detected(self):
         """Hooks without .bg. or __background should NOT be detected as background."""
-        from archivebox.hooks import is_background_hook
+        from archivebox.plugins.hooks import is_background_hook
 
         assert not is_background_hook("on_Snapshot__11_favicon.js")
 
     def test_foreground_py_hook_not_detected(self):
         """Python hooks without .bg. should NOT be detected as background."""
-        from archivebox.hooks import is_background_hook
+        from archivebox.plugins.hooks import is_background_hook
 
         assert not is_background_hook("on_Snapshot__50_wget.py")
 
@@ -254,7 +256,7 @@ class TestHookDiscovery:
 
     def test_normalize_hook_event_name_accepts_event_classes(self):
         """Hook discovery should normalize bus event class names to hook families."""
-        from archivebox import hooks as hooks_module
+        from archivebox.plugins import hooks as hooks_module
 
         assert hooks_module.normalize_hook_event_name("InstallEvent") == "Install"
         assert hooks_module.normalize_hook_event_name("BinaryRequestEvent") == "BinaryRequest"
@@ -263,7 +265,7 @@ class TestHookDiscovery:
 
     def test_normalize_hook_event_name_strips_event_suffix_for_lifecycle_events(self):
         """Lifecycle event names should normalize via simple suffix stripping."""
-        from archivebox import hooks as hooks_module
+        from archivebox.plugins import hooks as hooks_module
 
         assert hooks_module.normalize_hook_event_name("BinaryEvent") == "Binary"
         assert hooks_module.normalize_hook_event_name("CrawlEvent") == "Crawl"
@@ -333,7 +335,7 @@ class TestHookDiscovery:
             tmp_path,
             plugins_dir,
             """
-            from archivebox import hooks as hooks_module
+            from archivebox.plugins import hooks as hooks_module
 
             hooks = hooks_module.discover_hooks("Snapshot", config={"CHROME_ENABLED": False, "WGET_ENABLED": True})
             emit([hook.parent.name for hook in hooks])
@@ -357,10 +359,11 @@ class TestHookDiscovery:
             tmp_path,
             plugins_dir,
             """
-            from archivebox import hooks as hooks_module
+            from archivebox.plugins import hooks as hooks_module
 
-            hooks_module.get_plugins.cache_clear()
-            emit(hooks_module.get_plugins())
+            from archivebox.plugins.discovery import get_plugins
+            get_plugins.cache_clear()
+            emit(get_plugins())
             """,
         )
         assert "env" in plugins
@@ -391,9 +394,10 @@ class TestHookDiscovery:
             tmp_path,
             plugins_dir,
             """
-            from archivebox import hooks as hooks_module
+            from archivebox.plugins import hooks as hooks_module
 
-            hooks_module.get_plugins.cache_clear()
+            from archivebox.plugins.discovery import get_plugins
+            get_plugins.cache_clear()
             hooks = hooks_module.discover_hooks("BinaryRequest", config={"PLUGINS": "singlefile"})
             emit([hook.name for hook in hooks])
             """,
@@ -409,9 +413,10 @@ class TestHookDiscovery:
             tmp_path,
             plugins_dir,
             """
-            from archivebox import hooks as hooks_module
+            from archivebox.plugins import hooks as hooks_module
 
-            hooks_module.get_plugins.cache_clear()
+            from archivebox.plugins.discovery import get_plugins
+            get_plugins.cache_clear()
             binary_hooks = hooks_module.discover_hooks("BinaryRequestEvent", filter_disabled=False)
             snapshot_hooks = hooks_module.discover_hooks("SnapshotEvent", filter_disabled=False)
             emit({
@@ -432,9 +437,10 @@ class TestHookDiscovery:
             tmp_path,
             plugins_dir,
             """
-            from archivebox import hooks as hooks_module
+            from archivebox.plugins import hooks as hooks_module
 
-            hooks_module.get_plugins.cache_clear()
+            from archivebox.plugins.discovery import get_plugins
+            get_plugins.cache_clear()
             emit({
                 "binary": [hook.name for hook in hooks_module.discover_hooks("BinaryEvent", filter_disabled=False)],
                 "crawl_cleanup": [
@@ -711,7 +717,7 @@ class TestPluginMetadata:
 @pytest.mark.django_db(transaction=True)
 def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_path):
     """Hook subprocesses must get a real NODE_MODULES_DIR even when NODE_PATH has multiple entries."""
-    from archivebox.hooks import run_hook
+    from archivebox.plugins.hooks import run_hook
 
     lib_dir = tmp_path / "lib"
     node_modules_dir = lib_dir / "npm" / "node_modules"

archivebox/tests/test_misc_checks.py

@@ -0,0 +1,40 @@
+import os
+import signal
+
+from archivebox.misc.checks import _migration_interrupt_message
+from archivebox.misc.checks import _exit_on_migration_interrupt
+
+
+def test_migration_interrupt_message_prints_resume_command_and_atomic_safety():
+    message = _migration_interrupt_message()
+
+    assert "Migration interrupted." in message
+    assert "Database migrations are atomic" in message
+    assert "no data loss has occurred" in message
+    assert "archivebox init" in message
+
+
+def test_migration_interrupt_message_before_apply_says_no_changes_applied():
+    message = _migration_interrupt_message(before_apply=True)
+
+    assert "cancelled before any changes were applied" in message
+    assert "archivebox init" in message
+
+
+def test_migration_interrupt_handler_exits_for_sigint_and_sigterm(monkeypatch):
+    def fake_exit(code):
+        raise SystemExit(code)
+
+    monkeypatch.setattr("archivebox.misc.checks.os._exit", fake_exit)
+
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        previous_handler = signal.getsignal(sig)
+        try:
+            with _exit_on_migration_interrupt():
+                assert signal.getsignal(sig) != previous_handler
+                os.kill(os.getpid(), sig)
+        except SystemExit as err:
+            assert err.code == 130
+        else:
+            raise AssertionError(f"{sig.name} should exit during migration auto-apply")
+        assert signal.getsignal(sig) == previous_handler