backup: save in-progress dev changes

archivebox/api/auth.py

@@ -7,7 +7,7 @@ from django.http import HttpRequest
 from django.contrib.auth import authenticate
 from django.contrib.auth.models import User
 
-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader
 from ninja.errors import HttpError
 
 
@@ -106,33 +106,6 @@ class QueryParamTokenAuth(APIKeyQuery):
         return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
 
 
-class UsernameAndPasswordAuth(HttpBasicAuth):
-    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
-
-    def authenticate(self, request: HttpRequest, username: str, password: str) -> User | None:
-        return _require_superuser(
-            auth_using_password(username=username, password=password, request=request),
-            request,
-            self.__class__.__name__,
-        )
-
-
-class DjangoSessionAuth:
-    """Allow authenticating with existing Django session cookies (same-origin only)."""
-
-    def __call__(self, request: HttpRequest) -> User | None:
-        return self.authenticate(request)
-
-    def authenticate(self, request: HttpRequest, **kwargs) -> User | None:
-        user = getattr(request, "user", None)
-        if isinstance(user, User) and user.is_authenticated:
-            setattr(request, "_api_auth_method", self.__class__.__name__)
-            if not user.is_superuser:
-                raise HttpError(403, "Valid session but User does not have permission (make sure user.is_superuser=True)")
-            return user
-        return None
-
-
 ### Enabled Auth Methods
 
 API_AUTH_METHODS = [

archivebox/api/v1_cli.py

@@ -61,6 +61,10 @@ class AddCommandSchema(Schema):
     snapshot_ids: list[str] | None = None
     tag: str = ""
     depth: int = 0
+    max_urls: int = 0
+    crawl_max_size: int = 0
+    crawl_timeout: int = 0
+    snapshot_max_size: int = 0
     parser: str = "auto"
     plugins: str = ""
     update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
@@ -122,6 +126,10 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         snapshot_ids=args.snapshot_ids,
         tag=args.tag,
         depth=args.depth,
+        max_urls=args.max_urls,
+        crawl_max_size=args.crawl_max_size,
+        crawl_timeout=args.crawl_timeout,
+        snapshot_max_size=args.snapshot_max_size,
         update=args.update,
         index_only=args.index_only,
         overwrite=args.overwrite,

archivebox/api/v1_core.py

@@ -11,8 +11,7 @@ from typing import Union, Any, Annotated
 from datetime import datetime, time
 
 from django.db import transaction
-from django.db.models import Model, Q, Sum
-from django.db.models.functions import Coalesce
+from django.db.models import Model, Q
 from django.http import HttpRequest, HttpResponse
 from django.http.multipartparser import MultiPartParser, MultiPartParserError
 from django.core.exceptions import ValidationError
@@ -29,6 +28,7 @@ from ninja.pagination import paginate, PaginationBase
 from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot, ArchiveResult, Tag
+from archivebox.core.permissions import public_snapshots_queryset
 from archivebox.api.auth import auth_using_token
 from archivebox.config.common import get_config
 from archivebox.core.host_utils import build_web_url
@@ -671,7 +671,7 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_archive_size(obj):
-        return int(getattr(obj, "output_size_sum", obj.archive_size) or 0)
+        return int(obj.archive_size or 0)
 
     @staticmethod
     def resolve_output_size(obj):
@@ -689,6 +689,7 @@ class SnapshotSchema(Schema):
 
 
 class SnapshotUpdateSchema(Schema):
+    action: str | None = None
     status: str | None = None
     retry_at: datetime | None = None
     tags: list[str] | None = None
@@ -766,7 +767,6 @@ def _filter_snapshots_for_rss(
         )
         .filter(bookmarked_at__lte=before_dt)
     )
-
     crawl_id = crawl_id.strip()
     if crawl_id:
         queryset = queryset.filter(crawl__id__icontains=crawl_id)
@@ -854,7 +854,7 @@ class SnapshotFilterSchema(FilterSchema):
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
     """List all Snapshot entries matching these filters."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0))
+    queryset = Snapshot.objects.all()
     return filters.filter(queryset).distinct()
 
 
@@ -880,7 +880,7 @@ def get_snapshots_rss(
 def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
     """Get a specific Snapshot by id."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0))
+    queryset = Snapshot.objects.all()
     try:
         return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
@@ -963,8 +963,24 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
 
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
+    action = payload.pop("action", None)
     tags = payload.pop("tags", None)
 
+    if action:
+        if action == "pause":
+            snapshot.pause()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        if action in ("resume", "unpause"):
+            snapshot.resume()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        if action == "cancel":
+            snapshot.cancel()
+            setattr(request, "with_archiveresults", False)
+            return snapshot
+        raise HttpError(400, f"Invalid action: {action}")
+
     if "status" in payload:
         if payload["status"] not in Snapshot.StatusChoices.values:
             raise HttpError(400, f"Invalid status: {payload['status']}")
@@ -980,7 +996,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
     if tags is not None:
         snapshot.save_tags(normalize_tag_list(tags))
 
-    snapshot.save(update_fields=update_fields)
+    if payload.get("status") == Snapshot.StatusChoices.SEALED:
+        snapshot.cancel()
+    else:
+        snapshot.save(update_fields=update_fields)
     setattr(request, "with_archiveresults", False)
     return snapshot
 
@@ -1164,6 +1183,40 @@ class TagSnapshotResponseSchema(Schema):
     tag_name: str
 
 
+def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
+    snapshot_ref = str(snapshot_ref or "").strip().lower()
+    if not snapshot_ref:
+        raise HttpError(400, "Snapshot id is required")
+
+    snapshot_qs = Snapshot.objects.only("id")
+    is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
+    if is_full_uuid:
+        try:
+            return snapshot_qs.get(pk=snapshot_ref)
+        except (Snapshot.DoesNotExist, ValueError):
+            pass
+
+    if len(snapshot_ref) >= 14:
+        try:
+            return snapshot_qs.get(timestamp=snapshot_ref)
+        except Snapshot.DoesNotExist:
+            pass
+        except Snapshot.MultipleObjectsReturned:
+            snapshot = snapshot_qs.filter(timestamp=snapshot_ref).first()
+            if snapshot is not None:
+                return snapshot
+
+    try:
+        return snapshot_qs.get(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref))
+    except Snapshot.DoesNotExist:
+        raise HttpError(404, "Snapshot not found") from None
+    except Snapshot.MultipleObjectsReturned:
+        snapshot = snapshot_qs.filter(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref)).first()
+        if snapshot is None:
+            raise HttpError(404, "Snapshot not found")
+        return snapshot
+
+
 @router.get("/tags/search/", response=TagSearchResponseSchema, url_name="search_tags")
 def search_tags(
     request: HttpRequest,
@@ -1195,10 +1248,7 @@ def search_tags(
 
 
 def _public_tag_listing_enabled() -> bool:
-    config = get_config()
-    if config.PUBLIC_SNAPSHOTS_LIST is not None:
-        return config.PUBLIC_SNAPSHOTS_LIST
-    return config.PUBLIC_INDEX
+    return get_config().PUBLIC_INDEX
 
 
 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
@@ -1223,8 +1273,13 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     if not _request_has_tag_autocomplete_access(request):
         raise HttpError(401, "Authentication required")
 
-    tags = list(get_matching_tags(q, with_snapshot_counts=False)[: 50 if not q else 20])
-    add_snapshot_counts(tags)
+    public_only = not getattr(request.user, "is_authenticated", False) and not getattr(request, "_api_token", None)
+    queryset = get_matching_tags(q, with_snapshot_counts=False)
+    public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
+    if public_only:
+        queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()
+    tags = list(queryset[: 50 if not q else 20])
+    add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
 
     return {
         "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags],
@@ -1308,19 +1363,7 @@ def tag_snapshots_export(request: HttpRequest, tag_id: int):
 @router.post("/tags/add-to-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_add_to_snapshot")
 def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
     """Add a tag to a snapshot. Creates the tag if it doesn't exist."""
-    # Get the snapshot
-    try:
-        snapshot = Snapshot.objects.get(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        )
-    except Snapshot.DoesNotExist:
-        raise HttpError(404, "Snapshot not found")
-    except Snapshot.MultipleObjectsReturned:
-        snapshot = Snapshot.objects.filter(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        ).first()
-        if snapshot is None:
-            raise HttpError(404, "Snapshot not found")
+    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
 
     # Get or create the tag
     if data.tag_name:
@@ -1352,19 +1395,7 @@ def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
 @router.post("/tags/remove-from-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_remove_from_snapshot")
 def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
     """Remove a tag from a snapshot."""
-    # Get the snapshot
-    try:
-        snapshot = Snapshot.objects.get(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        )
-    except Snapshot.DoesNotExist:
-        raise HttpError(404, "Snapshot not found")
-    except Snapshot.MultipleObjectsReturned:
-        snapshot = Snapshot.objects.filter(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        ).first()
-        if snapshot is None:
-            raise HttpError(404, "Snapshot not found")
+    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
 
     # Get the tag
     if data.tag_id:

archivebox/api/v1_crawls.py

@@ -14,7 +14,6 @@ from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot
 from archivebox.crawls.models import Crawl
-from archivebox.config.common import get_config
 
 from .auth import API_AUTH_METHODS
 
@@ -33,22 +32,15 @@ class CrawlSchema(Schema):
 
     status: str
     retry_at: datetime | None
+    is_paused: bool
 
     urls: str
     max_depth: int
-    max_urls: int
-    crawl_max_size: int
-    snapshot_max_size: int
-    crawl_max_concurrent_snapshots: int
     tags_str: str
     config: dict
 
     # snapshots: List[SnapshotSchema]
 
-    @staticmethod
-    def resolve_crawl_max_concurrent_snapshots(obj):
-        return int(get_config(crawl=obj).CRAWL_MAX_CONCURRENT_SNAPSHOTS)
-
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -68,6 +60,7 @@ class CrawlSchema(Schema):
 
 
 class CrawlUpdateSchema(Schema):
+    action: str | None = None
     status: str | None = None
     retry_at: datetime | None = None
     tags: list[str] | None = None
@@ -77,10 +70,6 @@ class CrawlUpdateSchema(Schema):
 class CrawlCreateSchema(Schema):
     urls: list[str]
     max_depth: int = 0
-    max_urls: int = 0
-    crawl_max_size: int = 0
-    snapshot_max_size: int = 0
-    crawl_max_concurrent_snapshots: int | None = None
     tags: list[str] | None = None
     tags_str: str = ""
     label: str = ""
@@ -113,25 +102,12 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
         raise HttpError(400, "At least one URL is required")
     if data.max_depth not in (0, 1, 2, 3, 4):
         raise HttpError(400, "max_depth must be between 0 and 4")
-    if data.max_urls < 0:
-        raise HttpError(400, "max_urls must be >= 0")
-    if data.crawl_max_size < 0:
-        raise HttpError(400, "crawl_max_size must be >= 0")
-    if data.snapshot_max_size < 0:
-        raise HttpError(400, "snapshot_max_size must be >= 0")
-    if data.crawl_max_concurrent_snapshots is not None and data.crawl_max_concurrent_snapshots < 1:
-        raise HttpError(400, "crawl_max_concurrent_snapshots must be >= 1")
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
-    if data.crawl_max_concurrent_snapshots is not None:
-        config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = data.crawl_max_concurrent_snapshots
     crawl = Crawl.objects.create(
         urls="\n".join(urls),
         max_depth=data.max_depth,
-        max_urls=data.max_urls,
-        crawl_max_size=data.crawl_max_size,
-        snapshot_max_size=data.snapshot_max_size,
         tags_str=",".join(tags),
         label=data.label,
         notes=data.notes,
@@ -167,6 +143,19 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
 
+    action = payload.pop("action", None)
+    if action:
+        if action == "pause":
+            crawl.pause()
+            return crawl
+        if action in ("resume", "unpause"):
+            crawl.resume()
+            return crawl
+        if action == "cancel":
+            crawl.cancel()
+            return crawl
+        raise HttpError(400, f"Invalid action: {action}")
+
     tags = payload.pop("tags", None)
     tags_str = payload.pop("tags_str", None)
     if tags is not None or tags_str is not None:
@@ -186,19 +175,7 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
         update_fields.append("retry_at")
 
     if payload.get("status") == Crawl.StatusChoices.SEALED:
-        cancelled_at = timezone.now()
-        crawl.retry_at = None
-        if "retry_at" not in update_fields:
-            update_fields.append("retry_at")
-        crawl.save(update_fields=update_fields)
-        Snapshot.objects.filter(
-            crawl=crawl,
-            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
-        ).update(
-            status=Snapshot.StatusChoices.SEALED,
-            retry_at=None,
-            modified_at=cancelled_at,
-        )
+        crawl.cancel()
     else:
         crawl.save(update_fields=update_fields)
     return crawl

archivebox/base_models/admin.py

@@ -117,7 +117,7 @@ class KeyValueWidget(forms.Widget):
         config_meta_json = json.dumps(config_options)
 
         html = f'''
-        <div id="{widget_id}_container" class="key-value-editor" style="max-width: 700px;">
+        <div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
             <datalist id="{widget_id}_keys">
                 {datalist_options}
             </datalist>
@@ -606,7 +606,7 @@ class KeyValueWidget(forms.Widget):
                         var newRow = document.createElement('div');
                         newRow.className = 'key-value-row';
                         newRow.style.cssText = 'margin-bottom: 6px;';
-                        newRow.innerHTML = '<div style="display: flex; gap: 8px; align-items: center;">' +
+                        newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
                             '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
                             'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
                             '<input type="text" class="kv-value" placeholder="value" ' +
@@ -672,7 +672,7 @@ class KeyValueWidget(forms.Widget):
     def _render_row(self, widget_id: str, key: str, value: str) -> str:
         return f'''
             <div class="key-value-row" style="margin-bottom: 6px;">
-                <div style="display: flex; gap: 8px; align-items: center;">
+                <div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
                     <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
                            style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
                     <input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"

archivebox/cli/__init__.py

@@ -165,15 +165,13 @@ def cli(ctx, help=False):
                     os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
                     if "--reload" in sys.argv:
                         os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
-                        from archivebox.config.common import get_config
-
-                        os.environ["ARCHIVEBOX_RUNSERVER_PIDFILE"] = str(get_config().TMP_DIR / "runserver.pid")
 
             from archivebox.config.django import setup_django
-            from archivebox.misc.checks import check_data_folder
+            from archivebox.misc.checks import check_data_folder, check_migrations
 
             setup_django()
             check_data_folder()
+            check_migrations(auto_apply=True)
         except Exception as e:
             print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]", file=sys.stderr)
             if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand

archivebox/cli/archivebox_add.py

@@ -52,6 +52,7 @@ def add(
     depth: int | str = 0,
     max_urls: int = 0,
     crawl_max_size: int | str = 0,
+    crawl_timeout: int = 0,
     snapshot_max_size: int | str = 0,
     crawl_max_concurrent_snapshots: int | None = None,
     tag: str = "",
@@ -83,6 +84,7 @@ def add(
     depth = int(depth)
     max_urls = int(max_urls or 0)
     crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
+    crawl_timeout = int(crawl_timeout or 0)
     snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
     config = get_config()
     crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
@@ -96,6 +98,8 @@ def add(
         raise ValueError("max_urls must be >= 0")
     if crawl_max_size < 0:
         raise ValueError("crawl_max_size must be >= 0")
+    if crawl_timeout < 0:
+        raise ValueError("crawl_timeout must be >= 0")
     if snapshot_max_size < 0:
         raise ValueError("snapshot_max_size must be >= 0")
     if crawl_max_concurrent_snapshots < 1:
@@ -158,6 +162,10 @@ def add(
             and crawl_max_concurrent_snapshots != int(effective_persona_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS)
             else {}
         ),
+        **({"CRAWL_MAX_URLS": max_urls} if max_urls else {}),
+        **({"CRAWL_MAX_SIZE": crawl_max_size} if crawl_max_size else {}),
+        **({"CRAWL_TIMEOUT": crawl_timeout} if crawl_timeout else {}),
+        **({"SNAPSHOT_MAX_SIZE": snapshot_max_size} if snapshot_max_size else {}),
         **({"PARSER": parser} if parser != "auto" else {}),
         **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
         **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
@@ -166,9 +174,6 @@ def add(
     crawl = Crawl.objects.create(
         urls=urls_content,
         max_depth=depth,
-        max_urls=max_urls,
-        crawl_max_size=crawl_max_size,
-        snapshot_max_size=snapshot_max_size,
         tags_str=tag,
         persona_id=persona_obj.id,
         label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
@@ -274,6 +279,7 @@ def add(
 )
 @click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
 @click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
+@click.option("--crawl-timeout", type=int, default=0, help="Maximum total crawl runtime in seconds (0 = unlimited)")
 @click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
 @click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
 @click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
@@ -297,6 +303,8 @@ def main(**kwargs):
         raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
     if int(kwargs.get("max_urls") or 0) < 0:
         raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
+    if int(kwargs.get("crawl_timeout") or 0) < 0:
+        raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout")
     try:
         kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
     except ValueError as err:

archivebox/cli/archivebox_extract.py

@@ -32,6 +32,7 @@ __command__ = "archivebox extract"
 
 import sys
 from collections import defaultdict
+from itertools import product
 
 import rich_click as click
 
@@ -45,7 +46,6 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     through the shared crawl runner with the corresponding plugin selected.
     """
     from rich import print as rprint
-    from django.utils import timezone
     from archivebox.core.models import ArchiveResult
     from archivebox.services.runner import run_crawl
 
@@ -58,19 +58,29 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
 
     try:
+        was_paused = archiveresult.snapshot.is_paused
         archiveresult.reset_for_retry()
         snapshot = archiveresult.snapshot
-        snapshot.status = snapshot.StatusChoices.QUEUED
-        snapshot.retry_at = timezone.now()
-        snapshot.save(update_fields=["status", "retry_at", "modified_at"])
-
+        if not was_paused:
+            snapshot.queue_for_extraction()
+        else:
+            # A paused snapshot may still accept explicit maintenance for one
+            # ArchiveResult, but this path must not transition it back to
+            # queued/startable work.
+            snapshot.save(update_fields=["retry_at", "modified_at"])
         crawl = snapshot.crawl
-        if crawl.status != crawl.StatusChoices.STARTED:
-            crawl.status = crawl.StatusChoices.QUEUED
-        crawl.retry_at = timezone.now()
-        crawl.save(update_fields=["status", "retry_at", "modified_at"])
+        if not crawl.claim_processing_lock(lock_seconds=10):
+            rprint(
+                f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]",
+                file=sys.stderr,
+            )
+            return 1
 
-        run_crawl(str(crawl.id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        try:
+            run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        finally:
+            if was_paused:
+                snapshot.restore_paused_scheduler_marker()
         archiveresult.refresh_from_db()
 
         if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
@@ -98,6 +108,7 @@ def run_plugins(
     plugins: str = "",
     wait: bool = True,
     emit_results: bool = True,
+    show_progress: bool = True,
 ) -> int:
     """
     Run plugins on Snapshots from input.
@@ -118,7 +129,9 @@ def run_plugins(
         TYPE_ARCHIVERESULT,
     )
     from archivebox.core.models import Snapshot
+    from archivebox.core.models import ArchiveResult
     from archivebox.services.runner import run_crawl
+    from abx_dl.models import discover_plugins
 
     is_tty = sys.stdout.isatty()
 
@@ -145,7 +158,7 @@ def run_plugins(
         if record_type == TYPE_SNAPSHOT:
             snapshot_id = record.get("id")
             if snapshot_id:
-                snapshot_ids.add(snapshot_id)
+                snapshot_ids.add(str(snapshot_id))
             elif record.get("url"):
                 # Look up by URL (get most recent if multiple exist)
                 snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
@@ -157,41 +170,124 @@ def run_plugins(
         elif record_type == TYPE_ARCHIVERESULT:
             snapshot_id = record.get("snapshot_id")
             if snapshot_id:
-                snapshot_ids.add(snapshot_id)
+                snapshot_ids.add(str(snapshot_id))
                 plugin_name = record.get("plugin")
                 if plugin_name and not plugins_list:
                     requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
 
         elif "id" in record:
             # Assume it's a snapshot ID
-            snapshot_ids.add(record["id"])
+            snapshot_ids.add(str(record["id"]))
 
     if not snapshot_ids:
         rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
         return 1
 
-    # Get snapshots and ensure they have pending ArchiveResults
-    processed_count = 0
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-        except Snapshot.DoesNotExist:
-            rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
-            continue
+    existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id"))
+    existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots}
+    existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots}
+    missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids)
+    for snapshot_id in missing_snapshot_ids:
+        rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
 
-        requested_plugin_names = set(plugins_list) | requested_plugins_by_snapshot.get(str(snapshot.id), set())
-        for plugin_name in requested_plugin_names:
-            existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first()
-            if existing_result:
-                existing_result.reset_for_retry()
+    # Queue only the target plugin rows. Bulk updates keep large reindex runs
+    # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start.
+    requested_pairs: set[tuple[str, str]] = set()
+    if plugins_list:
+        requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list))
+    else:
+        requested_pairs.update(
+            (snapshot_id, plugin_name)
+            for snapshot_id, plugin_names in requested_plugins_by_snapshot.items()
+            if snapshot_id in existing_snapshot_ids
+            for plugin_name in plugin_names
+        )
+    plugins_by_name = discover_plugins()
+    requested_rows: set[tuple[str, str, str]] = set()
+    for snapshot_id, plugin_name in requested_pairs:
+        plugin = plugins_by_name.get(plugin_name)
+        hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
+        if hooks:
+            requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
+        else:
+            requested_rows.add((snapshot_id, plugin_name, ""))
 
-        # Reset snapshot status to allow processing
-        if snapshot.status == Snapshot.StatusChoices.SEALED:
-            snapshot.status = Snapshot.StatusChoices.STARTED
-            snapshot.retry_at = timezone.now()
-            snapshot.save()
+    reset_fields = {
+        "status": ArchiveResult.StatusChoices.QUEUED,
+        "output_str": "",
+        "output_json": None,
+        "output_files": {},
+        "output_size": 0,
+        "output_mimetypes": "",
+        "start_ts": None,
+        "end_ts": None,
+        "modified_at": timezone.now(),
+    }
+    if plugins_list:
+        ArchiveResult.objects.filter(snapshot_id__in=existing_snapshot_ids, plugin__in=plugins_list).update(**reset_fields)
+    elif requested_plugins_by_snapshot:
+        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
+        for snapshot_id, plugin_names in requested_plugins_by_snapshot.items():
+            if snapshot_id in existing_snapshot_ids:
+                for plugin_name in plugin_names:
+                    snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
+        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(**reset_fields)
+    existing_rows = set(
+        ArchiveResult.objects.filter(
+            snapshot_id__in=existing_snapshot_ids,
+            plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+        ).values_list("snapshot_id", "plugin", "hook_name"),
+    )
+    missing_rows = requested_rows - {(str(snapshot_id), plugin_name, hook_name) for snapshot_id, plugin_name, hook_name in existing_rows}
+    if missing_rows:
+        ArchiveResult.objects.bulk_create(
+            [
+                ArchiveResult(
+                    snapshot_id=snapshot_id,
+                    plugin=plugin_name,
+                    hook_name=hook_name,
+                    status=ArchiveResult.StatusChoices.QUEUED,
+                )
+                for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
+            ],
+            batch_size=500,
+        )
 
-        processed_count += 1
+    processed_count = len(existing_snapshot_ids)
+    queue_at = timezone.now()
+    if existing_snapshot_ids:
+        if requested_rows:
+            # Targeted ArchiveResult retries use retry_at as the scheduling
+            # signal and keep sealed snapshots sealed so extractors are not
+            # re-run outside the explicitly queued plugin rows. Paused snapshots
+            # also keep status=paused here: `retry_at` only asks the orchestrator
+            # to process the queued plugin rows, and run_due_snapshot restores
+            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
+            Snapshot.objects.filter(id__in=existing_snapshot_ids).update(
+                retry_at=queue_at,
+                modified_at=queue_at,
+            )
+        else:
+            # No plugin rows were requested, so this is a full snapshot retry.
+            Snapshot.objects.filter(id__in=existing_snapshot_ids).update(
+                status=Snapshot.StatusChoices.QUEUED,
+                retry_at=queue_at,
+                current_step=0,
+                modified_at=queue_at,
+            )
+    if existing_crawl_ids and not requested_rows:
+        from archivebox.crawls.models import Crawl
+
+        Crawl.objects.filter(id__in=existing_crawl_ids).exclude(status=Crawl.StatusChoices.STARTED).update(
+            status=Crawl.StatusChoices.QUEUED,
+            retry_at=queue_at,
+            modified_at=queue_at,
+        )
+        Crawl.objects.filter(id__in=existing_crawl_ids, status=Crawl.StatusChoices.STARTED).update(
+            retry_at=queue_at,
+            modified_at=queue_at,
+        )
 
     if processed_count == 0:
         rprint("[red]No snapshots to process[/red]", file=sys.stderr)
@@ -203,14 +299,19 @@ def run_plugins(
     if wait:
         rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
         snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id in snapshot_ids:
-            try:
-                snapshot = Snapshot.objects.only("id", "crawl_id").get(id=snapshot_id)
-            except Snapshot.DoesNotExist:
-                continue
-            snapshot_ids_by_crawl[str(snapshot.crawl_id)].add(str(snapshot.id))
+        for snapshot_id, crawl_id in existing_snapshots:
+            snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))
 
         for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
+            from archivebox.crawls.models import Crawl
+
+            crawl = Crawl.objects.get(id=crawl_id)
+            if not crawl.claim_processing_lock(lock_seconds=10):
+                rprint(
+                    f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]",
+                    file=sys.stderr,
+                )
+                return 1
             selected_plugins = (
                 plugins_list
                 or sorted(
@@ -222,6 +323,7 @@ def run_plugins(
                 crawl_id,
                 snapshot_ids=sorted(crawl_snapshot_ids),
                 selected_plugins=selected_plugins,
+                show_progress=show_progress,
             )
 
     if not emit_results:

archivebox/cli/archivebox_persona.py

@@ -31,12 +31,8 @@ import os
 import sys
 import shutil
 import platform
-import subprocess
-import tempfile
-import json
 from pathlib import Path
 from collections.abc import Iterable
-from collections import OrderedDict
 
 import rich_click as click
 from rich import print as rprint
@@ -214,136 +210,6 @@ CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"}
 # Cookie Extraction via CDP
 # =============================================================================
 
-NETSCAPE_COOKIE_HEADER = [
-    "# Netscape HTTP Cookie File",
-    "# https://curl.se/docs/http-cookies.html",
-    "# This file was generated by ArchiveBox persona cookie extraction",
-    "#",
-    "# Format: domain\\tincludeSubdomains\\tpath\\tsecure\\texpiry\\tname\\tvalue",
-    "",
-]
-
-
-def _parse_netscape_cookies(path: Path) -> "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]":
-    cookies = OrderedDict()
-    if not path.exists():
-        return cookies
-
-    for line in path.read_text().splitlines():
-        if not line or line.startswith("#"):
-            continue
-        parts = line.split("\t")
-        if len(parts) < 7:
-            continue
-        domain, include_subdomains, cookie_path, secure, expiry, name, value = parts[:7]
-        key = (domain, cookie_path, name)
-        cookies[key] = (domain, include_subdomains, cookie_path, secure, expiry, name, value)
-    return cookies
-
-
-def _write_netscape_cookies(path: Path, cookies: "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]") -> None:
-    lines = list(NETSCAPE_COOKIE_HEADER)
-    for cookie in cookies.values():
-        lines.append("\t".join(cookie))
-    path.write_text("\n".join(lines) + "\n")
-
-
-def _merge_netscape_cookies(existing_file: Path, new_file: Path) -> None:
-    existing = _parse_netscape_cookies(existing_file)
-    new = _parse_netscape_cookies(new_file)
-    for key, cookie in new.items():
-        existing[key] = cookie
-    _write_netscape_cookies(existing_file, existing)
-
-
-def extract_cookies_via_cdp(
-    user_data_dir: Path,
-    output_file: Path,
-    profile_dir: str | None = None,
-    chrome_binary: str | None = None,
-) -> bool:
-    """
-    Launch Chrome with the given user data dir and extract cookies via CDP.
-
-    Returns True if successful, False otherwise.
-    """
-    from archivebox.config.common import get_config
-
-    # Find the cookie extraction script
-    chrome_plugin_dir = Path(__file__).parent.parent / "plugins" / "chrome"
-    extract_script = chrome_plugin_dir / "extract_cookies.js"
-
-    if not extract_script.exists():
-        rprint(f"[yellow]Cookie extraction script not found at {extract_script}[/yellow]", file=sys.stderr)
-        return False
-
-    # Get node modules dir
-    node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules"
-
-    # Set up environment
-    env = os.environ.copy()
-    env["NODE_MODULES_DIR"] = str(node_modules_dir)
-    env["CHROME_USER_DATA_DIR"] = str(user_data_dir)
-    env["CHROME_HEADLESS"] = "true"
-    if chrome_binary:
-        env["CHROME_BINARY"] = str(chrome_binary)
-    output_path = output_file
-    temp_output = None
-    temp_dir = None
-    if output_file.exists():
-        temp_dir = Path(tempfile.mkdtemp(prefix="ab_cookies_"))
-        temp_output = temp_dir / "cookies.txt"
-        output_path = temp_output
-    if profile_dir:
-        extra_arg = f"--profile-directory={profile_dir}"
-        existing_extra = env.get("CHROME_ARGS_EXTRA", "").strip()
-        args_list = []
-        if existing_extra:
-            if existing_extra.startswith("["):
-                try:
-                    parsed = json.loads(existing_extra)
-                    if isinstance(parsed, list):
-                        args_list.extend(str(x) for x in parsed)
-                except Exception:
-                    args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
-            else:
-                args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
-        args_list.append(extra_arg)
-        env["CHROME_ARGS_EXTRA"] = json.dumps(args_list)
-
-    env["COOKIES_OUTPUT_FILE"] = str(output_path)
-
-    try:
-        result = subprocess.run(
-            ["node", str(extract_script)],
-            env=env,
-            capture_output=True,
-            text=True,
-            timeout=60,
-        )
-
-        if result.returncode == 0:
-            if temp_output and temp_output.exists():
-                _merge_netscape_cookies(output_file, temp_output)
-            return True
-        else:
-            rprint(f"[yellow]Cookie extraction failed: {result.stderr}[/yellow]", file=sys.stderr)
-            return False
-
-    except subprocess.TimeoutExpired:
-        rprint("[yellow]Cookie extraction timed out[/yellow]", file=sys.stderr)
-        return False
-    except FileNotFoundError:
-        rprint("[yellow]Node.js not found. Cannot extract cookies.[/yellow]", file=sys.stderr)
-        return False
-    except Exception as e:
-        rprint(f"[yellow]Cookie extraction error: {e}[/yellow]", file=sys.stderr)
-        return False
-    finally:
-        if temp_dir and temp_dir.exists():
-            shutil.rmtree(temp_dir, ignore_errors=True)
-
-
 # =============================================================================
 # Validation Helpers
 # =============================================================================

archivebox/cli/archivebox_run.py

@@ -40,6 +40,7 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox run"
 
+import os
 import sys
 from collections import defaultdict
 
@@ -111,10 +112,10 @@ def process_stdin_records() -> int:
                     crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if crawl:
-                    crawl.retry_at = timezone.now()
-                    if crawl.status not in [Crawl.StatusChoices.SEALED]:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save()
+                    crawl.update_and_requeue(
+                        status=Crawl.StatusChoices.QUEUED,
+                        retry_at=timezone.now(),
+                    )
                     full_crawl_ids.add(str(crawl.id))
                     run_all_plugins_for_crawl.add(str(crawl.id))
                     output_records.append(crawl.to_json())
@@ -132,15 +133,7 @@ def process_stdin_records() -> int:
                     snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
 
                 if snapshot:
-                    snapshot.retry_at = timezone.now()
-                    if snapshot.status not in [Snapshot.StatusChoices.SEALED]:
-                        snapshot.status = Snapshot.StatusChoices.QUEUED
-                    snapshot.save()
-                    crawl = snapshot.crawl
-                    crawl.retry_at = timezone.now()
-                    if crawl.status != Crawl.StatusChoices.STARTED:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
+                    snapshot.queue_for_extraction()
                     crawl_id = str(snapshot.crawl_id)
                     snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                     run_all_plugins_for_crawl.add(crawl_id)
@@ -177,15 +170,7 @@ def process_stdin_records() -> int:
                         snapshot = None
 
                 if snapshot:
-                    snapshot.retry_at = timezone.now()
-                    if snapshot.status != Snapshot.StatusChoices.STARTED:
-                        snapshot.status = Snapshot.StatusChoices.QUEUED
-                    snapshot.save(update_fields=["status", "retry_at", "modified_at"])
-                    crawl = snapshot.crawl
-                    crawl.retry_at = timezone.now()
-                    if crawl.status != Crawl.StatusChoices.STARTED:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
+                    snapshot.queue_for_extraction()
                     crawl_id = str(snapshot.crawl_id)
                     snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                     if plugin_name:
@@ -236,6 +221,13 @@ def process_stdin_records() -> int:
     targeted_crawl_ids = full_crawl_ids | set(snapshot_ids_by_crawl)
     if targeted_crawl_ids:
         for crawl_id in sorted(targeted_crawl_ids):
+            try:
+                crawl = Crawl.objects.get(id=crawl_id)
+            except Crawl.DoesNotExist:
+                continue
+            if not crawl.claim_processing_lock(lock_seconds=10):
+                rprint(f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]", file=sys.stderr)
+                return 1
             run_crawl(
                 crawl_id,
                 snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
@@ -253,17 +245,20 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
 
     Returns exit code (0 = success, 1 = error).
     """
-    from django.utils import timezone
+    from archivebox.config import CONSTANTS
     from archivebox.machine.models import Machine, Process
-    from archivebox.services.runner import cleanup_orchestrator_state, run_pending_crawls
+    from archivebox.services.supervision_service import healthy_orchestrator
+    from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls
 
-    cleanup_orchestrator_state(include_chrome=True)
+    recover_orchestrator_state(include_chrome=True)
     Machine.current()
+    existing = healthy_orchestrator(data_dir=CONSTANTS.DATA_DIR)
     current = Process.current()
-    if current.process_type != Process.TypeChoices.ORCHESTRATOR:
-        current.process_type = Process.TypeChoices.ORCHESTRATOR
-        current.save(update_fields=["process_type", "modified_at"])
-
+    existing_pid = existing.get("pid") if isinstance(existing, dict) else getattr(existing, "pid", None)
+    if existing_pid and existing_pid != os.getpid():
+        rprint(f"[green][*] Existing ArchiveBox orchestrator pid={existing_pid} is already running.[/green]", file=sys.stderr)
+        return 0
+    current.mark_running(process_type=Process.TypeChoices.ORCHESTRATOR, pwd=str(CONSTANTS.DATA_DIR), timeout=0)
     try:
         run_pending_crawls(daemon=daemon, crawl_id=crawl_id)
         return 0
@@ -275,9 +270,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
     finally:
         current.refresh_from_db()
         if current.status != Process.StatusChoices.EXITED:
-            current.status = Process.StatusChoices.EXITED
-            current.ended_at = current.ended_at or timezone.now()
-            current.save(update_fields=["status", "ended_at", "modified_at"])
+            current.mark_exited()
 
 
 @click.command()
@@ -328,11 +321,15 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
 
 def run_snapshot_worker(snapshot_id: str) -> int:
     from archivebox.core.models import Snapshot
-    from archivebox.services.runner import run_crawl
+    from archivebox.services.runner import run_due_snapshot
+    from django.utils import timezone
 
     try:
         snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)])
+        if snapshot.retry_at is None:
+            Snapshot.objects.filter(pk=snapshot.pk).update(retry_at=timezone.now(), modified_at=timezone.now())
+            snapshot.refresh_from_db()
+        run_due_snapshot(snapshot, lock_seconds=60)
         return 0
     except KeyboardInterrupt:
         return 0

archivebox/cli/archivebox_server.py

@@ -13,83 +13,6 @@ from archivebox.misc.util import docstring, enforce_types
 from archivebox.config.common import get_config
 
 
-def stop_existing_background_runner(*, machine, process_model, supervisor=None, stop_worker_fn=None, log=print) -> int:
-    """Stop any existing orchestrator process so the server can take ownership."""
-    running_runners = list(
-        process_model.objects.filter(
-            machine=machine,
-            status=process_model.StatusChoices.RUNNING,
-            process_type=process_model.TypeChoices.ORCHESTRATOR,
-        ).order_by("created_at"),
-    )
-
-    if not running_runners:
-        return 0
-
-    log("[yellow][*] Stopping existing ArchiveBox background runner...[/yellow]")
-
-    if supervisor is not None and stop_worker_fn is not None:
-        for worker_name in ("worker_runner", "worker_runner_watch"):
-            try:
-                stop_worker_fn(supervisor, worker_name)
-            except Exception:
-                pass
-
-    for proc in running_runners:
-        try:
-            proc.kill_tree(graceful_timeout=2.0)
-        except Exception:
-            try:
-                proc.terminate(graceful_timeout=2.0)
-            except Exception:
-                pass
-
-    return len(running_runners)
-
-
-def _read_supervisor_worker_command(worker_name: str) -> str:
-    from archivebox.workers.supervisord_util import WORKERS_DIR_NAME, get_sock_file
-
-    worker_conf = get_sock_file().parent / WORKERS_DIR_NAME / f"{worker_name}.conf"
-    if not worker_conf.exists():
-        return ""
-
-    for line in worker_conf.read_text().splitlines():
-        if line.startswith("command="):
-            return line.removeprefix("command=").strip()
-    return ""
-
-
-def _worker_command_matches_bind(command: str, host: str, port: str) -> bool:
-    if not command:
-        return False
-    return f"{host}:{port}" in command or (f"--bind={host}" in command and f"--port={port}" in command)
-
-
-def stop_existing_server_workers(*, supervisor, stop_worker_fn, host: str, port: str, log=print) -> int:
-    """Stop existing ArchiveBox web workers if they already own the requested bind."""
-    stopped = 0
-
-    for worker_name in ("worker_runserver", "worker_daphne"):
-        try:
-            proc = supervisor.getProcessInfo(worker_name) if supervisor else None
-        except Exception:
-            proc = None
-        if not isinstance(proc, dict) or proc.get("statename") != "RUNNING":
-            continue
-
-        command = _read_supervisor_worker_command(worker_name)
-        if not _worker_command_matches_bind(command, host, port):
-            continue
-
-        if stopped == 0:
-            log("[yellow][*] Taking over existing ArchiveBox web server on same port...[/yellow]")
-        stop_worker_fn(supervisor, worker_name)
-        stopped += 1
-
-    return stopped
-
-
 @enforce_types
 def server(
     runserver_args: Iterable[str] | None = None,
@@ -144,60 +67,17 @@ def server(
         pass
 
     from archivebox.workers.supervisord_util import (
-        get_existing_supervisord_process,
-        get_worker,
-        stop_worker,
         start_server_workers,
+        stop_existing_supervisord_process,
         is_port_in_use,
     )
-    from archivebox.machine.models import Machine, Process
-
-    machine = Machine.current()
-    supervisor = get_existing_supervisord_process()
-    stop_existing_background_runner(
-        machine=machine,
-        process_model=Process,
-        supervisor=supervisor,
-        stop_worker_fn=stop_worker,
+    from archivebox.machine.models import Process
+    from archivebox.services.supervision_service import (
+        command_owns_runtime_stack,
+        current_command,
+        standby_until_runtime_stack_needed,
     )
-    if supervisor:
-        stop_existing_server_workers(
-            supervisor=supervisor,
-            stop_worker_fn=stop_worker,
-            host=host,
-            port=port,
-        )
-
-    # Check if port is already in use
-    if is_port_in_use(host, int(port)):
-        print(f"[red][X] Error: Port {port} is already in use[/red]")
-        print(f"    Another process (possibly daphne or runserver) is already listening on {host}:{port}")
-        print("    Stop the conflicting process or choose a different port")
-        sys.exit(1)
-
-    supervisor = get_existing_supervisord_process()
-    if supervisor:
-        server_worker_name = "worker_runserver" if run_in_debug else "worker_daphne"
-        server_proc = get_worker(supervisor, server_worker_name)
-        server_state = server_proc.get("statename") if isinstance(server_proc, dict) else None
-        if server_state == "RUNNING":
-            runner_proc = get_worker(supervisor, "worker_runner")
-            runner_watch_proc = get_worker(supervisor, "worker_runner_watch")
-            runner_state = runner_proc.get("statename") if isinstance(runner_proc, dict) else None
-            runner_watch_state = runner_watch_proc.get("statename") if isinstance(runner_watch_proc, dict) else None
-            print("[red][X] Error: ArchiveBox server is already running[/red]")
-            print(
-                f"    [green]√[/green] Web server ({server_worker_name}) is RUNNING on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
-            )
-            if runner_state == "RUNNING":
-                print("    [green]√[/green] Background runner (worker_runner) is RUNNING")
-            if runner_watch_state == "RUNNING":
-                print("    [green]√[/green] Reload watcher (worker_runner_watch) is RUNNING")
-            print()
-            print("[yellow]To stop the existing server, run:[/yellow]")
-            print('    pkill -f "archivebox server"')
-            print("    pkill -f supervisord")
-            sys.exit(1)
+    from archivebox.core.shutdown_util import foreground_shutdown_signals
 
     if run_in_debug:
         print("[green][+] Starting ArchiveBox webserver in DEBUG mode...[/green]")
@@ -211,7 +91,42 @@ def server(
     )
     print("    > Writing ArchiveBox error log to ./logs/errors.log")
     print()
-    start_server_workers(host=host, port=port, daemonize=daemonize, debug=run_in_debug, reload=reload, nothreading=nothreading)
+    bind_url = f"http://{host}:{port}"
+    command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url)
+
+    try:
+        with foreground_shutdown_signals():
+            while True:
+                standby_until_runtime_stack_needed(command, data_dir=config.DATA_DIR)
+                sys.stdout.write(f"[*] ArchiveBox server parent pid={os.getpid()} is now running the orchestrator and server...\n")
+                sys.stdout.flush()
+                stop_existing_supervisord_process()
+                if is_port_in_use(host, int(port)):
+                    print(f"[red][X] Error: Port {port} is already in use[/red]")
+                    print(f"    Another process outside this ArchiveBox runtime is listening on {host}:{port}")
+                    sys.exit(1)
+
+                result = start_server_workers(
+                    host=host,
+                    port=port,
+                    daemonize=daemonize,
+                    debug=run_in_debug,
+                    reload=reload,
+                    nothreading=nothreading,
+                    keep_running=lambda: command_owns_runtime_stack(command, data_dir=config.DATA_DIR),
+                    should_stop_supervisord=lambda: command_owns_runtime_stack(command, data_dir=config.DATA_DIR),
+                )
+                if not command_owns_runtime_stack(command, data_dir=config.DATA_DIR):
+                    print("[yellow][*] Another ArchiveBox command took over the runtime stack; standing by.[/yellow]")
+                    continue
+                if result == "exited":
+                    print("[yellow][*] Runtime stack exited while this parent is still leader; restarting...[/yellow]")
+                    continue
+                break
+    except KeyboardInterrupt:
+        pass
+    finally:
+        command.mark_exited()
     print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
 
 

archivebox/cli/archivebox_snapshot.py

@@ -35,8 +35,7 @@ from collections.abc import Iterable
 
 import rich_click as click
 from rich import print as rprint
-from django.db.models import Case, IntegerField, Q, Sum, When
-from django.db.models.functions import Coalesce
+from django.db.models import Case, IntegerField, Q, QuerySet, When
 
 from archivebox.cli.cli_utils import apply_filters
 
@@ -179,26 +178,17 @@ def create_snapshots(
 # =============================================================================
 
 
-def list_snapshots(
+def build_snapshot_queryset(
+    *,
     status: str | None = None,
     url__icontains: str | None = None,
     url__istartswith: str | None = None,
     tag: str | None = None,
     crawl_id: str | None = None,
-    limit: int | None = None,
     sort: str | None = None,
-    csv: str | None = None,
-    with_headers: bool = False,
     search: str | None = None,
     query: str | None = None,
-) -> int:
-    """
-    List Snapshots as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
+) -> QuerySet:
     from archivebox.core.models import Snapshot
     from archivebox.search import (
         get_default_search_mode,
@@ -207,24 +197,17 @@ def list_snapshots(
         query_search_index,
     )
 
-    if with_headers and not csv:
-        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
-        return 2
-
-    is_tty = sys.stdout.isatty() and not csv
-
     queryset = Snapshot.objects.order_by("-created_at")
+    queryset = apply_filters(
+        queryset,
+        {
+            "status": status,
+            "url__icontains": url__icontains,
+            "url__istartswith": url__istartswith,
+            "crawl_id": crawl_id,
+        },
+    )
 
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "url__icontains": url__icontains,
-        "url__istartswith": url__istartswith,
-        "crawl_id": crawl_id,
-    }
-    queryset = apply_filters(queryset, filter_kwargs)
-
-    # Tag filter requires special handling (M2M)
     if tag:
         queryset = queryset.filter(tags__name__iexact=tag)
 
@@ -265,6 +248,48 @@ def list_snapshots(
     if sort:
         queryset = queryset.order_by(sort)
 
+    return queryset
+
+
+def list_snapshots(
+    status: str | None = None,
+    url__icontains: str | None = None,
+    url__istartswith: str | None = None,
+    tag: str | None = None,
+    crawl_id: str | None = None,
+    limit: int | None = None,
+    sort: str | None = None,
+    csv: str | None = None,
+    with_headers: bool = False,
+    search: str | None = None,
+    query: str | None = None,
+) -> int:
+    """
+    List Snapshots as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import Snapshot
+
+    if with_headers and not csv:
+        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
+        return 2
+
+    is_tty = sys.stdout.isatty() and not csv
+
+    queryset = build_snapshot_queryset(
+        status=status,
+        url__icontains=url__icontains,
+        url__istartswith=url__istartswith,
+        tag=tag,
+        crawl_id=crawl_id,
+        sort=sort,
+        search=search,
+        query=query,
+    )
+
     if not is_tty:
         if limit:
             limited_ids = list(queryset.values_list("id", flat=True)[:limit])
@@ -273,7 +298,7 @@ def list_snapshots(
                 output_field=IntegerField(),
             )
             queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order)
-        queryset = queryset.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).prefetch_related("tags")
+        queryset = queryset.prefetch_related("tags")
     elif limit:
         queryset = queryset[:limit]
 

archivebox/cli/archivebox_status.py

@@ -130,7 +130,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
         print("        [green]archivebox manage createsuperuser[/green]")
 
     print()
-    recent_snapshots = snapshots_qs.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).order_by(
+    recent_snapshots = snapshots_qs.order_by(
         "-downloaded_at",
         "-modified_at",
     )[:10]
@@ -141,7 +141,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
             (
                 "[grey53] "
                 f"   > {str(snapshot.downloaded_at)[:16]} "
-                f"[{snapshot.num_outputs} {('X', '√')[snapshot.status == Snapshot.StatusChoices.SEALED]} {printable_filesize(snapshot.output_size_sum or 0)}] "
+                f"[{snapshot.num_outputs} {('X', '√')[snapshot.status == Snapshot.StatusChoices.SEALED]} {printable_filesize(snapshot.output_size or 0)}] "
                 f'"{snapshot.title}": {snapshot.url}'
                 "[/grey53]"
             )[: config.TERM_WIDTH],

archivebox/cli/archivebox_update.py

@@ -3,15 +3,17 @@
 __package__ = "archivebox.cli"
 
 import os
+import asyncio
+import shlex
 import time
 
 from typing import TYPE_CHECKING, Any
-from collections.abc import Callable, Iterable
+from collections.abc import Iterable
 from pathlib import Path
 
 import rich_click as click
 from django.core.exceptions import ObjectDoesNotExist
-from django.db.models import Q, QuerySet
+from django.db.models import QuerySet
 
 from archivebox.misc.util import enforce_types, docstring
 
@@ -20,33 +22,6 @@ if TYPE_CHECKING:
     from archivebox.crawls.models import Crawl
 
 
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
 def _get_snapshot_crawl(snapshot: "Snapshot") -> "Crawl | None":
     try:
         return snapshot.crawl
@@ -73,17 +48,35 @@ def _build_filtered_snapshots_queryset(
     *,
     filter_patterns: Iterable[str],
     filter_type: str,
-    before: float | None,
-    after: float | None,
+    status: str | None = None,
+    url__icontains: str | None = None,
+    url__istartswith: str | None = None,
+    tag: str | None = None,
+    crawl_id: str | None = None,
+    limit: int | None = None,
+    sort: str | None = None,
+    search: str | None = None,
+    before: float | None = None,
+    after: float | None = None,
     resume: str | None = None,
 ):
-    from archivebox.core.models import Snapshot
     from datetime import datetime
+    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    snapshots = Snapshot.objects.all()
+    filter_patterns = tuple(filter_patterns)
+    snapshots = build_snapshot_queryset(
+        status=status,
+        url__icontains=url__icontains,
+        url__istartswith=url__istartswith,
+        tag=tag,
+        crawl_id=crawl_id,
+        sort=sort,
+        search=search,
+        query=" ".join(filter_patterns) if search else None,
+    )
 
-    if filter_patterns:
-        snapshots = _apply_pattern_filters(snapshots, list(filter_patterns), filter_type)
+    if filter_patterns and not search:
+        snapshots = snapshots.filter_by_patterns(list(filter_patterns), filter_type)
 
     if before:
         snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before))
@@ -91,8 +84,14 @@ def _build_filtered_snapshots_queryset(
         snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after))
     if resume:
         snapshots = snapshots.filter(timestamp__lte=resume)
+    if not sort:
+        snapshots = snapshots.order_by("-timestamp")
+    snapshots = snapshots.select_related("crawl")
+    if limit:
+        limited_ids = list(snapshots.values_list("id", flat=True)[:limit])
+        snapshots = snapshots.model.objects.filter(id__in=limited_ids).select_related("crawl")
 
-    return snapshots.select_related("crawl").order_by("-bookmarked_at")
+    return snapshots
 
 
 def reindex_snapshots(
@@ -100,53 +99,64 @@ def reindex_snapshots(
     *,
     search_plugins: list[str],
     batch_size: int,
-) -> dict[str, int]:
+    collect_ids: bool = False,
+) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
 
-    stats = {"processed": 0, "reconciled": 0, "queued": 0, "reindexed": 0}
+    stats: dict[str, Any] = {"processed": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
     records: list[dict[str, str]] = []
 
     total = snapshots.count()
     print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
 
-    for snapshot in snapshots.iterator(chunk_size=batch_size):
-        stats["processed"] += 1
+    def run_batch() -> None:
+        if not records:
+            return
+        batch_records = list(records)
+        # Index-only backfill intentionally queues only search ArchiveResult
+        # rows. The extract runner bumps Snapshot.retry_at so the orchestrator
+        # sees the maintenance work, but it does not change status away from
+        # PAUSED; run_due_snapshot restores retry_at=MAX after the targeted
+        # plugin rows finish.
+        exit_code = run_plugins(
+            args=(),
+            records=batch_records,
+            wait=False,
+            emit_results=False,
+            show_progress=False,
+        )
+        if exit_code != 0:
+            raise SystemExit(exit_code)
+        print(
+            f"    [{stats['processed']}/{total}] Queued {len(batch_records)} index jobs for orchestrator",
+        )
+        records.clear()
 
-        if _get_snapshot_crawl(snapshot) is None:
-            continue
+    for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size):
+        try:
+            stats["processed"] += 1
 
-        output_dir = Path(snapshot.output_dir)
-        has_directory = output_dir.exists() and output_dir.is_dir()
-        if has_directory:
-            snapshot.reconcile_with_index_json()
-            stats["reconciled"] += 1
+            if _get_snapshot_crawl(snapshot) is None:
+                continue
 
-        for plugin_name in search_plugins:
-            existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first()
-            if existing_result:
-                existing_result.reset_for_retry()
-            records.append(
-                {
-                    "type": "ArchiveResult",
-                    "snapshot_id": str(snapshot.id),
-                    "plugin": plugin_name,
-                },
-            )
-            stats["queued"] += 1
+            if collect_ids:
+                stats["snapshot_ids"].append(str(snapshot.id))
+            for plugin_name in search_plugins:
+                records.append(
+                    {
+                        "type": "ArchiveResult",
+                        "snapshot_id": str(snapshot.id),
+                        "plugin": plugin_name,
+                    },
+                )
+                stats["queued"] += 1
+            if len(records) >= batch_size:
+                run_batch()
+        except KeyboardInterrupt as err:
+            err.archivebox_resume = snapshot.timestamp
+            raise
 
-    if not records:
-        return stats
-
-    exit_code = run_plugins(
-        args=(),
-        records=records,
-        wait=True,
-        emit_results=False,
-    )
-    if exit_code != 0:
-        raise SystemExit(exit_code)
-
-    stats["reindexed"] = len(records)
+    run_batch()
     return stats
 
 
@@ -154,12 +164,21 @@ def reindex_snapshots(
 def update(
     filter_patterns: Iterable[str] = (),
     filter_type: str = "exact",
+    status: str | None = None,
+    url__icontains: str | None = None,
+    url__istartswith: str | None = None,
+    tag: str | None = None,
+    crawl_id: str | None = None,
+    limit: int | None = None,
+    sort: str | None = None,
+    search: str | None = None,
     before: float | None = None,
     after: float | None = None,
     resume: str | None = None,
     batch_size: int = 100,
     continuous: bool = False,
     index_only: bool = False,
+    migrate_only: bool = False,
 ) -> None:
     """
     Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving.
@@ -174,83 +193,173 @@ def update(
     """
 
     from rich import print
+    from archivebox.config import CONSTANTS
     from archivebox.config.django import setup_django
 
     setup_django()
+    from archivebox.machine.models import Process
+    from archivebox.services.supervision_service import current_command, ensure_daemon_stack
+    from archivebox.workers.supervisord_util import stop_existing_supervisord_process
 
-    from django.core.management import call_command
+    command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR)
+    is_filtered_update = any(
+        (
+            filter_patterns,
+            status,
+            url__icontains,
+            url__istartswith,
+            tag,
+            crawl_id,
+            limit,
+            sort,
+            search,
+            before,
+            after,
+        ),
+    )
+    touched_snapshot_ids: set[str] = set()
+
+    from archivebox.misc.checks import check_migrations
 
-    # Run migrations first to ensure DB schema is up-to-date
-    print("[*] Checking for pending migrations...")
     try:
-        call_command("migrate", "--no-input", verbosity=0)
-    except Exception as e:
-        print(f"[!] Warning: Migration check failed: {e}")
+        # Run migrations first to ensure DB schema is up-to-date
+        print("[*] Checking for pending migrations...")
+        check_migrations(auto_apply=True)
+        stop_existing_supervisord_process()
 
-    while True:
-        if index_only:
-            search_plugins = _get_search_indexing_plugins()
-            if not search_plugins:
-                print("[*] No search indexing plugins are available, nothing to backfill.")
+        while True:
+            do_migrate = migrate_only or not index_only
+            do_index = index_only or not migrate_only
+            do_run_until_idle = do_migrate or do_index
+
+            if do_migrate:
+                if filter_patterns or status or url__icontains or url__istartswith or tag or crawl_id or limit or sort or search or before or after:
+                    print("[*] Processing filtered snapshots from database...")
+                    stats = process_filtered_snapshots(
+                        filter_patterns=filter_patterns,
+                        filter_type=filter_type,
+                        status=status,
+                        url__icontains=url__icontains,
+                        url__istartswith=url__istartswith,
+                        tag=tag,
+                        crawl_id=crawl_id,
+                        limit=limit,
+                        sort=sort,
+                        search=search,
+                        before=before,
+                        after=after,
+                        resume=resume,
+                        batch_size=batch_size,
+                        queue_for_archiving=do_run_until_idle,
+                    )
+                    print_stats(stats)
+                    touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                else:
+                    stats_combined = {"phase1": {}, "phase2": {}}
+
+                    print("[*] Phase 1: Draining old archive/ directories (0.8.x → 0.9.x migration)...")
+                    stats_combined["phase1"] = drain_old_archive_dirs(
+                        resume_from=resume,
+                        batch_size=batch_size,
+                    )
+
+                    print("[*] Phase 2: Processing all database snapshots (most recent first)...")
+                    stats_combined["phase2"] = process_all_db_snapshots(batch_size=batch_size, resume=resume)
+                    print_combined_stats(stats_combined)
+
+            if do_index:
+                ensure_daemon_stack(reason="search indexing")
+                search_plugins = _get_search_indexing_plugins()
+                if not search_plugins:
+                    print("[*] No search indexing plugins are available, nothing to backfill.")
+                else:
+                    snapshots = _build_filtered_snapshots_queryset(
+                        filter_patterns=filter_patterns,
+                        filter_type=filter_type,
+                        status=status,
+                        url__icontains=url__icontains,
+                        url__istartswith=url__istartswith,
+                        tag=tag,
+                        crawl_id=crawl_id,
+                        limit=limit,
+                        sort=sort,
+                        search=search,
+                        before=before,
+                        after=after,
+                        resume=resume,
+                    )
+                    stats = reindex_snapshots(
+                        snapshots,
+                        search_plugins=search_plugins,
+                        batch_size=batch_size,
+                        collect_ids=is_filtered_update,
+                    )
+                    print_index_stats(stats)
+                    touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+
+            if do_run_until_idle:
+                print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
+                from archivebox.cli.archivebox_run import run_runner, run_snapshot_worker
+
+                if is_filtered_update:
+                    if not touched_snapshot_ids:
+                        print("[*] No matching snapshots queued work for the runner.")
+                    for snapshot_id in sorted(touched_snapshot_ids):
+                        exit_code = run_snapshot_worker(snapshot_id)
+                        if exit_code != 0:
+                            raise SystemExit(exit_code)
+                else:
+                    exit_code = run_runner(daemon=False)
+                    if exit_code != 0:
+                        raise SystemExit(exit_code)
+
+            if not continuous:
                 break
 
-            if not (filter_patterns or before or after):
-                print("[*] Phase 1: Draining old archive/ directories (0.8.x → 0.9.x migration)...")
-                drain_old_archive_dirs(
-                    resume_from=resume,
-                    batch_size=batch_size,
-                )
-
-            snapshots = _build_filtered_snapshots_queryset(
-                filter_patterns=filter_patterns,
-                filter_type=filter_type,
-                before=before,
-                after=after,
-                resume=resume,
-            )
-            stats = reindex_snapshots(
-                snapshots,
-                search_plugins=search_plugins,
-                batch_size=batch_size,
-            )
-            print_index_stats(stats)
-        elif filter_patterns or before or after:
-            # Filtered mode: query DB only
-            print("[*] Processing filtered snapshots from database...")
-            stats = process_filtered_snapshots(
-                filter_patterns=filter_patterns,
-                filter_type=filter_type,
-                before=before,
-                after=after,
-                resume=resume,
-                batch_size=batch_size,
-            )
-            print_stats(stats)
-        else:
-            # Full mode: drain old dirs + process DB
-            stats_combined = {"phase1": {}, "phase2": {}}
-
-            print("[*] Phase 1: Draining old archive/ directories (0.8.x → 0.9.x migration)...")
-            stats_combined["phase1"] = drain_old_archive_dirs(
-                resume_from=resume,
-                batch_size=batch_size,
-            )
-
-            print("[*] Phase 2: Processing all database snapshots (most recent first)...")
-            stats_combined["phase2"] = process_all_db_snapshots(batch_size=batch_size, resume=resume)
-
-            # Phase 3: Deduplication (disabled for now)
-            # print('[*] Phase 3: Deduplicating...')
-            # stats_combined['deduplicated'] = Snapshot.find_and_merge_duplicates()
-
-            print_combined_stats(stats_combined)
-
-        if not continuous:
-            break
-
-        print("[yellow]Sleeping 60s before next pass...[/yellow]")
-        time.sleep(60)
-        resume = None
+            print("[yellow]Sleeping 60s before next pass...[/yellow]")
+            time.sleep(60)
+            resume = None
+    except (KeyboardInterrupt, asyncio.CancelledError) as err:
+        exact_resume = getattr(err, "archivebox_resume", None)
+        resume_cmd = ["archivebox", "update"]
+        if migrate_only:
+            resume_cmd.append("--migrate-only")
+        if index_only:
+            resume_cmd.append("--index-only")
+        if batch_size != 100:
+            resume_cmd.extend(["--batch-size", str(batch_size)])
+        if exact_resume or resume:
+            resume_cmd.extend(["--resume", str(exact_resume or resume)])
+        if before is not None:
+            resume_cmd.extend(["--before", str(before)])
+        if after is not None:
+            resume_cmd.extend(["--after", str(after)])
+        if filter_type != "exact":
+            resume_cmd.extend(["--filter-type", filter_type])
+        if status:
+            resume_cmd.extend(["--status", status])
+        if url__icontains:
+            resume_cmd.extend(["--url__icontains", url__icontains])
+        if url__istartswith:
+            resume_cmd.extend(["--url__istartswith", url__istartswith])
+        if tag:
+            resume_cmd.extend(["--tag", tag])
+        if crawl_id:
+            resume_cmd.extend(["--crawl-id", crawl_id])
+        if limit:
+            resume_cmd.extend(["--limit", str(limit)])
+        if sort:
+            resume_cmd.extend(["--sort", sort])
+        if search:
+            resume_cmd.extend(["--search", search])
+        resume_cmd.extend(str(pattern) for pattern in filter_patterns)
+        print("\n[red][X] archivebox update interrupted.[/red]")
+        print("[yellow]Hint: resume this idempotent update with:[/yellow]")
+        print(f"    [green]{' '.join(shlex.quote(part) for part in resume_cmd)}[/green]")
+        raise SystemExit(130)
+    finally:
+        command.mark_exited()
+        stop_existing_supervisord_process()
 
 
 def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100) -> dict[str, int]:
@@ -269,11 +378,9 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
     from archivebox.core.models import Snapshot
     from archivebox.config.common import get_config
     from archivebox.crawls.models import Crawl
-    from django.db import transaction
     from django.utils import timezone
 
-    stats = {"processed": 0, "migrated": 0, "skipped": 0, "invalid": 0}
-    crawl_output_dirs: dict[str, Path] = {}
+    stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0}
     crawl_url_lines: dict[str, list[str]] = {}
     crawl_url_sets: dict[str, set[str]] = {}
     dirty_crawl_ids: set[str] = set()
@@ -283,18 +390,27 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
     if not archive_dir.exists():
         return stats
 
-    for crawl in Crawl.objects.filter(label__startswith="[migration] orphaned").iterator():
-        url_entries = crawl._iter_url_lines()
-        existing_urls = {url for _raw_line, url in url_entries if url}
-        lines = (crawl.urls or "").splitlines()
-        changed = False
-        for url in crawl.snapshot_set.order_by("timestamp").values_list("url", flat=True):
-            if url not in existing_urls:
-                lines.append(url)
-                existing_urls.add(url)
-                changed = True
-        if changed:
-            Crawl.objects.filter(pk=crawl.pk).update(urls="\n".join(lines), modified_at=timezone.now())
+    last_crawl_id = None
+    while True:
+        crawl_qs = Crawl.objects.filter(label__startswith="[migration] orphaned").order_by("id")
+        if last_crawl_id is not None:
+            crawl_qs = crawl_qs.filter(id__gt=last_crawl_id)
+        crawl_batch = list(crawl_qs[:batch_size])
+        if not crawl_batch:
+            break
+        for crawl in crawl_batch:
+            last_crawl_id = crawl.id
+            url_entries = crawl._iter_url_lines()
+            existing_urls = {url for _raw_line, url in url_entries if url}
+            lines = (crawl.urls or "").splitlines()
+            changed = False
+            for url in crawl.snapshot_set.order_by("timestamp").values_list("url", flat=True):
+                if url not in existing_urls:
+                    lines.append(url)
+                    existing_urls.add(url)
+                    changed = True
+            if changed:
+                Crawl.objects.filter(pk=crawl.pk).update(urls="\n".join(lines), modified_at=timezone.now())
 
     # Scan for real directories only (skip symlinks - they're already migrated)
     all_entries = list(os.scandir(archive_dir))
@@ -329,29 +445,17 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
                 continue
 
             try:
+                snapshot.status = Snapshot.StatusChoices.SEALED
+                snapshot.retry_at = timezone.now()
                 Snapshot.objects.bulk_create([snapshot])
-                snapshot.migrate_filesystem_to_current_version(source_dir=entry_path, config=runtime_config)
                 Snapshot.objects.filter(pk=snapshot.pk).update(
-                    fs_version=snapshot.fs_version,
+                    status=Snapshot.StatusChoices.SEALED,
+                    retry_at=snapshot.retry_at,
                 )
-                migration_cleanup = getattr(snapshot, "_pending_fs_migration_cleanup", None)
-                new_dir = None
-                if migration_cleanup:
-                    old_dir, new_dir = migration_cleanup
-                    transaction.on_commit(
-                        lambda old_dir=old_dir, new_dir=new_dir, snapshot=snapshot: snapshot._cleanup_old_migration_dir(old_dir, new_dir),
-                    )
-                    delattr(snapshot, "_pending_fs_migration_cleanup")
 
                 crawl = _get_snapshot_crawl(snapshot)
-                crawl_dir = None
                 if crawl is not None:
                     crawl_cache_key = str(crawl.id)
-                    crawl_dir = crawl_output_dirs.get(crawl_cache_key)
-                    if crawl_dir is None:
-                        crawl_dir = Path(crawl.output_dir)
-                        crawl_output_dirs[crawl_cache_key] = crawl_dir
-
                     existing_urls = crawl_url_sets.get(crawl_cache_key)
                     if existing_urls is None:
                         url_entries = crawl._iter_url_lines()
@@ -363,9 +467,8 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
                         existing_urls.add(snapshot.url)
                         dirty_crawl_ids.add(crawl_cache_key)
 
-                snapshot.ensure_crawl_symlink(crawl_dir=crawl_dir, snapshot_dir=new_dir)
-                stats["migrated"] += 1
-                print(f"    [{stats['processed']}] Imported orphaned snapshot: {entry_path.name}")
+                stats["queued"] += 1
+                print(f"    [{stats['processed']}] Imported orphaned snapshot and queued migration: {entry_path.name}")
             except Exception as e:
                 stats["skipped"] += 1
                 print(f"    [{stats['processed']}] Skipped (error: {e}): {entry_path.name}")
@@ -377,7 +480,9 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
         if not has_valid_crawl:
             # Create a new crawl (created_by will default to system user)
             crawl = Crawl.objects.create(urls=snapshot.url)
-            # Use queryset update to avoid triggering save() hooks
+            # Use queryset update to avoid save() hooks and keep the SQLite
+            # write to one statement while the migration loop does filesystem
+            # work outside any transaction.
             from archivebox.core.models import Snapshot as SnapshotModel
 
             SnapshotModel.objects.filter(pk=snapshot.pk).update(crawl=crawl)
@@ -386,32 +491,13 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
 
         # Check if needs migration (0.8.x → 0.9.x)
         try:
-            old_version = snapshot.fs_version
-            snapshot.migrate_filesystem_to_current_version(source_dir=entry_path, config=runtime_config)
-            if snapshot.fs_version != old_version or getattr(snapshot, "_pending_fs_migration_cleanup", None):
+            if snapshot.fs_migration_needed:
                 Snapshot.objects.filter(pk=snapshot.pk).update(
-                    fs_version=snapshot.fs_version,
+                    retry_at=timezone.now(),
+                    modified_at=timezone.now(),
                 )
-                migration_cleanup = getattr(snapshot, "_pending_fs_migration_cleanup", None)
-                new_dir = None
-                if migration_cleanup:
-                    old_dir, new_dir = migration_cleanup
-                    transaction.on_commit(
-                        lambda old_dir=old_dir, new_dir=new_dir, snapshot=snapshot: snapshot._cleanup_old_migration_dir(old_dir, new_dir),
-                    )
-                    delattr(snapshot, "_pending_fs_migration_cleanup")
-                crawl_dir = None
-                if snapshot.crawl_id:
-                    crawl_cache_key = str(snapshot.crawl_id)
-                    crawl_dir = crawl_output_dirs.get(crawl_cache_key)
-                    if crawl_dir is None:
-                        crawl = _get_snapshot_crawl(snapshot)
-                        if crawl is not None:
-                            crawl_dir = Path(crawl.output_dir)
-                            crawl_output_dirs[crawl_cache_key] = crawl_dir
-                snapshot.ensure_crawl_symlink(crawl_dir=crawl_dir, snapshot_dir=new_dir)
-                stats["migrated"] += 1
-                print(f"    [{stats['processed']}] Migrated: {entry_path.name}")
+                stats["queued"] += 1
+                print(f"    [{stats['processed']}] Queued filesystem migration: {entry_path.name}")
             else:
                 stats["skipped"] += 1
         except Exception as e:
@@ -425,7 +511,6 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
                     modified_at=timezone.now(),
                 )
             dirty_crawl_ids.clear()
-            transaction.commit()
 
     for crawl_id in tuple(dirty_crawl_ids):
         Crawl.objects.filter(pk=crawl_id).update(
@@ -433,7 +518,6 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
             modified_at=timezone.now(),
         )
     dirty_crawl_ids.clear()
-    transaction.commit()
     return stats
 
 
@@ -448,14 +532,21 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -
     No orphan detection needed - we trust 1:1 mapping between DB and filesystem
     after Phase 1 has drained all old archive/ directories.
     """
-    from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.config.common import get_config
+    import uuid
+    from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
-    from django.db import transaction
     from django.utils import timezone
 
-    stats = {"processed": 0, "reconciled": 0, "sealed": 0, "crawls_sealed": 0}
-    runtime_config = get_config()
+    stats = {
+        "processed": 0,
+        "scanned_dirs": 0,
+        "updated_json": 0,
+        "updated_db": 0,
+        "queued": 0,
+        "sealed": 0,
+        "crawls_sealed": 0,
+    }
+    current_fs_version = Snapshot._fs_current_version()
 
     queryset = Snapshot.objects.all()
     if resume:
@@ -463,89 +554,83 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -
     total = queryset.count()
     print(f"[*] Processing {total} snapshots from database (most recent first)...")
 
-    # Process from most recent to least recent
-    for snapshot in queryset.select_related("crawl__created_by").order_by("-bookmarked_at").iterator(chunk_size=batch_size):
-        stats["processed"] += 1
+    def update_in_batches(rows, *, label: str, **updates) -> int:
+        updated = 0
+        checked = 0
+        while True:
+            ids = list(rows.order_by("-timestamp").values_list("id", flat=True)[:batch_size])
+            if not ids:
+                if updated:
+                    print(f"    [{label}] complete: {updated} rows updated")
+                return updated
+            checked += len(ids)
+            print(f"    [{label}] updating next {len(ids)} rows (seen {checked})...")
+            # Each batch is one short UPDATE and is intentionally idempotent.
+            # If the command is interrupted, these rows no longer match on the
+            # next run and remaining rows continue from DB state.
+            updated += Snapshot.objects.filter(id__in=ids).update(**updates)
+            print(f"    [{label}] updated {updated} rows so far")
 
-        # Skip snapshots with missing crawl references (orphaned by migration errors)
-        if _get_snapshot_crawl(snapshot) is None:
-            continue
+    now = timezone.now()
+    updated_rows = update_in_batches(
+        queryset.exclude(
+            status__in=[
+                Snapshot.StatusChoices.QUEUED,
+                Snapshot.StatusChoices.STARTED,
+                Snapshot.StatusChoices.PAUSED,
+                Snapshot.StatusChoices.SEALED,
+            ],
+        ),
+        label="snapshot status normalization",
+        status=Snapshot.StatusChoices.SEALED,
+        retry_at=None,
+        modified_at=now,
+    )
+    stats["sealed"] += updated_rows
+    stats["updated_db"] += updated_rows
 
+    fs_version_rows = queryset.exclude(fs_version=current_fs_version)
+    stale_batch: list[tuple[uuid.UUID, uuid.UUID | None, str]] = []
+
+    def queue_stale_fs_batch() -> None:
+        if not stale_batch:
+            return
+        now = timezone.now()
+        snapshot_ids = [snapshot_id for snapshot_id, _crawl_id, _timestamp in stale_batch]
+        # Do not bump fs_version here. The orchestrator calls Snapshot.save(),
+        # which performs the idempotent filesystem migration and commits the new
+        # fs_version in the same serialized worker path as normal crawls.
+        updated = Snapshot.objects.filter(id__in=snapshot_ids).update(
+            retry_at=now,
+            modified_at=now,
+        )
+        stats["processed"] += len(stale_batch)
+        stats["updated_db"] += updated
+        stats["queued"] += updated
+        print(f"    [{stats['processed']}/{total}] Queued {updated} filesystem migrations for orchestrator...")
+        stale_batch.clear()
+
+    for snapshot in fs_version_rows.only("id", "crawl_id", "timestamp").order_by("-timestamp").paged_iterator(chunk_size=batch_size):
         try:
-            # Check if snapshot has a directory on disk
-            from pathlib import Path
+            stale_batch.append((snapshot.id, snapshot.crawl_id, snapshot.timestamp))
+            if len(stale_batch) >= batch_size:
+                queue_stale_fs_batch()
+        except KeyboardInterrupt as err:
+            err.archivebox_resume = snapshot.timestamp
+            raise
+    queue_stale_fs_batch()
 
-            output_dir = Path(snapshot.get_storage_path_for_version(snapshot.fs_version, config=runtime_config))
-            has_directory = output_dir.exists() and output_dir.is_dir()
-            current_fs_version = Snapshot._fs_current_version()
-            update_values = {
-                "status": Snapshot.StatusChoices.SEALED,
-                "retry_at": None,
-            }
-
-            # Only reconcile if directory exists (don't create empty directories for orphans)
-            if has_directory:
-                old_title = snapshot.title
-                snapshot.reconcile_with_index_json(output_dir=output_dir, update_existing_archive_results=False)
-                metadata_updates = []
-                for archiveresult in ArchiveResult.objects.filter(snapshot=snapshot).only(
-                    "id",
-                    "snapshot_id",
-                    "plugin",
-                    "output_str",
-                    "output_files",
-                    "output_size",
-                    "output_mimetypes",
-                    "modified_at",
-                ):
-                    if archiveresult.update_output_metadata_from_filesystem(snapshot_dir=output_dir, save=False):
-                        metadata_updates.append(archiveresult)
-                if metadata_updates:
-                    ArchiveResult.objects.bulk_update(
-                        metadata_updates,
-                        ["output_files", "output_size", "output_mimetypes", "modified_at"],
-                        batch_size=batch_size,
-                    )
-                if snapshot.title != old_title:
-                    update_values["title"] = snapshot.title
-                    update_values["modified_at"] = timezone.now()
-
-            # Clean up invalid field values from old migrations
-            if not isinstance(snapshot.current_step, int):
-                update_values["current_step"] = 0
-
-            if snapshot.fs_migration_needed:
-                legacy_dir = snapshot.get_storage_path_for_version("0.8.0", config=runtime_config)
-                current_dir = snapshot.get_storage_path_for_version(current_fs_version, config=runtime_config)
-                if legacy_dir.exists() or current_dir.exists():
-                    snapshot.migrate_filesystem_to_current_version(config=runtime_config)
-                    update_values["fs_version"] = snapshot.fs_version
-                    Snapshot.objects.filter(pk=snapshot.pk).update(**update_values)
-                else:
-                    update_values["fs_version"] = current_fs_version
-                    Snapshot.objects.filter(pk=snapshot.pk).update(**update_values)
-            else:
-                Snapshot.objects.filter(pk=snapshot.pk).update(**update_values)
-
-            stats["reconciled"] += 1 if has_directory else 0
-            stats["sealed"] += 1
-        except Exception as e:
-            # Skip snapshots that can't be processed (e.g., missing crawl)
-            print(f"    [!] Skipping snapshot {snapshot.id}: {e}")
-            continue
-
-        if stats["processed"] % batch_size == 0:
-            transaction.commit()
-            print(f"    [{stats['processed']}/{total}] Processed...")
-
-    transaction.commit()
     now = timezone.now()
     stats["crawls_sealed"] = (
         Crawl.objects.filter(
             status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED],
         )
         .exclude(
-            snapshot_set__status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
+            snapshot_set__status__in=[
+                Snapshot.StatusChoices.QUEUED,
+                Snapshot.StatusChoices.STARTED,
+                Snapshot.StatusChoices.PAUSED,
+            ],
         )
         .update(
             status=Crawl.StatusChoices.SEALED,
@@ -553,26 +638,44 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -
             modified_at=now,
         )
     )
+    stats["updated_db"] += stats["crawls_sealed"]
     return stats
 
 
 def process_filtered_snapshots(
     filter_patterns: Iterable[str],
     filter_type: str,
+    status: str | None,
+    url__icontains: str | None,
+    url__istartswith: str | None,
+    tag: str | None,
+    crawl_id: str | None,
+    limit: int | None,
+    sort: str | None,
+    search: str | None,
     before: float | None,
     after: float | None,
     resume: str | None,
     batch_size: int,
-) -> dict[str, int]:
+    queue_for_archiving: bool = True,
+) -> dict[str, Any]:
     """Process snapshots matching filters (DB query only)."""
-    from django.db import transaction
+    from archivebox.core.models import Snapshot
     from django.utils import timezone
 
-    stats = {"processed": 0, "reconciled": 0, "queued": 0}
+    stats: dict[str, Any] = {"processed": 0, "updated_json": 0, "updated_db": 0, "queued": 0, "snapshot_ids": []}
 
     snapshots = _build_filtered_snapshots_queryset(
         filter_patterns=filter_patterns,
         filter_type=filter_type,
+        status=status,
+        url__icontains=url__icontains,
+        url__istartswith=url__istartswith,
+        tag=tag,
+        crawl_id=crawl_id,
+        limit=limit,
+        sort=sort,
+        search=search,
         before=before,
         after=after,
         resume=resume,
@@ -581,7 +684,7 @@ def process_filtered_snapshots(
     total = snapshots.count()
     print(f"[*] Found {total} matching snapshots")
 
-    for snapshot in snapshots.select_related("crawl").iterator(chunk_size=batch_size):
+    for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size):
         stats["processed"] += 1
 
         # Skip snapshots with missing crawl references
@@ -589,30 +692,40 @@ def process_filtered_snapshots(
             continue
 
         try:
-            # Reconcile index.json with DB
-            snapshot.reconcile_with_index_json()
-
-            # Clean up invalid field values from old migrations
+            stats["snapshot_ids"].append(str(snapshot.id))
+            update_values = {}
             if not isinstance(snapshot.current_step, int):
-                snapshot.current_step = 0
+                update_values["current_step"] = 0
+            if queue_for_archiving:
+                update_values.update(
+                    {
+                        "status": Snapshot.StatusChoices.QUEUED,
+                        "retry_at": timezone.now(),
+                        "modified_at": timezone.now(),
+                    },
+                )
+            if update_values:
+                # update() is intentionally used instead of save(); save()
+                # runs output-dir hooks, which must not happen while SQLite
+                # is holding the write lock for this state change. Index-only
+                # maintenance goes through reindex_snapshots/run_plugins instead
+                # so paused snapshots keep status=paused while only their
+                # targeted search ArchiveResult rows run.
+                Snapshot.objects.filter(pk=snapshot.pk).update(**update_values)
+                stats["updated_db"] += 1
 
-            # Queue for archiving
-            snapshot.status = Snapshot.StatusChoices.QUEUED
-            snapshot.retry_at = timezone.now()
-            snapshot.save()
-
-            stats["reconciled"] += 1
-            stats["queued"] += 1
+            stats["queued"] += 1 if queue_for_archiving else 0
+        except KeyboardInterrupt as err:
+            err.archivebox_resume = snapshot.timestamp
+            raise
         except Exception as e:
             # Skip snapshots that can't be processed
             print(f"    [!] Skipping snapshot {snapshot.id}: {e}")
             continue
 
         if stats["processed"] % batch_size == 0:
-            transaction.commit()
             print(f"    [{stats['processed']}/{total}] Processed...")
 
-    transaction.commit()
     return stats
 
 
@@ -622,9 +735,10 @@ def print_stats(stats: dict):
 
     print(f"""
 [green]Update Complete[/green]
-  Processed:   {stats["processed"]}
-  Reconciled:  {stats["reconciled"]}
-  Queued:      {stats["queued"]}
+  Scanned rows:     {stats["processed"]}
+  Updated JSON:     {stats.get("updated_json", 0)}
+  Updated DB rows:  {stats.get("updated_db", 0)}
+  Queued snapshots: {stats["queued"]}
 """)
 
 
@@ -639,16 +753,17 @@ def print_combined_stats(stats_combined: dict):
 [green]Archive Update Complete[/green]
 
 Phase 1 (Drain Old Dirs):
-  Checked:     {s1.get("processed", 0)}
-  Migrated:    {s1.get("migrated", 0)}
-  Skipped:     {s1.get("skipped", 0)}
-  Invalid:     {s1.get("invalid", 0)}
+  Scanned dirs:     {s1.get("processed", 0)}
+  Moved files:      {s1.get("migrated", 0)}
+  Skipped dirs:     {s1.get("skipped", 0)}
+  Invalid dirs:     {s1.get("invalid", 0)}
 
 Phase 2 (Process DB):
-  Processed:   {s2.get("processed", 0)}
-  Reconciled:  {s2.get("reconciled", 0)}
-  Sealed:      {s2.get("sealed", 0)}
-  Crawls:      {s2.get("crawls_sealed", 0)} sealed
+  Scanned dirs:     {s2.get("scanned_dirs", 0)}
+  Updated JSON:     {s2.get("updated_json", 0)}
+  Updated DB rows:  {s2.get("updated_db", 0)}
+  Sealed snapshots: {s2.get("sealed", 0)}
+  Sealed crawls:    {s2.get("crawls_sealed", 0)}
 """)
 
 
@@ -657,21 +772,28 @@ def print_index_stats(stats: dict[str, Any]) -> None:
 
     print(f"""
 [green]Search Reindex Complete[/green]
-  Processed:   {stats["processed"]}
-  Reconciled:  {stats["reconciled"]}
-  Queued:      {stats["queued"]}
-  Reindexed:   {stats["reindexed"]}
+  Scanned rows:      {stats["processed"]}
+  Queued index jobs: {stats["queued"]}
 """)
 
 
 @click.command()
 @click.option("--resume", type=str, help="Resume from timestamp")
+@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
+@click.option("--url__icontains", help="Filter by URL contains")
+@click.option("--url__istartswith", help="Filter by URL starts with")
+@click.option("--tag", "-t", help="Filter by tag name")
+@click.option("--crawl-id", help="Filter by crawl ID")
+@click.option("--limit", "-n", type=int, help="Limit number of snapshots to update")
+@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
+@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for positional query")
 @click.option("--before", type=float, help="Only snapshots before timestamp")
 @click.option("--after", type=float, help="Only snapshots after timestamp")
-@click.option("--filter-type", "-t", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
+@click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
 @click.option("--batch-size", type=int, default=100, help="Commit every N snapshots")
 @click.option("--continuous", is_flag=True, help="Run continuously as background worker")
 @click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
+@click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
 @click.argument("filter_patterns", nargs=-1)
 @docstring(update.__doc__)
 def main(**kwargs):

archivebox/config/common.py

@@ -139,8 +139,6 @@ class ServerConfig(BaseConfigSet):
     # CUSTOM_TEMPLATES_DIR: Path          = Field(default=None)  # this is now a constant
 
     PUBLIC_INDEX: bool = Field(default=True)
-    PUBLIC_SNAPSHOTS: bool = Field(default=True)
-    PUBLIC_SNAPSHOTS_LIST: bool | None = Field(default=None)
     PUBLIC_ADD_VIEW: bool = Field(default=False)
 
     ADMIN_USERNAME: str | None = Field(default=None)
@@ -254,6 +252,7 @@ class ArchivingConfig(BaseConfigSet):
     MAX_DEPTH: int = Field(default=0)
     CRAWL_MAX_URLS: int = Field(default=0)
     CRAWL_MAX_SIZE: int = Field(default=0)
+    CRAWL_TIMEOUT: int = Field(default=0, description="Maximum total crawl runtime in seconds (0 = unlimited).")
     CRAWL_MAX_CONCURRENT_SNAPSHOTS: int = Field(
         default=4,
         description="Maximum number of snapshots to archive concurrently within one crawl.",
@@ -274,6 +273,10 @@ class ArchivingConfig(BaseConfigSet):
     SAVE_DENYLIST: dict[str, list[str]] = Field(default={})
 
     DEFAULT_PERSONA: str = Field(default="Default")
+    PERMISSIONS: str = Field(
+        default="public",
+        description="Snapshot visibility: public lists and serves content, unlisted serves direct links only, private requires admin login.",
+    )
     DELETE_AFTER: str = Field(
         default="0",
         description=(
@@ -310,6 +313,14 @@ class ArchivingConfig(BaseConfigSet):
             return "0"
         return str(value).strip() or "0"
 
+    @field_validator("PERMISSIONS", mode="before")
+    @classmethod
+    def validate_permissions(cls, value):
+        normalized = str(value or "public").strip().lower()
+        if normalized not in {"public", "unlisted", "private"}:
+            raise ValueError("PERMISSIONS must be one of: public, unlisted, private.")
+        return normalized
+
     @property
     def URL_ALLOWLIST_PTN(self) -> re.Pattern | None:
         return re.compile(self.URL_ALLOWLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS) if self.URL_ALLOWLIST else None
@@ -561,20 +572,7 @@ def get_config(
             machine = None
 
     if persona is None and crawl is not None:
-        from archivebox.personas.models import Persona
-
-        persona_id = crawl.persona_id
-        if persona_id:
-            persona = Persona.objects.filter(id=persona_id).first()
-            if persona is None:
-                raise Persona.DoesNotExist(f"Crawl {crawl.id} references missing Persona {persona_id}")
-
-        if persona is None:
-            crawl_config = crawl.config or {}
-            default_persona_name = str(crawl_config.get("DEFAULT_PERSONA") or "").strip()
-            if default_persona_name:
-                persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default")
-                persona.ensure_dirs()
+        persona = crawl.resolve_persona()
 
     config_data: ConfigPayload = dict(defaults or {})
     config_data.update(ArchiveBoxConfig().model_dump(mode="json"))

archivebox/config/django.py

@@ -28,14 +28,6 @@ STDERR = Console(stderr=True)
 logging.CONSOLE = CONSOLE
 
 
-def setup_django_minimal():
-    # sys.path.append(str(CONSTANTS.PACKAGE_DIR))
-    # os.environ.setdefault('ARCHIVEBOX_DATA_DIR', str(CONSTANTS.DATA_DIR))
-    # os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
-    # django.setup()
-    raise Exception("dont use this anymore")
-
-
 DJANGO_SET_UP = False
 
 

archivebox/core/admin_archiveresults.py

@@ -105,6 +105,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         "failed": ("#991b1b", "#fee2e2"),  # red
         "queued": ("#6b7280", "#f3f4f6"),  # gray
         "started": ("#92400e", "#fef3c7"),  # amber
+        "paused": ("#1d4ed8", "#dbeafe"),  # blue
         "backoff": ("#92400e", "#fef3c7"),
         "skipped": ("#475569", "#f1f5f9"),
         "noresults": ("#475569", "#f1f5f9"),
@@ -152,7 +153,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
             '''
 
         # Truncate output for display
-        full_output = result.output_str or "-"
+        full_output = result.output_str_for_display() or "-"
         output_display = full_output[:60]
         if len(full_output) > 60:
             output_display += "..."
@@ -718,12 +719,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
         return format_html(
             '<a href="{}" class="output-link">↗️</a><pre>{}</pre>',
             build_snapshot_url(snapshot_id, output_path, request=request, config=config),
-            result.output_str,
+            result.output_str_for_display(),
         )
 
     @admin.display(description="Output", ordering="output_str")
     def output_str_display(self, result):
-        output_text = str(result.output_str or "").strip()
+        output_text = str(result.output_str_for_display() or "").strip()
         if not output_text:
             return "-"
 
@@ -787,7 +788,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
         snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1]
         output_html = format_html(
             '<pre style="display: inline-block">{}</pre><br/>',
-            result.output_str,
+            result.output_str_for_display(),
         )
         snapshot_id = str(result.snapshot_id)
         request = getattr(self, "request", None)

archivebox/core/admin_snapshots.py

@@ -1,15 +1,24 @@
 __package__ = "archivebox.core"
 
-from functools import lru_cache
+import asyncio
 import json
+import threading
+from copy import copy
+from functools import lru_cache
+from queue import Full, Queue
+from types import SimpleNamespace
+from urllib.parse import urlsplit
+from uuid import UUID
 
 from django.contrib import admin, messages
-from django.urls import path
+from django.urls import path, reverse
 from django.shortcuts import get_object_or_404, redirect
-from django.utils.html import format_html
+from django.core.cache import cache
+from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed, QueryDict, StreamingHttpResponse
+from django.utils import timezone
+from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
-from django.db.models import Q, Sum, Count, Prefetch
-from django.db.models.functions import Coalesce
+from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch
 from django import forms
 from django.template import Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
@@ -18,8 +27,9 @@ from archivebox.config.common import get_config
 from archivebox.misc.util import htmldecode, urldecode
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.misc.logging_util import printable_filesize
-from archivebox.search.admin import SearchResultsAdminMixin
+from archivebox.search.admin import SEARCH_RESULT_CACHE_TTL, SearchResultsAdminMixin, SearchResultsChangeList, get_admin_search_cache_key
 from archivebox.core.host_utils import build_snapshot_url, build_web_url
+from archivebox.core.tag_utils import get_or_create_tag
 from archivebox.hooks import discover_hooks, get_plugin_icon, get_plugin_name, get_plugins
 
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
@@ -27,13 +37,27 @@ from archivebox.workers.tasks import bg_archive_snapshots, bg_add
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
 from archivebox.core.admin_archiveresults import render_archiveresults_list
+from archivebox.core.permissions import (
+    PERMISSIONS_CHOICES,
+    PERMISSIONS_PRIVATE,
+    PERMISSIONS_PUBLIC,
+    PERMISSIONS_UNLISTED,
+    get_snapshot_permissions,
+)
 from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
 from archivebox.crawls.models import Crawl
+from archivebox.personas.models import Persona
 
 
 # GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
 GLOBAL_CONTEXT = {}
 
+SNAPSHOT_PERMISSION_META = {
+    PERMISSIONS_PUBLIC: ("👥", "Public", "#047857", "#d1fae5"),
+    PERMISSIONS_UNLISTED: ("🔗", "Unlisted", "#1d4ed8", "#dbeafe"),
+    PERMISSIONS_PRIVATE: ("🔒", "Private", "#991b1b", "#fee2e2"),
+}
+
 
 @lru_cache(maxsize=1)
 def _plugin_sort_order() -> dict[str, int]:
@@ -51,22 +75,12 @@ class SnapshotActionForm(ActionForm):
         )
 
     def clean_tags(self):
-        """Parse comma-separated tag names into Tag objects."""
+        """Parse comma-separated tag names without touching the DB."""
         tags_str = self.cleaned_data.get("tags", "")
         if not tags_str:
             return []
 
-        tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
-        tags = []
-        for name in tag_names:
-            tag, _ = Tag.objects.get_or_create(
-                name__iexact=name,
-                defaults={"name": name},
-            )
-            # Use the existing tag if found by case-insensitive match
-            tag = Tag.objects.filter(name__iexact=name).first() or tag
-            tags.append(tag)
-        return tags
+        return [name.strip() for name in tags_str.split(",") if name.strip()]
 
     # TODO: allow selecting actions for specific extractor plugins? is this useful?
     # plugin = forms.ChoiceField(
@@ -95,6 +109,268 @@ class TagNameListFilter(admin.SimpleListFilter):
         return queryset
 
 
+class SnapshotPermissionsListFilter(admin.SimpleListFilter):
+    title = "permission"
+    parameter_name = "permissions"
+
+    def lookups(self, request, model_admin):
+        return PERMISSIONS_CHOICES
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value:
+            global_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower()
+            has_overrides = (
+                Snapshot.objects.filter(permissions__gt="").exists()
+                or Crawl.objects.filter(permissions__gt="").exists()
+                or Persona.objects.filter(permissions__gt="").exists()
+            )
+            if not has_overrides:
+                return queryset if value == global_permissions else queryset.none()
+
+            persona_query = Q(crawl__persona_id__in=self.persona_ids_for_value())
+            if global_permissions == value:
+                valid_persona_ids = Persona.objects.values_list("id", flat=True)
+                persona_query |= Q(crawl__persona_id__isnull=True) | ~Q(crawl__persona_id__in=valid_persona_ids)
+            return queryset.filter(
+                Q(permissions=value)
+                | (Q(permissions__isnull=True) & Q(crawl__permissions=value))
+                | (Q(permissions__isnull=True) & Q(crawl__permissions__isnull=True) & persona_query)
+            )
+        return queryset
+
+    def persona_ids_for_value(self):
+        global_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower()
+        query = Q(permissions=self.value())
+        if global_permissions == self.value():
+            query |= Q(permissions__isnull=True)
+        return Persona.objects.filter(query).values_list("id", flat=True)
+
+
+class SnapshotStatusListFilter(admin.SimpleListFilter):
+    title = "snapshot status"
+    parameter_name = "snapshot_status"
+
+    def lookups(self, request, model_admin):
+        return Snapshot.StatusChoices.choices
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value in Snapshot.StatusChoices.values:
+            return queryset.filter(status=value)
+        return queryset
+
+
+class SnapshotDepthListFilter(admin.SimpleListFilter):
+    title = "depth"
+    parameter_name = "depth_bucket"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("0", "0 root"),
+            ("1", "1"),
+            ("2", "2"),
+            ("3plus", "3+"),
+        )
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value == "0":
+            return queryset.filter(depth=0)
+        if value == "1":
+            return queryset.filter(depth=1)
+        if value == "2":
+            return queryset.filter(depth=2)
+        if value == "3plus":
+            return queryset.filter(depth__gte=3)
+        return queryset
+
+
+class SnapshotRelationListFilter(admin.SimpleListFilter):
+    title = "crawl position"
+    parameter_name = "position"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("root", "Root URL"),
+            ("discovered", "Discovered URL"),
+            ("has_children", "Has discovered URLs"),
+            ("no_children", "No discovered URLs"),
+        )
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value == "root":
+            return queryset.filter(parent_snapshot__isnull=True)
+        if value == "discovered":
+            return queryset.filter(parent_snapshot__isnull=False)
+        if value in {"has_children", "no_children"}:
+            child_snapshots = Snapshot.objects.filter(parent_snapshot_id=OuterRef("pk"))
+            queryset = queryset.annotate(has_child_snapshots=Exists(child_snapshots))
+            return queryset.filter(has_child_snapshots=value == "has_children")
+        return queryset
+
+
+class SnapshotArchiveStateListFilter(admin.SimpleListFilter):
+    title = "archive state"
+    parameter_name = "archive_state"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("downloaded", "Downloaded"),
+            ("not_downloaded", "Not downloaded"),
+            ("has_output", "Has saved files"),
+            ("empty_output", "No saved files"),
+            ("has_title", "Has title"),
+            ("missing_title", "Missing title"),
+        )
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value == "downloaded":
+            return queryset.filter(downloaded_at__isnull=False)
+        if value == "not_downloaded":
+            return queryset.filter(downloaded_at__isnull=True)
+        if value == "has_output":
+            return queryset.filter(output_size__gt=0)
+        if value == "empty_output":
+            return queryset.filter(output_size=0)
+        if value == "has_title":
+            return queryset.exclude(Q(title__isnull=True) | Q(title=""))
+        if value == "missing_title":
+            return queryset.filter(Q(title__isnull=True) | Q(title=""))
+        return queryset
+
+
+class SnapshotSizeListFilter(admin.SimpleListFilter):
+    title = "size"
+    parameter_name = "size"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("1gb", ">1GB"),
+            ("500mb", ">500MB"),
+            ("250mb", ">250MB"),
+            ("100mb", ">100MB"),
+            ("50mb", ">50MB"),
+            ("25mb", ">25MB"),
+        )
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        thresholds = {
+            "1gb": 1024 * 1024 * 1024,
+            "500mb": 500 * 1024 * 1024,
+            "250mb": 250 * 1024 * 1024,
+            "100mb": 100 * 1024 * 1024,
+            "50mb": 50 * 1024 * 1024,
+            "25mb": 25 * 1024 * 1024,
+        }
+        if value in thresholds:
+            return queryset.filter(output_size__gt=thresholds[value])
+        return queryset
+
+
+class SnapshotRetryListFilter(admin.SimpleListFilter):
+    title = "retry"
+    parameter_name = "retry"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("due", "Due now"),
+            ("future", "Scheduled later"),
+            ("none", "No retry time"),
+        )
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value == "due":
+            return queryset.filter(retry_at__isnull=False, retry_at__lte=timezone.now())
+        if value == "future":
+            return queryset.filter(retry_at__gt=timezone.now())
+        if value == "none":
+            return queryset.filter(retry_at__isnull=True)
+        return queryset
+
+
+class SnapshotResultHealthListFilter(admin.SimpleListFilter):
+    title = "ArchiveResult status"
+    parameter_name = "archiveresult_status"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("none", "No ArchiveResults"),
+            ("has_results", "Has ArchiveResults"),
+            ("succeeded", ">50% succeeded"),
+            ("failed", ">50% failed"),
+            ("running", ">50% running"),
+            ("pending", ">50% queued"),
+            ("backoff", ">50% waiting to retry"),
+            ("noresults", ">50% noresults"),
+        )
+
+    def queryset(self, request, queryset):
+        value = self.value()
+        if value:
+            results = ArchiveResult.objects.filter(snapshot_id=OuterRef("pk"))
+            if value == "none":
+                return queryset.annotate(has_results=Exists(results)).filter(has_results=False)
+            if value == "has_results":
+                return queryset.annotate(has_results=Exists(results)).filter(has_results=True)
+            status_by_value = {
+                "succeeded": ArchiveResult.StatusChoices.SUCCEEDED,
+                "failed": ArchiveResult.StatusChoices.FAILED,
+                "running": ArchiveResult.StatusChoices.STARTED,
+                "pending": ArchiveResult.StatusChoices.QUEUED,
+                "backoff": ArchiveResult.StatusChoices.BACKOFF,
+                "noresults": ArchiveResult.StatusChoices.NORESULTS,
+            }
+            if value in status_by_value:
+                queryset = queryset.annotate(
+                    total_results=Count("archiveresult"),
+                    matching_results=Count(
+                        "archiveresult",
+                        filter=Q(archiveresult__status=status_by_value[value]),
+                    ),
+                )
+                return queryset.filter(matching_results__gt=F("total_results") / 2)
+        return queryset
+
+
+class SnapshotChangeList(SearchResultsChangeList):
+    def __init__(self, request, *args, **kwargs):
+        super().__init__(request, *args, **kwargs)
+        resolver_name = getattr(getattr(request, "resolver_match", None), "url_name", "")
+        self.embedded_changelist = request.GET.get("_embedded") == "crawl"
+        self.snapshot_is_grid_view = not self.embedded_changelist and (resolver_name == "grid" or request.path.rstrip("/").endswith("/grid"))
+
+    def get_results(self, request):
+        super().get_results(request)
+        if request.GET.get("_embedded") == "crawl":
+            self.full_result_count = self.result_count
+        else:
+            self.full_result_count = self.model_admin.get_paginator(request, self.model._default_manager.all().order_by(), self.list_per_page).count
+        self.show_full_result_count = True
+
+        snapshot_ids = [obj.pk for obj in self.result_list]
+        if snapshot_ids:
+            results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
+            seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
+            rows = (
+                ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
+                .order_by("snapshot_id", "plugin")
+                .values_list("snapshot_id", "plugin", "status", "output_size")
+            )
+            for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
+                if plugin in seen_plugins[snapshot_id]:
+                    continue
+                seen_plugins[snapshot_id].add(plugin)
+                results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
+
+            for obj in self.result_list:
+                obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+
+
 class SnapshotAdminForm(forms.ModelForm):
     """Custom form for Snapshot admin with tag editor widget."""
 
@@ -104,6 +380,12 @@ class SnapshotAdminForm(forms.ModelForm):
         widget=TagEditorWidget(),
         help_text="Type tag names and press Enter or Space to add. Click × to remove.",
     )
+    permissions_config = forms.ChoiceField(
+        label="Permissions",
+        choices=PERMISSIONS_CHOICES,
+        required=True,
+        help_text="Per-snapshot visibility. Matching the crawl/persona default clears the per-snapshot override.",
+    )
 
     class Meta:
         model = Snapshot
@@ -116,9 +398,18 @@ class SnapshotAdminForm(forms.ModelForm):
             self.initial["tags_editor"] = ",".join(
                 sorted(tag.name for tag in self.instance.tags.all()),
             )
+            self.initial["permissions_config"] = get_snapshot_permissions(self.instance)
 
     def save(self, commit=True):
         instance = super().save(commit=False)
+        permissions = self.cleaned_data["permissions_config"]
+        inherited_permissions = str(get_config(crawl=instance.crawl, resolve_plugins=False).PERMISSIONS).strip().lower()
+        config = dict(instance.config or {})
+        if permissions == inherited_permissions:
+            config.pop("PERMISSIONS", None)
+        else:
+            config["PERMISSIONS"] = permissions
+        instance.config = config
 
         # Handle tags_editor field
         if commit:
@@ -150,7 +441,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     form = SnapshotAdminForm
     raw_id_fields = ("crawl", "parent_snapshot")
     list_select_related = ()
-    list_display = ("created_at", "preview_icon", "title_str", "tags_inline", "status_with_progress", "files", "size_with_stats")
+    list_display = ("permissions_badge", "created_at", "preview_icon", "title_str", "tags_inline", "status_with_progress", "files", "size_with_stats")
+    list_display_links = ("created_at",)
     sort_fields = ("title_str", "created_at", "status", "crawl")
     readonly_fields = (
         "admin_actions",
@@ -165,7 +457,20 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         "archiveresults_list",
     )
     search_fields = ("id", "url", "timestamp", "title", "tags__name")
-    list_filter = ("created_at", "downloaded_at", "archiveresult__status", "crawl__created_by", TagNameListFilter)
+    list_filter = (
+        SnapshotPermissionsListFilter,
+        SnapshotStatusListFilter,
+        SnapshotResultHealthListFilter,
+        SnapshotDepthListFilter,
+        SnapshotRelationListFilter,
+        SnapshotArchiveStateListFilter,
+        SnapshotSizeListFilter,
+        SnapshotRetryListFilter,
+        "created_at",
+        "downloaded_at",
+        "crawl__created_by",
+        TagNameListFilter,
+    )
 
     fieldsets = (
         (
@@ -192,7 +497,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         (
             "Tags",
             {
-                "fields": ("tags_editor",),
+                "fields": ("tags_editor", "permissions_config"),
                 "classes": ("card",),
             },
         ),
@@ -244,7 +549,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     ordering = ["-timestamp"]
     actions = ["add_tags", "remove_tags", "resnapshot_snapshot", "update_snapshots", "overwrite_snapshots", "delete_snapshots"]
     inlines = []  # Removed TagInline, using TagEditorWidget instead
-    list_per_page = 40
+    list_per_page = 50
 
     action_form = SnapshotActionForm
     paginator = AcceleratedPaginator
@@ -252,6 +557,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     save_on_top = True
     show_full_result_count = False
 
+    def get_changelist(self, request, **kwargs):
+        return SnapshotChangeList
+
+    def get_ordering(self, request):
+        if request.GET.get("o"):
+            return []
+        return super().get_ordering(request)
+
     def change_view(self, request, object_id, form_url="", extra_context=None):
         request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         extra_context = extra_context or {}
@@ -262,8 +575,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         self.request = request
         request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
-        self.list_per_page = min(max(5, request.archivebox_config.SNAPSHOTS_PER_PAGE), 25)
+        embedded_changelist = request.GET.get("_embedded") == "crawl"
+        if embedded_changelist:
+            try:
+                requested_per_page = int(request.GET.get("per_page", "200"))
+            except ValueError:
+                requested_per_page = 200
+            self.list_per_page = min(max(200, requested_per_page), 500)
+        else:
+            self.list_per_page = min(max(50, request.archivebox_config.SNAPSHOTS_PER_PAGE), 500)
         extra_context = extra_context or {}
+        extra_context["embedded_changelist"] = embedded_changelist
         extra_context["CONFIG"] = request.archivebox_config
         try:
             try:
@@ -281,6 +603,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         actions.pop("delete_selected", None)
         return actions
 
+    def lookup_allowed(self, lookup, value, request=None):
+        if lookup in {"crawl__id__exact", "crawl_id__exact", "crawl_id"}:
+            return True
+        return super().lookup_allowed(lookup, value, request=request)
+
     def get_snapshot_view_url(self, obj: Snapshot) -> str:
         request = getattr(self, "request", None)
         return build_snapshot_url(str(obj.id), request=request, config=getattr(request, "archivebox_config", None))
@@ -296,10 +623,185 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         urls = super().get_urls()
         custom_urls = [
             path("grid/", self.admin_site.admin_view(self.grid_view), name="grid"),
+            path("search-stream/", self.admin_site.admin_view(self.search_stream_view), name="core_snapshot_search_stream"),
             path("<path:object_id>/redo-failed/", self.admin_site.admin_view(self.redo_failed_view), name="core_snapshot_redo_failed"),
+            path("<path:object_id>/set-permissions/", self.admin_site.admin_view(self.set_permissions_view), name="core_snapshot_set_permissions"),
         ]
         return custom_urls + urls
 
+    def search_stream_view(self, request):
+        from archivebox.search import iter_query_search_ids
+
+        query = (request.GET.get("q") or "").strip()
+        from archivebox.search import get_search_mode, get_search_mode_base
+
+        search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
+        if not query:
+            return StreamingHttpResponse((), content_type="text/plain")
+
+        search_url = request.GET.get("search_url") or request.get_full_path()
+        target_url = urlsplit(search_url)
+        target_get = QueryDict(target_url.query, mutable=True)
+        for key in ("q", "search_mode", "p", "search_url"):
+            target_get.pop(key, None)
+
+        filter_request = copy(request)
+        filter_request.path = target_url.path or request.path
+        filter_request.path_info = target_url.path or request.path_info
+        filter_request.GET = target_get
+        filter_request.archivebox_config = getattr(request, "archivebox_config", None)
+
+        # Build the same filtered base queryset the changelist uses, but with
+        # the search params stripped. The stream then intersects each wave with
+        # this queryset before writing IDs into the short-lived cache.
+        current_request = getattr(self, "request", None)
+        try:
+            base_queryset = self.get_changelist_instance(filter_request).queryset
+        finally:
+            self.request = current_request
+
+        async def snapshot_ids():
+            seen = set()
+            ids = []
+            last_sent = 0
+            stream_batch_size = 100
+            stream_padding = " " * 4096
+            cache_key = get_admin_search_cache_key(request, search_url)
+            cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
+            yield f"0{stream_padding}\n"
+            queue = Queue(maxsize=8)
+            stop_event = threading.Event()
+
+            def emit(item):
+                while not stop_event.is_set():
+                    try:
+                        queue.put(item, timeout=0.1)
+                        return
+                    except Full:
+                        continue
+
+            def run_search():
+                nonlocal last_sent
+                iterator = None
+                try:
+                    search_mode_base = get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None))
+                    iterator = (
+                        self.iter_meta_search_ids(query, base_queryset)
+                        if search_mode_base == "meta"
+                        else self.iter_backend_search_ids(
+                            iter_query_search_ids(query, search_mode=search_mode, config=getattr(request, "archivebox_config", None)),
+                            base_queryset,
+                        )
+                    )
+                    for snapshot_id in iterator:
+                        if stop_event.is_set():
+                            break
+                        snapshot_id = str(snapshot_id).strip().lower()
+                        if len(snapshot_id.replace("-", "")) != 32 or snapshot_id in seen:
+                            continue
+                        seen.add(snapshot_id)
+                        ids.append(snapshot_id)
+                        if len(ids) - last_sent >= stream_batch_size:
+                            cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
+                            last_sent = len(ids)
+                            emit(f"{last_sent}{stream_padding}\n")
+                    if not stop_event.is_set() and len(ids) != last_sent:
+                        cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL)
+                        emit(f"{len(ids)}{stream_padding}\n")
+                except BaseException as err:
+                    emit(err)
+                finally:
+                    if iterator is not None:
+                        try:
+                            iterator.close()
+                        except AttributeError:
+                            pass
+                    cache.set(cache_key, {"ids": ids, "done": True}, SEARCH_RESULT_CACHE_TTL)
+                    emit(None)
+
+            threading.Thread(target=run_search, name="admin-snapshot-search-stream", daemon=True).start()
+            try:
+                while True:
+                    item = await asyncio.to_thread(queue.get)
+                    if item is None:
+                        break
+                    if isinstance(item, BaseException):
+                        raise item
+                    yield item
+            finally:
+                stop_event.set()
+
+        response = StreamingHttpResponse(snapshot_ids(), content_type="text/plain")
+        response["X-Accel-Buffering"] = "no"
+        return response
+
+    def iter_meta_search_ids(self, query, queryset):
+        seen = set()
+        try:
+            snapshot_id = UUID(query)
+        except ValueError:
+            snapshot_id = None
+        if snapshot_id:
+            for pk in queryset.filter(pk=snapshot_id).values_list("pk", flat=True):
+                seen.add(pk)
+                yield pk
+
+        for wave in (
+            Q(timestamp__startswith=query) | Q(url__istartswith=query) | Q(title__istartswith=query),
+            Q(url__icontains=query),
+            Q(title__icontains=query),
+            Q(tags__name__icontains=query),
+        ):
+            for pk in queryset.filter(wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500):
+                if pk in seen:
+                    continue
+                seen.add(pk)
+                yield pk
+
+    def iter_backend_search_ids(self, iterator, queryset):
+        batch = []
+        seen = set()
+
+        def flush_batch():
+            valid = {str(pk) for pk in queryset.filter(pk__in=batch).values_list("pk", flat=True)}
+            for snapshot_id in batch:
+                if snapshot_id in valid and snapshot_id not in seen:
+                    seen.add(snapshot_id)
+                    yield snapshot_id
+
+        for snapshot_id in iterator:
+            snapshot_id = str(snapshot_id).strip().lower()
+            if len(snapshot_id.replace("-", "")) != 32:
+                continue
+            batch.append(snapshot_id)
+            if len(batch) >= 200:
+                yield from flush_batch()
+                batch = []
+        if batch:
+            yield from flush_batch()
+
+    def set_permissions_view(self, request, object_id):
+        if request.method != "POST":
+            return HttpResponseNotAllowed(["POST"])
+
+        permissions = (request.POST.get("permissions") or "").strip().lower()
+        if permissions not in dict(PERMISSIONS_CHOICES):
+            return HttpResponseBadRequest("Invalid permissions value")
+
+        snapshot = get_object_or_404(Snapshot.objects.select_related("crawl"), pk=object_id)
+        config = dict(snapshot.config or {})
+        inherited_permissions = str(get_config(crawl=snapshot.crawl, resolve_plugins=False).PERMISSIONS).strip().lower()
+        if permissions == inherited_permissions:
+            config.pop("PERMISSIONS", None)
+        else:
+            config["PERMISSIONS"] = permissions
+
+        # Keep the quick-edit write to one targeted UPDATE so SQLite only holds
+        # the write lock for the permission/config change itself.
+        Snapshot.objects.filter(pk=snapshot.pk).update(config=config, modified_at=timezone.now())
+        icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
+        return JsonResponse({"permissions": permissions, "icon": icon, "label": label, "fg": fg, "bg": bg})
+
     def redo_failed_view(self, request, object_id):
         snapshot = get_object_or_404(Snapshot, pk=object_id)
 
@@ -327,42 +829,56 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def get_queryset(self, request):
         self.request = request
         ordering_fields = self._get_ordering_fields(request)
-        needs_size_sort = "size_with_stats" in ordering_fields
         needs_files_sort = "files" in ordering_fields
         needs_tags_sort = "tags_inline" in ordering_fields
         is_change_view = getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_change"
+        request.archivebox_default_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower()
 
-        prefetch_qs = ArchiveResult.objects.only(
-            "id",
-            "snapshot_id",
-            "plugin",
-            "status",
-            "output_size",
-            "output_files",
-            "output_str",
-        )
-        if not is_change_view:
-            prefetch_qs = prefetch_qs.filter(Q(status="succeeded"))
-
-        qs = (
-            super()
-            .get_queryset(request)
-            .defer("config", "notes")
-            .prefetch_related(
-                Prefetch("crawl", queryset=Crawl.objects.select_related("created_by")),
-                "tags",
-                Prefetch("archiveresult_set", queryset=prefetch_qs),
-            )
-        )
-
-        if needs_size_sort:
-            qs = qs.annotate(
-                output_size_sum=Coalesce(
-                    Sum("archiveresult__output_size"),
-                    0,
+        prefetches = [
+            Prefetch(
+                "crawl",
+                queryset=Crawl.objects.only(
+                    "id",
+                    "permissions",
+                    "persona_id",
+                    "status",
+                    "created_by_id",
+                ).prefetch_related("created_by"),
+            ),
+            "tags",
+        ]
+        if is_change_view:
+            prefetches.append(
+                Prefetch(
+                    "archiveresult_set",
+                    queryset=ArchiveResult.objects.only(
+                        "id",
+                        "snapshot_id",
+                        "plugin",
+                        "status",
+                        "output_size",
+                    ),
                 ),
             )
 
+        qs = super().get_queryset(request)
+        if is_change_view:
+            qs = qs.defer("notes")
+        else:
+            qs = qs.only(
+                "id",
+                "created_at",
+                "url",
+                "timestamp",
+                "bookmarked_at",
+                "crawl_id",
+                "title",
+                "status",
+                "fs_version",
+                "output_size",
+                "permissions",
+            )
+        qs = qs.prefetch_related(*prefetches)
         if needs_files_sort:
             qs = qs.annotate(
                 ar_succeeded_count=Count(
@@ -375,6 +891,66 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
         return qs
 
+    @admin.display(description="👁", ordering="permissions")
+    def permissions_badge(self, obj):
+        request = self.request
+        permissions = getattr(obj, "snapshot_permissions", None)
+        if permissions is None:
+            if obj.permissions:
+                permissions = obj.permissions
+            elif obj.crawl.permissions:
+                permissions = obj.crawl.permissions
+            elif obj.crawl.persona_id:
+                persona_permissions = getattr(request, "archivebox_persona_permissions", None)
+                if persona_permissions is None:
+                    persona_permissions = {
+                        str(persona.id): persona.permissions or request.archivebox_default_permissions
+                        for persona in Persona.objects.only("id", "permissions")
+                    }
+                    request.archivebox_persona_permissions = persona_permissions
+                permissions = persona_permissions.get(str(obj.crawl.persona_id), request.archivebox_default_permissions)
+            else:
+                permissions = request.archivebox_default_permissions
+        icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
+        menu_items = format_html_join(
+            "",
+            (
+                '<button type="button" class="snapshot-permissions-menu-item{}" data-permissions="{}">'
+                '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+                "<span>{}</span>"
+                "</button>"
+            ),
+            (
+                (
+                    " is-active" if choice_value == permissions else "",
+                    choice_value,
+                    choice_fg,
+                    choice_bg,
+                    choice_icon,
+                    choice_label,
+                )
+                for choice_value, choice_label in PERMISSIONS_CHOICES
+                for choice_icon, _choice_title, choice_fg, choice_bg in [SNAPSHOT_PERMISSION_META[choice_value]]
+            ),
+        )
+        return format_html(
+            '<span class="snapshot-permissions-quick" data-current-permissions="{}" data-permissions-url="{}">'
+            '<button type="button" class="snapshot-permissions-button snapshot-permissions-{}" title="{}" aria-label="Change snapshot permissions: {}" aria-expanded="false">'
+            '<span class="snapshot-permissions-icon" aria-hidden="true" style="color:{}; background:{};">{}</span>'
+            "</button>"
+            '<span class="snapshot-permissions-menu" role="menu" hidden>{}</span>'
+            "</span>",
+            permissions,
+            reverse(f"{self.admin_site.name}:core_snapshot_set_permissions", args=[obj.pk]),
+            permissions,
+            label,
+            label,
+            fg,
+            bg,
+            icon,
+            menu_items,
+        )
+
     @admin.display(description="Imported Timestamp")
     def imported_timestamp(self, obj):
         context = RequestContext(
@@ -555,15 +1131,15 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Tags", ordering="tag_count")
     def tags_inline(self, obj):
-        widget = InlineTagEditorWidget(snapshot_id=str(obj.pk))
+        widget = InlineTagEditorWidget(snapshot_id=str(obj.pk), editable=True)
         tags = self._get_prefetched_tags(obj)
         tags_html = widget.render(
-            name=f"tags_{obj.pk}",
+            name=f"tags_inline_{obj.pk}",
             value=tags if tags is not None else obj.tags.all(),
-            attrs={"id": f"tags_{obj.pk}"},
+            attrs={"id": f"tags_inline_{obj.pk}"},
             snapshot_id=str(obj.pk),
         )
-        return mark_safe(f'<span class="tags-inline-editor">{tags_html}</span>')
+        return mark_safe(f'<span class="tags-inline-editor tags-inline-editor--compact">{tags_html}</span>')
 
     @admin.display(description="Tags")
     def tags_badges(self, obj):
@@ -716,13 +1292,13 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def files(self, obj):
         results = self._get_prefetched_results(obj)
         if results is None:
-            results = obj.archiveresult_set.only("plugin", "status", "output_files", "output_str")
+            results = obj.archiveresult_set.only("plugin", "status", "output_size")
 
         plugins_with_output: dict[str, ArchiveResult] = {}
         for result in results:
             if result.status != ArchiveResult.StatusChoices.SUCCEEDED:
                 continue
-            if not (result.output_files or str(result.output_str or "").strip()):
+            if not result.output_size:
                 continue
             plugins_with_output.setdefault(result.plugin, result)
 
@@ -734,15 +1310,21 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             key=lambda result: (_plugin_sort_order().get(result.plugin, 9999), result.plugin),
         )
         visible_results = sorted_results[:14]
-        output = [
-            format_html(
-                '<a href="{}" class="exists-True" title="{}">{}</a>',
-                f"/{obj.archive_path_from_db}/{result.plugin}/",
-                result.plugin,
-                get_plugin_icon(result.plugin),
+        output = []
+        request = getattr(self, "request", None)
+        config = getattr(request, "archivebox_config", None)
+        for result in visible_results:
+            icon = mark_safe(get_plugin_icon(result.plugin))
+            if not icon.strip():
+                continue
+            output.append(
+                format_html(
+                    '<a href="{}" class="exists-True" title="{}">{}</a>',
+                    build_web_url(f"/{obj.archive_path_from_db}/{result.plugin}/", request=request, config=config),
+                    result.plugin,
+                    icon,
+                ),
             )
-            for result in visible_results
-        ]
         if len(sorted_results) > len(visible_results):
             output.append(
                 format_html(
@@ -788,6 +1370,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         status_colors = {
             "queued": ("#f59e0b", "#fef3c7"),  # amber
             "started": ("#3b82f6", "#dbeafe"),  # blue
+            "paused": ("#1d4ed8", "#dbeafe"),  # blue
             "sealed": ("#10b981", "#d1fae5"),  # green
             "succeeded": ("#10b981", "#d1fae5"),  # green
             "failed": ("#ef4444", "#fee2e2"),  # red
@@ -840,7 +1423,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(
         description="Size",
-        ordering="output_size_sum",
+        ordering="output_size",
     )
     def size_with_stats(self, obj):
         """Show archive size with output size from archive results."""
@@ -902,14 +1485,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         pending = max(total - succeeded - failed - running - skipped - noresults, 0)
         completed = succeeded + failed + skipped + noresults
         percent = int((completed / total * 100) if total > 0 else 0)
-        is_sealed = obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED)
-        output_size = None
-
-        if hasattr(obj, "output_size_sum"):
-            output_size = obj.output_size_sum or 0
-        else:
-            output_size = sum(r.output_size or 0 for r in results)
-
+        is_sealed = obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED)
         stats = {
             "total": total,
             "succeeded": succeeded,
@@ -919,13 +1495,15 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             "skipped": skipped,
             "noresults": noresults,
             "percent": percent,
-            "output_size": output_size or 0,
+            "output_size": obj.output_size or 0,
             "is_sealed": is_sealed,
         }
         obj._admin_progress_stats = stats
         return stats
 
     def _get_prefetched_results(self, obj):
+        if "_admin_archiveresults" in obj.__dict__:
+            return obj.__dict__["_admin_archiveresults"]
         if hasattr(obj, "_prefetched_objects_cache") and "archiveresult_set" in obj._prefetched_objects_cache:
             return obj.archiveresult_set.all()
         return None
@@ -1009,31 +1587,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return format_html('<span style="color: {};">{}</span>', color, h)
 
     def grid_view(self, request, extra_context=None):
-
-        # cl = self.get_changelist_instance(request)
-
-        # Save before monkey patching to restore for changelist list view
-        admin_cls = type(self)
-        saved_change_list_template = admin_cls.change_list_template
-        saved_list_per_page = admin_cls.list_per_page
-        saved_list_max_show_all = admin_cls.list_max_show_all
-
-        # Monkey patch here plus core_tags.py
-        admin_cls.change_list_template = "private_index_grid.html"
-        config = getattr(request, "archivebox_config", None) or get_config()
-        request.archivebox_config = config
-        admin_cls.list_per_page = config.SNAPSHOTS_PER_PAGE
-        admin_cls.list_max_show_all = admin_cls.list_per_page
-
-        # Call monkey patched view
-        rendered_response = self.changelist_view(request, extra_context=extra_context)
-
-        # Restore values
-        admin_cls.change_list_template = saved_change_list_template
-        admin_cls.list_per_page = saved_list_per_page
-        admin_cls.list_max_show_all = saved_list_max_show_all
-
-        return rendered_response
+        extra_context = extra_context or {}
+        extra_context["snapshot_is_grid_view"] = True
+        return self.changelist_view(request, extra_context=extra_context)
 
     # for debugging, uncomment this to print all requests:
     # def changelist_view(self, request, extra_context=None):
@@ -1123,28 +1679,24 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             messages.warning(request, "No tags specified.")
             return
 
-        # Parse comma-separated tag names and get/create Tag objects
         tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
         tags = []
         for name in tag_names:
-            tag, _ = Tag.objects.get_or_create(
-                name__iexact=name,
-                defaults={"name": name},
+            tag, _ = get_or_create_tag(
+                name,
+                created_by=request.user if request.user.is_authenticated else None,
             )
-            tag = Tag.objects.filter(name__iexact=name).first() or tag
             tags.append(tag)
 
         # Get snapshot IDs efficiently (works with select_across for all pages)
         snapshot_ids = list(queryset.values_list("id", flat=True))
         num_snapshots = len(snapshot_ids)
 
-        print("[+] Adding tags", [t.name for t in tags], "to", num_snapshots, "Snapshots")
-
-        # Bulk create M2M relationships (1 query per tag, not per snapshot)
         for tag in tags:
             SnapshotTag.objects.bulk_create(
-                [SnapshotTag(snapshot_id=sid, tag=tag) for sid in snapshot_ids],
-                ignore_conflicts=True,  # Skip if relationship already exists
+                [SnapshotTag(snapshot_id=sid, tag_id=tag.pk) for sid in snapshot_ids],
+                ignore_conflicts=True,
+                batch_size=1000,
             )
 
         messages.success(
@@ -1181,9 +1733,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         num_snapshots = len(snapshot_ids)
         tag_ids = [t.pk for t in tags]
 
-        print("[-] Removing tags", [t.name for t in tags], "from", num_snapshots, "Snapshots")
-
-        # Bulk delete M2M relationships (1 query total, not per snapshot)
         deleted_count, _ = SnapshotTag.objects.filter(
             snapshot_id__in=snapshot_ids,
             tag_id__in=tag_ids,

archivebox/core/apps.py

@@ -25,18 +25,6 @@ class CoreConfig(AppConfig):
         if "makemigrations" not in sys.argv:
             from archivebox.core import models  # noqa: F401
 
-        pidfile = os.environ.get("ARCHIVEBOX_RUNSERVER_PIDFILE")
-        if pidfile:
-            should_write_pid = True
-            if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":
-                should_write_pid = os.environ.get(DJANGO_AUTORELOAD_ENV) == "true"
-            if should_write_pid:
-                try:
-                    with open(pidfile, "w") as handle:
-                        handle.write(str(os.getpid()))
-                except Exception:
-                    pass
-
         def _should_prepare_runtime() -> bool:
             if os.environ.get("ARCHIVEBOX_RUNSERVER") == "1":
                 if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":
@@ -45,6 +33,13 @@ class CoreConfig(AppConfig):
             return False
 
         if _should_prepare_runtime():
-            from archivebox.machine.models import Machine
+            from archivebox.config import CONSTANTS
+            from archivebox.machine.models import Process
 
-            Machine.current()
+            Process.current().mark_running(
+                process_type=Process.TypeChoices.WORKER,
+                worker_type="worker_runserver",
+                pwd=str(CONSTANTS.DATA_DIR),
+                url=os.environ.get("ARCHIVEBOX_RUNSERVER_BIND_URL") or "",
+                timeout=0,
+            )

archivebox/core/forms.py

@@ -1,7 +1,9 @@
 __package__ = "archivebox.core"
 
 import json
+import re
 from collections.abc import Iterable, Mapping
+from decimal import Decimal, InvalidOperation, ROUND_CEILING
 from pathlib import Path
 from typing import Any
 
@@ -13,6 +15,7 @@ from taggit.utils import edit_string_for_tags, parse_tags
 from archivebox.base_models.admin import KeyValueWidget
 from archivebox.crawls.schedule_utils import validate_schedule
 from archivebox.config.common import get_config, parse_delete_after
+from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.hooks import get_plugins, discover_plugin_configs, get_plugin_icon
 from archivebox.personas.models import Persona
@@ -146,6 +149,7 @@ HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
     "search_backend_sqlite",
     "ssl",
 }
+TIMEOUT_INPUT_PATTERN = r"(0|[1-9][0-9]*|[0-9]+(?:\.[0-9]+)?\s*(?:s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours))"
 
 
 def get_plugin_choices():
@@ -509,6 +513,13 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         widget=forms.Textarea(
             attrs={
                 "data-url-regex": URL_REGEX.pattern,
+                "placeholder": (
+                    "Enter URLs to archive, as one per line, CSV, JSON, or embedded in text "
+                    "(e.g. markdown, HTML, etc.). Examples:\n"
+                    "https://example.com\n"
+                    "https://news.ycombinator.com,https://news.google.com\n"
+                    "[ArchiveBox](https://github.com/ArchiveBox/ArchiveBox)"
+                ),
             },
         ),
         required=True,
@@ -526,7 +537,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         widget=forms.RadioSelect(attrs={"class": "depth-selection"}),
     )
     max_urls = forms.IntegerField(
-        label="Max URLs",
+        label="Max crawl URLs",
         required=False,
         min_value=0,
         initial=0,
@@ -548,6 +559,29 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
+    crawl_timeout = forms.CharField(
+        label="Max crawl time",
+        required=False,
+        initial=0,
+        widget=forms.TextInput(
+            attrs={
+                "pattern": TIMEOUT_INPUT_PATTERN,
+                "title": "Use 0, integer seconds, or a duration like 1.5m or 1hr. Non-zero values must be greater than 10 seconds.",
+                "placeholder": "0, 300, 1.5m, or 1hr",
+            },
+        ),
+    )
+    timeout = forms.CharField(
+        label="Max subtask time",
+        required=False,
+        widget=forms.TextInput(
+            attrs={
+                "pattern": TIMEOUT_INPUT_PATTERN,
+                "title": "Use integer seconds or a duration like 1.5m or 1hr. Non-zero values must be greater than 10 seconds.",
+                "placeholder": "60, 1.5m, or 1hr",
+            },
+        ),
+    )
     snapshot_max_size = forms.CharField(
         label="Max snapshot size",
         required=False,
@@ -569,7 +603,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         ),
     )
     crawl_max_concurrent_snapshots = forms.IntegerField(
-        label="Max concurrent snapshots",
+        label="Max in parallel",
         required=False,
         min_value=1,
         widget=forms.NumberInput(
@@ -657,6 +691,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         empty_label=None,
         to_field_name="name",
     )
+    permissions = forms.ChoiceField(
+        label="Permissions",
+        choices=PERMISSIONS_CHOICES,
+        initial="public",
+        required=True,
+    )
     index_only = forms.BooleanField(
         label="Index only dry run (add crawl but don't archive yet)",
         initial=False,
@@ -670,23 +710,45 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
     )
 
     def __init__(self, *args, **kwargs):
+        self.request = kwargs.pop("request", None)
+        self.can_override_crawl_config = bool(self.request and is_admin_user(self.request))
         super().__init__(*args, **kwargs)
 
         default_persona = Persona.get_or_create_default()
-        default_config = get_config(persona=default_persona)
-        self.fields["persona"].queryset = Persona.objects.order_by("name")
-        self.fields["persona"].initial = default_persona.name
+        persona_queryset = Persona.objects.order_by("name")
+        if not self.can_override_crawl_config:
+            persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
+        self.fields["persona"].queryset = persona_queryset
+
+        selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first()
+        default_config = get_config(persona=selected_persona) if selected_persona else get_config()
+        if selected_persona:
+            self.fields["persona"].initial = selected_persona.name
+        self.fields["permissions"].initial = default_config.PERMISSIONS
+        self.fields["timeout"].initial = default_config.TIMEOUT
         self.fields["crawl_max_concurrent_snapshots"].initial = default_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
         self.fields["delete_after"].initial = default_config.DELETE_AFTER
 
-        selected_persona = default_persona
         if self.is_bound:
-            selected_persona = Persona.objects.filter(name=str(self.data.get(self.add_prefix("persona")) or "")).first() or default_persona
-        self.build_plugin_groups(get_config(persona=selected_persona))
+            selected_persona = persona_queryset.filter(name=str(self.data.get(self.add_prefix("persona")) or "")).first() or selected_persona
+        if self.can_override_crawl_config:
+            self.build_plugin_groups(get_config(persona=selected_persona) if selected_persona else get_config())
+        else:
+            all_plugins = get_plugins()
+            for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
+                get_choice_field(self, field_name).choices = [(p, p) for p in all_plugins if p in plugin_names]
+            get_choice_field(self, "other_plugins").choices = [(p, p) for p in all_plugins]
+            self.plugin_groups = []
 
     def clean(self):
         cleaned_data = super().clean() or {}
 
+        if not self.can_override_crawl_config:
+            cleaned_data["plugins"] = []
+            cleaned_data["plugin_config"] = {}
+            cleaned_data["config"] = {}
+            return cleaned_data
+
         # Combine all plugin groups into single list
         all_selected_plugins = []
         for field in [
@@ -731,6 +793,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
             "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
             "same_domain_only": bool(value.get("same_domain_only")),
+            "subpaths_only": bool(value.get("subpaths_only")),
         }
 
     def clean_max_urls(self):
@@ -749,6 +812,37 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError("Max crawl size must be 0 or a positive number of bytes.")
         return value
 
+    def clean_crawl_timeout(self):
+        return self._clean_timeout_seconds(self.cleaned_data.get("crawl_timeout"), "Max crawl time", blank_value=0)
+
+    def clean_timeout(self):
+        return self._clean_timeout_seconds(self.cleaned_data.get("timeout"), "Max subtask time", blank_value=None)
+
+    def _clean_timeout_seconds(self, raw_value, field_label: str, *, blank_value):
+        raw_value = str(raw_value or "").strip().lower()
+        if not raw_value:
+            return blank_value
+        if raw_value.isdigit():
+            value = int(raw_value)
+        else:
+            match = re.fullmatch(r"(\d+(?:\.\d+)?)\s*(s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours)", raw_value)
+            if not match:
+                raise forms.ValidationError(f"{field_label} must be seconds or a duration like 1.5m or 1hr.")
+            amount_str, unit = match.groups()
+            try:
+                amount = Decimal(amount_str)
+            except InvalidOperation as err:
+                raise forms.ValidationError(f"{field_label} must be seconds or a duration like 1.5m or 1hr.") from err
+            multiplier = 1
+            if unit in {"m", "min", "mins", "minute", "minutes"}:
+                multiplier = 60
+            elif unit in {"h", "hr", "hrs", "hour", "hours"}:
+                multiplier = 60 * 60
+            value = int((amount * multiplier).to_integral_value(rounding=ROUND_CEILING))
+        if 0 < value <= 10:
+            raise forms.ValidationError(f"{field_label} must be 0 or greater than 10 seconds.")
+        return value
+
     def clean_snapshot_max_size(self):
         raw_value = str(self.cleaned_data.get("snapshot_max_size") or "").strip()
         if not raw_value:

archivebox/core/host_utils.py

@@ -206,11 +206,6 @@ def get_public_base_url(request=None, config: dict[str, Any] | None = None, **co
     return _build_base_url_for_host(get_public_host(config=config), request=request, config=config)
 
 
-# Backwards-compat aliases (archive == web)
-def get_archive_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return get_web_base_url(request=request, config=config, **config_kwargs)
-
-
 def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:
@@ -233,14 +228,6 @@ def build_web_url(path: str = "", request=None, config: dict[str, Any] | None =
     return _build_url(get_web_base_url(request, config=config, **config_kwargs), path)
 
 
-def build_api_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_api_base_url(request, config=config, **config_kwargs), path)
-
-
-def build_archive_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    return _build_url(get_archive_base_url(request, config=config, **config_kwargs), path)
-
-
 def build_snapshot_url(snapshot_id: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     return _build_url(get_snapshot_base_url(snapshot_id, request=request, config=config, **config_kwargs), path)
 

archivebox/core/middleware.py

@@ -104,7 +104,7 @@ def CacheControlMiddleware(get_response):
                 if config is None:
                     config = get_config(resolve_plugins=False)
                     request.archivebox_config = config
-                policy = "public" if config.PUBLIC_SNAPSHOTS else "private"
+                policy = "private" if config.PERMISSIONS == "private" else "public"
                 response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
                 # print('Set Cache-Control header to', response['Cache-Control'])
         return response

archivebox/core/migrations/0041_snapshot_permissions.py

@@ -0,0 +1,19 @@
+# Generated by Django 6.0.5 on 2026-05-28 07:25
+
+import django.db.models.fields.json
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0040_archiveresult_delete_at_snapshot_delete_at'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='snapshot',
+            name='permissions',
+            field=models.GeneratedField(db_index=True, db_persist=True, expression=django.db.models.fields.json.KeyTextTransform('PERMISSIONS', 'config'), output_field=models.CharField(max_length=16, null=True)),
+        ),
+    ]

archivebox/core/migrations/0042_snapshot_output_size.py

@@ -0,0 +1,39 @@
+# Generated by Django 6.0.5 on 2026-05-28 08:22
+
+from django.db import migrations, models
+from django.db.models import Sum
+
+
+def backfill_snapshot_output_size(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    batch = []
+    rows = ArchiveResult.objects.values("snapshot_id").annotate(total_size=Sum("output_size")).order_by()
+    for row in rows.iterator(chunk_size=2000):
+        batch.append(Snapshot(id=row["snapshot_id"], output_size=row["total_size"] or 0))
+        if len(batch) >= 2000:
+            Snapshot.objects.bulk_update(batch, ["output_size"], batch_size=2000)
+            batch = []
+    if batch:
+        Snapshot.objects.bulk_update(batch, ["output_size"], batch_size=2000)
+
+
+def clear_snapshot_output_size(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    Snapshot.objects.update(output_size=0)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0041_snapshot_permissions'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='snapshot',
+            name='output_size',
+            field=models.BigIntegerField(db_index=True, default=0, editable=False, help_text='Total bytes of all ArchiveResult output files'),
+        ),
+        migrations.RunPython(backfill_snapshot_output_size, clear_snapshot_output_size),
+    ]

archivebox/core/migrations/0043_archiveresult_retry_at.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.5 on 2026-05-28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0042_snapshot_output_size"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="archiveresult",
+            name="retry_at",
+            field=models.DateTimeField(blank=True, db_index=True, default=None, null=True),
+        ),
+    ]

archivebox/core/migrations/0044_alter_archiveresult_status_alter_snapshot_status.py

@@ -0,0 +1,23 @@
+# Generated by Django 6.0.5 on 2026-05-28 12:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0043_archiveresult_retry_at'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='status',
+            field=models.CharField(choices=[('queued', 'Queued'), ('started', 'Started'), ('paused', 'Paused'), ('backoff', 'Waiting to retry'), ('succeeded', 'Succeeded'), ('failed', 'Failed'), ('skipped', 'Skipped'), ('noresults', 'No Results')], db_index=True, default='queued', max_length=16),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='status',
+            field=models.CharField(choices=[('queued', 'Queued'), ('started', 'Started'), ('paused', 'Paused'), ('sealed', 'Sealed')], db_index=True, default='queued', max_length=15),
+        ),
+    ]

archivebox/core/models.py

@@ -13,8 +13,9 @@ from urllib.parse import urlparse
 
 from statemachine import State, registry
 
-from django.db import models
-from django.db.models import Q, QuerySet
+from django.db import models, transaction
+from django.db.models import Q, QuerySet, Sum
+from django.db.models.fields.json import KT
 from django.utils.functional import cached_property
 from django.utils.text import slugify
 from django.utils import timezone
@@ -27,7 +28,7 @@ from django.utils.safestring import mark_safe
 
 from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config
-from archivebox.misc.system import get_dir_size, atomic_write
+from archivebox.misc.system import atomic_write
 from archivebox.misc.util import (
     MAX_URL_LENGTH,
     parse_date,
@@ -53,7 +54,7 @@ from archivebox.base_models.models import (
     ModelWithHealthStats,
     get_or_create_system_user_pk,
 )
-from archivebox.workers.models import ModelWithStateMachine, BaseStateMachine
+from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine
 from archivebox.workers.tasks import bg_archive_snapshot
 from archivebox.crawls.models import Crawl
 from archivebox.machine.models import Binary
@@ -148,6 +149,101 @@ class SnapshotTag(models.Model):
 class SnapshotQuerySet(models.QuerySet):
     """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
 
+    def paged_iterator(self, chunk_size: int = 500):
+        """
+        Iterate snapshots using bounded keyset pages instead of one streaming cursor.
+
+        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
+        cursor open until the full queryset is exhausted. That is fine for
+        read-only exports, but update/migration code does filesystem work and
+        writes while iterating; a long-lived read cursor there can stretch lock
+        waits across thousands of rows. This respects the queryset's existing
+        filters, order_by(), select_related(), and prefetch_related() state; if
+        no ordering is defined, it falls back to primary-key order.
+        """
+        pk_field = self.model._meta.pk.name
+        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
+
+        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        ordering = []
+        for term in raw_ordering:
+            descending = term.startswith("-")
+            field_name = term[1:] if descending else term
+            if field_name == "pk":
+                field_name = pk_field
+            ordering.append(f"-{field_name}" if descending else field_name)
+
+        ordered_field_names = [term[1:] if term.startswith("-") else term for term in ordering]
+        try:
+            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
+                offset = 0
+                while True:
+                    batch = list(self[offset : offset + chunk_size])
+                    if not batch:
+                        break
+                    yield from batch
+                    offset += chunk_size
+                return
+        except Exception:
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if getattr(field, "unique", False))}
+        if not any(field_name in unique_field_names for field_name in ordered_field_names):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        last_values = None
+        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
+        while True:
+            batch_qs = self.order_by(*ordering)
+            if last_values is not None:
+                page_filter = models.Q()
+                for idx, term in enumerate(ordering):
+                    descending = term.startswith("-")
+                    field_name = term[1:] if descending else term
+                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
+                    comparison = "lt" if descending else "gt"
+                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
+                batch_qs = batch_qs.filter(page_filter)
+
+            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
+            if not batch_rows:
+                break
+
+            pk_idx = value_field_names.index(pk_field)
+            snapshot_ids = [row[pk_idx] for row in batch_rows]
+            snapshots_by_id = {snapshot.pk: snapshot for snapshot in self.filter(pk__in=snapshot_ids).order_by()}
+
+            for row in batch_rows:
+                snapshot_id = row[pk_idx]
+                snapshot = snapshots_by_id.get(snapshot_id)
+                if snapshot is not None:
+                    yield snapshot
+
+            last_values = batch_rows[-1][: len(ordered_field_names)]
+
     # =========================================================================
     # Filtering Methods
     # =========================================================================
@@ -345,6 +441,14 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         default=ModelWithStateMachine.StatusChoices.QUEUED,
     )
     config = models.JSONField(default=dict, null=False, blank=False, editable=True)
+    permissions = models.GeneratedField(
+        expression=KT("config__PERMISSIONS"),
+        output_field=models.CharField(max_length=16, null=True),
+        db_persist=True,
+        db_index=True,
+        editable=False,
+    )
+    output_size = models.BigIntegerField(default=0, db_index=True, editable=False, help_text="Total bytes of all ArchiveResult output files")
     notes = models.TextField(blank=True, null=False, default="")
     # output_dir is computed via @cached_property from fs_version and get_storage_path_for_version()
 
@@ -389,6 +493,69 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def __str__(self):
         return f"[{self.id}] {self.url[:64]}"
 
+    def update_and_requeue(self, **kwargs) -> bool:
+        """
+        Update this Snapshot through the shared retry_at ownership path.
+
+        Any non-final Snapshot work means the parent Crawl must also be visible
+        to the runner. Keep that invariant here so CLI/admin callers do not
+        hand-edit the parent Crawl state every time they retry a hook.
+        """
+        updated = super().update_and_requeue(**kwargs)
+        if not updated:
+            return False
+
+        next_status = kwargs.get("status", self.status)
+        if next_status not in (self.StatusChoices.QUEUED, self.StatusChoices.STARTED) or not self.crawl_id:
+            return True
+
+        crawl = self.crawl
+        crawl_status = crawl.StatusChoices.STARTED if crawl.status == crawl.StatusChoices.STARTED else crawl.StatusChoices.QUEUED
+        crawl.update_and_requeue(
+            status=crawl_status,
+            retry_at=kwargs.get("retry_at") or timezone.now(),
+        )
+        return True
+
+    def queue_for_extraction(self, *, when=None) -> bool:
+        """Queue this Snapshot for the runner using the normal state path."""
+        return self.update_and_requeue(
+            status=self.StatusChoices.QUEUED,
+            retry_at=when or timezone.now(),
+            current_step=0,
+        )
+
+    def pause(self, *, save: bool = True) -> bool:
+        paused = super().pause(save=save)
+        if paused and self.pk:
+            ArchiveResult.pause_queryset(self.archiveresult_set.all())
+        return paused
+
+    def resume(self, *, when: datetime | None = None, save: bool = True) -> bool:
+        resumed = super().resume(when=when, save=save)
+        if resumed and self.pk:
+            ArchiveResult.resume_queryset(self.archiveresult_set.all(), when=when)
+        return resumed
+
+    def restore_paused_scheduler_marker(self) -> None:
+        """
+        Keep explicit maintenance from accidentally resuming paused snapshots.
+
+        Targeted jobs such as `archivebox update --index-only` may bump
+        retry_at so the orchestrator can run only queued search ArchiveResult
+        rows. After that maintenance pass, the lifecycle must remain PAUSED and
+        retry_at must go back to MAX until a real resume transition happens.
+        """
+        type(self).objects.filter(pk=self.pk, status=self.StatusChoices.PAUSED).update(
+            retry_at=RETRY_AT_MAX,
+            modified_at=timezone.now(),
+        )
+
+    def cancel(self) -> None:
+        self.status = self.StatusChoices.SEALED
+        self.retry_at = None
+        self.save(update_fields=["status", "retry_at", "modified_at"])
+
     def get_delete_after_config_value(self):
         return get_config(snapshot=self).DELETE_AFTER
 
@@ -1989,22 +2156,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @cached_property
     def archive_size(self):
-        if hasattr(self, "output_size_sum"):
-            return int(self.output_size_sum or 0)
-
-        prefetched_results = None
-        if hasattr(self, "_prefetched_objects_cache"):
-            prefetched_results = self._prefetched_objects_cache.get("archiveresult_set")
-        if prefetched_results:
-            return sum(result.output_size or result.output_size_from_files() for result in prefetched_results)
-
-        stats = self.archiveresult_set.aggregate(result_count=models.Count("id"), total_size=models.Sum("output_size"))
-        if stats["result_count"]:
-            return int(stats["total_size"] or 0)
-        try:
-            return get_dir_size(self.output_dir)[0]
-        except Exception:
-            return 0
+        return int(self.output_size or 0)
 
     def save_tags(self, tags: Iterable[str] = ()) -> None:
         tags_id = [Tag.objects.get_or_create(name=tag)[0].pk for tag in tags if tag.strip()]
@@ -2245,9 +2397,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         update_fields = []
 
         if queue_for_extraction:
-            snapshot.status = Snapshot.StatusChoices.QUEUED
+            if snapshot.status != Snapshot.StatusChoices.PAUSED:
+                snapshot.status = Snapshot.StatusChoices.QUEUED
+                update_fields.append("status")
             snapshot.retry_at = timezone.now()
-            update_fields.extend(["status", "retry_at"])
+            update_fields.append("retry_at")
 
         # Update additional fields if provided
         for field_name in ("depth", "parent_snapshot_id", "crawl_id", "bookmarked_at", "created_at", "downloaded_at"):
@@ -2384,6 +2538,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             ],
         )
         legacy_result_count = retryable_results.filter(hook_name="").count()
+        now = timezone.now()
         count = retryable_results.exclude(hook_name="").update(
             status=ArchiveResult.StatusChoices.QUEUED,
             output_str="",
@@ -2393,19 +2548,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             output_mimetypes="",
             start_ts=None,
             end_ts=None,
+            modified_at=now,
         )
 
         if count + legacy_result_count > 0:
-            self.status = self.StatusChoices.QUEUED
-            self.retry_at = timezone.now()
-            self.current_step = 0  # Reset to step 0 for retry
-            self.save(update_fields=["status", "retry_at", "current_step", "modified_at"])
-
-            crawl = self.crawl
-            if crawl.status != crawl.StatusChoices.STARTED:
-                crawl.status = crawl.StatusChoices.QUEUED
-            crawl.retry_at = timezone.now()
-            crawl.save(update_fields=["status", "retry_at", "modified_at"])
+            self.queue_for_extraction(when=now)
 
         return count + legacy_result_count
 
@@ -2961,13 +3108,21 @@ class SnapshotMachine(BaseStateMachine):
     # States
     queued = State(value=Snapshot.StatusChoices.QUEUED, initial=True)
     started = State(value=Snapshot.StatusChoices.STARTED)
+    paused = State(value=Snapshot.StatusChoices.PAUSED)
     sealed = State(value=Snapshot.StatusChoices.SEALED, final=True)
 
     # Tick Event (polled by workers)
-    tick = queued.to.itself(unless="can_start") | queued.to(started, cond="can_start") | started.to(sealed, cond="is_finished")
+    tick = (
+        queued.to.itself(unless="can_start")
+        | queued.to(started, cond="can_start")
+        | started.to(sealed, cond="is_finished")
+        | paused.to.itself()
+    )
 
     # Manual event (can also be triggered by last ArchiveResult finishing)
     seal = started.to(sealed)
+    pause_requested = queued.to(paused) | started.to(paused)
+    resume_requested = paused.to(queued)
 
     snapshot: Snapshot
 
@@ -2986,6 +3141,13 @@ class SnapshotMachine(BaseStateMachine):
             status=Snapshot.StatusChoices.QUEUED,
         )
 
+    @paused.enter
+    def enter_paused(self):
+        self.snapshot.update_and_requeue(
+            retry_at=RETRY_AT_MAX,
+            status=Snapshot.StatusChoices.PAUSED,
+        )
+
     @started.enter
     def enter_started(self):
         """Just mark as started. The shared runner creates ArchiveResults and runs hooks."""
@@ -2995,8 +3157,6 @@ class SnapshotMachine(BaseStateMachine):
 
     @sealed.enter
     def enter_sealed(self):
-        import sys
-
         # Clean up background hooks
         self.snapshot.cleanup()
 
@@ -3005,19 +3165,19 @@ class SnapshotMachine(BaseStateMachine):
             status=Snapshot.StatusChoices.SEALED,
         )
 
-        print(f"[cyan]  ✅ SnapshotMachine.enter_sealed() - sealed {self.snapshot.url}[/cyan]", file=sys.stderr)
-
         # Check if this is the last snapshot for the parent crawl - if so, seal the crawl
         if self.snapshot.crawl:
             crawl = self.snapshot.crawl
             remaining_active = Snapshot.objects.filter(
                 crawl=crawl,
-                status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
+                status__in=[
+                    Snapshot.StatusChoices.QUEUED,
+                    Snapshot.StatusChoices.STARTED,
+                    Snapshot.StatusChoices.PAUSED,
+                ],
             ).count()
 
             if remaining_active == 0 and crawl.status == crawl.StatusChoices.STARTED:
-                print(f"[cyan]🔒 All snapshots sealed for crawl {crawl.id}, sealing crawl[/cyan]", file=sys.stderr)
-                # Seal the parent crawl
                 cast(Any, crawl).sm.seal()
 
 
@@ -3025,6 +3185,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
     class StatusChoices(models.TextChoices):
         QUEUED = "queued", "Queued"
         STARTED = "started", "Started"
+        PAUSED = "paused", "Paused"
         BACKOFF = "backoff", "Waiting to retry"
         SUCCEEDED = "succeeded", "Succeeded"
         FAILED = "failed", "Failed"
@@ -3053,6 +3214,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
             "noresults": cls.StatusChoices.NORESULTS,
             "queued": cls.StatusChoices.QUEUED,
             "started": cls.StatusChoices.STARTED,
+            "paused": cls.StatusChoices.PAUSED,
             "backoff": cls.StatusChoices.BACKOFF,
         }.get(str(status or "").strip().lower(), cls.StatusChoices.FAILED)
 
@@ -3100,6 +3262,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
     end_ts = models.DateTimeField(default=None, null=True, blank=True)
 
     status = models.CharField(max_length=16, choices=StatusChoices.choices, default=StatusChoices.QUEUED, db_index=True)
+    retry_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True)
     notes = models.TextField(blank=True, null=False, default="")
     # output_dir is computed via @property from snapshot.output_dir / plugin
 
@@ -3123,6 +3286,22 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
     def __str__(self):
         return f"[{self.id}] {self.snapshot.url[:64]} -> {self.plugin}"
 
+    @staticmethod
+    def _format_output_line_for_display(line: str) -> str:
+        raw_line = str(line or "")
+        stripped = raw_line.strip()
+        if not stripped or "://" in stripped or not stripped.startswith(("/", "~/")):
+            return raw_line
+        try:
+            data_dir = CONSTANTS.DATA_DIR.expanduser().resolve(strict=False)
+            rel_path = Path(stripped).expanduser().resolve(strict=False).relative_to(data_dir)
+        except (OSError, ValueError):
+            return raw_line
+        return f"{raw_line[: len(raw_line) - len(raw_line.lstrip())]}./{rel_path}{raw_line[len(raw_line.rstrip()):]}"
+
+    def output_str_for_display(self) -> str:
+        return "\n".join(self._format_output_line_for_display(line) for line in str(self.output_str or "").splitlines())
+
     def get_delete_after_config_value(self):
         return get_config(archiveresult=self).DELETE_AFTER
 
@@ -3225,6 +3404,13 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
             return None
 
     def save(self, *args, **kwargs):
+        is_new = self._state.adding
+        update_fields = kwargs.get("update_fields")
+        refresh_snapshot_size = is_new or update_fields is None or "output_size" in update_fields or "snapshot" in update_fields or "snapshot_id" in update_fields
+        old_snapshot_id = None
+        if refresh_snapshot_size and not is_new:
+            old_snapshot_id = type(self).objects.filter(pk=self.pk).values_list("snapshot_id", flat=True).first()
+
         update_fields = kwargs.get("update_fields")
         if self.delete_at is None:
             self.set_delete_at_from_config()
@@ -3234,6 +3420,9 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
         # Skip ModelWithOutputDir.save() to avoid creating index.json in plugin directories
         # Call the Django Model.save() directly instead
         models.Model.save(self, *args, **kwargs)
+        if refresh_snapshot_size:
+            snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, self.snapshot_id) if snapshot_id}
+            transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids))
 
         # if is_new:
         #     from archivebox.misc.logging_util import log_worker_event
@@ -3250,6 +3439,19 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
         #         },
         #     )
 
+    def delete(self, *args, **kwargs):
+        snapshot_id = self.snapshot_id
+        deleted = super().delete(*args, **kwargs)
+        if snapshot_id:
+            transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes({snapshot_id}))
+        return deleted
+
+    @staticmethod
+    def refresh_snapshot_output_sizes(snapshot_ids):
+        for snapshot_id in snapshot_ids:
+            total_size = ArchiveResult.objects.filter(snapshot_id=snapshot_id).aggregate(total_size=Sum("output_size"))["total_size"] or 0
+            Snapshot.objects.filter(pk=snapshot_id).update(output_size=total_size)
+
     @cached_property
     def snapshot_dir(self):
         return Path(self.snapshot.output_dir)
@@ -3267,6 +3469,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
 
     def reset_for_retry(self, *, save: bool = True) -> None:
         self.status = self.StatusChoices.QUEUED
+        self.retry_at = None
         self.output_str = ""
         self.output_json = None
         self.output_files = {}
@@ -3278,6 +3481,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
             self.save(
                 update_fields=[
                     "status",
+                    "retry_at",
                     "output_str",
                     "output_json",
                     "output_files",
@@ -3289,6 +3493,48 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
                 ],
             )
 
+    @property
+    def is_paused(self) -> bool:
+        return self.status == self.StatusChoices.PAUSED
+
+    @classmethod
+    def pause_queryset(cls, queryset) -> int:
+        return queryset.exclude(status__in=[*cls.FINAL_STATES, cls.StatusChoices.PAUSED]).update(
+            status=cls.StatusChoices.PAUSED,
+            retry_at=RETRY_AT_MAX,
+            modified_at=timezone.now(),
+        )
+
+    @classmethod
+    def resume_queryset(cls, queryset, *, when: datetime | None = None) -> int:
+        return queryset.filter(status=cls.StatusChoices.PAUSED).update(
+            status=cls.StatusChoices.QUEUED,
+            retry_at=when or timezone.now(),
+            modified_at=timezone.now(),
+        )
+
+    def pause(self, *, save: bool = True) -> bool:
+        if self.status in self.FINAL_STATES:
+            return False
+        if self.is_paused:
+            return False
+        self.status = self.StatusChoices.PAUSED
+        self.retry_at = RETRY_AT_MAX
+        if save:
+            self.pause_queryset(type(self).objects.filter(pk=self.pk))
+            self.refresh_from_db()
+        return True
+
+    def resume(self, *, when: datetime | None = None, save: bool = True) -> bool:
+        if not self.is_paused:
+            return False
+        self.status = self.StatusChoices.QUEUED
+        self.retry_at = when or timezone.now()
+        if save:
+            self.resume_queryset(type(self).objects.filter(pk=self.pk), when=self.retry_at)
+            self.refresh_from_db()
+        return True
+
     @property
     def plugin_module(self) -> Any | None:
         # Hook scripts are now used instead of Python plugin modules
@@ -3370,7 +3616,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
             return False
 
         snapshot_dir = Path(snapshot_dir or self.snapshot.output_dir)
-        exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid", "cmd.sh"}
+        exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"}
         output_files: dict[str, dict[str, Any]] = {}
         mime_sizes: dict[str, int] = defaultdict(int)
         total_size = 0
@@ -3505,7 +3751,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
         plugin_name: str | None = None,
         output_file_map: dict[str, dict[str, Any]] | None = None,
     ) -> str | None:
-        ignored = {"stdout.log", "stderr.log", "hook.pid", "listener.pid", "cmd.sh"}
+        ignored = {"stdout.log", "stderr.log", "hook.pid", "listener.pid"}
         candidates = [
             path
             for path in output_file_paths
@@ -3731,15 +3977,14 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
             self.save()
             return
 
-        # Read and parse JSONL output from stdout.log
-        stdout_file = plugin_dir / "stdout.log"
         records = []
         process = self.process_record
         if process:
             records = extract_records_from_process(process)
 
         if not records:
-            stdout = stdout_file.read_text() if stdout_file.exists() else ""
+            stdout_file = plugin_dir / "stdout.log"
+            stdout = stdout_file.read_text(errors="replace") if stdout_file.exists() else ""
             records = Process.parse_records_from_text(stdout)
 
         # Find ArchiveResult record and update status/output from it
@@ -3785,7 +4030,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
                 self.output_str = "Hook did not output ArchiveResult record"
 
         # Walk filesystem and populate output_files, output_size, output_mimetypes
-        exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid", "cmd.sh"}
+        exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"}
         mime_sizes = defaultdict(int)
         total_size = 0
         output_files = {}

archivebox/core/permissions.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from django.db.models import Q, QuerySet
+from django.http import HttpRequest
+
+from archivebox.config.common import get_config
+
+PERMISSIONS_PUBLIC = "public"
+PERMISSIONS_UNLISTED = "unlisted"
+PERMISSIONS_PRIVATE = "private"
+PERMISSIONS_CHOICES = (
+    (PERMISSIONS_PUBLIC, "Public"),
+    (PERMISSIONS_UNLISTED, "Unlisted"),
+    (PERMISSIONS_PRIVATE, "Private"),
+)
+
+
+def is_admin_user(request: HttpRequest) -> bool:
+    user = request.user
+    return bool(user.is_authenticated and user.is_active and user.is_staff)
+
+
+def get_snapshot_permissions(snapshot) -> str:
+    try:
+        return str(get_config(snapshot=snapshot, resolve_plugins=False).PERMISSIONS).strip().lower()
+    except Exception:
+        return PERMISSIONS_PRIVATE
+
+
+def can_view_snapshot(request: HttpRequest, snapshot) -> bool:
+    permissions = get_snapshot_permissions(snapshot)
+    return permissions in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED} or is_admin_user(request)
+
+
+def _persona_ids_for_permissions(allowed_permissions: set[str]) -> list[str]:
+    from archivebox.personas.models import Persona
+
+    fallback_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower()
+    personas = Persona.objects.only("id", "config")
+    return [
+        str(persona.id)
+        for persona in personas
+        if (persona.permissions or fallback_permissions) in allowed_permissions
+    ]
+
+
+def filter_personas_by_permissions(queryset: QuerySet, allowed_permissions: set[str]) -> QuerySet:
+    return queryset.filter(id__in=_persona_ids_for_permissions(allowed_permissions))
+
+
+def filter_snapshots_by_permissions(queryset: QuerySet, *, direct: bool = False, allowed_permissions: set[str] | None = None) -> QuerySet:
+    from archivebox.crawls.models import Crawl
+    from archivebox.personas.models import Persona
+
+    allowed_permissions = allowed_permissions or ({PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED} if direct else {PERMISSIONS_PUBLIC})
+    fallback_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower()
+    has_overrides = (
+        queryset.model.objects.filter(permissions__gt="").exists()
+        or Crawl.objects.filter(permissions__gt="").exists()
+        or Persona.objects.filter(permissions__gt="").exists()
+    )
+    if not has_overrides:
+        return queryset if fallback_permissions in allowed_permissions else queryset.none()
+
+    allowed_persona_ids = _persona_ids_for_permissions(allowed_permissions)
+    valid_persona_ids = [str(persona_id) for persona_id in Persona.objects.values_list("id", flat=True)]
+    fallback_query = Q(crawl__persona_id__in=allowed_persona_ids)
+    if fallback_permissions in allowed_permissions:
+        fallback_query |= Q(crawl__persona_id__isnull=True) | ~Q(crawl__persona_id__in=valid_persona_ids)
+    inherited_query = Q(crawl__permissions__in=sorted(allowed_permissions)) | (Q(crawl__permissions__isnull=True) & fallback_query)
+    return queryset.filter(
+        Q(permissions__in=sorted(allowed_permissions)) | (Q(permissions__isnull=True) & inherited_query),
+    )
+
+
+def public_snapshots_queryset(queryset: QuerySet) -> QuerySet:
+    return filter_snapshots_by_permissions(queryset, direct=False)
+
+
+def direct_snapshots_queryset(request: HttpRequest, queryset: QuerySet) -> QuerySet:
+    return queryset if is_admin_user(request) else filter_snapshots_by_permissions(queryset, direct=True)

archivebox/core/recovery_util.py

@@ -0,0 +1,349 @@
+from __future__ import annotations
+
+from datetime import timedelta
+
+from django.utils import timezone
+from rich.console import Console
+
+
+def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]:
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.machine.models import Process
+    from django.db.models import Exists, OuterRef, Q, Subquery, Value
+    from django.db.models.functions import Coalesce
+
+    now = timezone.now()
+    stuck_cutoff = now - timedelta(hours=12)
+    recovery_console = Console(stderr=True, highlight=False)
+    cleaned = {
+        "stale_processes": Process.cleanup_stale_running(),
+        "orphaned_processes": Process.cleanup_orphaned_workers(),
+        "orphaned_chrome": Process.cleanup_orphaned_chrome() if include_chrome else 0,
+        "queued_crawls_unlocked": 0,
+        "sealed_crawl_locks_cleared": 0,
+        "sealed_snapshots": 0,
+        "unlocked_snapshots": 0,
+        "requeued_snapshots": 0,
+        "queued_snapshots_unlocked": 0,
+        "sealed_snapshot_locks_cleared": 0,
+        "requeued_archiveresults": 0,
+        "sealed_crawls": 0,
+        "unlocked_crawls": 0,
+        "requeued_crawls": 0,
+        "sealed_queued_snapshots": 0,
+        "sealed_queued_crawls": 0,
+    }
+
+    any_archiveresults = ArchiveResult.objects.filter(snapshot_id=OuterRef("pk"))
+    unfinished_archiveresults = any_archiveresults.exclude(status__in=ArchiveResult.FINAL_STATES)
+    recent_snapshots = Snapshot.objects.filter(crawl_id=OuterRef("pk"), modified_at__gt=stuck_cutoff)
+    recent_archiveresults = ArchiveResult.objects.filter(snapshot__crawl_id=OuterRef("pk"), modified_at__gt=stuck_cutoff)
+    recent_archiveresult_processes = Process.objects.filter(
+        archiveresult__snapshot__crawl_id=OuterRef("pk"),
+        modified_at__gt=stuck_cutoff,
+    )
+    recent_crawl_snapshots_for_snapshot = Snapshot.objects.filter(crawl_id=OuterRef("crawl_id"), modified_at__gt=stuck_cutoff)
+    recent_crawl_archiveresults_for_snapshot = ArchiveResult.objects.filter(
+        snapshot__crawl_id=OuterRef("crawl_id"),
+        modified_at__gt=stuck_cutoff,
+    )
+    recent_crawl_archiveresult_processes_for_snapshot = Process.objects.filter(
+        archiveresult__snapshot__crawl_id=OuterRef("crawl_id"),
+        modified_at__gt=stuck_cutoff,
+    )
+
+    # Stale-only repair: if a queued snapshot/crawl already has only final
+    # projected result rows and the whole crawl has been quiet for >12hr, it
+    # was likely interrupted after hook completion but before state sealing.
+    # Never run this on fresh rows: queued work is normal during direct
+    # reindex/extract and while a daemon runner is active.
+    stale_finished_snapshot_ids = (
+        Snapshot.objects.filter(
+            status=Snapshot.StatusChoices.QUEUED,
+            modified_at__lte=stuck_cutoff,
+            crawl__modified_at__lte=stuck_cutoff,
+        )
+        .filter(Q(retry_at__isnull=True) | Q(retry_at__lte=stuck_cutoff))
+        .annotate(
+            has_results=Exists(any_archiveresults),
+            has_unfinished_results=Exists(unfinished_archiveresults),
+            has_recent_snapshot=Exists(recent_crawl_snapshots_for_snapshot),
+            has_recent_archiveresult=Exists(recent_crawl_archiveresults_for_snapshot),
+            has_recent_archiveresult_process=Exists(recent_crawl_archiveresult_processes_for_snapshot),
+        )
+        .filter(
+            has_results=True,
+            has_unfinished_results=False,
+            has_recent_snapshot=False,
+            has_recent_archiveresult=False,
+            has_recent_archiveresult_process=False,
+        )
+        .values_list("id", flat=True)
+    )
+    cleaned["sealed_queued_snapshots"] = Snapshot.objects.filter(id__in=stale_finished_snapshot_ids).update(
+        status=Snapshot.StatusChoices.SEALED,
+        retry_at=None,
+        downloaded_at=Coalesce("downloaded_at", Value(now)),
+    )
+    unrecoverable_active_child_snapshots = (
+        Snapshot.objects.filter(
+            crawl_id=OuterRef("pk"),
+            status__in=[
+                Snapshot.StatusChoices.QUEUED,
+                Snapshot.StatusChoices.STARTED,
+                Snapshot.StatusChoices.PAUSED,
+            ],
+        )
+        .annotate(
+            has_results=Exists(any_archiveresults),
+            has_unfinished_results=Exists(unfinished_archiveresults),
+        )
+        .filter(
+            Q(status=Snapshot.StatusChoices.STARTED)
+            | Q(modified_at__gt=stuck_cutoff)
+            | Q(retry_at__gt=stuck_cutoff)
+            | Q(has_results=False)
+            | Q(has_unfinished_results=True),
+        )
+    )
+    cleaned["sealed_queued_crawls"] = (
+        Crawl.objects.filter(
+            status=Crawl.StatusChoices.QUEUED,
+            snapshot_set__isnull=False,
+            modified_at__lte=stuck_cutoff,
+        )
+        .filter(Q(retry_at__isnull=True) | Q(retry_at__lte=stuck_cutoff))
+        .annotate(
+            has_unrecoverable_active_child=Exists(unrecoverable_active_child_snapshots),
+            has_recent_snapshot=Exists(recent_snapshots),
+            has_recent_archiveresult=Exists(recent_archiveresults),
+            has_recent_archiveresult_process=Exists(recent_archiveresult_processes),
+        )
+        .filter(
+            has_unrecoverable_active_child=False,
+            has_recent_snapshot=False,
+            has_recent_archiveresult=False,
+            has_recent_archiveresult_process=False,
+        )
+        .update(
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
+            modified_at=now,
+        )
+    )
+
+    stale_crawls = (
+        Crawl.objects.filter(
+            status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED],
+            modified_at__lte=stuck_cutoff,
+        )
+        .filter(Q(retry_at__isnull=True) | Q(retry_at__lte=now))
+        .annotate(
+            has_recent_snapshot=Exists(recent_snapshots),
+            has_recent_archiveresult=Exists(recent_archiveresults),
+            has_recent_archiveresult_process=Exists(recent_archiveresult_processes),
+        )
+        .filter(has_recent_snapshot=False, has_recent_archiveresult=False, has_recent_archiveresult_process=False)
+        .order_by("modified_at")[:10]
+    )
+    stale_crawl_messages = []
+    for crawl in stale_crawls:
+        if not Process.objects.filter(
+            pwd__contains=str(crawl.id),
+            status=Process.StatusChoices.RUNNING,
+            modified_at__gt=stuck_cutoff,
+        ).exists():
+            stale_crawl_messages.append(
+                f"{crawl.id} status={crawl.status} retry_at={crawl.retry_at} modified_at={crawl.modified_at}",
+            )
+    if stale_crawl_messages:
+        recovery_console.print(
+            "[red]❌ Orchestrator recovery found stuck active crawl invariant violation; refusing to continue.[/red]",
+        )
+        raise RuntimeError(
+            "Stuck crawl invariant violated: active crawls had no crawl/snapshot/result/process changes for >12hr: "
+            + "; ".join(stale_crawl_messages),
+        )
+
+    running_archiveresults = ArchiveResult.objects.filter(
+        snapshot_id=OuterRef("pk"),
+        status=ArchiveResult.StatusChoices.STARTED,
+        process__status=Process.StatusChoices.RUNNING,
+    )
+    unfinished_archiveresult_statuses = [
+        ArchiveResult.StatusChoices.QUEUED,
+        ArchiveResult.StatusChoices.STARTED,
+        ArchiveResult.StatusChoices.PAUSED,
+        ArchiveResult.StatusChoices.BACKOFF,
+    ]
+    running_unfinished_archiveresults = ArchiveResult.objects.filter(
+        snapshot_id=OuterRef("pk"),
+        status__in=unfinished_archiveresult_statuses,
+        process__status=Process.StatusChoices.RUNNING,
+    )
+    unfinished_without_running_archiveresults = ArchiveResult.objects.filter(
+        snapshot_id=OuterRef("pk"),
+        status__in=[
+            ArchiveResult.StatusChoices.QUEUED,
+            ArchiveResult.StatusChoices.STARTED,
+            ArchiveResult.StatusChoices.BACKOFF,
+        ],
+    ).exclude(
+        status=ArchiveResult.StatusChoices.PAUSED,
+    ).exclude(
+        process__status=Process.StatusChoices.RUNNING,
+    )
+    active_child_snapshots = Snapshot.objects.filter(
+        crawl_id=OuterRef("pk"),
+        status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED],
+    )
+    due_child_snapshots = active_child_snapshots.exclude(status=Snapshot.StatusChoices.PAUSED).filter(
+        Q(retry_at__isnull=True) | Q(retry_at__lte=now),
+    )
+    next_future_child_retry = Subquery(
+        active_child_snapshots.filter(retry_at__gt=now).order_by("retry_at").values("retry_at")[:1],
+    )
+
+    # Broken lock repair: QUEUED rows with retry_at=NULL are invisible to the
+    # queue. Set only the scheduling field so the runner owns the next tick.
+    cleaned["queued_crawls_unlocked"] = Crawl.objects.filter(
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at__isnull=True,
+    ).update(retry_at=now, modified_at=now)
+    cleaned["queued_snapshots_unlocked"] = Snapshot.objects.filter(
+        status=Snapshot.StatusChoices.QUEUED,
+        retry_at__isnull=True,
+    ).update(retry_at=now, modified_at=now)
+
+    # ArchiveResult has no retry_at scheduler; BACKOFF is a legacy/impossible
+    # persisted state here, so move it back to QUEUED for the snapshot runner.
+    cleaned["requeued_archiveresults"] = ArchiveResult.objects.filter(
+        status=ArchiveResult.StatusChoices.BACKOFF,
+    ).update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
+    # Impossible state repair: STARTED ArchiveResults without a live Process
+    # have no owner left to emit completion. Requeue only the result row; the
+    # snapshot/crawl schedulers will pick up normal retry processing.
+    cleaned["requeued_archiveresults"] += (
+        ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED)
+        .exclude(process__status=Process.StatusChoices.RUNNING)
+        .update(status=ArchiveResult.StatusChoices.QUEUED, process=None, modified_at=now)
+    )
+
+    started_snapshots = Snapshot.objects.filter(
+        status=Snapshot.StatusChoices.STARTED,
+        retry_at__isnull=True,
+    )
+
+    # Normal transition: the snapshot has finished all known extractor work,
+    # but the process died before the state machine got to seal it.
+    finished_snapshot_ids = (
+        started_snapshots.annotate(
+            has_results=Exists(any_archiveresults),
+            has_unfinished_results=Exists(unfinished_archiveresults),
+        )
+        .filter(has_results=True, has_unfinished_results=False)
+        .values_list("id", flat=True)
+    )
+    for snapshot in Snapshot.objects.filter(id__in=finished_snapshot_ids).select_related("crawl").iterator(chunk_size=100):
+        snapshot.sm.seal()
+        cleaned["sealed_snapshots"] += 1
+
+    # Broken lock repair: STARTED + retry_at=NULL means "owned by an active
+    # runner". If no ArchiveResult has a live process anymore, only unlock it.
+    # The existing runner will pick the row up through the normal queue path.
+    cleaned["unlocked_snapshots"] = (
+        started_snapshots.annotate(has_running_results=Exists(running_archiveresults))
+        .filter(has_running_results=False)
+        .update(
+            retry_at=now,
+            modified_at=now,
+        )
+    )
+
+    # Impossible state repair: a SEALED snapshot with a still-running child is
+    # active, not final. Reflect that without starting duplicate work.
+    cleaned["requeued_snapshots"] += (
+        Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED)
+        .annotate(has_running_unfinished_results=Exists(running_unfinished_archiveresults))
+        .filter(has_running_unfinished_results=True)
+        .update(
+            status=Snapshot.StatusChoices.STARTED,
+            retry_at=None,
+            modified_at=now,
+        )
+    )
+
+    # Impossible state repair: SEALED snapshots should not contain unfinished
+    # ArchiveResults. There is no valid state-machine transition from final
+    # back to queued, so repair only the fields needed for the runner to retry.
+    cleaned["requeued_snapshots"] += (
+        Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED)
+        .annotate(has_unfinished_results_without_running=Exists(unfinished_without_running_archiveresults))
+        .filter(has_unfinished_results_without_running=True)
+        .update(
+            status=Snapshot.StatusChoices.QUEUED,
+            retry_at=now,
+            modified_at=now,
+        )
+    )
+
+    # Normal transition: a started crawl has no active snapshots left.
+    finished_crawl_ids = (
+        Crawl.objects.filter(status=Crawl.StatusChoices.STARTED, retry_at__isnull=True)
+        .exclude(
+            snapshot_set__status__in=[
+                Snapshot.StatusChoices.QUEUED,
+                Snapshot.StatusChoices.STARTED,
+                Snapshot.StatusChoices.PAUSED,
+            ],
+        )
+        .values_list("id", flat=True)
+    )
+    for crawl in Crawl.objects.filter(id__in=finished_crawl_ids).iterator(chunk_size=100):
+        crawl.sm.seal()
+        cleaned["sealed_crawls"] += 1
+
+    # Broken lock repair: STARTED + retry_at=NULL with unfinished snapshots is
+    # recoverable by unlocking the crawl. Do not create snapshots or results.
+    due_started_crawls = (
+        Crawl.objects.filter(status=Crawl.StatusChoices.STARTED, retry_at__isnull=True)
+        .annotate(has_due_child=Exists(due_child_snapshots))
+        .filter(has_due_child=True)
+    )
+    cleaned["unlocked_crawls"] = due_started_crawls.update(retry_at=now, modified_at=now)
+    future_started_crawls = (
+        Crawl.objects.filter(status=Crawl.StatusChoices.STARTED, retry_at__isnull=True)
+        .annotate(has_active_child=Exists(active_child_snapshots), has_due_child=Exists(due_child_snapshots), next_child_retry=next_future_child_retry)
+        .filter(has_active_child=True, has_due_child=False)
+    )
+    cleaned["unlocked_crawls"] += future_started_crawls.update(retry_at=Coalesce("next_child_retry", Value(now)), modified_at=now)
+
+    cleaned["requeued_crawls"] = 0
+
+    warning_recoveries = {
+        "stale_processes": "marked stale running Process row(s) exited",
+        "orphaned_processes": "marked orphaned worker/hook Process row(s) exited",
+        "orphaned_chrome": "terminated orphaned Chrome process(es)",
+        "sealed_snapshots": "sealed started Snapshot row(s) whose ArchiveResults were already final",
+        "unlocked_snapshots": "unlocked started Snapshot row(s) whose owner process was gone",
+        "sealed_crawls": "sealed started Crawl row(s) with no active Snapshots",
+        "unlocked_crawls": "unlocked started Crawl row(s) with pending child Snapshots",
+    }
+    error_recoveries = {
+        "queued_crawls_unlocked": "repaired queued Crawl row(s) with retry_at=NULL",
+        "queued_snapshots_unlocked": "repaired queued Snapshot row(s) with retry_at=NULL",
+        "requeued_archiveresults": "requeued ArchiveResult row(s) left in BACKOFF",
+        "requeued_snapshots": "reopened sealed Snapshot row(s) with unfinished ArchiveResults",
+        "requeued_crawls": "reopened sealed Crawl row(s) with active child Snapshots",
+        "sealed_queued_snapshots": "sealed stale queued Snapshot row(s) whose ArchiveResults were already final",
+        "sealed_queued_crawls": "sealed stale queued Crawl row(s) whose Snapshots were already final",
+    }
+    for key, message in warning_recoveries.items():
+        if cleaned[key]:
+            recovery_console.print(f"[yellow]⚠️ Orchestrator recovery: {cleaned[key]} {message}.[/yellow]")
+    for key, message in error_recoveries.items():
+        if cleaned[key]:
+            recovery_console.print(f"[red]❌ Orchestrator invariant repair: {cleaned[key]} {message}.[/red]")
+
+    return cleaned

archivebox/core/settings.py

@@ -229,7 +229,7 @@ SQLITE_JOURNAL_MODE = os.environ.get("ARCHIVEBOX_SQLITE_JOURNAL_MODE", "WAL")
 SQLITE_MMAP_SIZE = os.environ.get("ARCHIVEBOX_SQLITE_MMAP_SIZE", "0" if CONSTANTS.IN_DOCKER else "134217728")
 
 SQLITE_CONNECTION_OPTIONS = {
-    "ENGINE": "django.db.backends.sqlite3",
+    "ENGINE": "archivebox.core.sqlite_backend",
     "TIME_ZONE": CONSTANTS.TIMEZONE,
     "OPTIONS": {
         # https://gcollazo.com/optimal-sqlite-settings-for-django/
@@ -237,7 +237,13 @@ SQLITE_CONNECTION_OPTIONS = {
         # https://docs.djangoproject.com/en/5.1/ref/databases/#setting-pragma-options
         "timeout": 30,
         "check_same_thread": False,
-        "transaction_mode": "IMMEDIATE",
+        # Keep SQLite on Django's default deferred transaction mode. BEGIN
+        # IMMEDIATE grabs the write lock as soon as atomic() opens, which is
+        # exactly what hurts ArchiveBox on large collections where Python code
+        # may do filesystem work before the actual row write. Deferred BEGIN
+        # keeps writes statement-scoped unless a caller explicitly opens a
+        # transaction around multiple writes.
+        "transaction_mode": None,
         "init_command": (
             "PRAGMA foreign_keys=ON;"
             "PRAGMA busy_timeout = 30000;"

archivebox/core/shutdown_util.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import signal
+import subprocess
+import sys
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+
+import psutil
+
+
+@dataclass
+class ShutdownSignalState:
+    """Tracks the exact OS signal that asked a foreground command to exit."""
+
+    signal_name: str | None = None
+
+
+def configured_stopwaitsecs(workers: list[dict[str, str]] | tuple[dict[str, str], ...], *, default: int = 5, buffer: int = 5) -> int:
+    """Return a deterministic shutdown bound from generated worker definitions."""
+
+    stop_grace_seconds = default
+    for worker in workers:
+        try:
+            stop_grace_seconds = max(stop_grace_seconds, int(worker.get("stopwaitsecs") or default) + buffer)
+        except (TypeError, ValueError):
+            continue
+    return stop_grace_seconds
+
+
+def wait_popen_and_kill_children(
+    proc: subprocess.Popen,
+    children: list[psutil.Process],
+    *,
+    timeout: float,
+    kill_timeout: float = 2.0,
+) -> None:
+    """Wait for a Popen parent and then hard-kill any surviving descendants."""
+
+    try:
+        proc.wait(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        proc.wait(timeout=kill_timeout)
+    kill_remaining_processes(children, timeout=kill_timeout)
+
+
+def wait_psutil_and_kill_children(
+    proc: psutil.Process,
+    children: list[psutil.Process],
+    *,
+    timeout: float,
+    kill_timeout: float = 2.0,
+) -> None:
+    """Wait for a psutil parent and then hard-kill any surviving descendants."""
+
+    try:
+        if proc.status() == psutil.STATUS_ZOMBIE:
+            # Another ArchiveBox foreground parent owns this Popen and must reap
+            # it. By the time supervisord is a zombie it has already stopped
+            # accepting work, so the caller can clear stale pid/socket files
+            # without blocking for a process it cannot reap itself.
+            kill_remaining_processes(children, timeout=kill_timeout)
+            return
+        proc.wait(timeout=timeout)
+    except psutil.TimeoutExpired:
+        proc.kill()
+    kill_remaining_processes(children, timeout=kill_timeout)
+    try:
+        proc.wait(timeout=kill_timeout)
+    except (psutil.NoSuchProcess, psutil.TimeoutExpired):
+        pass
+
+
+def kill_remaining_processes(processes: list[psutil.Process], *, timeout: float = 2.0) -> None:
+    _gone, alive = psutil.wait_procs(processes, timeout=timeout)
+    for process in alive:
+        try:
+            process.kill()
+        except psutil.NoSuchProcess:
+            pass
+    psutil.wait_procs(alive, timeout=timeout)
+
+
+@contextmanager
+def foreground_shutdown_signals(
+    handled_signals: tuple[signal.Signals, ...] = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM),
+) -> Iterator[ShutdownSignalState]:
+    """Install foreground signal handlers that print an immediate exit notice.
+
+    Some log-tail loops intentionally swallow KeyboardInterrupt so that callers
+    can centralize cleanup in finally blocks. The handler writes the signal name
+    immediately, then raises KeyboardInterrupt to break out of the blocking read.
+    """
+
+    state = ShutdownSignalState()
+    previous_handlers = {sig: signal.getsignal(sig) for sig in handled_signals}
+
+    def raise_keyboard_interrupt(signum, _frame):
+        state.signal_name = signal.Signals(signum).name
+        sys.stdout.write(f"\n[🛑] Got {state.signal_name}, stopping gracefully...\n")
+        sys.stdout.flush()
+        raise KeyboardInterrupt
+
+    try:
+        for sig in handled_signals:
+            signal.signal(sig, raise_keyboard_interrupt)
+        yield state
+    finally:
+        for sig, previous_handler in previous_handlers.items():
+            signal.signal(sig, previous_handler)

archivebox/core/sqlite_backend/base.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import sqlite3
+import time
+from collections.abc import Mapping
+from itertools import tee
+import re
+
+from django.db.backends.sqlite3.base import DatabaseWrapper as DjangoSQLiteDatabaseWrapper
+from django.db.backends.sqlite3.base import SQLiteCursorWrapper as DjangoSQLiteCursorWrapper
+
+
+def _is_locked_error(error: BaseException) -> bool:
+    from django.db import OperationalError
+
+    return isinstance(error, (sqlite3.OperationalError, OperationalError)) and "database is locked" in str(error).lower()
+
+
+def _format_sql(query: str, params=None) -> str:
+    compact = " ".join(str(query).split())
+    match = re.match(r'^(INSERT INTO|UPDATE|DELETE FROM|SELECT) "?([A-Za-z0-9_]+)"?', compact, flags=re.IGNORECASE)
+    if match:
+        compact = f"{match.group(1).upper()} {match.group(2)}"
+    if params is not None:
+        if isinstance(params, str):
+            params_summary = params
+        elif isinstance(params, (tuple, list)):
+            preview = ", ".join(repr(param)[:60] for param in params[:4])
+            params_summary = f"{len(params)} params: {preview}"
+        elif isinstance(params, Mapping):
+            preview = ", ".join(f"{key}={repr(value)[:60]}" for key, value in list(params.items())[:4])
+            params_summary = f"{len(params)} params: {preview}"
+        else:
+            params_summary = repr(params)[:120]
+        compact = f"{compact} ({params_summary})"
+    return compact[:260]
+
+
+def _log_locked_database(query: str, params=None, *, attempt: int, elapsed: float) -> None:
+    from rich.console import Console
+
+    from archivebox.misc.db import sqlite_lock_holders
+
+    console = Console(stderr=True)
+    console.print(f"[yellow][*] SQLite database is locked for {elapsed:.0f}s; retrying in 5s... attempt={attempt}[/yellow]")
+    console.print(f"[yellow]    Query: {_format_sql(query, params)}[/yellow]")
+    holders = sqlite_lock_holders()
+    if holders:
+        console.print("[yellow]    DB holders:[/yellow]")
+        for holder in holders[:8]:
+            console.print(f"[yellow]    - {holder}[/yellow]")
+        if len(holders) > 8:
+            console.print(f"[yellow]    ... {len(holders) - 8} more[/yellow]")
+    else:
+        console.print("[yellow]    No local process with index.sqlite3 open was visible to this user.[/yellow]")
+    if attempt == 1:
+        console.print(
+            "[dim]    SQLite does not expose the active SQL statement from another process; only local PIDs with the DB open can be shown.[/dim]",
+        )
+
+
+def _retry_locked_database(action, query: str, params=None):
+    attempt = 0
+    started_at = time.monotonic()
+    while True:
+        try:
+            return action()
+        except (sqlite3.OperationalError, Exception) as err:
+            if not _is_locked_error(err):
+                raise
+            attempt += 1
+            _log_locked_database(query, params, attempt=attempt, elapsed=time.monotonic() - started_at)
+            time.sleep(5.0)
+
+
+class SQLiteCursorWrapper(DjangoSQLiteCursorWrapper):
+    def execute(self, query, params=None):
+        if params is None:
+            return _retry_locked_database(lambda: super(SQLiteCursorWrapper, self).execute(query), query)
+        param_names = list(params) if isinstance(params, Mapping) else None
+        converted_query = self.convert_query(query, param_names=param_names)
+        return _retry_locked_database(
+            lambda: super(DjangoSQLiteCursorWrapper, self).execute(converted_query, params),
+            converted_query,
+            params,
+        )
+
+    def executemany(self, query, param_list):
+        peekable, param_list = tee(iter(param_list))
+        if (params := next(peekable, None)) and isinstance(params, Mapping):
+            param_names = list(params)
+        else:
+            param_names = None
+        converted_query = self.convert_query(query, param_names=param_names)
+        param_list = tuple(param_list)
+        return _retry_locked_database(
+            lambda: super(DjangoSQLiteCursorWrapper, self).executemany(converted_query, param_list),
+            converted_query,
+            f"{len(param_list)} parameter sets",
+        )
+
+
+class DatabaseWrapper(DjangoSQLiteDatabaseWrapper):
+    def create_cursor(self, name=None):
+        return self.connection.cursor(factory=SQLiteCursorWrapper)
+
+    def _commit(self):
+        return _retry_locked_database(lambda: super(DatabaseWrapper, self)._commit(), "COMMIT")
+
+    def _rollback(self):
+        return _retry_locked_database(lambda: super(DatabaseWrapper, self)._rollback(), "ROLLBACK")

archivebox/core/tag_utils.py

@@ -104,14 +104,17 @@ def get_matching_tags(
     return queryset
 
 
-def add_snapshot_counts(tags: list[Tag]) -> None:
+def add_snapshot_counts(tags: list[Tag], snapshot_queryset: QuerySet[Snapshot] | None = None) -> None:
     tag_ids = [tag.pk for tag in tags]
     if not tag_ids:
         return
 
+    queryset = SnapshotTag.objects.filter(tag_id__in=tag_ids)
+    if snapshot_queryset is not None:
+        queryset = queryset.filter(snapshot_id__in=snapshot_queryset.values("id"))
     counts = {
         row["tag_id"]: row["num_snapshots"]
-        for row in SnapshotTag.objects.filter(tag_id__in=tag_ids).values("tag_id").annotate(num_snapshots=Count("snapshot_id"))
+        for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id"))
     }
     for tag in tags:
         tag.num_snapshots = counts.get(tag.pk, 0)

archivebox/core/templatetags/core_tags.py

@@ -278,6 +278,14 @@ def file_size(num_bytes: int | float) -> str:
     return "{:3.1f} {}".format(num_bytes, "TB")
 
 
+@register.filter
+def intcomma(value: int | str | None) -> str:
+    try:
+        return f"{int(value or 0):,}"
+    except (TypeError, ValueError):
+        return str(value or "")
+
+
 def result_list(context, cl):
     """
     Monkey patched result

archivebox/core/urls.py

@@ -24,6 +24,7 @@ from archivebox.core.views import (
     AddView,
     WebAddView,
     HealthCheckView,
+    live_progress_screencast_frame_view,
     live_progress_view,
 )
 
@@ -69,6 +70,11 @@ urlpatterns = [
     path("accounts/login/", RedirectView.as_view(url="/admin/login/")),
     path("accounts/logout/", RedirectView.as_view(url="/admin/logout/")),
     path("accounts/", include("django.contrib.auth.urls")),
+    re_path(
+        r"^admin/live-progress/screencast/(?P<snapshot_id>[0-9a-fA-F-]{8,36})\.jpg$",
+        archivebox_admin.admin_view(live_progress_screencast_frame_view),
+        name="live_progress_screencast_frame",
+    ),
     path("admin/live-progress/", archivebox_admin.admin_view(live_progress_view), name="live_progress"),
     path("admin/", archivebox_admin.urls),
     path("api/", include("archivebox.api.urls"), name="api"),

archivebox/core/views.py

@@ -12,7 +12,7 @@ from pathlib import Path
 from urllib.parse import quote, urlparse
 
 from django.shortcuts import render, redirect
-from django.http import JsonResponse, HttpRequest, HttpResponse, Http404, HttpResponseForbidden, QueryDict
+from django.http import FileResponse, JsonResponse, HttpRequest, HttpResponse, Http404, HttpResponseForbidden, QueryDict
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.views import View
@@ -22,6 +22,7 @@ from django.db.models import CharField, Count, Q, Prefetch, Sum
 from django.db.models.functions import Cast
 from django.contrib import messages
 from django.contrib.auth.mixins import UserPassesTestMixin
+from django.core.signing import BadSignature, SignatureExpired, TimestampSigner
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.gzip import gzip_page
 from django.utils.decorators import method_decorator
@@ -37,9 +38,25 @@ from archivebox.config.configset import BaseConfigSet
 from archivebox.misc.util import base_url, htmlencode, ts_to_date_str, urldecode, without_fragment
 from archivebox.misc.serve_static import serve_static_with_byterange_support
 from archivebox.misc.logging_util import printable_filesize
-from archivebox.search import get_search_mode, prioritize_metadata_matches, query_search_index
+from archivebox.search import (
+    get_search_backend_display_name,
+    get_search_mode,
+    get_search_mode_backend,
+    get_search_mode_base,
+    get_search_mode_options,
+    prioritize_metadata_matches,
+    query_search_index,
+)
 
 from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.core.permissions import (
+    PERMISSIONS_PUBLIC,
+    can_view_snapshot,
+    direct_snapshots_queryset,
+    filter_personas_by_permissions,
+    is_admin_user,
+    public_snapshots_queryset,
+)
 from archivebox.core.host_utils import (
     build_admin_url,
     build_snapshot_url,
@@ -62,6 +79,7 @@ from archivebox.hooks import (
 
 ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
 LIVE_PLUGIN_BASE_URL = "/admin/environment/plugins/"
+SCREENCAST_SIGNER = TimestampSigner(salt="archivebox.live-progress.screencast")
 
 
 def _get_request_config(request: HttpRequest, *, resolve_plugins: bool = False):
@@ -175,7 +193,7 @@ class SnapshotView(View):
             "USES_SUBDOMAIN_ROUTING",
             "ADMIN_BASE_URL",
             "ARCHIVE_BASE_URL",
-            "PUBLIC_SNAPSHOTS",
+            "PERMISSIONS",
             "SERVER_SECURITY_MODE",
         }
         scoped_config_keys = set((getattr(snapshot, "config", None) or {}).keys())
@@ -278,6 +296,12 @@ class SnapshotView(View):
             "size": printable_filesize(output_size) if output_size else "pending",
             "status": "archived" if is_archived else "not yet archived",
             "status_color": "success" if is_archived else "danger",
+            "snapshot_permissions": str(runtime_config.PERMISSIONS).strip().lower(),
+            "snapshot_permissions_icon": {
+                "public": "👥",
+                "unlisted": "🔗",
+                "private": "🔒",
+            }[str(runtime_config.PERMISSIONS).strip().lower()],
             "bookmarked_date": snapshot.bookmarked_date,
             "downloaded_datestr": snapshot.downloaded_datestr,
             "num_outputs": snapshot.num_outputs,
@@ -298,10 +322,6 @@ class SnapshotView(View):
         return render(template_name="core/snapshot.html", request=request, context=context)
 
     def get(self, request, path):
-        request_config = _get_request_config(request)
-        if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS:
-            return _admin_login_redirect_or_forbidden(request)
-
         snapshot = None
 
         try:
@@ -318,6 +338,8 @@ class SnapshotView(View):
             try:
                 try:
                     snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug))
+                    if not can_view_snapshot(request, snapshot):
+                        return _admin_login_redirect_or_forbidden(request)
                     canonical_base = snapshot.url_path
                     if canonical_base != snapshot.legacy_archive_path:
                         target_path = f"/{canonical_base}/{archivefile or 'index.html'}"
@@ -377,7 +399,7 @@ class SnapshotView(View):
                         snap.url,
                         snap.title_stripped[:64] or "",
                     )
-                    for snap in Snapshot.objects.filter(timestamp__startswith=slug)
+                    for snap in direct_snapshots_queryset(request, Snapshot.objects.filter(timestamp__startswith=slug))
                     .only("url", "timestamp", "title", "bookmarked_at")
                     .order_by("-bookmarked_at")
                 )
@@ -436,7 +458,7 @@ class SnapshotView(View):
         # slug is a URL
         try:
             try:
-                snapshot = SnapshotView.find_snapshots_for_url(path).get()
+                snapshot = direct_snapshots_queryset(request, SnapshotView.find_snapshots_for_url(path)).get()
             except Snapshot.DoesNotExist:
                 raise
         except Snapshot.DoesNotExist:
@@ -457,7 +479,7 @@ class SnapshotView(View):
                 status=404,
             )
         except Snapshot.MultipleObjectsReturned:
-            snapshots = SnapshotView.find_snapshots_for_url(path)
+            snapshots = direct_snapshots_queryset(request, SnapshotView.find_snapshots_for_url(path))
             snapshot_hrefs = mark_safe("<br/>").join(
                 format_html(
                     '{} <code style="font-size: 0.8em">{}</code> <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
@@ -501,11 +523,6 @@ class SnapshotPathView(View):
         path: str = "",
         url: str | None = None,
     ):
-        request_config = _get_request_config(request)
-
-        if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS:
-            return _admin_login_redirect_or_forbidden(request)
-
         if username == "system":
             return redirect(request.path.replace("/system/", "/web/", 1))
 
@@ -517,7 +534,7 @@ class SnapshotPathView(View):
             requested_url = domain
 
         snapshot = None
-        snapshots_qs = Snapshot.objects.select_related("crawl", "crawl__created_by")
+        snapshots_qs = direct_snapshots_queryset(request, Snapshot.objects.select_related("crawl", "crawl__created_by"))
         if snapshot_id:
             try:
                 snapshot = snapshots_qs.get(pk=snapshot_id)
@@ -672,7 +689,27 @@ def _snapshot_sort_key(match_path: str, cache: dict[str, float]) -> tuple[float,
     return (cache[snapshot_id], match_path)
 
 
-def _latest_response_match(domain: str, rel_path: str, *, data_root: Path) -> tuple[Path, Path] | None:
+def _snapshot_id_from_replay_path(path: Path) -> str | None:
+    parts = path.parts
+    try:
+        responses_idx = parts.index("responses")
+    except ValueError:
+        return None
+    return parts[responses_idx - 1] if responses_idx > 0 else None
+
+
+def _replay_path_visible(request: HttpRequest, path: Path) -> bool:
+    snapshot_id = _snapshot_id_from_replay_path(path)
+    if not snapshot_id:
+        return False
+    snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first()
+    if not snapshot or not can_view_snapshot(request, snapshot):
+        return False
+    request.archivebox_config = get_config(snapshot=snapshot, resolve_plugins=False)
+    return True
+
+
+def _latest_response_match(request: HttpRequest, domain: str, rel_path: str, *, data_root: Path) -> tuple[Path, Path] | None:
     if not domain or not rel_path:
         return None
     domain = domain.split(":", 1)[0].lower()
@@ -685,8 +722,10 @@ def _latest_response_match(domain: str, rel_path: str, *, data_root: Path) -> tu
         return None
 
     sort_cache: dict[str, float] = {}
-    best = max(matches, key=lambda match_path: _snapshot_sort_key(match_path, sort_cache))
-    best_path = Path(best)
+    best_paths = sorted(matches, key=lambda match_path: _snapshot_sort_key(match_path, sort_cache), reverse=True)
+    best_path = next((Path(match_path) for match_path in best_paths if _replay_path_visible(request, Path(match_path))), None)
+    if best_path is None:
+        return None
     parts = best_path.parts
     try:
         responses_idx = parts.index("responses")
@@ -697,7 +736,7 @@ def _latest_response_match(domain: str, rel_path: str, *, data_root: Path) -> tu
     return responses_root, rel_to_root
 
 
-def _latest_responses_root(domain: str, *, data_root: Path) -> Path | None:
+def _latest_responses_root(request: HttpRequest, domain: str, *, data_root: Path) -> Path | None:
     if not domain:
         return None
     domain = domain.split(":", 1)[0].lower()
@@ -708,16 +747,19 @@ def _latest_responses_root(domain: str, *, data_root: Path) -> Path | None:
         return None
 
     sort_cache: dict[str, float] = {}
-    best = max(matches, key=lambda match_path: _snapshot_sort_key(match_path, sort_cache))
-    return Path(best)
+    best_paths = sorted(matches, key=lambda match_path: _snapshot_sort_key(match_path, sort_cache), reverse=True)
+    return next((Path(match_path) for match_path in best_paths if _replay_path_visible(request, Path(match_path))), None)
 
 
-def _latest_snapshot_for_domain(domain: str) -> Snapshot | None:
+def _latest_snapshot_for_domain(request: HttpRequest, domain: str) -> Snapshot | None:
     if not domain:
         return None
 
     requested_domain = domain.split(":", 1)[0].lower()
-    snapshots = SnapshotView.find_snapshots_for_url(f"https://{requested_domain}").order_by("-bookmarked_at", "-created_at", "-timestamp")
+    snapshots = direct_snapshots_queryset(
+        request,
+        SnapshotView.find_snapshots_for_url(f"https://{requested_domain}"),
+    ).order_by("-bookmarked_at", "-created_at", "-timestamp")
     for snapshot in snapshots:
         if Snapshot.extract_domain_from_url(snapshot.url).lower() == requested_domain:
             return snapshot
@@ -774,7 +816,8 @@ def _serve_responses_path(request, responses_root: Path, rel_path: str, show_ind
 
 
 def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
-    request_config = _get_request_config(request)
+    request_config = get_config(snapshot=snapshot, resolve_plugins=False)
+    request.archivebox_config = request_config
     snapshot._runtime_config = request_config
     rel_path = path or ""
     is_directory_request = bool(path) and path.endswith("/")
@@ -823,13 +866,13 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
         raise Http404
 
     domain = domain.lower()
-    match = _latest_response_match(domain, rel_path, data_root=request_config.USERS_DIR)
+    match = _latest_response_match(request, domain, rel_path, data_root=request_config.USERS_DIR)
     if not match and "." not in Path(rel_path).name:
         index_path = f"{rel_path.rstrip('/')}/index.html"
-        match = _latest_response_match(domain, index_path, data_root=request_config.USERS_DIR)
+        match = _latest_response_match(request, domain, index_path, data_root=request_config.USERS_DIR)
     if not match and "." not in Path(rel_path).name:
         html_path = f"{rel_path}.html"
-        match = _latest_response_match(domain, html_path, data_root=request_config.USERS_DIR)
+        match = _latest_response_match(request, domain, html_path, data_root=request_config.USERS_DIR)
 
     show_indexes = bool(request.GET.get("files"))
     if match:
@@ -838,14 +881,14 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
         if response is not None:
             return response
 
-    responses_root = _latest_responses_root(domain, data_root=request_config.USERS_DIR)
+    responses_root = _latest_responses_root(request, domain, data_root=request_config.USERS_DIR)
     if responses_root:
         response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
         if response is not None:
             return response
 
     if requested_root_index and not show_indexes:
-        snapshot = _latest_snapshot_for_domain(domain)
+        snapshot = _latest_snapshot_for_domain(request, domain)
         if snapshot:
             return SnapshotView.render_live_index(request, snapshot)
 
@@ -861,12 +904,12 @@ class SnapshotHostView(View):
 
     def get(self, request, snapshot_id: str, path: str = ""):
         request_config = _get_request_config(request)
-        if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS:
-            return _admin_login_redirect_or_forbidden(request)
         snapshot = _find_snapshot_by_ref(snapshot_id)
 
         if not snapshot:
             raise Http404
+        if not can_view_snapshot(request, snapshot):
+            return _admin_login_redirect_or_forbidden(request)
 
         canonical_host = get_snapshot_host(str(snapshot.id), config=request_config)
         if not host_matches(request.get_host(), canonical_host):
@@ -882,13 +925,11 @@ class SnapshotReplayView(View):
     """Serve snapshot directory contents on a one-domain replay path."""
 
     def get(self, request, snapshot_id: str, path: str = ""):
-        request_config = _get_request_config(request)
-        if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS:
-            return _admin_login_redirect_or_forbidden(request)
-
         snapshot = _find_snapshot_by_ref(snapshot_id)
         if not snapshot:
             raise Http404
+        if not can_view_snapshot(request, snapshot):
+            return _admin_login_redirect_or_forbidden(request)
 
         return _serve_snapshot_replay(request, snapshot, path)
 
@@ -897,9 +938,6 @@ class OriginalDomainHostView(View):
     """Serve responses from the most recent snapshot when using <domain>.<listen_host>/<path>."""
 
     def get(self, request, domain: str, path: str = ""):
-        request_config = _get_request_config(request)
-        if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS:
-            return _admin_login_redirect_or_forbidden(request)
         return _serve_original_domain_replay(request, domain, path)
 
 
@@ -907,9 +945,6 @@ class OriginalDomainReplayView(View):
     """Serve original-domain replay content on a one-domain replay path."""
 
     def get(self, request, domain: str, path: str = ""):
-        request_config = _get_request_config(request)
-        if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS:
-            return _admin_login_redirect_or_forbidden(request)
         return _serve_original_domain_replay(request, domain, path)
 
 
@@ -928,6 +963,8 @@ class PublicIndexView(ListView):
         runtime_config = getattr(self, "runtime_config", None)
         if runtime_config is None:
             self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=True)
+        search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
+        search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config)
         context = {
             **super().get_context_data(**kwargs),
             "VERSION": VERSION,
@@ -935,21 +972,27 @@ class PublicIndexView(ListView):
             "COMMIT_HASH": runtime_config.COMMIT_HASH,
             "FOOTER_INFO": runtime_config.FOOTER_INFO,
             "WEB_BASE_URL": build_web_url(request=self.request, config=runtime_config),
-            "search_mode": get_search_mode(self.request.GET.get("search_mode")),
+            "search_mode": search_mode,
+            "search_mode_options": get_search_mode_options(config=runtime_config),
+            "search_backend_label": get_search_backend_display_name(search_mode_backend) if search_mode_backend else "",
         }
+        context["show_search_index_hint"] = bool(
+            self.request.GET.get("q")
+            and get_search_mode_base(search_mode, config=runtime_config) == "deep"
+            and search_mode_backend
+            and getattr(context.get("paginator"), "count", 0) == 0
+        )
         for snapshot in context.get("object_list") or ():
             snapshot._icons_compact = True
             snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
             results = getattr(snapshot, "_prefetched_objects_cache", {}).get("archiveresult_set")
             if results is not None:
-                snapshot.output_size_sum = sum(result.output_size or 0 for result in results)
                 snapshot.num_outputs_cached = len(results)
         return context
 
     def get_queryset(self, **kwargs):
         qs = (
-            super()
-            .get_queryset(**kwargs)
+            public_snapshots_queryset(super().get_queryset(**kwargs))
             .prefetch_related(
                 Prefetch("crawl", queryset=Crawl.objects.select_related("created_by")),
                 "tags",
@@ -970,24 +1013,30 @@ class PublicIndexView(ListView):
         if not query:
             return qs
 
-        search_mode = get_search_mode(self.request.GET.get("search_mode"))
+        search_mode = get_search_mode(self.request.GET.get("search_mode"), config=getattr(self, "runtime_config", None))
 
         metadata_qs = qs.filter(
             Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
         )
-        if search_mode == "meta":
+        search_mode_base = get_search_mode_base(search_mode, config=getattr(self, "runtime_config", None))
+        search_mode_backend = get_search_mode_backend(search_mode, config=getattr(self, "runtime_config", None))
+        if search_mode_base == "meta":
             qs = metadata_qs
         else:
             try:
-                qs = prioritize_metadata_matches(
-                    qs,
-                    metadata_qs,
-                    query_search_index(query, search_mode=search_mode),
-                    ordering=self.ordering,
-                )
+                backend_qs = query_search_index(query, search_mode=search_mode)
+                if search_mode_backend:
+                    qs = qs.filter(pk__in=backend_qs.values("pk"))
+                else:
+                    qs = prioritize_metadata_matches(
+                        qs,
+                        metadata_qs,
+                        backend_qs,
+                        ordering=self.ordering,
+                    )
             except Exception as err:
                 print(f"[!] Error while using search backend: {err.__class__.__name__} {err}")
-                qs = metadata_qs
+                qs = qs.none() if search_mode_backend else metadata_qs
 
         return qs.distinct()
 
@@ -1015,12 +1064,16 @@ class AddView(UserPassesTestMixin, FormView):
 
         return super().get_initial()
 
+    def get_form_kwargs(self):
+        kwargs = super().get_form_kwargs()
+        kwargs["request"] = self.request
+        return kwargs
+
     def test_func(self):
         return _get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
 
     def _can_override_crawl_config(self) -> bool:
-        user = self.request.user
-        return bool(user.is_authenticated and (getattr(user, "is_superuser", False) or getattr(user, "is_staff", False)))
+        return is_admin_user(self.request)
 
     def _get_custom_config_overrides(self, form: AddLinkForm) -> dict:
         custom_config = form.cleaned_data.get("config") or {}
@@ -1034,35 +1087,55 @@ class AddView(UserPassesTestMixin, FormView):
         return custom_config
 
     def get_context_data(self, **kwargs):
-        from archivebox.personas.models import Persona
-
+        context = super().get_context_data(**kwargs)
         request_config = _get_request_config(self.request, resolve_plugins=True)
         required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
-        plugin_configs = discover_plugin_configs()
+        can_override_crawl_config = self._can_override_crawl_config()
+        plugin_configs = discover_plugin_configs() if can_override_crawl_config else {}
         sensitive_keys = {
             str(config_key)
             for schema in plugin_configs.values()
             for config_key, prop_schema in (schema.get("properties") or {}).items()
             if isinstance(prop_schema, dict) and prop_schema.get("x-sensitive")
         }
+        public_persona_config_keys = {
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS",
+            "DELETE_AFTER",
+            "PERMISSIONS",
+            "TIMEOUT",
+        }
+        persona_queryset = context["form"].fields["persona"].queryset
+        if not can_override_crawl_config:
+            persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
         persona_config_map = {}
-        for persona in Persona.objects.order_by("name"):
-            raw_config = {str(key): value for key, value in (persona.config or {}).items() if str(key) not in sensitive_keys}
+        for persona in persona_queryset.order_by("name"):
             effective_config = get_config(persona=persona)
+            if can_override_crawl_config:
+                raw_config = {str(key): value for key, value in (persona.config or {}).items() if str(key) not in sensitive_keys}
+                effective_config_json = {str(key): value for key, value in effective_config.items() if str(key) not in sensitive_keys}
+                binary_urls = get_plugin_config_binary_urls(effective_config)
+            else:
+                raw_config = {}
+                effective_config_json = {key: effective_config.get(key) for key in public_persona_config_keys}
+                binary_urls = {}
             persona_config_map[persona.name] = {
                 "config": raw_config,
-                "effective_config": {str(key): value for key, value in effective_config.items() if str(key) not in sensitive_keys},
-                "binary_urls": get_plugin_config_binary_urls(effective_config),
+                "effective_config": effective_config_json,
+                "binary_urls": binary_urls,
+            }
+        plugin_dependency_map = {}
+        if can_override_crawl_config:
+            plugin_dependency_map = {
+                plugin_name: [
+                    str(required_plugin).strip()
+                    for required_plugin in (schema.get("required_plugins") or [])
+                    if str(required_plugin).strip()
+                ]
+                for plugin_name, schema in plugin_configs.items()
+                if isinstance(schema.get("required_plugins"), list) and schema.get("required_plugins")
             }
-        plugin_dependency_map = {
-            plugin_name: [
-                str(required_plugin).strip() for required_plugin in (schema.get("required_plugins") or []) if str(required_plugin).strip()
-            ]
-            for plugin_name, schema in plugin_configs.items()
-            if isinstance(schema.get("required_plugins"), list) and schema.get("required_plugins")
-        }
         return {
-            **super().get_context_data(**kwargs),
+            **context,
             "title": "Create Crawl",
             # We can't just call request.build_absolute_uri in the template, because it would include query parameters
             "absolute_add_path": self.request.build_absolute_uri(self.request.path),
@@ -1071,6 +1144,7 @@ class AddView(UserPassesTestMixin, FormView):
             "required_search_plugin": required_search_plugin,
             "plugin_dependency_map_json": json.dumps(plugin_dependency_map, sort_keys=True),
             "persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str),
+            "can_override_crawl_config": can_override_crawl_config,
             "stdout": "",
         }
 
@@ -1083,20 +1157,27 @@ class AddView(UserPassesTestMixin, FormView):
         depth = int(form.cleaned_data["depth"])
         max_urls = int(form.cleaned_data.get("max_urls") or 0)
         crawl_max_size = int(form.cleaned_data.get("crawl_max_size") or 0)
+        crawl_timeout = int(form.cleaned_data.get("crawl_timeout") or 0)
+        timeout = form.cleaned_data.get("timeout")
         snapshot_max_size = int(form.cleaned_data.get("snapshot_max_size") or 0)
         delete_after = str(form.cleaned_data.get("delete_after") or "0").strip() or "0"
         crawl_max_concurrent_snapshots = int(form.cleaned_data["crawl_max_concurrent_snapshots"])
-        plugins = ",".join(form.cleaned_data.get("plugins", []))
-        schedule = form.cleaned_data.get("schedule", "").strip()
+        permissions = str(form.cleaned_data.get("permissions") or "public").strip().lower()
+        can_override_crawl_config = self._can_override_crawl_config()
+        plugins = ",".join(form.cleaned_data.get("plugins", [])) if can_override_crawl_config else ""
+        schedule = form.cleaned_data.get("schedule", "").strip() if can_override_crawl_config else ""
         persona = form.cleaned_data.get("persona")
-        index_only = form.cleaned_data.get("index_only", False)
+        index_only = form.cleaned_data.get("index_only", False) if can_override_crawl_config else False
         notes = form.cleaned_data.get("notes", "")
         url_filters = form.cleaned_data.get("url_filters") or {}
         plugin_config = form.cleaned_data.get("plugin_config") or {}
         if not isinstance(plugin_config, dict):
             plugin_config = {}
+        if not can_override_crawl_config:
+            plugin_config = {}
         custom_config = self._get_custom_config_overrides(form)
         custom_config.pop("DEFAULT_PERSONA", None)
+        custom_config.pop("PERMISSIONS", None)
         if persona:
             persona.ensure_dirs()
 
@@ -1132,6 +1213,18 @@ class AddView(UserPassesTestMixin, FormView):
             config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
         if delete_after != str(effective_config.DELETE_AFTER):
             config["DELETE_AFTER"] = delete_after
+        if permissions != str(effective_config.PERMISSIONS):
+            config["PERMISSIONS"] = permissions
+        if max_urls:
+            config["CRAWL_MAX_URLS"] = max_urls
+        if crawl_max_size:
+            config["CRAWL_MAX_SIZE"] = crawl_max_size
+        if crawl_timeout:
+            config["CRAWL_TIMEOUT"] = crawl_timeout
+        if timeout is not None and int(timeout) != int(effective_config.TIMEOUT):
+            config["TIMEOUT"] = int(timeout)
+        if snapshot_max_size:
+            config["SNAPSHOT_MAX_SIZE"] = snapshot_max_size
 
         # Merge custom config overrides
         config.update(plugin_config)
@@ -1144,9 +1237,6 @@ class AddView(UserPassesTestMixin, FormView):
         crawl = Crawl.objects.create(
             urls=urls_content,
             max_depth=depth,
-            max_urls=max_urls,
-            crawl_max_size=crawl_max_size,
-            snapshot_max_size=snapshot_max_size,
             tags_str=tag,
             notes=notes,
             label=f"{created_by_name}@{HOSTNAME}{self.request.path} {timestamp}",
@@ -1175,11 +1265,6 @@ class AddView(UserPassesTestMixin, FormView):
 
         ensure_background_runner()
 
-        # 4. start the Orchestrator & wait until it completes
-        #    ... orchestrator will create the root Snapshot, which creates pending ArchiveResults, which gets run by the ArchiveResultActors ...
-        # from archivebox.crawls.actors import CrawlActor
-        # from archivebox.core.actors import SnapshotActor, ArchiveResultActor
-
         return crawl
 
     def form_valid(self, form):
@@ -1191,7 +1276,7 @@ class AddView(UserPassesTestMixin, FormView):
 
         # Build success message with schedule link if created
         schedule_msg = ""
-        if schedule:
+        if schedule and crawl.schedule_id:
             schedule_msg = f" and <a href='{crawl.schedule.admin_change_url}'>scheduled to repeat {schedule}</a>"
 
         messages.success(
@@ -1207,7 +1292,10 @@ class AddView(UserPassesTestMixin, FormView):
 
 class WebAddView(AddView):
     def _latest_snapshot_for_url(self, requested_url: str):
-        return SnapshotView.find_snapshots_for_url(requested_url).order_by("-bookmarked_at", "-created_at", "-timestamp").first()
+        return direct_snapshots_queryset(
+            self.request,
+            SnapshotView.find_snapshots_for_url(requested_url),
+        ).order_by("-bookmarked_at", "-created_at", "-timestamp").first()
 
     def _normalize_add_url(self, requested_url: str) -> str:
         if requested_url.startswith(("http://", "https://")):
@@ -1263,10 +1351,13 @@ class WebAddView(AddView):
                 "depth": defaults_form.fields["depth"].initial or "0",
                 "max_urls": defaults_form.fields["max_urls"].initial or 0,
                 "crawl_max_size": defaults_form.fields["crawl_max_size"].initial or "0",
+                "crawl_timeout": defaults_form.fields["crawl_timeout"].initial or 0,
+                "timeout": defaults_form.fields["timeout"].initial or 0,
                 "snapshot_max_size": defaults_form.fields["snapshot_max_size"].initial or "0",
                 "delete_after": defaults_form.fields["delete_after"].initial or "0",
                 "crawl_max_concurrent_snapshots": defaults_form.fields["crawl_max_concurrent_snapshots"].initial,
                 "persona": defaults_form.fields["persona"].initial or "Default",
+                "permissions": defaults_form.fields["permissions"].initial or "public",
                 "config": "{}",
             },
         )
@@ -1295,6 +1386,37 @@ class HealthCheckView(View):
         return HttpResponse("OK", content_type="text/plain", status=200)
 
 
+def live_progress_screencast_frame_view(request, snapshot_id: str):
+    """Serve cache-only Chrome screencast frames through the admin app."""
+    if not is_admin_user(request):
+        return HttpResponseForbidden("Permission denied")
+
+    token = request.GET.get("token", "")
+    try:
+        if SCREENCAST_SIGNER.unsign(token, max_age=60) != str(snapshot_id):
+            return HttpResponseForbidden("Permission denied")
+    except (BadSignature, SignatureExpired):
+        return HttpResponseForbidden("Permission denied")
+
+    snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first()
+    if not snapshot:
+        raise Http404
+
+    live_root = (CONSTANTS.CACHE_DIR / "chrome_screencast").resolve()
+    frame_path = live_root / str(snapshot.id) / "latest.jpg"
+    try:
+        resolved_frame_path = frame_path.resolve(strict=True)
+    except FileNotFoundError:
+        raise Http404 from None
+    if not resolved_frame_path.is_file() or live_root not in resolved_frame_path.parents:
+        raise Http404
+
+    response = FileResponse(resolved_frame_path.open("rb"), content_type="image/jpeg")
+    response["Cache-Control"] = "no-store, max-age=0"
+    response["X-Content-Type-Options"] = "nosniff"
+    return response
+
+
 @gzip_page
 def live_progress_view(request):
     """Simple JSON endpoint for live progress status - used by admin progress monitor."""
@@ -1456,18 +1578,20 @@ def live_progress_view(request):
             else None
         )
         runner_worker = None
-        try:
-            from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+        orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running)
+        if not orchestrator_proc_running:
+            try:
+                from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
 
-            supervisor = get_existing_supervisord_process()
-            runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
-        except Exception:
-            runner_worker = None
+                supervisor = get_existing_supervisord_process(quiet=True)
+                runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
+            except Exception:
+                runner_worker = None
 
         runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"))
         runner_worker_pid = runner_worker.get("pid") if runner_worker else None
-        orchestrator_running = orchestrator_proc is not None or runner_worker_running
-        orchestrator_pid = orchestrator_proc.pid if orchestrator_proc else runner_worker_pid
+        orchestrator_running = orchestrator_proc_running or runner_worker_running
+        orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid
 
         def count_statuses(queryset, statuses) -> dict[str, int]:
             counts = {status: 0 for status in statuses}
@@ -1476,9 +1600,13 @@ def live_progress_view(request):
             return counts
 
         # Get model counts by status
-        crawl_status_counts = count_statuses(crawl_scope, (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED))
+        crawl_status_counts = count_statuses(
+            crawl_scope,
+            (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.PAUSED),
+        )
         crawls_pending = crawl_status_counts.get(Crawl.StatusChoices.QUEUED, 0)
         crawls_started = crawl_status_counts.get(Crawl.StatusChoices.STARTED, 0)
+        crawls_paused = crawl_status_counts.get(Crawl.StatusChoices.PAUSED, 0)
 
         # Get recent crawls (last 24 hours)
         from datetime import timedelta
@@ -1487,23 +1615,31 @@ def live_progress_view(request):
         recently_cancelled_after = now - timedelta(minutes=10)
         crawls_recent = crawl_scope.filter(created_at__gte=one_day_ago).count()
 
-        snapshot_status_counts = count_statuses(snapshot_scope, (Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED))
+        snapshot_status_counts = count_statuses(
+            snapshot_scope,
+            (Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED),
+        )
         snapshots_pending = snapshot_status_counts.get(Snapshot.StatusChoices.QUEUED, 0)
         snapshots_started = snapshot_status_counts.get(Snapshot.StatusChoices.STARTED, 0)
+        snapshots_paused = snapshot_status_counts.get(Snapshot.StatusChoices.PAUSED, 0)
 
         archiveresult_status_counts = count_statuses(
             archiveresult_scope,
             (
                 ArchiveResult.StatusChoices.QUEUED,
                 ArchiveResult.StatusChoices.STARTED,
+                ArchiveResult.StatusChoices.PAUSED,
             ),
         )
         archiveresults_pending = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
         archiveresults_started = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
+        archiveresults_paused = archiveresult_status_counts.get(ArchiveResult.StatusChoices.PAUSED, 0)
         archiveresults_succeeded = 0
         archiveresults_failed = 0
 
         # Build hierarchical active crawls with nested snapshots and archive results
+        max_progress_crawls = 3
+        max_progress_snapshots = 50
 
         active_crawl_fields = (
             "id",
@@ -1513,9 +1649,6 @@ def live_progress_view(request):
             "urls",
             "config",
             "max_depth",
-            "max_urls",
-            "crawl_max_size",
-            "snapshot_max_size",
             "tags_str",
             "persona_id",
             "status",
@@ -1525,18 +1658,23 @@ def live_progress_view(request):
             "created_by__username",
         )
         active_crawl_candidates = []
-        for status in (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED):
+        for status in (Crawl.StatusChoices.STARTED, Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.PAUSED, Crawl.StatusChoices.SEALED):
             status_qs = crawl_scope.filter(status=status)
             if status == Crawl.StatusChoices.SEALED:
                 status_qs = status_qs.filter(modified_at__gte=recently_cancelled_after)
             active_crawl_candidates.extend(
-                status_qs.values(*active_crawl_fields).order_by("-modified_at"),
+                status_qs.values(*active_crawl_fields).order_by("-modified_at")[:max_progress_crawls],
             )
+        crawl_status_priority = {
+            Crawl.StatusChoices.STARTED: 0,
+            Crawl.StatusChoices.QUEUED: 1,
+            Crawl.StatusChoices.PAUSED: 2,
+            Crawl.StatusChoices.SEALED: 3,
+        }
         active_crawls_list = sorted(
             {str(crawl["id"]): crawl for crawl in active_crawl_candidates}.values(),
-            key=lambda crawl: crawl["modified_at"],
-            reverse=True,
-        )
+            key=lambda crawl: (crawl_status_priority.get(crawl["status"], 9), -(crawl["modified_at"].timestamp() if crawl["modified_at"] else 0)),
+        )[:max_progress_crawls]
         for crawl in active_crawls_list:
             crawl["id"] = str(crawl["id"])
             if crawl["persona_id"]:
@@ -1558,6 +1696,10 @@ def live_progress_view(request):
                 persona_details_by_id[str(persona.id)] = persona_details
                 persona_details_by_name[persona.name] = persona_details
         active_crawl_ids = [crawl["id"] for crawl in active_crawls_list]
+        active_crawl_objects = {
+            str(crawl.id): crawl
+            for crawl in Crawl.objects.filter(id__in=active_crawl_ids).select_related("created_by")
+        }
         snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids}
         cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
         crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
@@ -1588,7 +1730,6 @@ def live_progress_view(request):
         process_records_by_crawl: dict[str, list[tuple[dict[str, object], object | None]]] = {}
         process_records_by_snapshot: dict[str, list[tuple[dict[str, object], object | None]]] = {}
         seen_process_records: set[str] = set()
-        active_snapshot_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED}
         recently_cancelled_snapshots_q = Q(
             status=Snapshot.StatusChoices.SEALED,
             downloaded_at__isnull=True,
@@ -1596,21 +1737,7 @@ def live_progress_view(request):
         )
         crawls_by_id = {str(crawl["id"]): crawl for crawl in active_crawls_list}
         snapshots = list(
-            active_snapshot_scope.filter(status=Snapshot.StatusChoices.QUEUED)
-            .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
-            .values(
-                "id_str",
-                "url",
-                "crawl_id_str",
-                "title",
-                "status",
-            )
-            .order_by("crawl_id", "modified_at"),
-        )
-        snapshots.extend(
-            active_snapshot_scope.filter(
-                Q(status__in=active_snapshot_statuses - {Snapshot.StatusChoices.QUEUED}) | recently_cancelled_snapshots_q,
-            )
+            active_snapshot_scope.filter(status=Snapshot.StatusChoices.STARTED)
             .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
             .values(
                 "id_str",
@@ -1625,8 +1752,42 @@ def live_progress_view(request):
                 "fs_version",
                 "status",
             )
-            .order_by("crawl_id", "status", "modified_at"),
+            .order_by("crawl_id", "-modified_at")[:max_progress_snapshots],
         )
+        remaining_snapshot_slots = max_progress_snapshots - len(snapshots)
+        if remaining_snapshot_slots > 0:
+            snapshots.extend(
+                active_snapshot_scope.filter(status=Snapshot.StatusChoices.QUEUED)
+                .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
+                .values(
+                    "id_str",
+                    "url",
+                    "crawl_id_str",
+                    "title",
+                    "status",
+                )
+                .order_by("crawl_id", "modified_at")[:remaining_snapshot_slots],
+            )
+        remaining_snapshot_slots = max_progress_snapshots - len(snapshots)
+        if remaining_snapshot_slots > 0:
+            snapshots.extend(
+                active_snapshot_scope.filter(recently_cancelled_snapshots_q)
+                .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
+                .values(
+                    "id_str",
+                    "created_at",
+                    "modified_at",
+                    "url",
+                    "timestamp",
+                    "bookmarked_at",
+                    "crawl_id_str",
+                    "title",
+                    "downloaded_at",
+                    "fs_version",
+                    "status",
+                )
+                .order_by("crawl_id", "-modified_at")[:remaining_snapshot_slots],
+            )
 
         def dashed_uuid(value: str) -> str:
             value = str(value)
@@ -1762,7 +1923,7 @@ def live_progress_view(request):
                 if proc["status"] == Process.StatusChoices.RUNNING
                 else (
                     "skipped"
-                    if proc["exit_code"] == PROCESS_EXIT_SKIPPED
+                    if proc["exit_code"] == PROCESS_EXIT_SKIPPED or (phase == "binary" and proc["exit_code"] not in (None, 0))
                     else ("failed" if proc["exit_code"] not in (None, 0) else "succeeded")
                 )
             )
@@ -1839,6 +2000,8 @@ def live_progress_view(request):
                 snapshot_favicon_url = ""
                 snapshot_preview_url = ""
                 snapshot_preview_link = ""
+                snapshot_screencast_url = ""
+                snapshot_screencast_link = ""
                 snapshot_fallback_urls: list[str] = []
                 result_by_plugin = {result.plugin: result for result in snapshot_results}
                 title_result = result_by_plugin.get("title")
@@ -1859,6 +2022,17 @@ def live_progress_view(request):
                 elif snapshot_favicon_url:
                     snapshot_preview_url = snapshot_favicon_url
 
+                if snapshot["status"] == Snapshot.StatusChoices.STARTED:
+                    live_preview_path = CONSTANTS.CACHE_DIR / "chrome_screencast" / str(snapshot["id"]) / "latest.jpg"
+                    try:
+                        live_preview_stat = live_preview_path.stat()
+                    except OSError:
+                        live_preview_stat = None
+                    if live_preview_stat and live_preview_stat.st_size > 0:
+                        token = SCREENCAST_SIGNER.sign(str(snapshot["id"]))
+                        snapshot_screencast_url = f"/admin/live-progress/screencast/{snapshot['id']}.jpg?v={live_preview_stat.st_mtime_ns}&token={quote(token)}"
+                        snapshot_screencast_link = snapshot_view_url(snapshot)
+
                 def plugin_sort_key(ar):
                     status_order = {
                         ArchiveResult.StatusChoices.STARTED: 0,
@@ -1989,6 +2163,9 @@ def live_progress_view(request):
                     if snapshot_preview_url:
                         snapshot_payload["preview_url"] = snapshot_preview_url
                         snapshot_payload["preview_link"] = snapshot_preview_link
+                    if snapshot_screencast_url:
+                        snapshot_payload["screencast_url"] = snapshot_screencast_url
+                        snapshot_payload["screencast_link"] = snapshot_screencast_link
                     if snapshot_fallback_urls:
                         snapshot_payload["preview_fallbacks"] = snapshot_fallback_urls
                     if snapshot_process_pids.get(str(snapshot["id"])):
@@ -2005,17 +2182,25 @@ def live_progress_view(request):
             persona_details = persona_details or persona_details_by_name.get(persona_name)
             crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
             avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
+            effective_crawl_config = get_config(crawl=active_crawl_objects[crawl_id])
+            max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0)
+            crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0)
+            crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0)
+            snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0)
 
             # Check if retry_at is in the future (would prevent worker from claiming)
             retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False
-            seconds_until_retry = int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
+            is_paused = active_crawl_objects[crawl_id].is_paused
+            seconds_until_retry = 0 if is_paused else int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
             crawl_worker_state = (
                 "running"
                 if crawl_process_pids.get(crawl_id)
                 or any(isinstance(snapshot, dict) and snapshot.get("worker_pid") for snapshot in active_snapshots_for_crawl)
                 else "waiting"
             )
-            if crawl["status"] == Crawl.StatusChoices.SEALED and cancelled_snapshots:
+            if is_paused:
+                crawl_worker_state = "paused"
+            elif crawl["status"] == Crawl.StatusChoices.SEALED and cancelled_snapshots:
                 crawl_worker_state = "cancelled"
             elif (
                 crawl["status"] == Crawl.StatusChoices.STARTED
@@ -2029,19 +2214,20 @@ def live_progress_view(request):
                     "id": crawl_id,
                     "label": (next((line.strip() for line in (crawl["urls"] or "").splitlines() if line.strip()), "") or crawl_id)[:60],
                     "status": crawl["status"],
+                    "is_paused": is_paused,
                     "started": crawl["created_at"].isoformat() if crawl["created_at"] else None,
                     "progress": crawl_progress,
                     "created_by": crawl["created_by__username"],
                     "persona": persona_name,
                     "persona_admin_url": persona_details["admin_url"] if persona_details else None,
                     "max_depth": crawl["max_depth"],
-                    "max_urls": crawl["max_urls"],
-                    "max_crawl_size": crawl["crawl_max_size"],
-                    "max_snapshot_size": crawl["snapshot_max_size"],
-                    "max_crawl_size_display": printable_filesize(crawl["crawl_max_size"]) if crawl["crawl_max_size"] else "unlimited",
-                    "max_snapshot_size_display": printable_filesize(crawl["snapshot_max_size"])
-                    if crawl["snapshot_max_size"]
-                    else "unlimited",
+                    "max_urls": max_urls,
+                    "max_crawl_size": crawl_max_size,
+                    "crawl_timeout": crawl_timeout,
+                    "max_snapshot_size": snapshot_max_size,
+                    "max_crawl_size_display": printable_filesize(crawl_max_size) if crawl_max_size else "unlimited",
+                    "crawl_timeout_display": f"{crawl_timeout}s" if crawl_timeout else "unlimited",
+                    "max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited",
                     "crawl_output_size": crawl_output_size,
                     "avg_snapshot_size": avg_snapshot_size,
                     "crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B",
@@ -2075,11 +2261,14 @@ def live_progress_view(request):
             "total_workers": total_workers,
             "crawls_pending": crawls_pending,
             "crawls_started": crawls_started,
+            "crawls_paused": crawls_paused,
             "crawls_recent": crawls_recent,
             "snapshots_pending": snapshots_pending,
             "snapshots_started": snapshots_started,
+            "snapshots_paused": snapshots_paused,
             "archiveresults_pending": archiveresults_pending,
             "archiveresults_started": archiveresults_started,
+            "archiveresults_paused": archiveresults_paused,
             "archiveresults_succeeded": archiveresults_succeeded,
             "archiveresults_failed": archiveresults_failed,
             "active_crawls": active_crawls,
@@ -2103,11 +2292,14 @@ def live_progress_view(request):
                 "total_workers": 0,
                 "crawls_pending": 0,
                 "crawls_started": 0,
+                "crawls_paused": 0,
                 "crawls_recent": 0,
                 "snapshots_pending": 0,
                 "snapshots_started": 0,
+                "snapshots_paused": 0,
                 "archiveresults_pending": 0,
                 "archiveresults_started": 0,
+                "archiveresults_paused": 0,
                 "archiveresults_succeeded": 0,
                 "archiveresults_failed": 0,
                 "active_crawls": [],

archivebox/core/widgets.py

@@ -402,7 +402,7 @@ class URLFiltersWidget(forms.Widget):
                 <div class="url-filters-column">
                     <div class="url-filter-label-row">
                         <label for="{widget_id}_allowlist" class="url-filter-label"><span class="url-filter-label-main">🟢 URL_ALLOWLIST</span></label>
-                        <span class="url-filter-label-note">Regex patterns or domains to exclude, one pattern per line.</span>
+                        <span class="url-filter-label-note">Regex patterns or domains to include, one pattern per line.</span>
                     </div>
                     <textarea id="{widget_id}_allowlist"
                               name="{name}_allowlist"
@@ -424,15 +424,20 @@ class URLFiltersWidget(forms.Widget):
                 <input type="checkbox" id="{widget_id}_same_domain_only" name="{name}_same_domain_only" value="1">
                 <span>Same domain only</span>
             </label>
+            <label class="url-filters-toggle" for="{widget_id}_subpaths_only">
+                <input type="checkbox" id="{widget_id}_subpaths_only" name="{name}_subpaths_only" value="1">
+                <span>Subpaths only</span>
+            </label>
             <div class="help-text">These values can be one regex pattern or domain per line. URL_DENYLIST takes precedence over URL_ALLOWLIST.</div>
             <script>
             (function() {{
                 var allowlistField = document.getElementById('{widget_id}_allowlist');
                 var denylistField = document.getElementById('{widget_id}_denylist');
                 var sameDomainOnly = document.getElementById('{widget_id}_same_domain_only');
+                var subpathsOnly = document.getElementById('{widget_id}_subpaths_only');
                 var sourceField = document.querySelector({json.dumps(self.source_selector)});
                 var lastAutoGeneratedAllowlist = '';
-                if (!allowlistField || !sameDomainOnly || !sourceField) {{
+                if (!allowlistField || !sameDomainOnly || !subpathsOnly || !sourceField) {{
                     return;
                 }}
 
@@ -463,6 +468,37 @@ class URLFiltersWidget(forms.Widget):
                     return '^https?://(' + domains.map(escapeRegex).join('|') + ')([:/]|$)';
                 }}
 
+                function buildSubpathRegex(paths) {{
+                    if (!paths.length) {{
+                        return '';
+                    }}
+                    return paths.map(function(item) {{
+                        if (item.path === '/') {{
+                            return '^https?://' + escapeRegex(item.host) + '([/?#]|$)';
+                        }}
+                        if (item.path.endsWith('/')) {{
+                            return '^https?://' + escapeRegex(item.host) + escapeRegex(item.path);
+                        }}
+                        return '^https?://' + escapeRegex(item.host) + escapeRegex(item.path) + '([/?#]|$)';
+                    }}).join('\\n');
+                }}
+
+                function getSubpathPrefix(parsed) {{
+                    var pathname = String(parsed.pathname || '/').replace(/\\/+/g, '/');
+                    if (!pathname || pathname === '/') {{
+                        return '/';
+                    }}
+                    if (pathname.endsWith('/')) {{
+                        return pathname;
+                    }}
+                    var lastSlash = pathname.lastIndexOf('/');
+                    var lastPart = pathname.slice(lastSlash + 1);
+                    if (lastPart.indexOf('.') !== -1) {{
+                        return pathname.slice(0, lastSlash + 1) || '/';
+                    }}
+                    return pathname;
+                }}
+
                 function getConfigEditorRows() {{
                     return document.getElementById('id_config_rows');
                 }}
@@ -534,7 +570,7 @@ class URLFiltersWidget(forms.Widget):
                 }}
 
                 function syncAllowlistFromUrls() {{
-                    if (!sameDomainOnly.checked) {{
+                    if (!sameDomainOnly.checked && !subpathsOnly.checked) {{
                         if (allowlistField.value.trim() === lastAutoGeneratedAllowlist) {{
                             allowlistField.value = '';
                             syncConfigEditor();
@@ -545,6 +581,7 @@ class URLFiltersWidget(forms.Widget):
 
                     var seen = Object.create(null);
                     var domains = [];
+                    var paths = [];
                     sourceField.value.split(/\\n+/).forEach(function(line) {{
                         var url = extractUrl(line);
                         if (!url) {{
@@ -554,20 +591,33 @@ class URLFiltersWidget(forms.Widget):
                             var parsed = new URL(url);
                             var domain = String(parsed.hostname || '').toLowerCase();
                             if (!domain || seen[domain]) {{
-                                return;
+                                domain = '';
+                            }}
+                            if (domain) {{
+                                seen[domain] = true;
+                                domains.push(domain);
+                            }}
+                            if (subpathsOnly.checked) {{
+                                var pathname = getSubpathPrefix(parsed);
+                                var hostAndPort = String(parsed.host || parsed.hostname || '').toLowerCase();
+                                var pathKey = hostAndPort + pathname;
+                                if (!hostAndPort || seen[pathKey]) {{
+                                    return;
+                                }}
+                                seen[pathKey] = true;
+                                paths.push({{ host: hostAndPort, path: pathname }});
                             }}
-                            seen[domain] = true;
-                            domains.push(domain);
                         }} catch (error) {{
                             return;
                         }}
                     }});
-                    lastAutoGeneratedAllowlist = buildHostRegex(domains);
+                    lastAutoGeneratedAllowlist = subpathsOnly.checked ? buildSubpathRegex(paths) : buildHostRegex(domains);
                     allowlistField.value = lastAutoGeneratedAllowlist;
                     syncConfigEditor();
                 }}
 
                 sameDomainOnly.addEventListener('change', syncAllowlistFromUrls);
+                subpathsOnly.addEventListener('change', syncAllowlistFromUrls);
                 sourceField.addEventListener('input', syncAllowlistFromUrls);
                 sourceField.addEventListener('change', syncAllowlistFromUrls);
                 allowlistField.addEventListener('input', syncConfigEditor);
@@ -592,6 +642,7 @@ class URLFiltersWidget(forms.Widget):
             "allowlist": data.get(f"{name}_allowlist", ""),
             "denylist": data.get(f"{name}_denylist", ""),
             "same_domain_only": data.get(f"{name}_same_domain_only") in ("1", "on", "true"),
+            "subpaths_only": data.get(f"{name}_subpaths_only") in ("1", "on", "true"),
         }
 
 

archivebox/crawls/admin.py

@@ -1,14 +1,19 @@
 __package__ = "archivebox.crawls"
 
+from copy import copy
+from urllib.parse import urlencode
+
 from django import forms
+from django.core.paginator import Paginator
 from django.http import JsonResponse, HttpRequest, HttpResponseNotAllowed
 from django.shortcuts import get_object_or_404, redirect
+from django.template.loader import render_to_string
 from django.urls import path, reverse
 from django.utils.html import escape, format_html, format_html_join
 from django.utils import timezone
 from django.utils.safestring import mark_safe
 from django.contrib import admin, messages
-from django.db.models import Count, IntegerField, OuterRef, Prefetch, Q, Subquery, Value
+from django.db.models import Case, CharField, Count, IntegerField, OuterRef, Prefetch, Q, Subquery, Value, When
 from django.db.models.functions import Coalesce
 
 
@@ -16,9 +21,12 @@ from django_object_actions import action
 
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
+from archivebox.config.common import get_config
 from archivebox.core.models import Snapshot
-from archivebox.core.widgets import TagEditorWidget
+from archivebox.core.permissions import PERMISSIONS_PRIVATE, PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED
+from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.crawls.models import Crawl, CrawlSchedule
+from archivebox.personas.models import Persona
 
 
 class MaxDepthListFilter(admin.SimpleListFilter):
@@ -35,24 +43,118 @@ class MaxDepthListFilter(admin.SimpleListFilter):
         return queryset
 
 
-def render_snapshots_list(snapshots_qs, limit=20, crawl=None):
+def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50, prefix="snapshots"):
     """Render a nice inline list view of snapshots with status, title, URL, and progress."""
 
-    snapshots = snapshots_qs.order_by("-created_at")[:limit].annotate(
+    query_param = f"{prefix}_q"
+    status_param = f"{prefix}_status"
+    page_param = f"{prefix}_page"
+    query = (request.GET.get(query_param, "") if request is not None else "").strip()
+    status_filter = (request.GET.get(status_param, "") if request is not None else "").strip()
+    valid_statuses = {choice[0] for choice in Snapshot.StatusChoices.choices}
+
+    filtered_qs = snapshots_qs
+    if query:
+        id_query = query.replace("-", "")
+        filtered_qs = filtered_qs.filter(Q(id__icontains=id_query) | Q(url__icontains=query) | Q(title__icontains=query))
+    if status_filter in valid_statuses:
+        filtered_qs = filtered_qs.filter(status=status_filter)
+
+    global_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower()
+    persona_ids_by_permissions = {
+        PERMISSIONS_PUBLIC: [],
+        PERMISSIONS_UNLISTED: [],
+        PERMISSIONS_PRIVATE: [],
+    }
+    for persona in Persona.objects.only("id", "permissions"):
+        persona_ids_by_permissions[persona.permissions or global_permissions].append(str(persona.id))
+
+    snapshots_qs = filtered_qs.order_by("-created_at").annotate(
         total_results=Count("archiveresult"),
         succeeded_results=Count("archiveresult", filter=Q(archiveresult__status="succeeded")),
         failed_results=Count("archiveresult", filter=Q(archiveresult__status="failed")),
         started_results=Count("archiveresult", filter=Q(archiveresult__status="started")),
         skipped_results=Count("archiveresult", filter=Q(archiveresult__status="skipped")),
+        snapshot_permissions=Case(
+            When(permissions=PERMISSIONS_PUBLIC, then=Value(PERMISSIONS_PUBLIC)),
+            When(permissions=PERMISSIONS_UNLISTED, then=Value(PERMISSIONS_UNLISTED)),
+            When(permissions=PERMISSIONS_PRIVATE, then=Value(PERMISSIONS_PRIVATE)),
+            When(crawl__permissions=PERMISSIONS_PUBLIC, then=Value(PERMISSIONS_PUBLIC)),
+            When(crawl__permissions=PERMISSIONS_UNLISTED, then=Value(PERMISSIONS_UNLISTED)),
+            When(crawl__permissions=PERMISSIONS_PRIVATE, then=Value(PERMISSIONS_PRIVATE)),
+            When(crawl__persona_id__in=persona_ids_by_permissions[PERMISSIONS_PUBLIC], then=Value(PERMISSIONS_PUBLIC)),
+            When(crawl__persona_id__in=persona_ids_by_permissions[PERMISSIONS_UNLISTED], then=Value(PERMISSIONS_UNLISTED)),
+            When(crawl__persona_id__in=persona_ids_by_permissions[PERMISSIONS_PRIVATE], then=Value(PERMISSIONS_PRIVATE)),
+            default=Value(global_permissions),
+            output_field=CharField(),
+        ),
     )
 
+    page_number = request.GET.get(page_param, 1) if request is not None else 1
+    paginator = Paginator(snapshots_qs, page_size)
+    page_obj = paginator.get_page(page_number)
+    snapshots = page_obj.object_list
+    total_count = paginator.count
+
+    def querystring(**updates):
+        if request is None:
+            return "#"
+        params = request.GET.copy()
+        for key, value in updates.items():
+            if value in (None, ""):
+                params.pop(key, None)
+            else:
+                params[key] = str(value)
+        return f"?{params.urlencode()}" if params else "?"
+
+    preserved_inputs = ""
+    if request is not None:
+        managed_params = {query_param, status_param, page_param}
+        preserved_inputs = "".join(
+            f'<input type="hidden" name="{escape(key)}" value="{escape(value)}">'
+            for key, values in request.GET.lists()
+            if key not in managed_params
+            for value in values
+        )
+
+    status_options = "".join(
+        f'<option value="{escape(value)}"{" selected" if status_filter == value else ""}>{escape(label)}</option>'
+        for value, label in Snapshot.StatusChoices.choices
+    )
+
+    controls = f"""
+        <div class="crawl-snapshots-toolbar" style="display: flex; gap: 10px; align-items: center; justify-content: space-between; flex-wrap: wrap; padding: 10px 12px; background: #f8fafc; border-bottom: 1px solid #e2e8f0;">
+            <form method="get" style="display: flex; gap: 8px; align-items: center; flex: 1 1 540px; margin: 0;">
+                {preserved_inputs}
+                <input type="search" name="{query_param}" value="{escape(query)}" placeholder="Filter snapshots by title, URL, or ID"
+                       style="min-width: 260px; flex: 1 1 360px; padding: 7px 10px; border: 1px solid #cbd5e1; border-radius: 6px;">
+                <select name="{status_param}" style="max-width: 170px; padding: 7px 10px; border: 1px solid #cbd5e1; border-radius: 6px;">
+                    <option value="">All statuses</option>
+                    {status_options}
+                </select>
+                <input type="hidden" name="{page_param}" value="1">
+                <button type="submit" class="button" style="padding: 7px 12px;">Filter</button>
+                {f'<a href="{querystring(**{query_param: None, status_param: None, page_param: None})}" style="font-size: 12px; color: #64748b;">Clear</a>' if query or status_filter else ""}
+            </form>
+            <div style="font-size: 12px; color: #64748b; white-space: nowrap;">
+                {page_obj.start_index() if total_count else 0}-{page_obj.end_index() if total_count else 0} of {total_count}
+            </div>
+        </div>
+    """
+
     if not snapshots:
-        return mark_safe('<div style="color: #666; font-style: italic; padding: 8px 0;">No Snapshots yet...</div>')
+        return mark_safe(f"""
+            <div data-crawl-snapshots-list style="border: 1px solid #ddd; border-radius: 6px; overflow: hidden; max-width: 100%;">
+                {controls}
+                <div style="color: #666; font-style: italic; padding: 12px;">No Snapshots found.</div>
+            </div>
+        """)
 
     # Status colors matching Django admin and progress monitor
     status_colors = {
         "queued": ("#6c757d", "#f8f9fa"),  # gray
         "started": ("#856404", "#fff3cd"),  # amber
+        "paused": ("#1d4ed8", "#dbeafe"),  # blue
         "sealed": ("#155724", "#d4edda"),  # green
         "failed": ("#721c24", "#f8d7da"),  # red
     }
@@ -61,6 +163,17 @@ def render_snapshots_list(snapshots_qs, limit=20, crawl=None):
     for snapshot in snapshots:
         status = snapshot.status or "queued"
         color, bg = status_colors.get(status, ("#6c757d", "#f8f9fa"))
+        permissions = snapshot.snapshot_permissions
+        permission_icon = {
+            PERMISSIONS_PUBLIC: "👁",
+            PERMISSIONS_UNLISTED: "🔗",
+            PERMISSIONS_PRIVATE: "🔒",
+        }[permissions]
+        permission_fg, permission_bg = {
+            PERMISSIONS_PUBLIC: ("#047857", "#d1fae5"),
+            PERMISSIONS_UNLISTED: ("#1d4ed8", "#dbeafe"),
+            PERMISSIONS_PRIVATE: ("#991b1b", "#fee2e2"),
+        }[permissions]
 
         # Calculate progress
         total = snapshot.total_results
@@ -123,6 +236,9 @@ def render_snapshots_list(snapshots_qs, limit=20, crawl=None):
                                  font-size: 11px; font-weight: 500; text-transform: uppercase;
                                  color: {color}; background: {bg};">{status}</span>
                 </td>
+                <td style="padding: 6px 8px; white-space: nowrap; text-align: center;">
+                    <span title="{permissions}" style="display:inline-flex; align-items:center; justify-content:center; width:22px; height:22px; border-radius:999px; font-size:12px; color:{permission_fg}; background:{permission_bg};">{permission_icon}</span>
+                </td>
                 <td style="padding: 6px 8px; white-space: nowrap;">
                     <a href="/{snapshot.archive_path}/" style="text-decoration: none;">
                         <img src="/{snapshot.archive_path}/favicon.ico"
@@ -158,23 +274,24 @@ def render_snapshots_list(snapshots_qs, limit=20, crawl=None):
             </tr>
         ''')
 
-    total_count = snapshots_qs.count()
-    footer = ""
-    if total_count > limit:
-        footer = f"""
-            <tr>
-                <td colspan="6" style="padding: 8px; text-align: center; color: #666; font-size: 12px; background: #f8f9fa;">
-                    Showing {limit} of {total_count} snapshots
-                </td>
-            </tr>
+    pagination = ""
+    if paginator.num_pages > 1:
+        pagination = f"""
+            <div style="display: flex; gap: 10px; align-items: center; justify-content: center; padding: 10px 12px; background: #f8fafc; border-top: 1px solid #e2e8f0; font-size: 12px;">
+                {"<a class='button' style='padding: 5px 10px;' href='" + querystring(**{page_param: page_obj.previous_page_number()}) + "'>Previous</a>" if page_obj.has_previous() else "<span style='color:#94a3b8;'>Previous</span>"}
+                <span style="color: #64748b;">Page {page_obj.number} of {paginator.num_pages}</span>
+                {"<a class='button' style='padding: 5px 10px;' href='" + querystring(**{page_param: page_obj.next_page_number()}) + "'>Next</a>" if page_obj.has_next() else "<span style='color:#94a3b8;'>Next</span>"}
+            </div>
         """
 
     return mark_safe(f"""
         <div data-crawl-snapshots-list style="border: 1px solid #ddd; border-radius: 6px; overflow: hidden; max-width: 100%;">
+            {controls}
             <table style="width: 100%; border-collapse: collapse; font-size: 13px;">
                 <thead>
                     <tr style="background: #f5f5f5; border-bottom: 2px solid #ddd;">
                         <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Status</th>
+                        <th style="padding: 8px 4px; text-align: center; font-weight: 600; color: #333; width: 22px;">🔒</th>
                         <th style="padding: 8px; text-align: left; font-weight: 600; color: #333; width: 24px;"></th>
                         <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">Title</th>
                         <th style="padding: 8px; text-align: left; font-weight: 600; color: #333;">URL</th>
@@ -187,9 +304,9 @@ def render_snapshots_list(snapshots_qs, limit=20, crawl=None):
                 </thead>
                 <tbody>
                     {"".join(rows)}
-                    {footer}
                 </tbody>
             </table>
+            {pagination}
         </div>
         {
         '''
@@ -260,117 +377,13 @@ def render_snapshots_list(snapshots_qs, limit=20, crawl=None):
     """)
 
 
-class URLFiltersWidget(forms.Widget):
-    def render(self, name, value, attrs=None, renderer=None):
-        value = value if isinstance(value, dict) else {}
-        widget_id = (attrs or {}).get("id", name)
-        allowlist = escape(value.get("allowlist", "") or "")
-        denylist = escape(value.get("denylist", "") or "")
-
-        return mark_safe(f'''
-            <div id="{widget_id}_container" style="min-width: 420px;">
-                <input type="hidden" name="{name}" value="">
-                <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px;">
-                    <div>
-                        <label for="{widget_id}_allowlist" style="display: block; font-weight: 600; margin-bottom: 4px;">Allowlist</label>
-                        <textarea id="{widget_id}_allowlist" name="{name}_allowlist" rows="3"
-                                  style="width: 100%; font-family: monospace; font-size: 12px;"
-                                  placeholder="example.com&#10;*.example.com">{allowlist}</textarea>
-                    </div>
-                    <div>
-                        <label for="{widget_id}_denylist" style="display: block; font-weight: 600; margin-bottom: 4px;">Denylist</label>
-                        <textarea id="{widget_id}_denylist" name="{name}_denylist" rows="3"
-                                  style="width: 100%; font-family: monospace; font-size: 12px;"
-                                  placeholder="static.example.com">{denylist}</textarea>
-                    </div>
-                </div>
-                <label style="display: inline-flex; align-items: center; gap: 6px; margin-top: 8px; font-weight: 500;">
-                    <input type="checkbox" id="{widget_id}_same_domain_only" name="{name}_same_domain_only" value="1">
-                    Same domain only
-                </label>
-                <p style="color: #666; font-size: 11px; margin: 6px 0 0 0;">
-                    Enter domains, wildcards, or regex patterns. Denylist takes precedence over allowlist.
-                </p>
-                <script>
-                    (function() {{
-                        if (window.__archiveboxUrlFilterEditors && window.__archiveboxUrlFilterEditors['{widget_id}']) {{
-                            return;
-                        }}
-                        window.__archiveboxUrlFilterEditors = window.__archiveboxUrlFilterEditors || {{}};
-                        window.__archiveboxUrlFilterEditors['{widget_id}'] = true;
-
-                        var urlsField = document.getElementById('id_urls');
-                        var allowlistField = document.getElementById('{widget_id}_allowlist');
-                        var sameDomainOnly = document.getElementById('{widget_id}_same_domain_only');
-
-                        function extractUrl(line) {{
-                            var trimmed = (line || '').trim();
-                            if (!trimmed || trimmed.charAt(0) === '#') {{
-                                return '';
-                            }}
-                            if (trimmed.charAt(0) === '{{') {{
-                                try {{
-                                    var record = JSON.parse(trimmed);
-                                    return String(record.url || '').trim();
-                                }} catch (error) {{
-                                    return '';
-                                }}
-                            }}
-                            return trimmed;
-                        }}
-
-                        function syncAllowlistFromUrls() {{
-                            if (!urlsField || !allowlistField || !sameDomainOnly || !sameDomainOnly.checked) {{
-                                return;
-                            }}
-                            var domains = [];
-                            var seen = Object.create(null);
-                            urlsField.value.split(/\\n+/).forEach(function(line) {{
-                                var url = extractUrl(line);
-                                if (!url) {{
-                                    return;
-                                }}
-                                try {{
-                                    var parsed = new URL(url);
-                                    var domain = (parsed.hostname || '').toLowerCase();
-                                    if (domain && !seen[domain]) {{
-                                        seen[domain] = true;
-                                        domains.push(domain);
-                                    }}
-                                }} catch (error) {{
-                                    return;
-                                }}
-                            }});
-                            allowlistField.value = domains.join('\\n');
-                        }}
-
-                        if (sameDomainOnly) {{
-                            sameDomainOnly.addEventListener('change', syncAllowlistFromUrls);
-                        }}
-                        if (urlsField) {{
-                            urlsField.addEventListener('input', syncAllowlistFromUrls);
-                            urlsField.addEventListener('change', syncAllowlistFromUrls);
-                        }}
-                    }})();
-                </script>
-            </div>
-        ''')
-
-    def value_from_datadict(self, data, files, name):
-        return {
-            "allowlist": data.get(f"{name}_allowlist", ""),
-            "denylist": data.get(f"{name}_denylist", ""),
-            "same_domain_only": data.get(f"{name}_same_domain_only") in ("1", "on", "true"),
-        }
-
-
 class URLFiltersField(forms.Field):
-    widget = URLFiltersWidget
+    widget = URLFiltersWidget(source_selector="#id_urls")
 
     def to_python(self, value):
         if isinstance(value, dict):
             return value
-        return {"allowlist": "", "denylist": "", "same_domain_only": False}
+        return {"allowlist": "", "denylist": "", "same_domain_only": False, "subpaths_only": False}
 
 
 class CrawlAdminForm(forms.ModelForm):
@@ -416,6 +429,7 @@ class CrawlAdminForm(forms.ModelForm):
             "allowlist": config.get("URL_ALLOWLIST", ""),
             "denylist": config.get("URL_DENYLIST", ""),
             "same_domain_only": False,
+            "subpaths_only": False,
         }
 
     def clean_tags_editor(self):
@@ -439,16 +453,18 @@ class CrawlAdminForm(forms.ModelForm):
             "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))),
             "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))),
             "same_domain_only": bool(value.get("same_domain_only")),
+            "subpaths_only": bool(value.get("subpaths_only")),
         }
 
     def save(self, commit=True):
         instance = super().save(commit=False)
         instance.tags_str = self.cleaned_data.get("tags_editor", "")
-        url_filters = self.cleaned_data.get("url_filters") or {}
-        instance.set_url_filters(
-            url_filters.get("allowlist", ""),
-            url_filters.get("denylist", ""),
-        )
+        if f"{self.add_prefix('url_filters')}_allowlist" in self.data or f"{self.add_prefix('url_filters')}_denylist" in self.data:
+            url_filters = self.cleaned_data.get("url_filters") or {}
+            instance.set_url_filters(
+                url_filters.get("allowlist", ""),
+                url_filters.get("denylist", ""),
+            )
         if commit:
             instance.save()
             instance.apply_crawl_config_filters()
@@ -460,15 +476,16 @@ class CrawlAdminForm(forms.ModelForm):
 
 class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
     form = CrawlAdminForm
+    change_form_template = "admin/crawls/crawl/change_form.html"
     list_select_related = ()
     list_display = (
         "id",
         "created_at",
         "created_by",
         "max_depth",
-        "max_urls",
-        "crawl_max_size",
-        "snapshot_max_size",
+        "stop_reason_badge",
+        "pause_control",
+        "resume_control",
         "label",
         "notes",
         "urls_preview",
@@ -483,9 +500,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         "created_at",
         "created_by",
         "max_depth",
-        "max_urls",
-        "crawl_max_size",
-        "snapshot_max_size",
         "label",
         "notes",
         "schedule_str",
@@ -496,9 +510,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         "id",
         "created_by__username",
         "max_depth",
-        "max_urls",
-        "crawl_max_size",
-        "snapshot_max_size",
         "label",
         "notes",
         "schedule_id",
@@ -506,56 +517,33 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         "urls",
     )
 
-    readonly_fields = ("created_at", "modified_at", "snapshots")
+    readonly_fields = ("created_at", "modified_at", "stop_reason_display")
 
     fieldsets = (
         (
             "URLs",
             {
-                "fields": ("urls",),
+                "fields": ("urls", "url_filters"),
                 "classes": ("card", "wide"),
             },
         ),
         (
-            "Info",
+            "Overview",
             {
-                "fields": ("label", "notes", "tags_editor"),
-                "classes": ("card",),
+                "fields": (
+                    ("label", "status", "retry_at", "schedule", "created_by", "created_at", "modified_at"),
+                    ("max_depth",),
+                    ("stop_reason_display",),
+                    ("notes", "tags_editor"),
+                ),
+                "classes": ("card", "wide", "crawl-admin-overview"),
             },
         ),
         (
-            "Settings",
+            "Config",
             {
-                "fields": (("max_depth", "max_urls", "crawl_max_size", "snapshot_max_size"), "url_filters", "config"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Status",
-            {
-                "fields": ("status", "retry_at"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Relations",
-            {
-                "fields": ("schedule", "created_by"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Snapshots",
-            {
-                "fields": ("snapshots",),
-                "classes": ("card", "wide"),
+                "fields": ("config",),
+                "classes": ("card", "wide", "crawl-admin-config"),
             },
         ),
     )
@@ -563,36 +551,26 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         (
             "URLs",
             {
-                "fields": ("urls",),
+                "fields": ("urls", "url_filters"),
                 "classes": ("card", "wide"),
             },
         ),
         (
-            "Info",
+            "Overview",
             {
-                "fields": ("label", "notes", "tags_editor"),
-                "classes": ("card",),
+                "fields": (
+                    ("label", "status", "retry_at", "schedule", "created_by"),
+                    ("max_depth",),
+                    ("notes", "tags_editor"),
+                ),
+                "classes": ("card", "wide", "crawl-admin-overview"),
             },
         ),
         (
-            "Settings",
+            "Config",
             {
-                "fields": (("max_depth", "max_urls", "crawl_max_size", "snapshot_max_size"), "url_filters", "config"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Status",
-            {
-                "fields": ("status", "retry_at"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Relations",
-            {
-                "fields": ("schedule", "created_by"),
-                "classes": ("card",),
+                "fields": ("config",),
+                "classes": ("card", "wide", "crawl-admin-config"),
             },
         ),
     )
@@ -600,9 +578,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
     list_filter = (MaxDepthListFilter, "schedule", "created_by", "status", "retry_at")
     ordering = ["-created_at", "-retry_at"]
     list_per_page = 50
-    actions = ["delete_selected_batched"]
+    actions = ["pause_selected_crawls", "resume_selected_crawls", "delete_selected_batched"]
     change_actions = ["recrawl"]
 
+    class Media:
+        css = {"all": ("admin/crawls/crawl_change.css",)}
+        js = ("admin/crawls/crawl_admin.js",)
+
     def get_queryset(self, request):
         """Keep joins page-local while computing per-row snapshot counts in the page query."""
         snapshot_count = (
@@ -623,6 +605,20 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
             )
         )
 
+    def change_view(self, request, object_id, form_url="", extra_context=None):
+        self.request = request
+        crawl = self.get_object(request, object_id)
+        extra_context = {
+            **(extra_context or {}),
+            "crawl_stop_reason": crawl.limit_stop_reason() if crawl else "",
+            "crawl_snapshots_changelist": self.snapshots_changelist(crawl) if crawl else "",
+        }
+        return super().change_view(request, object_id, form_url, extra_context)
+
+    def add_view(self, request, form_url="", extra_context=None):
+        self.request = request
+        return super().add_view(request, form_url, extra_context)
+
     def get_fieldsets(self, request, obj=None):
         return self.fieldsets if obj else self.add_fieldsets
 
@@ -658,6 +654,29 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
         messages.success(request, f"Successfully deleted {total} crawls ({deleted_count} total objects including related records).")
 
+    @admin.action(description="Pause selected crawls")
+    def pause_selected_crawls(self, request, queryset):
+        paused = 0
+        for crawl in queryset.exclude(status=Crawl.StatusChoices.SEALED).iterator(chunk_size=100):
+            paused += int(crawl.pause())
+        if paused:
+            messages.success(request, f"Paused {paused} crawl(s). The runner will stop scheduling new work on the next sweep.")
+        else:
+            messages.warning(request, "No active crawls were selected to pause.")
+
+    @admin.action(description="Resume selected crawls")
+    def resume_selected_crawls(self, request, queryset):
+        resumed = 0
+        for crawl in queryset.iterator(chunk_size=100):
+            if crawl.status == Crawl.StatusChoices.SEALED:
+                crawl.status = Crawl.StatusChoices.PAUSED
+                crawl.save(update_fields=["status", "modified_at"])
+            resumed += int(crawl.resume())
+        if resumed:
+            messages.success(request, f"Resumed {resumed} crawl(s). The runner will pick them up on the next sweep.")
+        else:
+            messages.warning(request, "No paused or sealed crawls were selected to resume.")
+
     @action(label="Recrawl", description="Create a new crawl with the same settings")
     def recrawl(self, request, obj):
         """Duplicate this crawl as a new crawl with the same URLs and settings."""
@@ -670,9 +689,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         new_crawl = Crawl.objects.create(
             urls=obj.urls,
             max_depth=obj.max_depth,
-            max_urls=obj.max_urls,
-            crawl_max_size=obj.crawl_max_size,
-            snapshot_max_size=obj.snapshot_max_size,
             tags_str=obj.tags_str,
             config=obj.config,
             schedule=obj.schedule,
@@ -687,6 +703,39 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
         return redirect("admin:crawls_crawl_change", new_crawl.id)
 
+    @admin.display(description="Stop Reason")
+    def stop_reason_display(self, obj):
+        reason = obj.limit_stop_reason() if obj else ""
+        if not reason:
+            return mark_safe('<span class="crawl-stop-reason crawl-stop-reason--empty">None</span>')
+        return format_html('<span class="crawl-stop-reason">{}</span>', reason)
+
+    @admin.display(description="Stop Reason")
+    def stop_reason_badge(self, obj):
+        return self.stop_reason_display(obj)
+
+    @admin.display(description="Resume")
+    def resume_control(self, obj):
+        if obj.status != Crawl.StatusChoices.SEALED and not obj.is_paused:
+            return mark_safe('<span class="crawl-resume-muted">-</span>')
+        reason = "paused" if obj.is_paused else (obj.limit_stop_reason() or "sealed")
+        return format_html(
+            '<button type="button" class="button crawl-resume-row" data-crawl-id="{}" title="Resume crawl. Stop reason: {}">Resume</button>',
+            obj.pk,
+            reason,
+        )
+
+    @admin.display(description="Pause")
+    def pause_control(self, obj):
+        if obj.status == Crawl.StatusChoices.SEALED:
+            return mark_safe('<span class="crawl-resume-muted">-</span>')
+        if obj.is_paused:
+            return mark_safe('<span class="crawl-resume-muted">Paused</span>')
+        return format_html(
+            '<button type="button" class="button crawl-pause-row" data-crawl-id="{}" title="Pause crawl">Pause</button>',
+            obj.pk,
+        )
+
     def num_snapshots(self, obj):
         # Use cached annotation from get_queryset to avoid N+1
         count = getattr(obj, "num_snapshots_cached", None)
@@ -694,8 +743,40 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
             count = obj.snapshot_set.count()
         return count
 
-    def snapshots(self, obj):
-        return render_snapshots_list(obj.snapshot_set.all(), crawl=obj)
+    @admin.display(description="Snapshots")
+    def snapshots_changelist(self, obj):
+        request = getattr(self, "request", None)
+        snapshot_changelist = reverse("admin:core_snapshot_changelist")
+        scoped_params = {"crawl_id": str(obj.pk)}
+        full_url = f"{snapshot_changelist}?{urlencode(scoped_params)}"
+        if request is None:
+            return format_html('<a class="button" href="{}">Open snapshots changelist</a>', full_url)
+
+        snapshot_admin = self.admin_site._registry[Snapshot]
+        changelist_request = copy(request)
+        changelist_request.method = "GET"
+        changelist_request.path = snapshot_changelist
+        changelist_request.GET = request.GET.copy()
+        changelist_request.GET.update(
+            {
+                **scoped_params,
+                "_embedded": "crawl",
+                "per_page": "200",
+            },
+        )
+        changelist_request.POST = request.POST.copy()
+        changelist_request.POST.clear()
+
+        response = snapshot_admin.changelist_view(
+            changelist_request,
+            extra_context={"embedded_changelist": True},
+        )
+        context = {
+            **response.context_data,
+            "snapshot_changelist_url": full_url,
+            "crawl": obj,
+        }
+        return mark_safe(render_to_string("admin/crawls/crawl/snapshots_changelist.html", context, request=request))
 
     def delete_snapshot_view(self, request: HttpRequest, object_id: str, snapshot_id: str):
         if request.method != "POST":
@@ -832,6 +913,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
     actions = ["delete_selected"]
 
     def get_queryset(self, request):
+        self.request = request
         return (
             super()
             .get_queryset(request)
@@ -842,6 +924,10 @@ class CrawlScheduleAdmin(BaseModelAdmin):
             )
         )
 
+    def change_view(self, request, object_id, form_url="", extra_context=None):
+        self.request = request
+        return super().change_view(request, object_id, form_url, extra_context)
+
     def get_fieldsets(self, request, obj=None):
         if obj is None:
             return tuple(fieldset for fieldset in self.fieldsets if fieldset[0] not in {"Crawls", "Snapshots"})
@@ -879,7 +965,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
 
     def snapshots(self, obj):
         crawl_ids = obj.crawl_set.values_list("pk", flat=True)
-        return render_snapshots_list(Snapshot.objects.filter(crawl_id__in=crawl_ids))
+        return render_snapshots_list(Snapshot.objects.filter(crawl_id__in=crawl_ids), request=getattr(self, "request", None), prefix="schedule_snapshots")
 
 
 def register_admin(admin_site):

archivebox/crawls/migrations/0011_move_crawl_limits_to_config.py

@@ -0,0 +1,38 @@
+from django.db import migrations
+
+
+def move_limit_fields_to_config(apps, schema_editor):
+    Crawl = apps.get_model("crawls", "Crawl")
+    rows = Crawl.objects.values("id", "config", "max_urls", "crawl_max_size", "snapshot_max_size").iterator(chunk_size=1000)
+    for row in rows:
+        config = dict(row["config"] or {})
+        if row["max_urls"]:
+            config["CRAWL_MAX_URLS"] = row["max_urls"]
+        if row["crawl_max_size"]:
+            config["CRAWL_MAX_SIZE"] = row["crawl_max_size"]
+        if row["snapshot_max_size"]:
+            config["SNAPSHOT_MAX_SIZE"] = row["snapshot_max_size"]
+        if config != (row["config"] or {}):
+            Crawl.objects.filter(id=row["id"]).update(config=config)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("crawls", "0010_crawl_delete_at"),
+    ]
+
+    operations = [
+        migrations.RunPython(move_limit_fields_to_config, migrations.RunPython.noop),
+        migrations.RemoveField(
+            model_name="crawl",
+            name="max_urls",
+        ),
+        migrations.RemoveField(
+            model_name="crawl",
+            name="crawl_max_size",
+        ),
+        migrations.RemoveField(
+            model_name="crawl",
+            name="snapshot_max_size",
+        ),
+    ]

archivebox/crawls/migrations/0012_drop_stale_crawl_timeout_column.py

@@ -0,0 +1,22 @@
+from django.db import migrations
+
+
+def drop_stale_crawl_timeout_column(apps, schema_editor):
+    table_name = "crawls_crawl"
+    column_name = "crawl_timeout"
+    connection = schema_editor.connection
+    with connection.cursor() as cursor:
+        columns = {column.name for column in connection.introspection.get_table_description(cursor, table_name)}
+        if column_name not in columns:
+            return
+        schema_editor.execute(f"ALTER TABLE {schema_editor.quote_name(table_name)} DROP COLUMN {schema_editor.quote_name(column_name)}")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("crawls", "0011_move_crawl_limits_to_config"),
+    ]
+
+    operations = [
+        migrations.RunPython(drop_stale_crawl_timeout_column, migrations.RunPython.noop),
+    ]

archivebox/crawls/migrations/0013_crawl_permissions.py

@@ -0,0 +1,19 @@
+# Generated by Django 6.0.5 on 2026-05-28 07:25
+
+import django.db.models.fields.json
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('crawls', '0012_drop_stale_crawl_timeout_column'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='crawl',
+            name='permissions',
+            field=models.GeneratedField(db_index=True, db_persist=True, expression=django.db.models.fields.json.KeyTextTransform('PERMISSIONS', 'config'), output_field=models.CharField(max_length=16, null=True)),
+        ),
+    ]

archivebox/crawls/migrations/0014_crawl_persona_fk.py

@@ -0,0 +1,41 @@
+# Generated by hand on 2026-05-28
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+def clear_stale_persona_ids(apps, _schema_editor):
+    Crawl = apps.get_model("crawls", "Crawl")
+    Persona = apps.get_model("personas", "Persona")
+    Crawl.objects.filter(persona_id__isnull=False).exclude(
+        persona_id__in=Persona.objects.values_list("id", flat=True),
+    ).update(persona_id=None)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("crawls", "0013_crawl_permissions"),
+        ("personas", "0003_persona_permissions"),
+    ]
+
+    operations = [
+        migrations.RunPython(clear_stale_persona_ids, migrations.RunPython.noop),
+        migrations.RenameField(
+            model_name="crawl",
+            old_name="persona_id",
+            new_name="persona",
+        ),
+        migrations.AlterField(
+            model_name="crawl",
+            name="persona",
+            field=models.ForeignKey(
+                blank=True,
+                db_column="persona_id",
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="crawls",
+                to="personas.persona",
+            ),
+        ),
+    ]

archivebox/crawls/migrations/0015_alter_crawl_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.5 on 2026-05-28 12:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('crawls', '0014_crawl_persona_fk'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='crawl',
+            name='status',
+            field=models.CharField(choices=[('queued', 'Queued'), ('started', 'Started'), ('paused', 'Paused'), ('sealed', 'Sealed')], db_index=True, default='queued', max_length=15),
+        ),
+    ]

archivebox/crawls/models.py

@@ -14,6 +14,7 @@ from urllib.parse import urlparse
 
 from django.db import IntegrityError, models, transaction
 from django.db.models import Q
+from django.db.models.fields.json import KT
 from django.core.exceptions import ValidationError
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.conf import settings
@@ -31,7 +32,7 @@ from archivebox.base_models.models import (
     ModelWithHealthStats,
     get_or_create_system_user_pk,
 )
-from archivebox.workers.models import ModelWithStateMachine, BaseStateMachine
+from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine
 from archivebox.crawls.schedule_utils import next_run_for_schedule, validate_schedule
 from archivebox.misc.util import validate_url_length
 
@@ -101,9 +102,6 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
             urls=template.urls,
             config=template.config or {},
             max_depth=template.max_depth,
-            max_urls=template.max_urls,
-            crawl_max_size=template.crawl_max_size,
-            snapshot_max_size=template.snapshot_max_size,
             tags_str=template.tags_str,
             persona_id=template.persona_id,
             label=label,
@@ -123,24 +121,23 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     urls = models.TextField(blank=False, null=False, help_text="Newline-separated list of URLs to crawl")
     config = models.JSONField(default=dict, null=True, blank=True)
+    permissions = models.GeneratedField(
+        expression=KT("config__PERMISSIONS"),
+        output_field=models.CharField(max_length=16, null=True),
+        db_persist=True,
+        db_index=True,
+        editable=False,
+    )
     max_depth = models.PositiveSmallIntegerField(default=0, validators=[MinValueValidator(0), MaxValueValidator(4)])
-    max_urls = models.IntegerField(
-        default=0,
-        validators=[MinValueValidator(0)],
-        help_text="Maximum number of URLs to snapshot for this crawl (0 = unlimited).",
-    )
-    crawl_max_size = models.BigIntegerField(
-        default=0,
-        validators=[MinValueValidator(0)],
-        help_text="Maximum total archived output size in bytes for this crawl (0 = unlimited).",
-    )
-    snapshot_max_size = models.BigIntegerField(
-        default=0,
-        validators=[MinValueValidator(0)],
-        help_text="Maximum archived output size in bytes for each snapshot (0 = unlimited).",
-    )
     tags_str = models.CharField(max_length=1024, blank=True, null=False, default="")
-    persona_id = models.UUIDField(null=True, blank=True)
+    persona = models.ForeignKey(
+        "personas.Persona",
+        db_column="persona_id",
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="crawls",
+    )
     label = models.CharField(max_length=64, blank=True, null=False, default="")
     notes = models.TextField(blank=True, null=False, default="")
     schedule = models.ForeignKey(CrawlSchedule, on_delete=models.SET_NULL, null=True, blank=True, editable=True)
@@ -189,6 +186,59 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         return get_config(crawl=self).DELETE_AFTER
 
+    def pause(self, *, save: bool = True) -> bool:
+        paused = super().pause(save=save)
+        if paused and self.pk:
+            from archivebox.core.models import ArchiveResult, Snapshot
+
+            active_snapshots = self.snapshot_set.filter(
+                status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
+            )
+            active_snapshots.update(
+                status=Snapshot.StatusChoices.PAUSED,
+                retry_at=RETRY_AT_MAX,
+                modified_at=timezone.now(),
+            )
+            ArchiveResult.pause_queryset(ArchiveResult.objects.filter(snapshot__crawl=self))
+        return paused
+
+    def resume(self, *, when=None, save: bool = True) -> bool:
+        resumed = super().resume(when=when, save=save)
+        if resumed and self.pk:
+            from archivebox.core.models import ArchiveResult, Snapshot
+
+            resume_at = when or timezone.now()
+            active_snapshots = self.snapshot_set.filter(
+                status=Snapshot.StatusChoices.PAUSED,
+            )
+            active_snapshots.update(
+                status=Snapshot.StatusChoices.QUEUED,
+                retry_at=resume_at,
+                modified_at=timezone.now(),
+            )
+            ArchiveResult.resume_queryset(ArchiveResult.objects.filter(snapshot__crawl=self), when=resume_at)
+        return resumed
+
+    def cancel(self) -> None:
+        from archivebox.core.models import Snapshot
+
+        cancelled_at = timezone.now()
+        self.status = self.StatusChoices.SEALED
+        self.retry_at = None
+        self.save(update_fields=["status", "retry_at", "modified_at"])
+        Snapshot.objects.filter(
+            crawl=self,
+            status__in=[
+                Snapshot.StatusChoices.QUEUED,
+                Snapshot.StatusChoices.STARTED,
+                Snapshot.StatusChoices.PAUSED,
+            ],
+        ).update(
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=None,
+            modified_at=cancelled_at,
+        )
+
     @classmethod
     def missing_delete_at_candidates(cls):
         from archivebox.personas.models import Persona
@@ -199,27 +249,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     def save(self, *args, **kwargs):
         update_fields = kwargs.get("update_fields")
         sync_tags = update_fields is None or "tags_str" in update_fields
+        old_crawl = type(self).objects.filter(pk=self.pk).first() if self.pk else None
         previous_tag_names = set()
-        if sync_tags and self.pk:
-            previous_tags_str = type(self).objects.filter(pk=self.pk).values_list("tags_str", flat=True).first()
-            previous_tag_names = set(self.parse_tag_names(previous_tags_str or ""))
+        if sync_tags and old_crawl is not None:
+            previous_tag_names = set(self.parse_tag_names(old_crawl.tags_str or ""))
 
         config = dict(self.config or {})
-        if self.max_urls > 0:
-            config["CRAWL_MAX_URLS"] = self.max_urls
-        else:
-            config.pop("CRAWL_MAX_URLS", None)
-
-        if self.crawl_max_size > 0:
-            config["CRAWL_MAX_SIZE"] = self.crawl_max_size
-        else:
-            config.pop("CRAWL_MAX_SIZE", None)
-
-        if self.snapshot_max_size > 0:
-            config["SNAPSHOT_MAX_SIZE"] = self.snapshot_max_size
-        else:
-            config.pop("SNAPSHOT_MAX_SIZE", None)
-
         if "CRAWL_MAX_CONCURRENT_SNAPSHOTS" in config:
             raw_concurrency = config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]
             if raw_concurrency in (None, ""):
@@ -342,9 +377,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             "urls": self.urls,
             "status": self.status,
             "max_depth": self.max_depth,
-            "max_urls": self.max_urls,
-            "crawl_max_size": self.crawl_max_size,
-            "snapshot_max_size": self.snapshot_max_size,
+            "config": self.config or {},
             "tags_str": self.tags_str,
             "label": self.label,
             "created_at": self.created_at.isoformat() if self.created_at else None,
@@ -386,9 +419,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         crawl = Crawl.objects.create(
             urls=urls,
             max_depth=record.get("max_depth", record.get("depth", 0)),
-            max_urls=record.get("max_urls", 0),
-            crawl_max_size=record.get("crawl_max_size", 0),
-            snapshot_max_size=record.get("snapshot_max_size", 0),
+            config=record.get("config") or {},
             tags_str=record.get("tags_str", record.get("tags", "")),
             label=record.get("label", ""),
             status=Crawl.StatusChoices.QUEUED,
@@ -551,7 +582,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         filtered_snapshots = [
             snapshot
             for snapshot in self.snapshot_set.filter(
-                status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
+                status__in=[
+                    Snapshot.StatusChoices.QUEUED,
+                    Snapshot.StatusChoices.STARTED,
+                    Snapshot.StatusChoices.PAUSED,
+                ],
             ).only("pk", "url", "status")
             if not self.url_passes_filters(snapshot.url, snapshot=snapshot, use_effective_config=False)
         ]
@@ -603,18 +638,24 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         return len(urls)
 
     def remaining_url_capacity(self) -> int | None:
-        if self.max_urls <= 0:
+        from archivebox.config.common import get_config
+
+        max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
+        if max_urls <= 0:
             return None
-        return max(self.max_urls - self.count_urls_for_limit(), 0)
+        return max(max_urls - self.count_urls_for_limit(), 0)
 
     def has_remaining_url_capacity(self) -> bool:
         remaining = self.remaining_url_capacity()
         return remaining is None or remaining > 0
 
     def remaining_snapshot_capacity(self) -> int | None:
-        if self.max_urls <= 0:
+        from archivebox.config.common import get_config
+
+        max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
+        if max_urls <= 0:
             return None
-        return max(self.max_urls - self.snapshot_set.count(), 0)
+        return max(max_urls - self.snapshot_set.count(), 0)
 
     def has_remaining_snapshot_capacity(self) -> bool:
         remaining = self.remaining_snapshot_capacity()
@@ -687,17 +728,27 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         if self.persona_id:
             persona = Persona.objects.filter(id=self.persona_id).first()
-            if persona is None:
-                raise Persona.DoesNotExist(f"Crawl {self.id} references missing Persona {self.persona_id}")
-            return persona
+            if persona is not None:
+                return persona
 
         default_persona_name = str((self.config or {}).get("DEFAULT_PERSONA") or "").strip()
         if default_persona_name:
             persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default")
+            persona.ensure_dirs()
             return persona
 
         return None
 
+    def limit_stop_reason(self) -> str:
+        from abx_dl.limits import CrawlLimitState
+        from archivebox.config.common import get_config
+
+        if not (self.output_dir / ".abx-dl" / "limits.json").exists():
+            return ""
+        config = get_config(crawl=self, include_machine=False)
+        config["CRAWL_DIR"] = str(self.output_dir)
+        return CrawlLimitState.from_config(config).get_stop_reason()
+
     def add_url(self, entry: dict) -> bool:
         """
         Add a URL to the crawl queue if not already present.
@@ -1132,17 +1183,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             The root Snapshot for this crawl, or None for system crawls that don't create snapshots
         """
         import time
-        from pathlib import Path
         from archivebox.hooks import run_hook, discover_hooks, process_hook_records, is_finite_background_hook
         from archivebox.config.common import get_config
         from archivebox.machine.models import Binary, Machine
 
-        # Debug logging to file (since stdout/stderr redirected to /dev/null in progress mode)
-        debug_log = Path("/tmp/archivebox_crawl_debug.log")
-        with open(debug_log, "a") as f:
-            f.write(f"\n=== Crawl.run() starting for {self.id} at {time.time()} ===\n")
-            f.flush()
-
         def get_runtime_config():
             config = get_config(crawl=self)
             if persona_runtime_overrides:
@@ -1177,9 +1221,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 self.urls.strip(),
             )
 
-            with open(debug_log, "a") as f:
-                f.write(f"Running hook: {hook.name}\n")
-                f.flush()
             hook_start = time.time()
             plugin_name = hook.parent.name
             output_dir = self.output_dir / plugin_name
@@ -1194,10 +1235,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 url=primary_url,
                 snapshot_id=str(self.id),
             )
-            with open(debug_log, "a") as f:
-                f.write(f"Hook {hook.name} completed with status={process.status}\n")
-                f.flush()
-
             hook_elapsed = time.time() - hook_start
             if hook_elapsed > 0.5:
                 print(f"[yellow]⏱️  Hook {hook.name} took {hook_elapsed:.2f}s[/yellow]")
@@ -1213,9 +1250,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             from archivebox.hooks import extract_records_from_process
 
             records = []
-            # Finite background hooks can exit before their stdout log is fully
-            # visible to our polling loop. Give successful hooks a brief chance
-            # to flush JSONL records before we move on to downstream hooks.
+            # Finite background hooks can exit before their completed Process
+            # metadata is visible. Give successful hooks a brief chance to
+            # flush JSONL stdout into the Process row before downstream hooks.
             for delay in (0.0, 0.05, 0.1, 0.25, 0.5):
                 if delay:
                     time.sleep(delay)
@@ -1284,14 +1321,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 for hook in provider_hooks:
                     resolved_binary_names.update(run_crawl_hook(hook))
 
-        # Discover and run on_Crawl hooks
-        with open(debug_log, "a") as f:
-            f.write("Discovering Crawl hooks...\n")
-            f.flush()
         hooks = discover_hooks("Crawl", config=get_runtime_config())
-        with open(debug_log, "a") as f:
-            f.write(f"Found {len(hooks)} hooks\n")
-            f.flush()
 
         for hook in hooks:
             hook_binary_names = run_crawl_hook(hook)
@@ -1309,20 +1339,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 leaked_count = leaked_snapshots.count()
                 leaked_snapshots.delete()
                 print(f"[yellow]⚠️  Removed {leaked_count} leaked snapshot(s) created during system crawl {system_task}[/yellow]")
-            with open(debug_log, "a") as f:
-                f.write(f"Skipping snapshot creation for system crawl: {system_task}\n")
-                f.write("=== Crawl.run() complete ===\n\n")
-                f.flush()
             return None
 
-        with open(debug_log, "a") as f:
-            f.write("Creating snapshots from URLs...\n")
-            f.flush()
-        created_snapshots = self.create_snapshots_from_urls()
-        with open(debug_log, "a") as f:
-            f.write(f"Created {len(created_snapshots)} snapshots\n")
-            f.write("=== Crawl.run() complete ===\n\n")
-            f.flush()
+        self.create_snapshots_from_urls()
 
         # Return first snapshot for this crawl (newly created or existing)
         # This ensures the crawl doesn't seal if snapshots exist, even if they weren't just created
@@ -1340,7 +1359,13 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
             return True
 
         # If snapshots exist, check if all are sealed
-        if snapshots.filter(status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED]).exists():
+        if snapshots.filter(
+            status__in=[
+                Snapshot.StatusChoices.QUEUED,
+                Snapshot.StatusChoices.STARTED,
+                Snapshot.StatusChoices.PAUSED,
+            ],
+        ).exists():
             return False
 
         return True
@@ -1426,13 +1451,16 @@ class CrawlMachine(BaseStateMachine):
     # States
     queued = State(value=Crawl.StatusChoices.QUEUED, initial=True)
     started = State(value=Crawl.StatusChoices.STARTED)
+    paused = State(value=Crawl.StatusChoices.PAUSED)
     sealed = State(value=Crawl.StatusChoices.SEALED, final=True)
 
     # Tick Event (polled by workers)
-    tick = queued.to.itself(unless="can_start") | queued.to(started, cond="can_start") | started.to(sealed, cond="is_finished")
+    tick = queued.to.itself(unless="can_start") | queued.to(started, cond="can_start") | started.to(sealed, cond="is_finished") | paused.to.itself()
 
     # Manual event (triggered by last Snapshot sealing)
     seal = started.to(sealed)
+    pause_requested = queued.to(paused) | started.to(paused)
+    resume_requested = paused.to(queued)
 
     def can_start(self) -> bool:
         if not self.crawl.urls:
@@ -1448,6 +1476,13 @@ class CrawlMachine(BaseStateMachine):
         """Check if all Snapshots for this crawl are finished."""
         return self.crawl.is_finished()
 
+    @queued.enter
+    def enter_queued(self):
+        self.crawl.update_and_requeue(
+            retry_at=timezone.now(),
+            status=Crawl.StatusChoices.QUEUED,
+        )
+
     @started.enter
     def enter_started(self):
         import sys
@@ -1482,6 +1517,13 @@ class CrawlMachine(BaseStateMachine):
             traceback.print_exc()
             raise
 
+    @paused.enter
+    def enter_paused(self):
+        self.crawl.update_and_requeue(
+            retry_at=RETRY_AT_MAX,
+            status=Crawl.StatusChoices.PAUSED,
+        )
+
     @sealed.enter
     def enter_sealed(self):
         # Clean up background hooks and run on_CrawlEnd hooks

archivebox/dead/archivebox_persona.py

@@ -0,0 +1,129 @@
+# ruff: noqa
+NETSCAPE_COOKIE_HEADER = [
+    "# Netscape HTTP Cookie File",
+    "# https://curl.se/docs/http-cookies.html",
+    "# This file was generated by ArchiveBox persona cookie extraction",
+    "#",
+    "# Format: domain\\tincludeSubdomains\\tpath\\tsecure\\texpiry\\tname\\tvalue",
+    "",
+]
+
+
+def _parse_netscape_cookies(path: Path) -> "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]":
+    cookies = OrderedDict()
+    if not path.exists():
+        return cookies
+
+    for line in path.read_text().splitlines():
+        if not line or line.startswith("#"):
+            continue
+        parts = line.split("\t")
+        if len(parts) < 7:
+            continue
+        domain, include_subdomains, cookie_path, secure, expiry, name, value = parts[:7]
+        key = (domain, cookie_path, name)
+        cookies[key] = (domain, include_subdomains, cookie_path, secure, expiry, name, value)
+    return cookies
+
+
+def _write_netscape_cookies(path: Path, cookies: "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]") -> None:
+    lines = list(NETSCAPE_COOKIE_HEADER)
+    for cookie in cookies.values():
+        lines.append("\t".join(cookie))
+    path.write_text("\n".join(lines) + "\n")
+
+
+def _merge_netscape_cookies(existing_file: Path, new_file: Path) -> None:
+    existing = _parse_netscape_cookies(existing_file)
+    new = _parse_netscape_cookies(new_file)
+    for key, cookie in new.items():
+        existing[key] = cookie
+    _write_netscape_cookies(existing_file, existing)
+
+
+def extract_cookies_via_cdp(
+    user_data_dir: Path,
+    output_file: Path,
+    profile_dir: str | None = None,
+    chrome_binary: str | None = None,
+) -> bool:
+    """
+    Launch Chrome with the given user data dir and extract cookies via CDP.
+
+    Returns True if successful, False otherwise.
+    """
+    from archivebox.config.common import get_config
+
+    # Find the cookie extraction script
+    chrome_plugin_dir = Path(__file__).parent.parent / "plugins" / "chrome"
+    extract_script = chrome_plugin_dir / "extract_cookies.js"
+
+    if not extract_script.exists():
+        rprint(f"[yellow]Cookie extraction script not found at {extract_script}[/yellow]", file=sys.stderr)
+        return False
+
+    # Get node modules dir
+    node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules"
+
+    # Set up environment
+    env = os.environ.copy()
+    env["NODE_MODULES_DIR"] = str(node_modules_dir)
+    env["CHROME_USER_DATA_DIR"] = str(user_data_dir)
+    env["CHROME_HEADLESS"] = "true"
+    if chrome_binary:
+        env["CHROME_BINARY"] = str(chrome_binary)
+    output_path = output_file
+    temp_output = None
+    temp_dir = None
+    if output_file.exists():
+        temp_dir = Path(tempfile.mkdtemp(prefix="ab_cookies_"))
+        temp_output = temp_dir / "cookies.txt"
+        output_path = temp_output
+    if profile_dir:
+        extra_arg = f"--profile-directory={profile_dir}"
+        existing_extra = env.get("CHROME_ARGS_EXTRA", "").strip()
+        args_list = []
+        if existing_extra:
+            if existing_extra.startswith("["):
+                try:
+                    parsed = json.loads(existing_extra)
+                    if isinstance(parsed, list):
+                        args_list.extend(str(x) for x in parsed)
+                except Exception:
+                    args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
+            else:
+                args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
+        args_list.append(extra_arg)
+        env["CHROME_ARGS_EXTRA"] = json.dumps(args_list)
+
+    env["COOKIES_OUTPUT_FILE"] = str(output_path)
+
+    try:
+        result = subprocess.run(
+            ["node", str(extract_script)],
+            env=env,
+            capture_output=True,
+            text=True,
+            timeout=60,
+        )
+
+        if result.returncode == 0:
+            if temp_output and temp_output.exists():
+                _merge_netscape_cookies(output_file, temp_output)
+            return True
+        else:
+            rprint(f"[yellow]Cookie extraction failed: {result.stderr}[/yellow]", file=sys.stderr)
+            return False
+
+    except subprocess.TimeoutExpired:
+        rprint("[yellow]Cookie extraction timed out[/yellow]", file=sys.stderr)
+        return False
+    except FileNotFoundError:
+        rprint("[yellow]Node.js not found. Cannot extract cookies.[/yellow]", file=sys.stderr)
+        return False
+    except Exception as e:
+        rprint(f"[yellow]Cookie extraction error: {e}[/yellow]", file=sys.stderr)
+        return False
+    finally:
+        if temp_dir and temp_dir.exists():
+            shutil.rmtree(temp_dir, ignore_errors=True)

archivebox/dead/auth.py

@@ -0,0 +1,26 @@
+# ruff: noqa
+class UsernameAndPasswordAuth(HttpBasicAuth):
+    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
+
+    def authenticate(self, request: HttpRequest, username: str, password: str) -> User | None:
+        return _require_superuser(
+            auth_using_password(username=username, password=password, request=request),
+            request,
+            self.__class__.__name__,
+        )
+
+
+class DjangoSessionAuth:
+    """Allow authenticating with existing Django session cookies (same-origin only)."""
+
+    def __call__(self, request: HttpRequest) -> User | None:
+        return self.authenticate(request)
+
+    def authenticate(self, request: HttpRequest, **kwargs) -> User | None:
+        user = getattr(request, "user", None)
+        if isinstance(user, User) and user.is_authenticated:
+            setattr(request, "_api_auth_method", self.__class__.__name__)
+            if not user.is_superuser:
+                raise HttpError(403, "Valid session but User does not have permission (make sure user.is_superuser=True)")
+            return user
+        return None

archivebox/dead/config_tags.py

@@ -1,3 +1,4 @@
+# ruff: noqa
 """Template tags for accessing config values in templates."""
 
 from typing import Any

archivebox/dead/db.py

@@ -0,0 +1,29 @@
+# ruff: noqa
+def list_migrations(out_dir: Path = DATA_DIR) -> list[tuple[bool, str]]:
+    """List all Django migrations and their status"""
+    from django.core.management import call_command
+
+    def showmigrations() -> StringIO:
+        out = StringIO()
+        call_command("showmigrations", list=True, stdout=out)
+        out.seek(0)
+        return out
+
+    out = retry_sqlite_locks(showmigrations, label="checking migrations")
+
+    migrations = []
+    for line in out.readlines():
+        if line.strip() and "]" in line:
+            status_str, name_str = line.strip().split("]", 1)
+            is_applied = "X" in status_str
+            migration_name = name_str.strip()
+            migrations.append((is_applied, migration_name))
+
+    return migrations
+
+
+def get_admins(out_dir: Path = DATA_DIR) -> list[Any]:
+    """Get list of superuser accounts"""
+    from django.contrib.auth.models import User
+
+    return list(User.objects.filter(is_superuser=True).exclude(username="system"))

archivebox/dead/debugging.py

@@ -1,3 +1,4 @@
+# ruff: noqa
 from functools import wraps
 from time import time
 

archivebox/dead/detect.py

@@ -0,0 +1,3 @@
+# ruff: noqa
+def get_host_immutable_info(host_info: dict[str, Any]) -> dict[str, Any]:
+    return {key: value for key, value in host_info.items() if key in ["guid", "net_mac", "os_family", "cpu_arch"]}

archivebox/dead/django.py

@@ -0,0 +1,7 @@
+# ruff: noqa
+def setup_django_minimal():
+    # sys.path.append(str(CONSTANTS.PACKAGE_DIR))
+    # os.environ.setdefault('ARCHIVEBOX_DATA_DIR', str(CONSTANTS.DATA_DIR))
+    # os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
+    # django.setup()
+    raise Exception("dont use this anymore")

archivebox/dead/folders.py

@@ -1,3 +1,4 @@
+# ruff: noqa
 """
 Folder utilities for ArchiveBox.
 

archivebox/dead/hooks.py

@@ -0,0 +1,35 @@
+# ruff: noqa
+class HookResult(TypedDict, total=False):
+    """Raw result from run_hook()."""
+
+    returncode: int
+    stdout: str
+    stderr: str
+    output_json: dict[str, Any] | None
+    output_files: list[dict[str, Any]]
+    duration_ms: int
+    hook: str
+    plugin: str  # Plugin name (directory name, e.g., 'wget', 'screenshot')
+    hook_name: str  # Full hook filename (e.g., 'on_Snapshot__50_wget.py')
+    # New fields for JSONL parsing
+    records: list[dict[str, Any]]  # Parsed JSONL records with 'type' field
+
+
+def get_config_defaults_from_plugins() -> dict[str, Any]:
+    """
+    Get default values for all plugin config options.
+
+    Returns:
+        Dict mapping config keys to their default values.
+        e.g., {'SAVE_WGET': True, 'WGET_TIMEOUT': 60, ...}
+    """
+    plugin_configs = discover_plugin_configs()
+    defaults = {}
+
+    for plugin_name, schema in plugin_configs.items():
+        properties = schema.get("properties", {})
+        for key, prop_schema in properties.items():
+            if "default" in prop_schema:
+                defaults[key] = prop_schema["default"]
+
+    return defaults

archivebox/dead/host_utils.py

@@ -0,0 +1,11 @@
+# ruff: noqa
+def get_archive_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    return get_web_base_url(request=request, config=config, **config_kwargs)
+
+
+def build_api_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    return _build_url(get_api_base_url(request, config=config, **config_kwargs), path)
+
+
+def build_archive_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    return _build_url(get_archive_base_url(request, config=config, **config_kwargs), path)

archivebox/dead/jsonl.py

@@ -0,0 +1,12 @@
+# ruff: noqa
+def write_records(records: Iterator[dict[str, Any]], stream: TextIO | None = None) -> int:
+    """
+    Write multiple JSONL records to stdout (or provided stream).
+
+    Returns count of records written.
+    """
+    count = 0
+    for record in records:
+        write_record(record, stream)
+        count += 1
+    return count

archivebox/dead/legacy.py

@@ -1,3 +1,4 @@
+# ruff: noqa
 """
 Legacy archive import utilities.
 

archivebox/dead/live_ui.py

@@ -1,3 +1,4 @@
+# ruff: noqa
 from abx_dl.cli import LiveBusUI
 
 __all__ = ["LiveBusUI"]

archivebox/dead/process_plugin.py

@@ -1,3 +1,4 @@
+# ruff: noqa
 __package__ = "archivebox.ideas"
 
 import asyncio

archivebox/dead/shutdown_util.py

@@ -0,0 +1,101 @@
+# ruff: noqa
+def pid_is_running(pid: int) -> bool:
+    """Return True when the OS still has a process for pid.
+
+    This intentionally does not inspect ArchiveBox state. It is used only for
+    foreground parent processes and stale pid files; orchestrator ownership
+    still belongs to the database state machine and retry_at locks.
+    """
+
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    return True
+
+
+def read_pid_file(pid_file: Path) -> int | None:
+    try:
+        return int(pid_file.read_text().strip())
+    except (FileNotFoundError, ValueError):
+        return None
+
+
+def unlink_pid_file_if_owner(pid_file: Path, pid: int) -> None:
+    """Remove a pid file only if it still points at the expected process."""
+
+    try:
+        if pid_file.read_text().strip() == str(pid):
+            pid_file.unlink(missing_ok=True)
+    except FileNotFoundError:
+        pass
+
+
+def wait_for_pid_exit(pid: int, *, timeout: float, interval: float = 0.1) -> bool:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if not pid_is_running(pid):
+            return True
+        time.sleep(interval)
+    return not pid_is_running(pid)
+
+
+def stop_pidfile_owner(
+    pid_file: Path,
+    *,
+    current_pid: int,
+    description: str,
+    graceful_timeout: float,
+    log: Callable[[str], object],
+    owner_matches: Callable[[int], bool] | None = None,
+    on_stale_pid: Callable[[], object] | None = None,
+    on_forced_stop: Callable[[], object] | None = None,
+) -> int:
+    """Stop a previous foreground owner recorded in pid_file.
+
+    This is for command-parent takeover only. It deliberately does not claim
+    Crawl/Snapshot work; crashed work is resumed by the existing retry_at/state
+    machine path after the parent process is gone.
+    """
+
+    pid = read_pid_file(pid_file)
+    if pid is None or pid == current_pid:
+        return 0
+
+    if not pid_is_running(pid):
+        pid_file.unlink(missing_ok=True)
+        if on_stale_pid is not None:
+            on_stale_pid()
+        return 0
+    if owner_matches is not None and not owner_matches(pid):
+        # PIDs can be reused after an unclean exit. A stale pidfile must never
+        # let one ArchiveBox collection stop a process owned by another
+        # collection or another app entirely.
+        pid_file.unlink(missing_ok=True)
+        if on_stale_pid is not None:
+            on_stale_pid()
+        return 0
+
+    log(f"[yellow][*] Stopping existing {description} pid={pid}...[/yellow]")
+    try:
+        os.kill(pid, signal.SIGTERM)
+    except ProcessLookupError:
+        pid_file.unlink(missing_ok=True)
+        if on_stale_pid is not None:
+            on_stale_pid()
+        return 0
+
+    if wait_for_pid_exit(pid, timeout=graceful_timeout):
+        pid_file.unlink(missing_ok=True)
+        return 1
+
+    try:
+        os.kill(pid, signal.SIGKILL)
+    except ProcessLookupError:
+        pass
+    if on_forced_stop is not None:
+        on_forced_stop()
+    pid_file.unlink(missing_ok=True)
+    return 1

archivebox/dead/supervision_service.py

@@ -0,0 +1,41 @@
+# ruff: noqa
+def ensure_single_orchestrator(*, data_dir: str | Path, takeover: bool, reason: str = ""):
+    from archivebox.machine.models import Machine, Process
+    from archivebox.workers.supervisord_util import (
+        RUNNER_WORKER,
+        get_existing_supervisord_process,
+        get_or_create_supervisord_process,
+        start_worker,
+        stop_worker,
+    )
+
+    existing = healthy_orchestrator(data_dir=data_dir)
+    if existing and not takeover:
+        if reason:
+            pid = existing.get("pid") if isinstance(existing, dict) else existing.pid
+            print(f"[green][*] {reason}; existing orchestrator pid={pid} will process it.[/green]")
+        return existing
+
+    supervisor = get_existing_supervisord_process() or get_or_create_supervisord_process(daemonize=False)
+    if existing and takeover:
+        print("[yellow][*] Taking over existing ArchiveBox orchestrator...[/yellow]")
+        try:
+            stop_worker(supervisor, RUNNER_WORKER["name"])
+        except Exception:
+            pass
+        for proc in Process.objects.filter(
+            machine=Machine.current(),
+            process_type=Process.TypeChoices.ORCHESTRATOR,
+            status=Process.StatusChoices.RUNNING,
+            pwd=str(data_dir),
+        ).order_by("created_at"):
+            if proc.is_running:
+                proc.terminate(graceful_timeout=2.0)
+
+    return start_worker(supervisor, RUNNER_WORKER)
+
+
+def wait_until_replaced_or_signal(command, *, process_type: str, data_dir: str | Path, url: str | None = None, interval: float = 2.0) -> None:
+    while command_is_newest(command, process_type=process_type, data_dir=data_dir, url=url):
+        command.heartbeat()
+        time.sleep(interval)

archivebox/dead/supervisord_util.py

@@ -0,0 +1,90 @@
+# ruff: noqa
+def follow(file, sleep_sec=0.1) -> Iterator[str]:
+    """Yield each line from a file as they are written.
+    `sleep_sec` is the time to sleep after empty reads."""
+    line = ""
+    while True:
+        tmp = file.readline()
+        if tmp is not None and tmp != "":
+            line += tmp
+            if line.endswith("\n"):
+                yield line
+                line = ""
+        elif sleep_sec:
+            time.sleep(sleep_sec)
+
+
+def tail_worker_logs(log_path: str):
+    get_or_create_supervisord_process(daemonize=False)
+
+    from rich.live import Live
+    from rich.table import Table
+
+    table = Table()
+    table.add_column("TS")
+    table.add_column("URL")
+
+    try:
+        with Live(table, refresh_per_second=1) as live:  # update 4 times a second to feel fluid
+            with open(log_path) as f:
+                for line in follow(f):
+                    if "://" in line:
+                        live.console.print(f"Working on: {line.strip()}")
+                    # table.add_row("123124234", line.strip())
+    except (KeyboardInterrupt, BrokenPipeError, OSError):
+        STDERR.print("\n[🛑] Got Ctrl+C, stopping gracefully...")
+    except SystemExit:
+        pass
+
+
+def watch_worker(supervisor, daemon_name, interval=5):
+    """loop continuously and monitor worker's health"""
+    while True:
+        proc = get_worker(supervisor, daemon_name)
+        if not proc:
+            raise Exception("Worker disappeared while running! " + daemon_name)
+
+        if proc["statename"] == "STOPPED":
+            return proc
+
+        if proc["statename"] == "RUNNING":
+            time.sleep(1)
+            continue
+
+        if proc["statename"] in ("STARTING", "BACKOFF", "FATAL", "EXITED", "STOPPING"):
+            print(f"[🦸‍♂️] WARNING: Worker {daemon_name} {proc['statename']} {proc['description']}")
+            time.sleep(interval)
+            continue
+
+
+def start_cli_workers(watch=False):
+    from archivebox.config.common import get_config
+
+    supervisor = get_or_create_supervisord_process(daemonize=False)
+
+    sonic_worker = get_sonic_supervisord_worker_from_plugin(get_config())
+    workers = [(RUNNER_WORKER, False)]
+    if sonic_worker is not None:
+        workers.insert(0, (sonic_worker, False))
+
+    sync_supervisord_workers(supervisor, workers, prune=True)
+
+    if watch:
+        try:
+            # Block on supervisord process - it will handle signals and stop children
+            if _supervisord_proc:
+                _supervisord_proc.wait()
+            else:
+                # Fallback to watching worker if no proc reference
+                watch_worker(supervisor, RUNNER_WORKER["name"])
+        except (KeyboardInterrupt, BrokenPipeError, OSError):
+            STDERR.print("\n[🛑] Got Ctrl+C, stopping gracefully...")
+        except SystemExit:
+            pass
+        except BaseException as e:
+            STDERR.print(f"\n[🛑] Got {e.__class__.__name__} exception, stopping gracefully...")
+        finally:
+            # Ensure supervisord and all children are stopped
+            stop_existing_supervisord_process()
+            time.sleep(1.0)  # Give processes time to fully terminate
+    return [RUNNER_WORKER]

archivebox/dead/system.py

@@ -0,0 +1,78 @@
+# ruff: noqa
+@enforce_types
+def chmod_file(path: str, cwd: str = "", config=None, **config_kwargs) -> None:
+    """chmod -R <permissions> <cwd>/<path>"""
+
+    root = Path(cwd or os.getcwd()) / path
+    if not os.access(root, os.R_OK):
+        raise Exception(f"Failed to chmod: {path} does not exist (did the previous step fail?)")
+
+    if not root.is_dir():
+        # path is just a plain file
+        config = config or get_config(**config_kwargs)
+        os.chmod(root, int(config.OUTPUT_PERMISSIONS, base=8))
+    else:
+        config = config or get_config(**config_kwargs)
+        for subpath in Path(path).glob("**/*"):
+            if subpath.is_dir():
+                # directories need execute permissions to be able to list contents
+                os.chmod(subpath, int(config.DIR_OUTPUT_PERMISSIONS, base=8))
+            else:
+                os.chmod(subpath, int(config.OUTPUT_PERMISSIONS, base=8))
+
+
+@enforce_types
+def copy_and_overwrite(from_path: str | Path, to_path: str | Path):
+    """copy a given file or directory to a given path, overwriting the destination"""
+
+    assert os.access(from_path, os.R_OK)
+
+    if Path(from_path).is_dir():
+        shutil.rmtree(to_path, ignore_errors=True)
+        shutil.copytree(from_path, to_path)
+    else:
+        with open(from_path, "rb") as src:
+            contents = src.read()
+        atomic_write(to_path, contents)
+
+
+class suppress_output:
+    """
+    A context manager for doing a "deep suppression" of stdout and stderr in
+    Python, i.e. will suppress all print, even if the print originates in a
+    compiled C/Fortran sub-function.
+
+    This will not suppress raised exceptions, since exceptions are printed
+    to stderr just before a script exits, and after the context manager has
+    exited (at least, I think that is why it lets exceptions through).
+
+    with suppress_stdout_stderr():
+        rogue_function()
+    """
+
+    def __init__(self, stdout=True, stderr=True):
+        # Open a pair of null files
+        # Save the actual stdout (1) and stderr (2) file descriptors.
+        self.stdout, self.stderr = stdout, stderr
+        if stdout:
+            self.null_stdout = os.open(os.devnull, os.O_RDWR)
+            self.real_stdout = os.dup(1)
+        if stderr:
+            self.null_stderr = os.open(os.devnull, os.O_RDWR)
+            self.real_stderr = os.dup(2)
+
+    def __enter__(self):
+        # Assign the null pointers to stdout and stderr.
+        if self.stdout:
+            os.dup2(self.null_stdout, 1)
+        if self.stderr:
+            os.dup2(self.null_stderr, 2)
+
+    def __exit__(self, *_):
+        # Re-assign the real stdout/stderr back to (1) and (2)
+        if self.stdout:
+            os.dup2(self.real_stdout, 1)
+            os.close(self.null_stdout)
+        if self.stderr:
+            os.dup2(self.real_stderr, 2)
+            os.close(self.null_stderr)

archivebox/dead/util.py

@@ -0,0 +1,114 @@
+# ruff: noqa
+def short_ts(ts: Any) -> str | None:
+    parsed = parse_date(ts)
+    return None if parsed is None else str(parsed.timestamp()).split(".")[0]
+
+
+def ts_to_iso(ts: Any) -> str | None:
+    parsed = parse_date(ts)
+    return None if parsed is None else parsed.isoformat()
+
+
+def is_static_file(url: str):
+    # TODO: the proper way is with MIME type detection + ext, not only extension
+    return extension(url).lower() in CONSTANTS.STATICFILE_EXTENSIONS
+
+
+@enforce_types
+def str_between(string: str, start: str, end: str | None = None) -> str:
+    """(<abc>12345</def>, <abc>, </def>)  ->  12345"""
+
+    content = string.split(start, 1)[-1]
+    if end is not None:
+        content = content.rsplit(end, 1)[0]
+
+    return content
+
+
+@enforce_types
+def get_headers(url: str, timeout: int | None = None, config=None, **config_kwargs) -> str:
+    """Download the contents of a remote url and return the headers"""
+    # TODO: get rid of this and use an abx pluggy hook instead
+
+    from archivebox.config.common import get_config
+
+    config = config or get_config(**config_kwargs)
+    timeout = timeout or config.TIMEOUT
+
+    try:
+        response = requests.head(
+            url,
+            headers={"User-Agent": config.USER_AGENT},
+            verify=config.CHECK_SSL_VALIDITY,
+            timeout=timeout,
+            allow_redirects=True,
+        )
+        if response.status_code >= 400:
+            raise RequestException
+    except ReadTimeout:
+        raise
+    except RequestException:
+        response = requests.get(
+            url,
+            headers={"User-Agent": config.USER_AGENT},
+            verify=config.CHECK_SSL_VALIDITY,
+            timeout=timeout,
+            stream=True,
+        )
+
+    return pyjson.dumps(
+        {
+            "URL": url,
+            "Status-Code": response.status_code,
+            "Elapsed": response.elapsed.total_seconds() * 1000,
+            "Encoding": str(response.encoding),
+            "Apparent-Encoding": response.apparent_encoding,
+            **dict(response.headers),
+        },
+        indent=4,
+    )
+
+
+def chrome_cleanup(config=None, **config_kwargs):
+    """
+    Cleans up any state or runtime files that Chrome leaves behind when killed by
+    a timeout or other error. Handles:
+    - All persona chrome_profile directories (via Persona.cleanup_chrome_all())
+    - Explicit CHROME_USER_DATA_DIR from config
+    - Legacy Docker chromium path
+    """
+    import os
+    from pathlib import Path
+    from archivebox.config.permissions import IN_DOCKER
+
+    # Clean up all persona chrome directories using Persona class
+    try:
+        from archivebox.personas.models import Persona
+
+        # Clean up all personas
+        Persona.cleanup_chrome_all()
+
+        # Also clean up the active persona's explicit CHROME_USER_DATA_DIR if set
+        # (in case it's a custom path not under PERSONAS_DIR)
+        from archivebox.config.common import get_config
+
+        config = config or get_config(**config_kwargs)
+        chrome_user_data_dir = config.get("CHROME_USER_DATA_DIR")
+        if chrome_user_data_dir:
+            singleton_lock = Path(chrome_user_data_dir) / "SingletonLock"
+            if os.path.lexists(singleton_lock):
+                try:
+                    singleton_lock.unlink()
+                except OSError:
+                    pass
+    except Exception:
+        pass  # Persona/config not available during early startup
+
+    # Legacy Docker cleanup (for backwards compatibility)
+    if IN_DOCKER:
+        singleton_lock = "/home/archivebox/.config/chromium/SingletonLock"
+        if os.path.lexists(singleton_lock):
+            try:
+                os.remove(singleton_lock)
+            except OSError:
+                pass

archivebox/dead/views.py

@@ -1 +1,2 @@
+# ruff: noqa
 # Create your views here.

archivebox/hooks.py

@@ -159,22 +159,6 @@ def normalize_hook_event_name(event_name: str) -> str | None:
     return normalized
 
 
-class HookResult(TypedDict, total=False):
-    """Raw result from run_hook()."""
-
-    returncode: int
-    stdout: str
-    stderr: str
-    output_json: dict[str, Any] | None
-    output_files: list[dict[str, Any]]
-    duration_ms: int
-    hook: str
-    plugin: str  # Plugin name (directory name, e.g., 'wget', 'screenshot')
-    hook_name: str  # Full hook filename (e.g., 'on_Snapshot__50_wget.py')
-    # New fields for JSONL parsing
-    records: list[dict[str, Any]]  # Parsed JSONL records with 'type' field
-
-
 def _model_output_dir_from_child_path(path: Path, marker: str) -> Path | None:
     """
     Infer the model output dir from a model dir or one of its plugin subdirs.
@@ -873,26 +857,6 @@ def discover_plugin_configs() -> dict[str, dict[str, Any]]:
     return configs
 
 
-def get_config_defaults_from_plugins() -> dict[str, Any]:
-    """
-    Get default values for all plugin config options.
-
-    Returns:
-        Dict mapping config keys to their default values.
-        e.g., {'SAVE_WGET': True, 'WGET_TIMEOUT': 60, ...}
-    """
-    plugin_configs = discover_plugin_configs()
-    defaults = {}
-
-    for plugin_name, schema in plugin_configs.items():
-        properties = schema.get("properties", {})
-        for key, prop_schema in properties.items():
-            if "default" in prop_schema:
-                defaults[key] = prop_schema["default"]
-
-    return defaults
-
-
 def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited: set[str] | None = None) -> PluginSpecialConfig:
     """
     Extract special config keys for a plugin following naming conventions.

archivebox/machine/detect.py

@@ -275,10 +275,6 @@ def get_host_stats() -> dict[str, Any]:
         return {}
 
 
-def get_host_immutable_info(host_info: dict[str, Any]) -> dict[str, Any]:
-    return {key: value for key, value in host_info.items() if key in ["guid", "net_mac", "os_family", "cpu_arch"]}
-
-
 def get_host_guid() -> str:
     return machineid.hashed_id("archivebox")
 

archivebox/machine/migrations/0017_shorten_process_progress_index_names.py

@@ -0,0 +1,26 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("machine", "0016_process_delete_at"),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name="process",
+            name="machine_pro_progress_recent_idx",
+        ),
+        migrations.RemoveIndex(
+            model_name="process",
+            name="machine_pro_progress_running_idx",
+        ),
+        migrations.AddIndex(
+            model_name="process",
+            index=models.Index(fields=["machine", "process_type", "-modified_at"], name="mach_proc_recent_idx"),
+        ),
+        migrations.AddIndex(
+            model_name="process",
+            index=models.Index(fields=["machine", "status", "process_type"], name="mach_proc_running_idx"),
+        ),
+    ]

archivebox/machine/migrations/0018_alter_process_process_type.py

@@ -0,0 +1,34 @@
+# Generated by ArchiveBox on 2026-05-28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("machine", "0017_shorten_process_progress_index_names"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="process",
+            name="process_type",
+            field=models.CharField(
+                choices=[
+                    ("supervisord", "Supervisord"),
+                    ("orchestrator", "Orchestrator"),
+                    ("server", "Server"),
+                    ("update", "Update"),
+                    ("add", "Add"),
+                    ("search", "Search"),
+                    ("worker", "Worker"),
+                    ("cli", "CLI"),
+                    ("hook", "Hook"),
+                    ("binary", "Binary"),
+                ],
+                db_index=True,
+                default="cli",
+                help_text="Type of process (cli, worker, orchestrator, binary, supervisord)",
+                max_length=16,
+            ),
+        ),
+    ]

archivebox/machine/models.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 __package__ = "archivebox.machine"
 
 import os
-import shlex
 import sys
 import uuid
 import socket
@@ -196,8 +195,10 @@ class Machine(ModelWithHealthStats):
         app_label = "machine"
 
     @classmethod
-    def current(cls) -> Machine:
+    def current(cls, refresh: bool = False) -> Machine:
         global _CURRENT_MACHINE
+        if refresh:
+            _CURRENT_MACHINE = None
         if _CURRENT_MACHINE:
             if timezone.now() < _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
                 return _CURRENT_MACHINE
@@ -335,7 +336,7 @@ class NetworkInterface(ModelWithHealthStats):
     @classmethod
     def current(cls, refresh: bool = False) -> NetworkInterface:
         global _CURRENT_INTERFACE
-        machine = Machine.current()
+        machine = Machine.current(refresh=refresh)
         if _CURRENT_INTERFACE:
             if (
                 not refresh
@@ -928,6 +929,10 @@ class Process(ModelWithDeleteAfter, models.Model):
     class TypeChoices(models.TextChoices):
         SUPERVISORD = "supervisord", "Supervisord"
         ORCHESTRATOR = "orchestrator", "Orchestrator"
+        SERVER = "server", "Server"
+        UPDATE = "update", "Update"
+        ADD = "add", "Add"
+        SEARCH = "search", "Search"
         WORKER = "worker", "Worker"
         CLI = "cli", "CLI"
         HOOK = "hook", "Hook"
@@ -1107,8 +1112,8 @@ class Process(ModelWithDeleteAfter, models.Model):
             models.Index(fields=["binary", "exit_code"]),
             models.Index(fields=["pid", "started_at"]),
             models.Index(fields=["process_type", "worker_type", "pwd", "started_at"]),
-            models.Index(fields=["machine", "process_type", "-modified_at"], name="machine_pro_progress_recent_idx"),
-            models.Index(fields=["machine", "status", "process_type"], name="machine_pro_progress_running_idx"),
+            models.Index(fields=["machine", "process_type", "-modified_at"], name="mach_proc_recent_idx"),
+            models.Index(fields=["machine", "status", "process_type"], name="mach_proc_running_idx"),
         ]
 
     def __str__(self) -> str:
@@ -1225,7 +1230,7 @@ class Process(ModelWithDeleteAfter, models.Model):
         """Parse JSONL records from this process's stdout."""
         stdout = self.stdout
         if not stdout and self.stdout_file and self.stdout_file.exists():
-            stdout = self.stdout_file.read_text()
+            stdout = self.stdout_file.read_text(errors="replace")
         return self.parse_records_from_text(stdout or "")
 
     @staticmethod
@@ -1259,6 +1264,50 @@ class Process(ModelWithDeleteAfter, models.Model):
         self.save()
         return True
 
+    def mark_running(
+        self,
+        *,
+        process_type: str | None = None,
+        pwd: str | Path | None = None,
+        url: str | None = None,
+        worker_type: str = "",
+        timeout: int | None = None,
+    ) -> None:
+        """Record the current process role without changing ownership state elsewhere."""
+        updates = ["status", "retry_at", "modified_at"]
+        self.status = self.StatusChoices.RUNNING
+        self.retry_at = None
+        if process_type is not None and self.process_type != process_type:
+            self.process_type = process_type
+            updates.append("process_type")
+        if worker_type and self.worker_type != worker_type:
+            self.worker_type = worker_type
+            updates.append("worker_type")
+        if pwd is not None and self.pwd != str(pwd):
+            self.pwd = str(pwd)
+            updates.append("pwd")
+        if url is not None and self.url != url:
+            self.url = url
+            updates.append("url")
+        if timeout is not None and self.timeout != timeout:
+            self.timeout = timeout
+            updates.append("timeout")
+        self.save(update_fields=updates)
+
+    def heartbeat(self) -> None:
+        """Touch modified_at so standby/leader selection can see this parent is alive."""
+        self.save(update_fields=["modified_at"])
+
+    def mark_exited(self, *, exit_code: int = 0) -> None:
+        """Mark a foreground/internal process row exited after command cleanup."""
+        if self.status == self.StatusChoices.EXITED and self.exit_code == exit_code:
+            return
+        self.status = self.StatusChoices.EXITED
+        self.exit_code = exit_code
+        self.ended_at = self.ended_at or timezone.now()
+        self.retry_at = None
+        self.save(update_fields=["status", "exit_code", "ended_at", "retry_at", "modified_at"])
+
     # =========================================================================
     # Process.current() and hierarchy methods
     # =========================================================================
@@ -1426,6 +1475,14 @@ class Process(ModelWithDeleteAfter, models.Model):
             return cls.TypeChoices.SUPERVISORD
         elif "runner_watch" in argv_str:
             return cls.TypeChoices.WORKER
+        elif "archivebox server" in argv_str:
+            return cls.TypeChoices.SERVER
+        elif "archivebox update" in argv_str:
+            return cls.TypeChoices.UPDATE
+        elif "archivebox add" in argv_str:
+            return cls.TypeChoices.ADD
+        elif "archivebox search" in argv_str or "archivebox list" in argv_str:
+            return cls.TypeChoices.SEARCH
         elif "archivebox run" in argv_str:
             return cls.TypeChoices.ORCHESTRATOR
         elif "archivebox" in argv_str:
@@ -1451,7 +1508,9 @@ class Process(ModelWithDeleteAfter, models.Model):
         if machine is not None:
             stale = stale.filter(machine=machine)
 
-        for proc in stale:
+        # Recovery can run against damaged DB state; stream rows so a large
+        # stale Process backlog cannot be materialized in memory at once.
+        for proc in stale.iterator(chunk_size=100):
             if proc.poll() is not None:
                 cleaned += 1
                 continue
@@ -1461,7 +1520,7 @@ class Process(ModelWithDeleteAfter, models.Model):
             if proc.started_at:
                 timeout_seconds = max(int(proc.timeout or 0), 0)
                 timeout_deadline = proc.started_at + timedelta(seconds=timeout_seconds) + PROCESS_TIMEOUT_GRACE
-                if timezone.now() >= timeout_deadline:
+                if timeout_seconds > 0 and timezone.now() >= timeout_deadline:
                     is_stale = True
 
             # Check if too old (PID definitely reused)
@@ -1483,7 +1542,7 @@ class Process(ModelWithDeleteAfter, models.Model):
                 proc.status = cls.StatusChoices.EXITED
                 proc.ended_at = proc.ended_at or timezone.now()
                 proc.exit_code = proc.exit_code if proc.exit_code is not None else 0
-                proc.save(update_fields=["status", "ended_at", "exit_code"])
+                proc.save(update_fields=["status", "ended_at", "exit_code", "modified_at"])
                 cleaned += 1
 
         return cleaned
@@ -1652,18 +1711,6 @@ class Process(ModelWithDeleteAfter, models.Model):
     # Lifecycle methods (launch, kill, poll, wait)
     # =========================================================================
 
-    @property
-    def pid_file(self) -> Path | None:
-        """Path to PID file for this process."""
-        runtime_dir = self.runtime_dir
-        return runtime_dir / "process.pid" if runtime_dir else None
-
-    @property
-    def cmd_file(self) -> Path | None:
-        """Path to cmd.sh script for this process."""
-        runtime_dir = self.runtime_dir
-        return runtime_dir / "cmd.sh" if runtime_dir else None
-
     @property
     def stdout_file(self) -> Path | None:
         """Path to stdout log."""
@@ -1694,7 +1741,7 @@ class Process(ModelWithDeleteAfter, models.Model):
 
     @property
     def runtime_dir(self) -> Path | None:
-        """Directory where this process stores runtime logs/pid/cmd metadata."""
+        """Directory where this process stores runtime stdout/stderr logs."""
         if not self.pwd:
             return None
 
@@ -1830,32 +1877,6 @@ class Process(ModelWithDeleteAfter, models.Model):
         for line in self.tail_stderr(lines=lines, follow=follow):
             print(line, file=sys.stderr, flush=True)
 
-    def _write_pid_file(self) -> None:
-        """Write PID file with mtime set to process start time."""
-        if self.pid and self.started_at and self.pid_file:
-            self.pid_file.parent.mkdir(parents=True, exist_ok=True)
-            # Write PID to file
-            self.pid_file.write_text(str(self.pid))
-            # Set mtime to process start time for validation
-            try:
-                start_time = self.started_at.timestamp()
-                os.utime(self.pid_file, (start_time, start_time))
-            except OSError:
-                pass  # mtime optional, validation degrades gracefully
-
-    def _write_cmd_file(self) -> None:
-        """Write cmd.sh script for debugging/validation."""
-        if self.cmd and self.cmd_file:
-            self.cmd_file.parent.mkdir(parents=True, exist_ok=True)
-
-            # Write executable shell script
-            script = "#!/bin/bash\n" + shlex.join(self.cmd) + "\n"
-            self.cmd_file.write_text(script)
-            try:
-                self.cmd_file.chmod(0o755)
-            except OSError:
-                pass
-
     def ensure_log_files(self) -> None:
         """Ensure stdout/stderr log files exist for this process."""
         runtime_dir = self.runtime_dir
@@ -1918,9 +1939,6 @@ class Process(ModelWithDeleteAfter, models.Model):
         # Use provided cwd or default to pwd
         working_dir = cwd or self.pwd
 
-        # Write cmd.sh for debugging
-        self._write_cmd_file()
-
         stdout_path = self.stdout_file
         stderr_path = self.stderr_file
         if stdout_path:
@@ -1956,8 +1974,6 @@ class Process(ModelWithDeleteAfter, models.Model):
             self.status = self.StatusChoices.RUNNING
             self.save()
 
-            self._write_pid_file()
-
             if not background:
                 try:
                     proc.wait(timeout=self.timeout)
@@ -1971,9 +1987,9 @@ class Process(ModelWithDeleteAfter, models.Model):
 
                 self.ended_at = timezone.now()
                 if stdout_path.exists():
-                    self.stdout = stdout_path.read_text()
+                    self.stdout = stdout_path.read_text(errors="replace")
                 if stderr_path.exists():
-                    self.stderr = stderr_path.read_text()
+                    self.stderr = stderr_path.read_text(errors="replace")
                 self.status = self.StatusChoices.EXITED
                 self.save()
 
@@ -2013,10 +2029,6 @@ class Process(ModelWithDeleteAfter, models.Model):
             self.status = self.StatusChoices.EXITED
             self.save()
 
-            # Clean up PID file
-            if self.pid_file and self.pid_file.exists():
-                self.pid_file.unlink(missing_ok=True)
-
             return True
         except (psutil.NoSuchProcess, psutil.AccessDenied, ProcessLookupError):
             # Process already exited between proc check and kill
@@ -2052,24 +2064,14 @@ class Process(ModelWithDeleteAfter, models.Model):
                     pass
             # Process exited - read output and copy to DB
             if self.stdout_file and self.stdout_file.exists():
-                self.stdout = self.stdout_file.read_text()
+                self.stdout = self.stdout_file.read_text(errors="replace")
                 # TODO: Uncomment to cleanup (keeping for debugging for now)
                 # self.stdout_file.unlink(missing_ok=True)
             if self.stderr_file and self.stderr_file.exists():
-                self.stderr = self.stderr_file.read_text()
+                self.stderr = self.stderr_file.read_text(errors="replace")
                 # TODO: Uncomment to cleanup (keeping for debugging for now)
                 # self.stderr_file.unlink(missing_ok=True)
 
-            # Clean up PID file (not needed for debugging)
-            if self.pid_file and self.pid_file.exists():
-                self.pid_file.unlink(missing_ok=True)
-
-            # TODO: Uncomment to cleanup cmd.sh (keeping for debugging for now)
-            # if self.pwd:
-            #     cmd_file = Path(self.pwd) / 'cmd.sh'
-            #     if cmd_file.exists():
-            #         cmd_file.unlink(missing_ok=True)
-
             # Try to get exit code from proc or default to unknown
             self.exit_code = self.exit_code if self.exit_code is not None else 0
             if self.exit_code == -1:
@@ -2433,12 +2435,14 @@ class Process(ModelWithDeleteAfter, models.Model):
             status=cls.StatusChoices.RUNNING,
         )
 
-        for proc in running_children:
+        # Recovery can run against damaged DB state; stream rows so a large
+        # orphaned Process backlog cannot be materialized in memory at once.
+        for proc in running_children.iterator(chunk_size=100):
             if not proc.is_running:
                 proc.status = cls.StatusChoices.EXITED
                 proc.ended_at = proc.ended_at or timezone.now()
                 proc.exit_code = proc.exit_code if proc.exit_code is not None else 0
-                proc.save(update_fields=["status", "ended_at", "exit_code"])
+                proc.save(update_fields=["status", "ended_at", "exit_code", "modified_at"])
                 cleaned += 1
                 continue
 
@@ -2447,14 +2451,21 @@ class Process(ModelWithDeleteAfter, models.Model):
             if root.id == proc.id and root.process_type in (cls.TypeChoices.WORKER, cls.TypeChoices.HOOK):
                 continue
 
-            # If root is an active orchestrator/cli, keep it
-            if root.process_type in (cls.TypeChoices.ORCHESTRATOR, cls.TypeChoices.CLI) and root.is_running:
+            # If root is an active ArchiveBox command/orchestrator, keep it.
+            if root.process_type in (
+                cls.TypeChoices.ORCHESTRATOR,
+                cls.TypeChoices.SERVER,
+                cls.TypeChoices.UPDATE,
+                cls.TypeChoices.ADD,
+                cls.TypeChoices.SEARCH,
+                cls.TypeChoices.CLI,
+            ) and root.is_running:
                 continue
 
             proc.status = cls.StatusChoices.EXITED
             proc.ended_at = proc.ended_at or timezone.now()
             proc.exit_code = proc.exit_code if proc.exit_code is not None else 0
-            proc.save(update_fields=["status", "ended_at", "exit_code"])
+            proc.save(update_fields=["status", "ended_at", "exit_code", "modified_at"])
             cleaned += 1
 
         if cleaned:

archivebox/misc/checks.py

@@ -2,6 +2,7 @@ __package__ = "archivebox.misc"
 
 import os
 import sys
+import time
 from pathlib import Path
 
 from rich import print
@@ -55,23 +56,43 @@ def check_data_folder(config=None, **config_kwargs) -> None:
     check_data_dir_permissions(config=config)
 
 
-def check_migrations():
+def check_migrations(*, blocking: bool = True, auto_apply: bool = False, cancel_delay: int = 3) -> list[str]:
     from archivebox import DATA_DIR
-    from archivebox.misc.db import list_migrations
+    from archivebox.misc.db import apply_migrations, pending_migrations
 
-    pending_migrations = [name for status, name in list_migrations() if not status]
+    pending = pending_migrations()
     is_migrating = any(arg in sys.argv for arg in ["makemigrations", "migrate", "init"])
 
-    if pending_migrations and not is_migrating:
-        print("[red][X] This collection was created with an older version of ArchiveBox and must be upgraded first.[/red]")
+    if pending and not is_migrating:
+        print("[red][X] This collection was created with an older version of ArchiveBox and must be upgraded first.[/red]", file=sys.stderr)
         print(f"    {DATA_DIR}", file=sys.stderr)
         print(file=sys.stderr)
         print(
-            f"    [violet]Hint:[/violet] To upgrade it to the latest version and apply the {len(pending_migrations)} pending migrations, run:",
+            f"    [violet]Hint:[/violet] To upgrade it to the latest version and apply the {len(pending)} pending migrations, run:",
             file=sys.stderr,
         )
         print("        archivebox init", file=sys.stderr)
-        raise SystemExit(3)
+        if auto_apply:
+            print(file=sys.stderr)
+            print(
+                f"[yellow][*] ArchiveBox will apply migrations automatically in {cancel_delay}s. Press CTRL+C to cancel.[/yellow]",
+                file=sys.stderr,
+            )
+            try:
+                time.sleep(cancel_delay)
+            except KeyboardInterrupt:
+                print("[red][X] Migration cancelled before any changes were applied.[/red]", file=sys.stderr)
+                raise SystemExit(130) from None
+
+            # Always delegate to Django's migration executor. It records each
+            # migration only after it succeeds, so power loss or SIGKILL leaves
+            # unapplied work visible here and the next startup resumes normally.
+            print("[yellow][*] Applying database migrations...[/yellow]", file=sys.stderr)
+            apply_migrations(stdout=sys.stderr, stderr=sys.stderr, verbosity=1)
+            return pending_migrations()
+        if blocking:
+            raise SystemExit(3)
+    return pending
 
 
 def check_io_encoding():

archivebox/misc/db.py

@@ -6,48 +6,181 @@ __package__ = "archivebox.misc"
 
 from io import StringIO
 from pathlib import Path
+from typing import TextIO
 from typing import Any
+import fcntl
+import importlib
+import time
+from collections.abc import Callable
+from contextlib import contextmanager
+from sqlite3 import OperationalError as SQLiteOperationalError
 
 from archivebox.config import DATA_DIR
 from archivebox.misc.util import enforce_types
 
 
-@enforce_types
-def list_migrations(out_dir: Path = DATA_DIR) -> list[tuple[bool, str]]:
-    """List all Django migrations and their status"""
-    from django.core.management import call_command
+def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:
+    parts = [str(part) for part in (cmdline or []) if str(part)]
+    if not parts:
+        return fallback
+    for marker in ("archivebox", "daphne", "gunicorn", "uvicorn", "supervisord", "sonic", "node"):
+        for idx, part in enumerate(parts):
+            if Path(part).name == marker or part == marker:
+                return " ".join([Path(parts[idx]).name, *parts[idx + 1 :]])[:220]
+    return " ".join([Path(parts[0]).name, *parts[1:]])[:220]
 
-    out = StringIO()
-    call_command("showmigrations", list=True, stdout=out)
-    out.seek(0)
 
-    migrations = []
-    for line in out.readlines():
-        if line.strip() and "]" in line:
-            status_str, name_str = line.strip().split("]", 1)
-            is_applied = "X" in status_str
-            migration_name = name_str.strip()
-            migrations.append((is_applied, migration_name))
+def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]:
+    import psutil
 
-    return migrations
+    db_path = db_path.resolve()
+    holders: list[str] = []
+    for proc in psutil.process_iter(["pid", "ppid", "name", "cmdline", "status"]):
+        try:
+            open_files = proc.open_files()
+        except (psutil.AccessDenied, psutil.NoSuchProcess, psutil.ZombieProcess):
+            continue
+        for open_file in open_files:
+            try:
+                open_path = Path(open_file.path).resolve()
+            except (OSError, RuntimeError):
+                continue
+            if open_path == db_path or open_path.name in {f"{db_path.name}-wal", f"{db_path.name}-shm", f"{db_path.name}-journal"}:
+                info = proc.info
+                cmdline = compact_command(info.get("cmdline"), fallback=info.get("name") or "")
+                holders.append(f"pid={info['pid']} ppid={info['ppid']} {info['status']} {cmdline}")
+                break
+    return holders
+
+
+def sqlite_lock_error(error: BaseException) -> bool:
+    return isinstance(error, SQLiteOperationalError) and "database is locked" in str(error).lower()
+
+
+def retry_sqlite_locks(action: Callable[[], Any], *, label: str, stderr: TextIO | None = None) -> Any:
+    from django.db import OperationalError, connections
+    from rich.console import Console
+
+    console = Console(file=stderr or None, stderr=stderr is None)
+    attempts = 0
+    while True:
+        try:
+            return action()
+        except OperationalError as err:
+            if "database is locked" not in str(err).lower():
+                raise
+        except SQLiteOperationalError as err:
+            if not sqlite_lock_error(err):
+                raise
+
+        attempts += 1
+        connections.close_all()
+        holders = sqlite_lock_holders()
+        console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in 5s...[/yellow]")
+        if holders:
+            console.print("[yellow]    DB holders:[/yellow]")
+            for holder in holders[:8]:
+                console.print(f"[yellow]    - {holder}[/yellow]")
+            if len(holders) > 8:
+                console.print(f"[yellow]    ... {len(holders) - 8} more[/yellow]")
+        else:
+            console.print("[yellow]    No local process with index.sqlite3 open was visible to this user.[/yellow]")
+        if attempts == 1:
+            console.print("[dim]    SQLite does not expose the active SQL statement from another process; only the owning local PIDs can be shown.[/dim]")
+        with console.status("[yellow]Waiting for SQLite database lock to clear...[/yellow]", spinner="dots"):
+            time.sleep(5.0)
+
+
+@contextmanager
+def migration_lock(stdout: TextIO | None = None):
+    from archivebox.config.paths import get_or_create_working_tmp_dir
+    from rich.console import Console
+
+    lock_path = get_or_create_working_tmp_dir(autofix=True, quiet=True) / "migrate.lock"
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    with lock_path.open("a+") as lock_file:
+        try:
+            fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            # Migrations on large SQLite collections can run for hours. Use a
+            # kernel lock with no timeout so parallel ArchiveBox commands queue
+            # behind the active migrate process instead of racing it.
+            console = Console(file=stdout or None, stderr=stdout is None)
+            with console.status("[yellow]Waiting for migration lock...[/yellow]", spinner="dots"):
+                while True:
+                    try:
+                        fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                        break
+                    except BlockingIOError:
+                        time.sleep(1.0)
+        try:
+            yield
+        finally:
+            fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
 
 
 @enforce_types
-def apply_migrations(out_dir: Path = DATA_DIR) -> list[str]:
+def pending_migrations(out_dir: Path = DATA_DIR) -> list[str]:
+    """Cheaply compare migration files to django_migrations without invoking migrate."""
+    from django.apps import apps
+    from django.db import connection
+    from django.db.migrations.loader import MigrationLoader
+
+    def applied_rows() -> set[tuple[str, str]]:
+        with connection.cursor() as cursor:
+            try:
+                cursor.execute("SELECT app, name FROM django_migrations")
+            except Exception as err:
+                if "no such table" in str(err).lower():
+                    return set()
+                raise
+            return {(str(app), str(name)) for app, name in cursor.fetchall()}
+
+    applied = retry_sqlite_locks(applied_rows, label="checking applied migrations")
+    disk_migrations: set[tuple[str, str]] = set()
+    for app_config in apps.get_app_configs():
+        module_name, explicit = MigrationLoader.migrations_module(app_config.label)
+        if module_name is None:
+            continue
+        try:
+            module = importlib.import_module(module_name)
+        except ModuleNotFoundError:
+            if explicit:
+                raise
+            continue
+        module_file = getattr(module, "__file__", None)
+        if not module_file:
+            continue
+        for migration_file in Path(module_file).parent.glob("[0-9][0-9][0-9][0-9]_*.py"):
+            disk_migrations.add((app_config.label, migration_file.stem))
+
+    return [f"{app}.{name}" for app, name in sorted(disk_migrations - applied)]
+
+
+@enforce_types
+def apply_migrations(out_dir: Path = DATA_DIR, stdout: TextIO | None = None, stderr: TextIO | None = None, verbosity: int = 1) -> list[str]:
     """Apply pending Django migrations"""
     from django.core.management import call_command
 
-    out1 = StringIO()
+    with migration_lock(stdout=stderr or stdout):
+        if not pending_migrations():
+            return []
 
-    call_command("migrate", interactive=False, database="default", stdout=out1)
-    out1.seek(0)
+        if stdout is not None:
+            retry_sqlite_locks(
+                lambda: call_command("migrate", interactive=False, database="default", stdout=stdout, stderr=stderr, verbosity=verbosity),
+                label="applying migrations",
+                stderr=stderr,
+            )
+            return []
 
-    return [line.strip() for line in out1.readlines() if line.strip()]
+        def migrate() -> StringIO:
+            out1 = StringIO()
+            call_command("migrate", interactive=False, database="default", stdout=out1, verbosity=verbosity)
+            out1.seek(0)
+            return out1
 
+        out1 = retry_sqlite_locks(migrate, label="applying migrations")
 
-@enforce_types
-def get_admins(out_dir: Path = DATA_DIR) -> list[Any]:
-    """Get list of superuser accounts"""
-    from django.contrib.auth.models import User
+        return [line.strip() for line in out1.readlines() if line.strip()]
 
-    return list(User.objects.filter(is_superuser=True).exclude(username="system"))

archivebox/misc/jsonl.py

@@ -164,15 +164,3 @@ def write_record(record: dict[str, Any], stream: TextIO | None = None) -> None:
     active_stream.write(json.dumps(record) + "\n")
     active_stream.flush()
 
-
-def write_records(records: Iterator[dict[str, Any]], stream: TextIO | None = None) -> int:
-    """
-    Write multiple JSONL records to stdout (or provided stream).
-
-    Returns count of records written.
-    """
-    count = 0
-    for record in records:
-        write_record(record, stream)
-        count += 1
-    return count

archivebox/misc/monkey_patches.py

@@ -35,13 +35,28 @@ warnings.filterwarnings("ignore", category=SyntaxWarning, module="sonic")
 class ModifiedAccessLogGenerator(access.AccessLogGenerator):
     """Clutge workaround until daphne uses the Python logging framework. https://github.com/django/daphne/pull/473/files"""
 
-    def write_entry(self, host, date, request, status=None, length=None, ident=None, user=None):
+    def __call__(self, protocol, action, details):
+        if protocol == "http" and action == "complete":
+            self.write_entry(
+                host=details["client"],
+                date=datetime.datetime.now(),
+                request="%(method)s %(path)s" % details,
+                status=details["status"],
+                length=details["size"],
+                time_taken=details.get("time_taken"),
+            )
+            return
+        return super().__call__(protocol, action, details)
+
+    def write_entry(self, host, date, request, status=None, length=None, ident=None, user=None, time_taken=None):
 
         # Ignore noisy requests to staticfiles / favicons / etc.
         if "GET /static/" in request:
             return
         if "GET /health/" in request:
             return
+        if "GET /admin/live-progress/" in request and (time_taken is None or time_taken < 1.0):
+            return
         if "GET /admin/jsi18n/" in request:
             return
         if request.endswith("/favicon.ico") or request.endswith("/robots.txt") or request.endswith("/screenshot.png"):

archivebox/misc/serve_static.py

@@ -93,7 +93,7 @@ def _resolve_archive_path(document_root: str | Path, rel_path: str) -> tuple[Pat
 
 def _cache_policy(config=None, **config_kwargs) -> str:
     config = config or get_config(resolve_plugins=False, **config_kwargs)
-    return "public" if config.PUBLIC_SNAPSHOTS else "private"
+    return "private" if config.PERMISSIONS == "private" else "public"
 
 
 def _render_mhtml_preview_document(filename: str, output_path: str) -> str:

archivebox/misc/system.py

@@ -3,7 +3,6 @@ __package__ = "archivebox.misc"
 
 import os
 import signal
-import shutil
 import sys
 
 from json import dump
@@ -124,43 +123,6 @@ def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool
     os.chmod(path, int(config.OUTPUT_PERMISSIONS, base=8))
 
 
-@enforce_types
-def chmod_file(path: str, cwd: str = "", config=None, **config_kwargs) -> None:
-    """chmod -R <permissions> <cwd>/<path>"""
-
-    root = Path(cwd or os.getcwd()) / path
-    if not os.access(root, os.R_OK):
-        raise Exception(f"Failed to chmod: {path} does not exist (did the previous step fail?)")
-
-    if not root.is_dir():
-        # path is just a plain file
-        config = config or get_config(**config_kwargs)
-        os.chmod(root, int(config.OUTPUT_PERMISSIONS, base=8))
-    else:
-        config = config or get_config(**config_kwargs)
-        for subpath in Path(path).glob("**/*"):
-            if subpath.is_dir():
-                # directories need execute permissions to be able to list contents
-                os.chmod(subpath, int(config.DIR_OUTPUT_PERMISSIONS, base=8))
-            else:
-                os.chmod(subpath, int(config.OUTPUT_PERMISSIONS, base=8))
-
-
-@enforce_types
-def copy_and_overwrite(from_path: str | Path, to_path: str | Path):
-    """copy a given file or directory to a given path, overwriting the destination"""
-
-    assert os.access(from_path, os.R_OK)
-
-    if Path(from_path).is_dir():
-        shutil.rmtree(to_path, ignore_errors=True)
-        shutil.copytree(from_path, to_path)
-    else:
-        with open(from_path, "rb") as src:
-            contents = src.read()
-        atomic_write(to_path, contents)
-
-
 @enforce_types
 def get_dir_size(path: str | Path, recursive: bool = True, pattern: str | None = None) -> tuple[int, int, int]:
     """get the total disk size of a given directory, optionally summing up
@@ -187,44 +149,3 @@ def get_dir_size(path: str | Path, recursive: bool = True, pattern: str | None =
         pass
     return num_bytes, num_dirs, num_files
 
-
-class suppress_output:
-    """
-    A context manager for doing a "deep suppression" of stdout and stderr in
-    Python, i.e. will suppress all print, even if the print originates in a
-    compiled C/Fortran sub-function.
-
-    This will not suppress raised exceptions, since exceptions are printed
-    to stderr just before a script exits, and after the context manager has
-    exited (at least, I think that is why it lets exceptions through).
-
-    with suppress_stdout_stderr():
-        rogue_function()
-    """
-
-    def __init__(self, stdout=True, stderr=True):
-        # Open a pair of null files
-        # Save the actual stdout (1) and stderr (2) file descriptors.
-        self.stdout, self.stderr = stdout, stderr
-        if stdout:
-            self.null_stdout = os.open(os.devnull, os.O_RDWR)
-            self.real_stdout = os.dup(1)
-        if stderr:
-            self.null_stderr = os.open(os.devnull, os.O_RDWR)
-            self.real_stderr = os.dup(2)
-
-    def __enter__(self):
-        # Assign the null pointers to stdout and stderr.
-        if self.stdout:
-            os.dup2(self.null_stdout, 1)
-        if self.stderr:
-            os.dup2(self.null_stderr, 2)
-
-    def __exit__(self, *_):
-        # Re-assign the real stdout/stderr back to (1) and (2)
-        if self.stdout:
-            os.dup2(self.real_stdout, 1)
-            os.close(self.null_stdout)
-        if self.stderr:
-            os.dup2(self.real_stderr, 2)
-            os.close(self.null_stderr)

archivebox/misc/util.py

@@ -16,7 +16,6 @@ from hashlib import sha256
 from urllib.parse import urlparse, quote, unquote
 from html import escape, unescape
 from datetime import datetime, timezone
-from requests.exceptions import RequestException, ReadTimeout
 
 from base32_crockford import encode as base32_encode
 from w3lib.encoding import html_body_declared_encoding, http_content_type_encoding
@@ -29,8 +28,6 @@ except ImportError:
     detect_encoding = lambda rawdata: "utf-8"
 
 
-from archivebox.config.constants import CONSTANTS
-
 from .logging import COLOR_DICT
 
 
@@ -61,21 +58,11 @@ htmlencode = lambda s: s and escape(s, quote=True)
 htmldecode = lambda s: s and unescape(s)
 
 
-def short_ts(ts: Any) -> str | None:
-    parsed = parse_date(ts)
-    return None if parsed is None else str(parsed.timestamp()).split(".")[0]
-
-
 def ts_to_date_str(ts: Any) -> str | None:
     parsed = parse_date(ts)
     return None if parsed is None else parsed.strftime("%Y-%m-%d %H:%M")
 
 
-def ts_to_iso(ts: Any) -> str | None:
-    parsed = parse_date(ts)
-    return None if parsed is None else parsed.isoformat()
-
-
 COLOR_REGEX = re.compile(r"\[(?P<arg_1>\d+)(;(?P<arg_2>\d+)(;(?P<arg_3>\d+))?)?m")
 
 
@@ -296,11 +283,6 @@ def parse_filesize_to_bytes(value: str | int | float | None) -> int:
     return int(amount * multiplier)
 
 
-def is_static_file(url: str):
-    # TODO: the proper way is with MIME type detection + ext, not only extension
-    return extension(url).lower() in CONSTANTS.STATICFILE_EXTENSIONS
-
-
 def enforce_types(func):
     """
     Enforce function arg and kwarg types at runtime using its python3 type hints
@@ -355,17 +337,6 @@ def docstring(text: str | None):
     return decorator
 
 
-@enforce_types
-def str_between(string: str, start: str, end: str | None = None) -> str:
-    """(<abc>12345</def>, <abc>, </def>)  ->  12345"""
-
-    content = string.split(start, 1)[-1]
-    if end is not None:
-        content = content.rsplit(end, 1)[0]
-
-    return content
-
-
 @enforce_types
 def parse_date(date: Any) -> datetime | None:
     """Parse unix timestamps, iso format, and human-readable strings"""
@@ -448,50 +419,6 @@ def download_url(url: str, timeout: int | None = None, config=None, **config_kwa
         return url.rsplit("/", 1)[-1]
 
 
-@enforce_types
-def get_headers(url: str, timeout: int | None = None, config=None, **config_kwargs) -> str:
-    """Download the contents of a remote url and return the headers"""
-    # TODO: get rid of this and use an abx pluggy hook instead
-
-    from archivebox.config.common import get_config
-
-    config = config or get_config(**config_kwargs)
-    timeout = timeout or config.TIMEOUT
-
-    try:
-        response = requests.head(
-            url,
-            headers={"User-Agent": config.USER_AGENT},
-            verify=config.CHECK_SSL_VALIDITY,
-            timeout=timeout,
-            allow_redirects=True,
-        )
-        if response.status_code >= 400:
-            raise RequestException
-    except ReadTimeout:
-        raise
-    except RequestException:
-        response = requests.get(
-            url,
-            headers={"User-Agent": config.USER_AGENT},
-            verify=config.CHECK_SSL_VALIDITY,
-            timeout=timeout,
-            stream=True,
-        )
-
-    return pyjson.dumps(
-        {
-            "URL": url,
-            "Status-Code": response.status_code,
-            "Elapsed": response.elapsed.total_seconds() * 1000,
-            "Encoding": str(response.encoding),
-            "Apparent-Encoding": response.apparent_encoding,
-            **dict(response.headers),
-        },
-        indent=4,
-    )
-
-
 @enforce_types
 def ansi_to_html(text: str) -> str:
     """
@@ -698,50 +625,3 @@ _test_url_strs = {
 for url_str, num_urls in _test_url_strs.items():
     assert len(list(find_all_urls(url_str))) == num_urls, f"{url_str} does not contain {num_urls} urls"
 
-
-### Chrome Helpers
-
-
-def chrome_cleanup(config=None, **config_kwargs):
-    """
-    Cleans up any state or runtime files that Chrome leaves behind when killed by
-    a timeout or other error. Handles:
-    - All persona chrome_profile directories (via Persona.cleanup_chrome_all())
-    - Explicit CHROME_USER_DATA_DIR from config
-    - Legacy Docker chromium path
-    """
-    import os
-    from pathlib import Path
-    from archivebox.config.permissions import IN_DOCKER
-
-    # Clean up all persona chrome directories using Persona class
-    try:
-        from archivebox.personas.models import Persona
-
-        # Clean up all personas
-        Persona.cleanup_chrome_all()
-
-        # Also clean up the active persona's explicit CHROME_USER_DATA_DIR if set
-        # (in case it's a custom path not under PERSONAS_DIR)
-        from archivebox.config.common import get_config
-
-        config = config or get_config(**config_kwargs)
-        chrome_user_data_dir = config.get("CHROME_USER_DATA_DIR")
-        if chrome_user_data_dir:
-            singleton_lock = Path(chrome_user_data_dir) / "SingletonLock"
-            if os.path.lexists(singleton_lock):
-                try:
-                    singleton_lock.unlink()
-                except OSError:
-                    pass
-    except Exception:
-        pass  # Persona/config not available during early startup
-
-    # Legacy Docker cleanup (for backwards compatibility)
-    if IN_DOCKER:
-        singleton_lock = "/home/archivebox/.config/chromium/SingletonLock"
-        if os.path.lexists(singleton_lock):
-            try:
-                os.remove(singleton_lock)
-            except OSError:
-                pass

archivebox/personas/admin.py

@@ -26,7 +26,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
         (
             "Persona",
             {
-                "fields": ("name", "created_by"),
+                "fields": ("name", "created_by", "permissions"),
                 "classes": ("card", "persona-card-primary"),
             },
         ),

archivebox/personas/forms.py

@@ -7,6 +7,7 @@ from django.utils.safestring import mark_safe
 
 from archivebox.config.common import get_config
 from archivebox.core.forms import PluginConfigFormMixin
+from archivebox.core.permissions import PERMISSIONS_CHOICES
 from archivebox.personas.importers import (
     PersonaImportResult,
     PersonaImportSource,
@@ -25,6 +26,12 @@ def _mode_label(title: str, description: str) -> str:
 
 
 class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
+    permissions = forms.ChoiceField(
+        label="Permissions",
+        choices=PERMISSIONS_CHOICES,
+        required=True,
+        help_text="Default visibility for crawls and snapshots that use this persona.",
+    )
     import_mode = forms.ChoiceField(
         required=False,
         initial="none",
@@ -102,6 +109,7 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
 
         self.fields["import_mode"].widget.attrs["class"] = "abx-import-mode"
         self.fields["import_discovered_profile"].widget.attrs["class"] = "abx-profile-picker"
+        self.fields["permissions"].initial = str((self.instance.config or {}).get("PERMISSIONS") or "public").strip().lower()
 
         if self.discovered_profiles:
             self.fields["import_discovered_profile"].choices = [
@@ -131,6 +139,7 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
         manual_config = cleaned_data.get("config") or {}
         if not isinstance(manual_config, dict):
             manual_config = {}
+        manual_config["PERMISSIONS"] = cleaned_data.get("permissions") or "public"
         plugin_config_overrides = self.clean_plugin_config_overrides(get_config())
         cleaned_data["plugin_config"] = plugin_config_overrides
         cleaned_data["config"] = {

archivebox/personas/migrations/0003_persona_permissions.py

@@ -0,0 +1,19 @@
+# Generated by Django 6.0.5 on 2026-05-28 07:25
+
+import django.db.models.fields.json
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('personas', '0002_alter_persona_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='persona',
+            name='permissions',
+            field=models.GeneratedField(db_index=True, db_persist=True, expression=django.db.models.fields.json.KeyTextTransform('PERMISSIONS', 'config'), output_field=models.CharField(max_length=16, null=True)),
+        ),
+    ]

archivebox/personas/models.py

@@ -19,6 +19,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from django.db import models
+from django.db.models.fields.json import KT
 from django.conf import settings
 from django.utils import timezone
 
@@ -83,6 +84,13 @@ class Persona(ModelWithConfig):
     name = models.CharField(max_length=64, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    permissions = models.GeneratedField(
+        expression=KT("config__PERMISSIONS"),
+        output_field=models.CharField(max_length=16, null=True),
+        db_persist=True,
+        db_index=True,
+        editable=False,
+    )
 
     class Meta(ModelWithConfig.Meta):
         app_label = "personas"

archivebox/search/__init__.py

@@ -28,6 +28,11 @@ from archivebox.config.common import get_config
 # Cache discovered backends to avoid repeated filesystem scans
 _search_backends_cache: dict | None = None
 SEARCH_MODES = ("meta", "contents", "deep")
+SEARCH_BACKEND_UI_NAMES = {
+    "rg": "ripgrep",
+    "sonic": "sonic",
+    "fts": "sqlite",
+}
 
 
 @contextmanager
@@ -54,14 +59,72 @@ def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: An
                 os.environ[key] = value
 
 
+def normalize_search_backend_name(backend_name: str | None) -> str:
+    return (backend_name or "").strip().lower().replace("-", "_")
+
+
+def get_search_backend_display_name(backend_name: str) -> str:
+    backend_name = normalize_search_backend_name(backend_name)
+    return next((ui_name for ui_name, canonical_name in SEARCH_BACKEND_UI_NAMES.items() if canonical_name == backend_name), backend_name)
+
+
 def get_default_search_mode(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
-    return "meta" if config.SEARCH_BACKEND_ENGINE == "ripgrep" else "contents"
+    backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
+    backends = get_available_backends()
+    if backend_name in backends:
+        return f"deep:{backend_name}"
+    if "ripgrep" in backends:
+        return "deep:ripgrep"
+    return "contents"
 
 
 def get_search_mode(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    normalized = (search_mode or "").strip().lower()
-    return normalized if normalized in SEARCH_MODES else get_default_search_mode(config=config, **config_kwargs)
+    normalized = (search_mode or "").strip().lower().replace(" ", "")
+    if normalized in SEARCH_MODES:
+        return normalized
+    if ":" in normalized:
+        mode, backend_name = normalized.split(":", 1)
+        backend_name = normalize_search_backend_name(backend_name)
+        if mode == "deep" and backend_name in get_available_backends():
+            return f"{mode}:{backend_name}"
+    return get_default_search_mode(config=config, **config_kwargs)
+
+
+def get_search_mode_base(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    return get_search_mode(search_mode, config=config, **config_kwargs).split(":", 1)[0]
+
+
+def get_search_mode_backend(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str | None:
+    normalized = get_search_mode(search_mode, config=config, **config_kwargs)
+    if ":" not in normalized:
+        return None
+    return normalized.split(":", 1)[1]
+
+
+def get_search_mode_options(config: dict[str, Any] | None = None, **config_kwargs: Any) -> list[dict[str, str]]:
+    config = config or get_config(**config_kwargs)
+    backends = get_available_backends()
+    configured_backend = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
+    backend_names = [
+        *([configured_backend] if configured_backend in backends else []),
+        *(name for name in sorted(backends) if name != configured_backend),
+    ]
+    options = [
+        {"value": "meta", "label": "meta"},
+        {"value": "contents", "label": "contents"},
+    ]
+    if backend_names:
+        options.extend(
+            {
+                "value": f"deep:{backend_name}",
+                "label": f"deep: {get_search_backend_display_name(backend_name)}",
+            }
+            for backend_name in backend_names
+        )
+    else:
+        options.append({"value": "deep", "label": "deep"})
+    return options
 
 
 def prioritize_metadata_matches(
@@ -127,7 +190,7 @@ def get_backend(config: dict[str, Any] | None = None, **config_kwargs: Any) -> A
     Falls back to 'ripgrep' if configured backend is not found.
     """
     config = config or get_config(**config_kwargs)
-    backend_name = config.SEARCH_BACKEND_ENGINE
+    backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
     backends = get_available_backends()
 
     if backend_name in backends:
@@ -158,25 +221,50 @@ def query_search_index(query: str, search_mode: str | None = None, config: dict[
         return Snapshot.objects.none()
 
     search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config)
-    if search_mode == "meta":
+    search_mode_base = get_search_mode_base(search_mode, config=config)
+    if search_mode_base == "meta":
         return Snapshot.objects.none()
+    from archivebox.services.supervision_service import ensure_daemon_stack
+
+    ensure_daemon_stack(reason="search query")
+    snapshot_pks = list(iter_query_search_ids(query, search_mode=search_mode, config=config))
+    return Snapshot.objects.filter(pk__in=list(dict.fromkeys(snapshot_pks)))
+
+
+def iter_query_search_ids(query: str, search_mode: str | None = None, config: dict[str, Any] | None = None, **config_kwargs: Any):
+    """Yield snapshot IDs from configured search backends as soon as each backend produces them."""
+    config = config or get_config(**config_kwargs)
+    if not config.USE_SEARCHING_BACKEND:
+        return
+
+    search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config)
+    search_mode_base = get_search_mode_base(search_mode, config=config)
+    forced_backend = get_search_mode_backend(search_mode, config=config)
+    if search_mode_base == "meta":
+        return
 
     backends = get_available_backends()
-    backend_names: list[str] = []
-    configured_backend = config.SEARCH_BACKEND_ENGINE
-    if search_mode == "deep":
-        if "ripgrep" in backends:
-            backend_names.append("ripgrep")
-        backend_names.extend(name for name in backends if name != "ripgrep")
+    configured_backend = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
+    if forced_backend:
+        if forced_backend not in backends:
+            raise RuntimeError(
+                f'Search backend "{forced_backend}" not found. Available backends: {list(backends) or "none"}',
+            )
+        backend_names = [forced_backend]
+    elif search_mode_base == "deep":
+        backend_names = [
+            *([configured_backend] if configured_backend in backends and configured_backend != "ripgrep" else []),
+            *(name for name in backends if name not in {configured_backend, "ripgrep"}),
+            *(["ripgrep"] if "ripgrep" in backends else []),
+        ]
     elif configured_backend in backends:
-        backend_names.append(configured_backend)
+        backend_names = [configured_backend]
     elif "ripgrep" in backends:
-        backend_names.append("ripgrep")
+        backend_names = ["ripgrep"]
     else:
         get_backend()
-        return Snapshot.objects.none()
+        return
 
-    snapshot_pks: list[str] = []
     errors: list[Exception] = []
     successful_backends = 0
     try:
@@ -184,14 +272,16 @@ def query_search_index(query: str, search_mode: str | None = None, config: dict[
             backend = backends[backend_name]
             try:
                 with search_backend_env(config=config):
-                    if backend_name == "ripgrep":
-                        snapshot_pks.extend(backend.search(query, search_mode=search_mode))
+                    if hasattr(backend, "iter_search"):
+                        yield from backend.iter_search(query, search_mode=search_mode_base)
+                    elif backend_name == "ripgrep":
+                        yield from backend.search(query, search_mode=search_mode_base)
                     else:
-                        snapshot_pks.extend(backend.search(query))
+                        yield from backend.search(query)
                 successful_backends += 1
             except Exception as err:
                 errors.append(err)
-                if search_mode != "deep":
+                if search_mode_base != "deep" or forced_backend:
                     raise
     except Exception as err:
         stderr()
@@ -201,9 +291,8 @@ def query_search_index(query: str, search_mode: str | None = None, config: dict[
         )
         raise
     else:
-        if not successful_backends and errors and search_mode == "deep":
+        if not successful_backends and errors and search_mode_base == "deep":
             raise errors[0]
-        return Snapshot.objects.filter(pk__in=list(dict.fromkeys(snapshot_pks)))
 
 
 @enforce_types

archivebox/search/admin.py

@@ -1,16 +1,69 @@
 __package__ = "archivebox.search"
 
-from django.contrib import messages
-from django.contrib import admin
-from django.contrib.admin.views.main import ChangeList, ORDER_VAR
+import hashlib
+import json
 
-from archivebox.search import get_default_search_mode, get_search_mode, prioritize_metadata_matches, query_search_index
+from django.contrib import admin
+from django.contrib.admin.views.main import ChangeList
+from django.core.cache import cache
+
+from archivebox.search import (
+    get_search_backend_display_name,
+    get_default_search_mode,
+    get_search_mode,
+    get_search_mode_backend,
+    get_search_mode_base,
+    get_search_mode_options,
+    query_search_index,
+)
+
+
+SEARCH_RESULT_CACHE_TTL = 60
+
+
+def get_admin_search_cache_key(request, url: str | None = None) -> str:
+    # Search streams publish IDs for one exact changelist URL. Keeping the URL
+    # whole makes sidebar filters, ordering, and user scope part of the key.
+    payload = json.dumps(
+        {
+            "user": str(request.user.pk or "anon"),
+            "url": url or request.get_full_path(),
+        },
+        sort_keys=True,
+    )
+    return f"abx:admin-search:{hashlib.sha256(payload.encode()).hexdigest()}"
+
+
+def get_cached_admin_search_ids(request) -> list[str] | None:
+    cached = cache.get(get_admin_search_cache_key(request))
+    if isinstance(cached, dict):
+        return cached.get("ids") or []
+    return None
 
 
 class SearchResultsChangeList(ChangeList):
+    def __init__(self, request, *args, **kwargs):
+        self.search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
+        self.search_mode_backend = get_search_mode_backend(self.search_mode, config=getattr(request, "archivebox_config", None))
+        self.search_backend_label = get_search_backend_display_name(self.search_mode_backend) if self.search_mode_backend else ""
+        super().__init__(request, *args, **kwargs)
+        self.embedded_changelist = request.GET.get("_embedded") == "crawl"
+
+    def get_results(self, request):
+        super().get_results(request)
+        self.show_search_index_hint = bool(
+            self.opts.model_name == "snapshot"
+            and self.query
+            and self.result_count == 0
+            and get_search_mode_base(self.search_mode, config=getattr(request, "archivebox_config", None)) == "deep"
+            and self.search_mode_backend
+        )
+
     def get_filters_params(self, params=None):
         lookup_params = super().get_filters_params(params)
         lookup_params.pop("search_mode", None)
+        lookup_params.pop("_embedded", None)
+        lookup_params.pop("per_page", None)
         return lookup_params
 
 
@@ -24,37 +77,36 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
         request = getattr(self, "request", None)
         return get_default_search_mode(config=getattr(request, "archivebox_config", None))
 
+    def get_search_mode_options(self):
+        request = getattr(self, "request", None)
+        return get_search_mode_options(config=getattr(request, "archivebox_config", None))
+
     def get_search_results(self, request, queryset, search_term: str):
         """Enhances the search queryset with results from the search backend"""
 
-        qs, use_distinct = super().get_search_results(request, queryset, search_term)
-
         search_term = search_term.strip()
         if not search_term:
-            return qs, use_distinct
-        search_mode = get_search_mode(request.GET.get("search_mode"))
-        if search_mode == "meta":
-            return qs, use_distinct
-        try:
-            deep_qsearch = None
-            if search_mode == "deep":
-                qsearch = query_search_index(search_term, search_mode="contents")
-                deep_qsearch = query_search_index(search_term, search_mode="deep")
-            else:
-                qsearch = query_search_index(search_term, search_mode=search_mode)
-            qs = prioritize_metadata_matches(
-                queryset,
-                qs,
-                qsearch,
-                deep_queryset=deep_qsearch,
-                ordering=() if not request.GET.get(ORDER_VAR) else None,
-            )
-        except Exception as err:
-            print(f"[!] Error while using search backend: {err.__class__.__name__} {err}")
-            messages.add_message(
-                request,
-                messages.WARNING,
-                f"Error from the search backend, only showing results from default admin search fields - Error: {err}",
-            )
+            return super().get_search_results(request, queryset, search_term)
+        search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
+        if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl":
+            cached_ids = get_cached_admin_search_ids(request)
+            if cached_ids is not None:
+                return queryset.filter(pk__in=cached_ids) if cached_ids else queryset.none(), False
+            return queryset.none(), False
 
-        return qs, True
+        if get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None)) == "meta":
+            qs, use_distinct = super().get_search_results(request, queryset, search_term)
+            return qs, use_distinct
+        if request.GET.get("_embedded") == "crawl":
+            try:
+                return queryset.filter(
+                    pk__in=query_search_index(
+                        search_term,
+                        search_mode=search_mode,
+                        config=getattr(request, "archivebox_config", None),
+                    ).values("pk"),
+                ), False
+            except Exception as err:
+                print(f"[!] Error while using search backend: {err.__class__.__name__} {err}")
+                return queryset.none(), False
+        return queryset.none(), False

archivebox/search/sonic_daemon.py

@@ -21,6 +21,9 @@ def register_sonic_daemon_event_handler(bus) -> None:
         from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
 
         daemon_event = SonicDaemonStartEvent.from_record(record)
+        if is_port_listening(daemon_event.host, daemon_event.port):
+            return
+
         supervisor = get_existing_supervisord_process()
         if supervisor is None:
             raise RuntimeError("Sonic search backend is required, but ArchiveBox supervisord is not running")
@@ -32,7 +35,6 @@ def register_sonic_daemon_event_handler(bus) -> None:
             raise RuntimeError(
                 f"Sonic search backend worker is {worker.get('statename')}: {worker.get('description')}",
             )
-        if not is_port_listening(daemon_event.host, daemon_event.port):
-            raise RuntimeError(f"Sonic search backend is not listening at {daemon_event.url}")
+        raise RuntimeError(f"Sonic search backend is not listening at {daemon_event.url}")
 
     bus.on(ProcessStdoutEvent, on_ProcessStdoutEvent__require_sonic_daemon)

archivebox/services/archive_result_service.py

@@ -17,7 +17,7 @@ from .process_service import parse_event_datetime
 
 
 def _collect_output_metadata(plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
-    exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid", "cmd.sh"}
+    exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"}
     output_files: dict[str, dict] = {}
     mime_sizes: dict[str, int] = defaultdict(int)
     total_size = 0

archivebox/services/crawl_service.py

@@ -22,6 +22,8 @@ class CrawlService(BaseService):
         from archivebox.crawls.models import Crawl
 
         crawl = await Crawl.objects.aget(id=self.crawl_id)
+        if crawl.is_paused:
+            return
         if crawl.status != Crawl.StatusChoices.SEALED:
             crawl.status = Crawl.StatusChoices.STARTED
         crawl.retry_at = None
@@ -31,6 +33,8 @@ class CrawlService(BaseService):
         from archivebox.crawls.models import Crawl
 
         crawl = await Crawl.objects.aget(id=self.crawl_id)
+        if crawl.is_paused:
+            return
         if crawl.status != Crawl.StatusChoices.SEALED:
             crawl.status = Crawl.StatusChoices.STARTED
         crawl.retry_at = None
@@ -41,8 +45,10 @@ class CrawlService(BaseService):
         from archivebox.core.models import Snapshot
 
         crawl = await Crawl.objects.aget(id=self.crawl_id)
+        if crawl.is_paused:
+            return
         is_finished = not await crawl.snapshot_set.filter(
-            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
+            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED],
         ).aexists()
         if is_finished:
             crawl.status = Crawl.StatusChoices.SEALED
@@ -59,8 +65,10 @@ class CrawlService(BaseService):
         from archivebox.core.models import Snapshot
 
         crawl = await Crawl.objects.aget(id=self.crawl_id)
+        if crawl.is_paused:
+            return
         is_finished = not await crawl.snapshot_set.filter(
-            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
+            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED],
         ).aexists()
         if not is_finished:
             if crawl.status != Crawl.StatusChoices.SEALED:

archivebox/services/runner.py

@@ -11,7 +11,6 @@ import sys
 import time
 from collections.abc import Mapping
 from contextlib import nullcontext
-from datetime import timedelta
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any
@@ -50,11 +49,13 @@ from abx_dl.orchestrator import (
 from abx_dl.services.process_service import ProcessService as HookProcessService
 from abx_dl.services.binary_service import BinaryService as HookBinaryService
 from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService
+from abx_dl.cli import LiveBusUI
 from abxbus import BaseEvent
 from abxbus.event_bus import EventBus, get_current_event, in_handler_context
 from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError
 
 from archivebox.config.configset import BaseConfigSet
+from archivebox.core.recovery_util import recover_orchestrator_state
 from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
 
 from .archive_result_service import ArchiveResultService
@@ -64,7 +65,6 @@ from .machine_service import MachineService
 from .process_service import ProcessService as PersistedProcessService
 from .snapshot_service import SnapshotService
 from .tag_service import TagService
-from .live_ui import LiveBusUI
 
 
 def _bus_name(prefix: str, identifier: str) -> str:
@@ -182,6 +182,7 @@ class CrawlRunner:
         snapshot_ids: list[str] | None = None,
         selected_plugins: list[str] | None = None,
         process_discovered_snapshots_inline: bool = True,
+        show_progress: bool = True,
     ):
         self.crawl = crawl
         self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0)
@@ -194,6 +195,7 @@ class CrawlRunner:
         CrawlService(self.bus, crawl_id=str(crawl.id))
         MachineService(self.bus)
         self.process_discovered_snapshots_inline = process_discovered_snapshots_inline
+        self.show_progress = show_progress
 
         async def ignore_snapshot(_snapshot_id: str) -> None:
             return None
@@ -224,7 +226,7 @@ class CrawlRunner:
     def _install_signal_handlers(self) -> list[tuple[signal.Signals, Any, bool]]:
         loop = asyncio.get_running_loop()
         installed: list[tuple[signal.Signals, Any, bool]] = []
-        for sig in (signal.SIGINT, signal.SIGTERM):
+        for sig in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM):
             previous = signal.getsignal(sig)
 
             def request_abort(sig=sig) -> None:
@@ -273,6 +275,12 @@ class CrawlRunner:
             return True
         return await Crawl.objects.filter(id=self.crawl.id, status=Crawl.StatusChoices.SEALED).aexists()
 
+    async def crawl_is_paused(self) -> bool:
+        from archivebox.crawls.models import Crawl
+
+        crawl = await Crawl.objects.only("status").aget(id=self.crawl.id)
+        return crawl.is_paused
+
     async def watch_for_cancelled_crawl(self, parent_event: BaseEvent, *, poll_interval: float = 1.0) -> None:
         while True:
             await asyncio.sleep(poll_interval)
@@ -285,6 +293,10 @@ class CrawlRunner:
     def runtime_plugins(self) -> dict[str, Plugin]:
         return filter_plugins(self.plugins, self.selected_plugins, include_providers=True) if self.selected_plugins else self.plugins
 
+    @property
+    def allow_paused_snapshot_maintenance(self) -> bool:
+        return bool(self.initial_snapshot_ids and self.selected_plugins)
+
     async def run(self) -> None:
         heartbeat = CrawlHeartbeat(
             Path(self.crawl_output_dir),
@@ -293,6 +305,7 @@ class CrawlRunner:
         )
         installed_signal_handlers = self._install_signal_handlers()
         root_snapshot_id: str | None = None
+        bus_destroyed = False
         try:
             self._run_task = asyncio.current_task()
             snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)()
@@ -301,24 +314,37 @@ class CrawlRunner:
             self.snapshot_semaphore = asyncio.Semaphore(max_concurrent_snapshots)
             live_ui = self._create_live_ui()
             with live_ui if live_ui is not None else nullcontext():
-                await heartbeat.start()
-                await _emit_machine_config(
-                    self.bus,
-                    config={
-                        **self.base_config,
-                        "ABX_RUNTIME": "archivebox",
-                    },
-                    derived_config=self.derived_config,
-                )
-                if snapshot_ids:
-                    root_snapshot_id = snapshot_ids[0]
-                    await self.run_crawl(root_snapshot_id, snapshot_ids)
+                try:
+                    await heartbeat.start()
+                    await _emit_machine_config(
+                        self.bus,
+                        config={
+                            **self.base_config,
+                            "ABX_RUNTIME": "archivebox",
+                        },
+                        derived_config=self.derived_config,
+                    )
+                    if snapshot_ids:
+                        root_snapshot_id = snapshot_ids[0]
+                        await self.run_crawl(root_snapshot_id, snapshot_ids)
+                finally:
+                    self._run_task = None
+                    self._restore_signal_handlers(installed_signal_handlers)
+                    await heartbeat.stop()
+                    await self.stop_snapshot_tasks()
+                    try:
+                        if not self._skip_wait_until_idle:
+                            await self.bus.wait_until_idle(timeout=30.0)
+                    finally:
+                        await self.bus.destroy(clear=False)
+                        bus_destroyed = True
         finally:
-            self._run_task = None
-            self._restore_signal_handlers(installed_signal_handlers)
-            await heartbeat.stop()
-            if not self._skip_wait_until_idle:
-                await self.bus.wait_until_idle(timeout=30.0)
+            if not bus_destroyed:
+                self._run_task = None
+                self._restore_signal_handlers(installed_signal_handlers)
+                await heartbeat.stop()
+                await self.stop_snapshot_tasks()
+                await self.bus.destroy(clear=False)
             if self._live_stream is not None:
                 try:
                     self._live_stream.close()
@@ -330,6 +356,8 @@ class CrawlRunner:
     async def enqueue_snapshot(self, snapshot_id: str, crawl_start_event: CrawlStartEvent | None = None) -> None:
         if await self.crawl_is_cancelled():
             return
+        if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance:
+            return
         task = self.snapshot_tasks.get(snapshot_id)
         if task is not None and not task.done():
             return
@@ -342,6 +370,15 @@ class CrawlRunner:
             task = asyncio.create_task(self.run_snapshot(snapshot_id), context=_runner_task_context())
         self.snapshot_tasks[snapshot_id] = task
 
+    async def stop_snapshot_tasks(self) -> None:
+        if not self.snapshot_tasks:
+            return
+        done, pending = await asyncio.wait(list(self.snapshot_tasks.values()), timeout=5.0)
+        for task in pending:
+            task.cancel()
+        await asyncio.gather(*done, *pending, return_exceptions=True)
+        self.snapshot_tasks.clear()
+
     async def wait_for_snapshot_tasks(self) -> None:
         task_errors: list[Exception] = []
         stop_scheduling = False
@@ -388,7 +425,10 @@ class CrawlRunner:
                 except Exception as err:
                     task_errors.append(err)
                     stop_scheduling = True
-            if self.snapshot_tasks and await self.crawl_is_cancelled():
+            if self.snapshot_tasks and (
+                await self.crawl_is_cancelled()
+                or (await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance)
+            ):
                 stop_scheduling = True
             if not stop_scheduling:
                 await self.enqueue_pending_snapshots_from_projection()
@@ -422,6 +462,8 @@ class CrawlRunner:
             return
         if await self.crawl_is_cancelled():
             return
+        if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance:
+            return
 
         await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)()
         config = await sync_to_async(lambda: get_config(crawl=self.crawl, include_machine=False), thread_sensitive=True)()
@@ -433,7 +475,7 @@ class CrawlRunner:
             return
         pending_snapshot_ids = await sync_to_async(
             lambda: list(
-                self.crawl.snapshot_set.exclude(status=Snapshot.StatusChoices.SEALED)
+                self.crawl.snapshot_set.filter(status=Snapshot.StatusChoices.QUEUED)
                 .exclude(id__in=active_snapshot_ids)
                 .filter(retry_at__lte=timezone.now())
                 .order_by("depth", "created_at")
@@ -447,6 +489,7 @@ class CrawlRunner:
 
     def load_run_state(self) -> list[str]:
         from archivebox.config.common import get_config
+        from archivebox.core.models import Snapshot
         from archivebox.hooks import discover_hooks
         from archivebox.machine.models import Machine, NetworkInterface, Process, _sanitize_machine_config
 
@@ -480,22 +523,34 @@ class CrawlRunner:
                 ),
             )
         if self.initial_snapshot_ids:
+            # Direct snapshot maintenance paths are allowed to name paused
+            # snapshots explicitly. The runner still requires selected_plugins
+            # later, so this does not restart the crawl lifecycle.
             return [str(snapshot_id) for snapshot_id in self.initial_snapshot_ids]
+        if self.crawl.is_paused:
+            return []
         pending_snapshots = list(
-            self.crawl.snapshot_set.exclude(status="sealed").order_by("depth", "created_at"),
+            self.crawl.snapshot_set.filter(status=Snapshot.StatusChoices.QUEUED)
+            .filter(retry_at__lte=timezone.now())
+            .order_by("depth", "created_at"),
         )
         if pending_snapshots:
             return [str(snapshot.id) for snapshot in pending_snapshots]
+        if self.crawl.snapshot_set.exclude(status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED]).exists():
+            return []
         created = self.crawl.create_snapshots_from_urls()
         snapshots = created or list(self.crawl.snapshot_set.filter(depth=0).order_by("created_at"))
         return [str(snapshot.id) for snapshot in snapshots]
 
     def finalize_run_state(self) -> None:
         from archivebox.crawls.models import Crawl
+        from archivebox.core.models import Snapshot
 
         if self.persona:
             self.persona.cleanup_runtime_for_crawl(self.crawl)
         crawl = Crawl.objects.get(id=self.crawl.id)
+        if crawl.is_paused:
+            return
         if crawl.is_finished():
             if crawl.status != Crawl.StatusChoices.SEALED:
                 if crawl.status == Crawl.StatusChoices.STARTED:
@@ -506,23 +561,33 @@ class CrawlRunner:
                         retry_at=None,
                     )
             return
+        active_snapshots = crawl.snapshot_set.filter(
+            status__in=[
+                Snapshot.StatusChoices.QUEUED,
+                Snapshot.StatusChoices.STARTED,
+                Snapshot.StatusChoices.PAUSED,
+            ],
+        )
+        next_snapshot_retry = active_snapshots.order_by("retry_at", "created_at").values_list("retry_at", flat=True).first()
         if crawl.status == Crawl.StatusChoices.SEALED:
             crawl.update_and_requeue(
                 status=Crawl.StatusChoices.QUEUED,
-                retry_at=timezone.now(),
+                retry_at=next_snapshot_retry or timezone.now(),
             )
             return
         elif crawl.status != Crawl.StatusChoices.STARTED:
             crawl.update_and_requeue(
                 status=Crawl.StatusChoices.STARTED,
-                retry_at=crawl.retry_at or timezone.now(),
+                retry_at=crawl.retry_at or next_snapshot_retry or timezone.now(),
             )
             return
         crawl.update_and_requeue(
-            retry_at=crawl.retry_at or timezone.now(),
+            retry_at=crawl.retry_at or next_snapshot_retry or timezone.now(),
         )
 
     def _create_live_ui(self) -> LiveBusUI | None:
+        if not self.show_progress:
+            return None
         stdout_is_tty = sys.stdout.isatty()
         stderr_is_tty = sys.stderr.isatty()
         interactive_tty = stdout_is_tty or stderr_is_tty
@@ -606,6 +671,8 @@ class CrawlRunner:
         from archivebox.hooks import collect_urls_from_plugins
 
         await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)()
+        if self.crawl.is_paused and not self.allow_paused_snapshot_maintenance:
+            return
         if int(snapshot_payload["depth"]) >= self.crawl.max_depth:
             return
 
@@ -623,7 +690,7 @@ class CrawlRunner:
             lambda: get_config(crawl=self.crawl, snapshot=parent_snapshot, include_machine=False),
             thread_sensitive=True,
         )()
-        if CrawlLimitState.from_config(config).get_stop_reason() == "crawl_max_size":
+        if CrawlLimitState.from_config(config).get_stop_reason() in ("crawl_max_size", "crawl_timeout"):
             return
 
         await sync_to_async(self.crawl.create_discovered_snapshots, thread_sensitive=True)(
@@ -650,7 +717,7 @@ class CrawlRunner:
         crawl_setup_phase_timeout = compute_phase_timeout(setup_hooks, config)
         install_phase_timeout = compute_install_phase_timeout(get_install_plugins(plugins), config)
         snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")]
-        max_snapshot_count = max(1, int(self.crawl.max_urls or len(snapshot_ids) or 1))
+        max_snapshot_count = max(1, int(config.get("CRAWL_MAX_URLS") or len(snapshot_ids) or 1))
         snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config) * max_snapshot_count
         crawl_cleanup_phase_timeout = crawl_setup_phase_timeout
         crawl_lifecycle_timeout = (
@@ -723,6 +790,8 @@ class CrawlRunner:
                     break
                 if await self.crawl_is_cancelled():
                     break
+                if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance:
+                    break
                 await self.enqueue_snapshot(snapshot_id)
             await self.wait_for_snapshot_tasks()
 
@@ -732,7 +801,9 @@ class CrawlRunner:
             cancel_watcher = asyncio.create_task(self.watch_for_cancelled_crawl(event))
             try:
                 try:
-                    if not await self.crawl_is_cancelled():
+                    if not await self.crawl_is_cancelled() and (
+                        not await self.crawl_is_paused() or self.allow_paused_snapshot_maintenance
+                    ):
                         await _run_event_now(
                             event.emit(
                                 CrawlSetupEvent(
@@ -745,7 +816,9 @@ class CrawlRunner:
                             ),
                             crawl_setup_phase_timeout,
                         )
-                    if not await self.crawl_is_cancelled():
+                    if not await self.crawl_is_cancelled() and (
+                        not await self.crawl_is_paused() or self.allow_paused_snapshot_maintenance
+                    ):
                         crawl_start_event = CrawlStartEvent(
                             url=snapshot["url"],
                             snapshot_id=snapshot["id"],
@@ -838,9 +911,13 @@ class CrawlRunner:
             if not isinstance(crawl_start_event, CrawlStartEvent):
                 raise RuntimeError("Snapshot events must be emitted from a CrawlStartEvent handler")
             snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(snapshot_id)
-            if snapshot["status"] == "sealed":
+            if snapshot["status"] == "sealed" and not self.selected_plugins:
+                await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id)
                 return
-            if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() == "crawl_max_size":
+            if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() in (
+                "crawl_max_size",
+                "crawl_timeout",
+            ):
                 await sync_to_async(self.seal_snapshot_due_to_limit, thread_sensitive=True)(snapshot_id)
                 return
             config = _normalize_runtime_config(snapshot["config"])
@@ -889,6 +966,9 @@ class CrawlRunner:
                     raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
                 await completed_snapshot.wait(timeout=snapshot_phase_timeout)
                 await completed_snapshot.event_results_list()
+                if snapshot["status"] == "sealed":
+                    await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id)
+                    return
                 await self.enqueue_discovered_snapshots_from_outputs(snapshot)
                 await sync_to_async(
                     lambda: (
@@ -898,6 +978,7 @@ class CrawlRunner:
                             status__in=[
                                 self.crawl.snapshot_set.model.StatusChoices.QUEUED,
                                 self.crawl.snapshot_set.model.StatusChoices.STARTED,
+                                self.crawl.snapshot_set.model.StatusChoices.PAUSED,
                             ],
                         ).exists()
                         else None
@@ -928,6 +1009,7 @@ def run_crawl(
     snapshot_ids: list[str] | None = None,
     selected_plugins: list[str] | None = None,
     process_discovered_snapshots_inline: bool = True,
+    show_progress: bool = True,
 ) -> None:
     from archivebox.crawls.models import Crawl
 
@@ -938,6 +1020,7 @@ def run_crawl(
             snapshot_ids=snapshot_ids,
             selected_plugins=selected_plugins,
             process_discovered_snapshots_inline=process_discovered_snapshots_inline,
+            show_progress=show_progress,
         ).run(),
     )
 
@@ -994,6 +1077,148 @@ def run_binary(binary_id: str) -> None:
     asyncio.run(_run_binary(binary_id))
 
 
+def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
+    from archivebox.core.models import ArchiveResult
+
+    queued_plugins = sorted(
+        set(
+            ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            )
+            .exclude(plugin="")
+            .values_list("plugin", flat=True),
+        ),
+    )
+    if queued_plugins:
+        return queued_plugins
+    return None
+
+
+def run_snapshot_maintenance(snapshot_id: str) -> bool:
+    from archivebox.core.models import ArchiveResult, Snapshot
+
+    snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
+    if snapshot is None:
+        return False
+    if snapshot.archiveresult_set.filter(status=ArchiveResult.StatusChoices.QUEUED).exists():
+        return False
+
+    # retry_at is the universal "tick me" signal. For already-sealed snapshots,
+    # a tick with no queued ArchiveResults is maintenance-only: run normal
+    # save/write side effects like lazy fs migration/json rewriting, then clear
+    # retry_at. Paused snapshots do not reach this helper while search/index
+    # plugin rows are queued; run_due_snapshot restores their paused scheduler
+    # marker after the targeted plugin rows finish.
+    snapshot.retry_at = None
+    snapshot.save(update_fields=["retry_at", "modified_at"])
+    snapshot.write_index_jsonl()
+    snapshot.write_json_details()
+    snapshot.write_html_details()
+    return True
+
+
+def run_due_crawl(crawl, *, lock_seconds: int) -> bool:
+    if crawl.is_paused:
+        return True
+    if crawl.status in (crawl.StatusChoices.QUEUED, crawl.StatusChoices.STARTED):
+        from archivebox.core.models import Snapshot
+
+        snapshot_count = crawl.snapshot_set.count()
+        due_nonsealed_snapshots = (
+            crawl.snapshot_set.filter(status=Snapshot.StatusChoices.QUEUED, retry_at__lte=timezone.now()).exists()
+        )
+        if snapshot_count and not due_nonsealed_snapshots:
+            crawl.retry_at = None
+            crawl.save(update_fields=["retry_at", "modified_at"])
+            return True
+        if not crawl.claim_processing_lock(lock_seconds=lock_seconds):
+            return False
+        run_crawl(str(crawl.id), process_discovered_snapshots_inline=True)
+        return True
+
+    if crawl.status == crawl.StatusChoices.SEALED:
+        crawl.retry_at = None
+        crawl.save(update_fields=["retry_at", "modified_at"])
+        return True
+
+    crawl.retry_at = None
+    crawl.save(update_fields=["retry_at", "modified_at"])
+    return True
+
+
+def run_due_snapshot(snapshot, *, lock_seconds: int) -> bool:
+    from archivebox.core.models import Snapshot
+
+    if snapshot.is_paused:
+        selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
+        if not selected_plugins:
+            # Paused is a real lifecycle state; retry_at=MAX is only the
+            # orchestrator selection marker. If a direct maintenance/update
+            # command bumps retry_at on a paused snapshot but there are no
+            # targeted ArchiveResult rows to run, restore the scheduler marker
+            # without changing status.
+            snapshot.restore_paused_scheduler_marker()
+            return True
+        if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
+            return False
+        try:
+            # Explicit maintenance, e.g. `archivebox update --index-only`, may
+            # need to run search/index hooks for a paused snapshot. That should
+            # not resume the crawl or make unrelated queued work runnable, so
+            # selected_plugins is required and the paused state is restored in
+            # the finally block below.
+            run_crawl(
+                str(snapshot.crawl_id),
+                snapshot_ids=[str(snapshot.id)],
+                selected_plugins=selected_plugins,
+                process_discovered_snapshots_inline=True,
+            )
+        finally:
+            # Targeted plugin rows can complete while the Snapshot remains
+            # paused. Put retry_at back at MAX so the orchestrator leaves the
+            # paused lifecycle alone until an explicit resume transition.
+            snapshot.restore_paused_scheduler_marker()
+        return True
+    if snapshot.status == Snapshot.StatusChoices.SEALED:
+        if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
+            return False
+        snapshot.refresh_from_db()
+        selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
+        if selected_plugins:
+            run_crawl(
+                str(snapshot.crawl_id),
+                snapshot_ids=[str(snapshot.id)],
+                selected_plugins=selected_plugins,
+                process_discovered_snapshots_inline=True,
+            )
+            return True
+        return run_snapshot_maintenance(str(snapshot.id))
+
+    if not snapshot.claim_processing_lock(lock_seconds=lock_seconds):
+        return False
+    run_crawl(
+        str(snapshot.crawl_id),
+        snapshot_ids=[str(snapshot.id)],
+        selected_plugins=queued_plugins_for_snapshot(str(snapshot.id)),
+        process_discovered_snapshots_inline=True,
+    )
+    return True
+
+
+def run_due_binary(binary, *, lock_seconds: int) -> bool:
+    binary_name = str(binary.name or "")
+    binary_path = Path(binary_name).expanduser()
+    if (binary_path.is_absolute() or binary_name.startswith("~")) and not binary_path.exists():
+        binary.retry_at = None
+        binary.save(update_fields=["retry_at", "modified_at"])
+        return True
+    if not binary.claim_processing_lock(lock_seconds=lock_seconds):
+        return False
+    run_binary(str(binary.id))
+    return True
+
+
 async def _run_install(plugin_names: list[str] | None = None) -> None:
     from archivebox.config.common import get_config
     from archivebox.machine.models import Machine, _sanitize_machine_config
@@ -1012,6 +1237,7 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
     MachineService(bus)
     await _emit_machine_config(bus, config=config, derived_config=derived_config)
     live_stream = None
+    bus_destroyed = False
 
     try:
         selected_plugins = filter_plugins(plugins, list(plugin_names), include_providers=True) if plugin_names else plugins
@@ -1068,20 +1294,28 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
                     plugins_label=plugins_label,
                 )
             with live_ui if live_ui is not None else nullcontext():
-                await abx_install_plugins(
-                    plugin_names=plugin_names,
-                    plugins=plugins,
-                    output_dir=output_dir,
-                    config_overrides=config,
-                    derived_config_overrides=derived_config,
-                    emit_jsonl=False,
-                    bus=bus,
-                    MachineService=None,
-                )
+                try:
+                    await abx_install_plugins(
+                        plugin_names=plugin_names,
+                        plugins=plugins,
+                        output_dir=output_dir,
+                        config_overrides=config,
+                        derived_config_overrides=derived_config,
+                        emit_jsonl=False,
+                        bus=bus,
+                        MachineService=None,
+                    )
+                finally:
+                    try:
+                        await bus.wait_until_idle()
+                    finally:
+                        await bus.destroy(clear=False)
+                        bus_destroyed = True
             if live_ui is not None:
                 live_ui.print_summary(output_dir=output_dir)
     finally:
-        await bus.wait_until_idle()
+        if not bus_destroyed:
+            await bus.destroy(clear=False)
         try:
             if live_stream is not None:
                 live_stream.close()
@@ -1093,219 +1327,16 @@ def run_install(*, plugin_names: list[str] | None = None) -> None:
     asyncio.run(_run_install(plugin_names=plugin_names))
 
 
-def recover_orphaned_crawls() -> int:
-    from archivebox.crawls.models import Crawl
-    from archivebox.core.models import Snapshot
-    from archivebox.machine.models import Process
-
-    active_crawl_ids: set[str] = set()
-    orphaned_crawls = list(
-        Crawl.objects.filter(
-            status=Crawl.StatusChoices.STARTED,
-            retry_at__isnull=True,
-        ).prefetch_related("snapshot_set"),
-    )
-    running_processes = (
-        Process.get_running()
-        .filter(
-            process_type__in=[
-                Process.TypeChoices.WORKER,
-                Process.TypeChoices.HOOK,
-                Process.TypeChoices.BINARY,
-            ],
-        )
-        .only("pwd")
-    )
-
-    for proc in running_processes:
-        if not proc.pwd:
-            continue
-        proc_pwd = Path(proc.pwd)
-        for crawl in orphaned_crawls:
-            matched_snapshot = None
-            for snapshot in crawl.snapshot_set.all():
-                try:
-                    proc_pwd.relative_to(snapshot.output_dir)
-                    matched_snapshot = snapshot
-                    break
-                except ValueError:
-                    continue
-            if matched_snapshot is not None:
-                active_crawl_ids.add(str(crawl.id))
-                break
-
-    recovered = 0
-    now = timezone.now()
-    for crawl in orphaned_crawls:
-        if str(crawl.id) in active_crawl_ids:
-            continue
-
-        snapshots = list(crawl.snapshot_set.all())
-        if not snapshots or all(snapshot.status == Snapshot.StatusChoices.SEALED for snapshot in snapshots):
-            if crawl.status == Crawl.StatusChoices.STARTED:
-                crawl.sm.seal()
-            else:
-                crawl.update_and_requeue(
-                    status=Crawl.StatusChoices.SEALED,
-                    retry_at=None,
-                )
-            recovered += 1
-            continue
-
-        crawl.update_and_requeue(
-            status=Crawl.StatusChoices.STARTED,
-            retry_at=now,
-        )
-        recovered += 1
-
-    return recovered
-
-
-def recover_orphaned_snapshots() -> int:
-    from archivebox.crawls.models import Crawl
-    from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.machine.models import Process
-    from django.db.models import Exists, OuterRef
-
-    active_snapshot_ids: set[str] = set()
-    now = timezone.now()
-    orphaned_snapshots = list(
-        Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED, retry_at__isnull=True)
-        .select_related("crawl")
-        .prefetch_related("archiveresult_set"),
-    )
-
-    queued_result_snapshot_ids = list(
-        ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED).values_list("snapshot_id", flat=True).distinct(),
-    )
-    if queued_result_snapshot_ids:
-        orphaned_snapshots.extend(
-            snapshot
-            for snapshot in Snapshot.objects.filter(id__in=queued_result_snapshot_ids)
-            .select_related("crawl")
-            .prefetch_related("archiveresult_set")
-            if snapshot.status == Snapshot.StatusChoices.SEALED
-        )
-
-    recent_active_crawl_ids = list(
-        Crawl.objects.filter(
-            status__in=[Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED],
-            modified_at__gte=now - timedelta(days=1),
-        )
-        .order_by("-modified_at")
-        .values_list("id", flat=True)[:1000],
-    )
-    if recent_active_crawl_ids:
-        orphaned_snapshots.extend(
-            Snapshot.objects.filter(
-                crawl_id__in=recent_active_crawl_ids,
-                status=Snapshot.StatusChoices.SEALED,
-                downloaded_at__isnull=False,
-            )
-            .annotate(has_results=Exists(ArchiveResult.objects.filter(snapshot_id=OuterRef("pk"))))
-            .filter(has_results=False)
-            .select_related("crawl")
-            .prefetch_related("archiveresult_set")
-            .order_by("-modified_at")[:1000],
-        )
-    running_processes = (
-        Process.get_running()
-        .filter(
-            process_type__in=[
-                Process.TypeChoices.WORKER,
-                Process.TypeChoices.HOOK,
-                Process.TypeChoices.BINARY,
-            ],
-        )
-        .only("pwd")
-    )
-
-    for proc in running_processes:
-        if not proc.pwd:
-            continue
-        proc_pwd = Path(proc.pwd)
-        for snapshot in orphaned_snapshots:
-            try:
-                proc_pwd.relative_to(snapshot.output_dir)
-                active_snapshot_ids.add(str(snapshot.id))
-                break
-            except ValueError:
-                continue
-
-    recovered = 0
-    for snapshot in orphaned_snapshots:
-        if str(snapshot.id) in active_snapshot_ids:
-            continue
-
-        results = list(snapshot.archiveresult_set.all())
-        if results and all(result.status in ArchiveResult.FINAL_STATES for result in results):
-            snapshot.downloaded_at = snapshot.downloaded_at or now
-            snapshot.save(update_fields=["downloaded_at", "modified_at"])
-            if snapshot.status == Snapshot.StatusChoices.STARTED:
-                snapshot.sm.seal()
-            else:
-                snapshot.update_and_requeue(
-                    status=Snapshot.StatusChoices.SEALED,
-                    retry_at=None,
-                )
-
-            crawl = snapshot.crawl
-            if crawl.is_finished() and crawl.status != Crawl.StatusChoices.SEALED:
-                if crawl.status == Crawl.StatusChoices.STARTED:
-                    crawl.sm.seal()
-                else:
-                    crawl.update_and_requeue(
-                        status=Crawl.StatusChoices.SEALED,
-                        retry_at=None,
-                    )
-            recovered += 1
-            continue
-
-        snapshot.update_and_requeue(
-            status=Snapshot.StatusChoices.QUEUED,
-            retry_at=now,
-        )
-
-        crawl = snapshot.crawl
-        crawl_status = crawl.status if crawl.status == Crawl.StatusChoices.STARTED else Crawl.StatusChoices.QUEUED
-        crawl.update_and_requeue(
-            status=crawl_status,
-            retry_at=now,
-        )
-        recovered += 1
-
-    return recovered
-
-
-def cleanup_orchestrator_state(*, recover: bool = True, include_chrome: bool = False) -> dict[str, int]:
-    from archivebox.machine.models import Process
-
-    cleaned = {
-        "stale_processes": Process.cleanup_stale_running(),
-        "orphaned_processes": Process.cleanup_orphaned_workers(),
-        "orphaned_chrome": Process.cleanup_orphaned_chrome() if include_chrome else 0,
-        "orphaned_snapshots": 0,
-        "orphaned_crawls": 0,
-    }
-    if recover:
-        cleaned["orphaned_snapshots"] = recover_orphaned_snapshots()
-        cleaned["orphaned_crawls"] = recover_orphaned_crawls()
-    return cleaned
-
-
 def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> int:
     from archivebox.crawls.models import Crawl, CrawlSchedule
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.machine.models import Binary, Process
 
+    crawl_claim_lock_seconds = 10
     last_recovery_at = 0.0
     last_retention_at = 0.0
     while True:
         now_monotonic = time.monotonic()
-        if daemon:
-            if now_monotonic - last_recovery_at >= 30.0:
-                cleanup_orchestrator_state()
-                last_recovery_at = now_monotonic
         if now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 model.delete_expired(batch_size=100)
@@ -1317,77 +1348,41 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) ->
                 if schedule.is_due(now):
                     schedule.enqueue(queued_at=now)
 
-        queued_crawls = Crawl.objects.filter(
-            retry_at__lte=timezone.now(),
-            status=Crawl.StatusChoices.QUEUED,
-        )
+        due_crawls = Crawl.objects.filter(retry_at__lte=timezone.now())
         if crawl_id:
-            queued_crawls = queued_crawls.filter(id=crawl_id)
-        queued_crawls = queued_crawls.order_by("retry_at", "created_at")
-
-        queued_crawl = queued_crawls.first()
-        if queued_crawl is not None:
-            if not queued_crawl.claim_processing_lock(lock_seconds=60):
+            due_crawls = due_crawls.filter(id=crawl_id)
+        due_crawl = due_crawls.order_by("retry_at", "created_at").first()
+        if due_crawl is not None:
+            if not run_due_crawl(due_crawl, lock_seconds=crawl_claim_lock_seconds):
                 continue
-            run_crawl(str(queued_crawl.id), process_discovered_snapshots_inline=True)
             continue
 
-        pending = Crawl.objects.filter(
-            retry_at__lte=timezone.now(),
-            status=Crawl.StatusChoices.STARTED,
-        )
+        due_snapshots = Snapshot.objects.filter(retry_at__lte=timezone.now()).select_related("crawl")
         if crawl_id:
-            pending = pending.filter(id=crawl_id)
-        pending = pending.order_by("retry_at", "created_at")
-
-        crawl = pending.first()
-        if crawl is not None:
-            if not crawl.claim_processing_lock(lock_seconds=60):
+            due_snapshots = due_snapshots.filter(crawl_id=crawl_id)
+        due_snapshot = due_snapshots.order_by("retry_at", "created_at").first()
+        if due_snapshot is not None:
+            if not run_due_snapshot(due_snapshot, lock_seconds=60):
                 continue
-            run_crawl(str(crawl.id), process_discovered_snapshots_inline=True)
             continue
 
         if crawl_id is None:
-            snapshot = (
-                Snapshot.objects.filter(retry_at__lte=timezone.now())
-                .exclude(status=Snapshot.StatusChoices.SEALED)
-                .exclude(crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED])
-                .select_related("crawl")
-                .order_by("retry_at", "created_at")
-                .first()
-            )
-            if snapshot is not None:
-                if not snapshot.claim_processing_lock(lock_seconds=60):
-                    continue
-                run_crawl(
-                    str(snapshot.crawl_id),
-                    snapshot_ids=[str(snapshot.id)],
-                    process_discovered_snapshots_inline=True,
-                )
-                continue
-
-        if crawl_id is None:
-            # Standalone binary backlog should not starve queued crawls or snapshots.
-            # Crawl.run() already claims and installs crawl-declared Binary rows as needed.
-            binary = (
+            due_binary = (
                 Binary.objects.filter(retry_at__lte=timezone.now())
                 .exclude(status=Binary.StatusChoices.INSTALLED)
                 .order_by("retry_at", "created_at")
                 .first()
             )
-            if binary is not None:
-                binary_name = str(binary.name or "")
-                binary_path = Path(binary_name).expanduser()
-                if (binary_path.is_absolute() or binary_name.startswith("~")) and not binary_path.exists():
-                    binary.retry_at = None
-                    binary.save(update_fields=["retry_at", "modified_at"])
+            if due_binary is not None:
+                if not run_due_binary(due_binary, lock_seconds=60):
                     continue
-                if not binary.claim_processing_lock(lock_seconds=60):
-                    continue
-                run_binary(str(binary.id))
                 continue
 
         if daemon:
+            now_monotonic = time.monotonic()
+            if now_monotonic - last_recovery_at >= 30.0:
+                recover_orchestrator_state()
+                last_recovery_at = now_monotonic
             time.sleep(2.0)
             continue
         return 0

archivebox/services/snapshot_service.py

@@ -24,6 +24,8 @@ class SnapshotService(BaseService):
         snapshot = await Snapshot.objects.filter(id=event.snapshot_id, crawl_id=self.crawl_id).afirst()
 
         if snapshot is not None:
+            if snapshot.is_paused:
+                return
             if snapshot.status == Snapshot.StatusChoices.QUEUED:
                 await sync_to_async(snapshot.sm.tick, thread_sensitive=True)()
                 await sync_to_async(snapshot.refresh_from_db, thread_sensitive=True)()
@@ -42,7 +44,7 @@ class SnapshotService(BaseService):
             snapshot.downloaded_at = snapshot.downloaded_at or timezone.now()
             await snapshot.asave(update_fields=["downloaded_at", "modified_at"])
             stop_reason = await sync_to_async(self._crawl_limit_stop_reason, thread_sensitive=True)(snapshot.crawl)
-            if snapshot.crawl_id and stop_reason == "crawl_max_size":
+            if snapshot.crawl_id and stop_reason in ("crawl_max_size", "crawl_timeout"):
                 await (
                     Snapshot.objects.filter(
                         crawl_id=snapshot.crawl_id,

archivebox/services/supervision_service.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import time
+from pathlib import Path
+
+from django.utils import timezone
+from rich import print
+
+
+def runtime_stack_owner_types():
+    from archivebox.machine.models import Process
+
+    return (
+        Process.TypeChoices.UPDATE,
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ORCHESTRATOR,
+        Process.TypeChoices.ADD,
+    )
+
+
+def current_command(process_type: str, *, data_dir: str | Path, url: str | None = None):
+    from archivebox.machine.models import Process
+
+    proc = Process.current()
+    proc.mark_running(process_type=process_type, pwd=str(data_dir), url=url, timeout=0)
+    return proc
+
+
+def live_processes(*, process_type: str, data_dir: str | Path, url: str | None = None):
+    from archivebox.machine.models import Machine, Process
+
+    Process.cleanup_stale_running(machine=Machine.current())
+    qs = Process.objects.filter(
+        machine=Machine.current(),
+        process_type=process_type,
+        status=Process.StatusChoices.RUNNING,
+        pwd=str(data_dir),
+    )
+    if url is not None:
+        qs = qs.filter(url=url)
+    return [proc for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50) if proc.is_running]
+
+
+def newest_live_process(*, process_type: str, data_dir: str | Path, url: str | None = None):
+    processes = live_processes(process_type=process_type, data_dir=data_dir, url=url)
+    return processes[0] if processes else None
+
+
+def command_is_newest(command, *, process_type: str, data_dir: str | Path, url: str | None = None) -> bool:
+    leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
+    return bool(leader and leader.id == command.id)
+
+
+def runtime_stack_owner(*, data_dir: str | Path):
+    from archivebox.machine.models import Machine, Process
+
+    Process.cleanup_stale_running(machine=Machine.current())
+    base_qs = Process.objects.filter(
+        machine=Machine.current(),
+        status=Process.StatusChoices.RUNNING,
+        pwd=str(data_dir),
+        process_type__in=runtime_stack_owner_types(),
+    )
+    for process_types in (
+        (Process.TypeChoices.UPDATE,),
+        (Process.TypeChoices.SERVER, Process.TypeChoices.ADD),
+        (Process.TypeChoices.ORCHESTRATOR,),
+    ):
+        qs = base_qs.filter(process_type__in=process_types)
+        for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+            if proc.is_running:
+                return proc
+    return None
+
+
+def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
+    owner = runtime_stack_owner(data_dir=data_dir)
+    return bool(owner and owner.id == command.id)
+
+
+def ensure_daemon_stack(*, reason: str = ""):
+    from archivebox.config.common import get_config
+    from archivebox.workers.supervisord_util import (
+        get_existing_supervisord_process,
+        get_or_create_supervisord_process,
+        get_sonic_supervisord_worker_from_plugin,
+        get_worker,
+        start_worker,
+    )
+
+    config = get_config()
+    sonic_worker = get_sonic_supervisord_worker_from_plugin(config)
+    if sonic_worker is None:
+        return None
+
+    from abx_plugins.plugins.search_backend_sonic.daemon import is_port_listening, prepare_sonic_daemon
+
+    sonic_event = prepare_sonic_daemon(config)
+    if is_port_listening(sonic_event.host, sonic_event.port):
+        return {
+            "name": sonic_event.worker_name,
+            "statename": "RUNNING",
+            "description": f"existing Sonic daemon at {sonic_event.url}",
+        }
+
+    supervisor = get_existing_supervisord_process() or get_or_create_supervisord_process(daemonize=False)
+    worker = get_worker(supervisor, sonic_worker["name"])
+    if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
+        return worker
+
+    if reason:
+        print(f"[yellow][*] Starting daemon stack for {reason}...[/yellow]")
+    return start_worker(supervisor, sonic_worker)
+
+
+def healthy_orchestrator(*, data_dir: str | Path):
+    from archivebox.machine.models import Machine, Process
+    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+
+    Process.cleanup_stale_running(machine=Machine.current())
+    supervisor = get_existing_supervisord_process()
+    worker = get_worker(supervisor, "worker_runner") if supervisor else None
+    if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
+        return worker
+
+    for proc in Process.objects.filter(
+        machine=Machine.current(),
+        process_type=Process.TypeChoices.ORCHESTRATOR,
+        status=Process.StatusChoices.RUNNING,
+        pwd=str(data_dir),
+    ).order_by("-created_at"):
+        if proc.is_running:
+            return proc
+    return None
+
+
+def standby_until_leader_needed(command, *, process_type: str, data_dir: str | Path, url: str | None = None, interval: float = 2.0) -> None:
+    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+
+    announced = False
+    while not command_is_newest(command, process_type=process_type, data_dir=data_dir, url=url):
+        reap_foreground_supervisord_process()
+        if not announced:
+            leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
+            leader_pid = leader.pid if leader else "unknown"
+            print(f"[yellow][*] Standing by; newer ArchiveBox parent pid={leader_pid} is running the orchestrator and server.[/yellow]")
+            announced = True
+        command.heartbeat()
+        time.sleep(interval)
+    command.modified_at = timezone.now()
+    command.save(update_fields=["modified_at"])
+
+
+def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> None:
+    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+
+    announced = False
+    while not command_owns_runtime_stack(command, data_dir=data_dir):
+        reap_foreground_supervisord_process()
+        if not announced:
+            owner = runtime_stack_owner(data_dir=data_dir)
+            owner_pid = owner.pid if owner else "unknown"
+            owner_type = owner.process_type if owner else "unknown"
+            print(f"[yellow][*] Standing by; ArchiveBox {owner_type} pid={owner_pid} owns the runtime stack.[/yellow]")
+            announced = True
+        command.heartbeat()
+        time.sleep(interval)
+    command.modified_at = timezone.now()
+    command.save(update_fields=["modified_at"])

archivebox/templates/admin/actions.html

@@ -1,5 +1,5 @@
-{% load i18n %}
-<div class="actions">
+{% load i18n core_tags %}
+<div class="actions {% if action_index|default:0 == 0 %}actions-top{% else %}actions-bottom{% endif %}">
   <div class="actions-left">
     {% block actions %}
       {% block actions-form %}
@@ -16,50 +16,42 @@
       {% endblock %}
       {% block actions-counter %}
       {% if actions_selection_counter %}
-          <span class="action-counter" data-actions-icnt="{{ cl.result_list|length }}">{{ selection_note }}</span>
+        <span class="action-summary" data-page-count="{{ cl.result_list|length }}" data-result-count="{{ cl.result_count }}" data-full-result-count="{{ cl.full_result_count|default:cl.result_count }}">
+          <span class="action-selected-count">0 / {{ cl.result_list|length|intcomma }} selected</span>
+          <span class="action-counter hidden" data-actions-icnt="{{ cl.result_list|length }}" style="display: none !important;" aria-hidden="true">{{ selection_note }}</span>
+          {% if cl.opts.model_name == 'snapshot' %}
+          <span class="action-total-count">
+            {% if cl.full_result_count and cl.full_result_count != cl.result_count %}
+              <span class="question hidden">
+                  <a role="button" href="#" title="{% translate "Select all matching rows across all pages" %}">{{ cl.result_count|intcomma }}</a>
+              </span>
+              <a class="action-match-count-static action-total-select" href="#" title="{% translate "Select all matching rows across all pages" %}">{{ cl.result_count|intcomma }}</a>
+              /
+              <a href="?" title="{% translate "Show all rows" %}">{{ cl.full_result_count|intcomma }}</a>
+              total
+            {% else %}
+              <span class="question hidden">
+                  <a role="button" href="#" title="{% translate "Select all matching rows across all pages" %}">{{ cl.result_count|intcomma }}</a>
+              </span>
+              <a class="action-total-reset" href="?" title="{% translate "Show all rows" %}">{{ cl.result_count|intcomma }}</a>
+              total
+            {% endif %}
+          </span>
+          {% endif %}
           {% if cl.result_count != cl.result_list|length %}
           <span class="all hidden">{{ selection_note_all }}</span>
+          {% if cl.opts.model_name != 'snapshot' %}
           <span class="question hidden">
               <a role="button" href="#" title="{% translate "Click here to select the objects across all pages" %}">{% blocktranslate with cl.result_count as total_count %}Select all {{ total_count }} {{ module_name }}{% endblocktranslate %}</a>
           </span>
+          {% endif %}
+          {% if cl.opts.model_name != 'snapshot' %}
           <span class="clear hidden"><a role="button" href="#">{% translate "Clear selection" %}</a></span>
           {% endif %}
+          {% endif %}
+        </span>
       {% endif %}
       {% endblock %}
     {% endblock %}
   </div>
-  {% if action_index|default:0 == 0 %}
-    {% if cl.has_filters or opts.model_name == 'snapshot' %}
-      <div class="actions-right">
-        {% if cl.has_filters %}
-          <button
-            type="button"
-            class="button"
-            id="changelist-toolbar-filter-toggle"
-          >
-            {% translate "Filters" %}
-          </button>
-        {% endif %}
-        {% if request.resolver_match.url_name == 'grid' %}
-          <button
-            type="button"
-            class="button"
-            id="snapshot-view-toggle"
-            onclick="window.location.href='{% url 'admin:core_snapshot_changelist' %}{% if request.GET.urlencode %}?{{ request.GET.urlencode }}{% endif %}'"
-          >
-            {% translate "List" %}
-          </button>
-        {% elif opts.model_name == 'snapshot' %}
-          <button
-            type="button"
-            class="button"
-            id="snapshot-view-toggle"
-            onclick="window.location.href='{% url 'admin:grid' %}{% if request.GET.urlencode %}?{{ request.GET.urlencode }}{% endif %}'"
-          >
-            {% translate "Grid" %}
-          </button>
-        {% endif %}
-      </div>
-    {% endif %}
-  {% endif %}
 </div>

archivebox/templates/admin/base.html

@@ -1216,6 +1216,28 @@
                 font-size: 15px;
             }
 
+            .search-empty-state {
+                padding: 28px 18px;
+                text-align: center;
+                color: #475569;
+                background: #f8fafc;
+                border-top: 1px solid #e2e8f0;
+            }
+
+            .search-empty-state p {
+                margin: 0;
+            }
+
+            .search-empty-state code {
+                display: inline-block;
+                margin-left: 4px;
+                padding: 2px 6px;
+                border-radius: 5px;
+                background: #e2e8f0;
+                color: #0f172a;
+                font-size: 13px;
+            }
+
             /* Date hierarchy */
             .xfull {
                 padding: 12px 16px;
@@ -1457,9 +1479,63 @@
             .actions .tag-inline-input {
                 min-width: 40px;
                 padding: 0;
+                margin: 0;
+                height: auto;
+                min-height: 0;
+                border: 0;
+                border-radius: 0;
+                box-shadow: none;
+                background: transparent;
                 font-size: 11px;
             }
 
+            .model-snapshot.change-list #changelist .actions-tags-with-buttons {
+                display: grid;
+                grid-template-columns: minmax(0, 1fr) 34px 34px;
+                gap: 0;
+                align-items: stretch;
+                border: 1px solid #cbd5e1;
+                border-radius: 8px;
+                background: #fff;
+                overflow: hidden;
+            }
+
+            .model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-editor-container {
+                width: auto;
+                max-width: none;
+                height: 34px;
+                min-height: 34px;
+                padding: 4px 9px;
+                border: 0;
+                border-radius: 0;
+                box-shadow: none;
+                overflow: hidden;
+            }
+
+            .model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-inline-input {
+                height: auto;
+                min-height: 0;
+                padding: 0;
+                margin: 0;
+                border: 0;
+                border-radius: 0;
+                box-shadow: none;
+                background: transparent;
+            }
+
+            .model-snapshot.change-list #changelist .actions-tags-with-buttons .button[name="add_tags"],
+            .model-snapshot.change-list #changelist .actions-tags-with-buttons .button[name="remove_tags"] {
+                width: 34px;
+                min-width: 34px;
+                height: 34px;
+                padding: 0;
+                margin: 0;
+                border: 0;
+                border-left: 1px solid #cbd5e1;
+                border-radius: 0;
+                justify-content: center;
+            }
+
 
             /* Container in list view title column */
             .tags-inline-editor {
@@ -1675,18 +1751,41 @@
             // change the admin actions button from a dropdown to buttons across
             function fix_actions() {
                 const container = $('div.actions')
+                if (container.find('.action-buttons').length) return
 
                 // too many actions to turn into buttons
                 if (container.find('select[name=action] option').length >= 11) return
 
                 // hide the empty default option thats just a placeholder with no value
-                container.find('label:nth-child(1), button[value=0]').hide()
+                container.find('label:nth-child(1), button[type=submit][name=index][value=0]').hide()
 
                 const buttons = $('<div></div>')
                     .insertAfter('div.actions button[type=submit]')
                     .css('display', 'inline')
                     .addClass('action-buttons');
 
+                function flushPendingActionTags() {
+                    const tagContainer = document.querySelector('.actions-tags')
+                    const input = tagContainer?.querySelector('.tag-inline-input')
+                    const hidden = tagContainer?.querySelector('input[type="hidden"][name="tags"]')
+                    const pending = (input?.value || '').trim()
+                    if (!pending || !hidden) return
+                    const seen = new Set()
+                    const tags = (hidden.value ? hidden.value.split(',') : [])
+                        .concat(pending.split(','))
+                        .map((tag) => tag.trim())
+                        .filter((tag) => {
+                            const key = tag.toLowerCase()
+                            if (!tag || seen.has(key)) return false
+                            seen.add(key)
+                            return true
+                        })
+                    hidden.value = tags.join(',')
+                    input.value = ''
+                    hidden.dispatchEvent(new Event('input', { bubbles: true }))
+                    hidden.dispatchEvent(new Event('change', { bubbles: true }))
+                }
+
                 // for each action in the dropdown, turn it into a button instead
                 container.find('select[name=action] option:gt(0)').each(function () {
                     const action_type = this.value
@@ -1699,7 +1798,11 @@
                             e.preventDefault()
                             e.stopPropagation()
 
-                            const num_selected = document.querySelector('.action-counter').innerText.split(' ')[0]
+                            const num_selected = (
+                                document.querySelector('.action-selected-count')?.textContent.split('/')[0].trim()
+                                || document.querySelector('.action-counter')?.textContent.split(' ')[0]
+                                || '0'
+                            )
 
                             if (action_type === 'overwrite_snapshots') {
                                 const message = (
@@ -1715,6 +1818,9 @@
                                 )
                                 if (!window.confirm(message)) return false
                             }
+                            if (action_type === 'add_tags' || action_type === 'remove_tags') {
+                                flushPendingActionTags()
+                            }
 
                             // select the action from the original Django admin dropdown
                             container.find('select[name=action]')
@@ -1729,6 +1835,14 @@
                         .appendTo(buttons)
                 })
                 console.log('Converted', buttons.children().length, 'admin actions from dropdown to buttons')
+                const tagContainer = document.querySelector('.actions-tags')
+                if (tagContainer) {
+                    const tagButtons = buttons.find('button[name="add_tags"], button[name="remove_tags"]')
+                    if (tagButtons.length) {
+                        tagContainer.classList.add('actions-tags-with-buttons')
+                        tagButtons.appendTo(tagContainer)
+                    }
+                }
                 if (window.jQuery && window.jQuery.fn.select2) {
                     window.jQuery('select[multiple]').select2();
                 }
@@ -1737,7 +1851,188 @@
                 const tagContainer = document.querySelector('.actions-tags');
                 if (!tagContainer) return;
                 const checked = document.querySelectorAll('#changelist-form input.action-select:checked').length;
-                tagContainer.style.display = checked > 0 ? 'inline-flex' : 'none';
+                tagContainer.style.display = tagContainer.classList.contains('actions-tags-with-buttons') || checked > 0 ? 'inline-flex' : 'none';
+            }
+            function setupActionSummary() {
+                const summary = document.querySelector('.action-summary')
+                if (!summary || summary.dataset.summaryReady) return
+                summary.dataset.summaryReady = '1'
+                const selectedCount = summary.querySelector('.action-selected-count')
+                const counter = summary.querySelector('.action-counter')
+                const formatter = new Intl.NumberFormat()
+                const update = function() {
+                    if (!selectedCount || !counter) return
+                    const match = counter.textContent.match(/(\d+)\s+of\s+(\d+)\s+selected/)
+                    const selectAcross = document.querySelector('div.actions input.select-across')?.value === '1'
+                    const selected = selectAcross
+                        ? Number(summary.dataset.resultCount || 0)
+                        : (match ? Number(match[1]) : document.querySelectorAll('#changelist-form input.action-select:checked').length)
+                    const pageCount = match ? Number(match[2]) : Number(summary.dataset.pageCount || 0)
+                    const selectedLimit = selectAcross ? Number(summary.dataset.resultCount || pageCount) : pageCount
+                    selectedCount.textContent = formatter.format(selected) + ' / ' + formatter.format(selectedLimit) + ' selected'
+                    summary.classList.toggle('action-summary-has-selection', selected > 0)
+                    summary.classList.toggle('action-summary-select-across', selectAcross)
+                    const totalCount = summary.querySelector('.action-total-count')
+                    if (totalCount) {
+                        totalCount.hidden = selectAcross
+                    }
+                }
+                new MutationObserver(update).observe(counter, { childList: true, characterData: true, subtree: true })
+                document.querySelector('#changelist-form')?.addEventListener('change', function() {
+                    window.setTimeout(update, 0)
+                })
+                summary.addEventListener('click', function(event) {
+                    const explicitSelectAll = event.target.closest('.action-total-select, .action-total-count .question a')
+                    const selectedTotalClick = event.target.closest('.action-total-reset') && summary.classList.contains('action-summary-has-selection')
+                    if (explicitSelectAll || selectedTotalClick) {
+                        const questionLink = summary.querySelector('.action-total-count .question a')
+                        const allToggle = document.getElementById('action-toggle')
+                        const pageCount = Number(summary.dataset.pageCount || 0)
+                        const resultCount = Number(summary.dataset.resultCount || pageCount)
+                        event.preventDefault()
+                        if (allToggle && !allToggle.checked) {
+                            allToggle.click()
+                        }
+                        if (questionLink && resultCount > pageCount) {
+                            questionLink.click()
+                        }
+                    }
+                    window.setTimeout(update, 0)
+                })
+                update()
+            }
+            function setupSearchModeSelect() {
+                const storageKey = 'archivebox-admin-search-mode'
+                const params = new URLSearchParams(window.location.search)
+                document.querySelectorAll('.search-mode-select').forEach(function(select) {
+                    if (select.dataset.searchModeReady) return
+                    select.dataset.searchModeReady = '1'
+                    const values = Array.from(select.options).map(option => option.value)
+                    const stored = localStorage.getItem(storageKey)
+                    if (!params.has('search_mode') && values.includes(stored)) {
+                        select.value = stored
+                        if ((params.get('q') || '').trim()) {
+                            params.set('search_mode', stored)
+                            params.delete('p')
+                            window.location.replace(window.location.pathname + '?' + params.toString() + window.location.hash)
+                            return
+                        }
+                    }
+                    select.addEventListener('change', function() {
+                        localStorage.setItem(storageKey, select.value)
+                        const search = select.closest('#changelist-search')
+                        if (search && search.dataset.embeddedSearch === '1') {
+                            submitEmbeddedChangelistSearch(search)
+                            return
+                        }
+                        if (search && search.tagName === 'FORM' && document.activeElement === select) {
+                            search.requestSubmit()
+                        }
+                    })
+                })
+            }
+            function submitEmbeddedChangelistSearch(search) {
+                const params = new URLSearchParams(window.location.search)
+                search.querySelectorAll('[name]').forEach(function(input) {
+                    const value = (input.value || '').trim()
+                    if (value) {
+                        params.set(input.name, value)
+                    } else {
+                        params.delete(input.name)
+                    }
+                })
+                params.delete('p')
+                const query = params.toString()
+                window.location.href = window.location.pathname + (query ? '?' + query : '') + window.location.hash
+            }
+            function setupEmbeddedChangelistSearch() {
+                document.querySelectorAll('#changelist-search[data-embedded-search="1"]').forEach(function(search) {
+                    if (search.dataset.searchReady) return
+                    search.dataset.searchReady = '1'
+                    search.querySelector('.changelist-search-submit')?.addEventListener('click', function() {
+                        submitEmbeddedChangelistSearch(search)
+                    })
+                    search.querySelector('#searchbar')?.addEventListener('keydown', function(event) {
+                        if (event.key === 'Enter') {
+                            event.preventDefault()
+                            submitEmbeddedChangelistSearch(search)
+                        }
+                    })
+                })
+            }
+            function setupSnapshotPermissionsQuickEdit() {
+                if (document.body.dataset.snapshotPermissionsReady) return
+                document.body.dataset.snapshotPermissionsReady = '1'
+                document.addEventListener('click', function(event) {
+                    const toggle = event.target.closest('.snapshot-permissions-button')
+                    const item = event.target.closest('.snapshot-permissions-menu-item')
+                    document.querySelectorAll('.snapshot-permissions-quick.is-open').forEach(function(openMenu) {
+                        if (!openMenu.contains(event.target)) {
+                            openMenu.classList.remove('is-open')
+                            openMenu.querySelector('.snapshot-permissions-button')?.setAttribute('aria-expanded', 'false')
+                            const menu = openMenu.querySelector('.snapshot-permissions-menu')
+                            if (menu) menu.hidden = true
+                        }
+                    })
+                    if (toggle) {
+                        event.preventDefault()
+                        const wrapper = toggle.closest('.snapshot-permissions-quick')
+                        const menu = wrapper?.querySelector('.snapshot-permissions-menu')
+                        if (!wrapper || !menu) return
+                        const isOpen = wrapper.classList.toggle('is-open')
+                        toggle.setAttribute('aria-expanded', isOpen ? 'true' : 'false')
+                        menu.hidden = !isOpen
+                        return
+                    }
+                    if (!item) return
+                    event.preventDefault()
+                    const wrapper = item.closest('.snapshot-permissions-quick')
+                    const permissions = item.dataset.permissions
+                    const csrf = document.querySelector('input[name="csrfmiddlewaretoken"]')?.value
+                    if (!wrapper || !permissions || !csrf || wrapper.dataset.saving === '1') return
+                    wrapper.dataset.saving = '1'
+                    const body = new URLSearchParams({permissions: permissions, csrfmiddlewaretoken: csrf})
+                    fetch(wrapper.dataset.permissionsUrl, {
+                        method: 'POST',
+                        headers: {'X-CSRFToken': csrf, 'Content-Type': 'application/x-www-form-urlencoded'},
+                        body: body.toString(),
+                    }).then(function(response) {
+                        if (!response.ok) throw new Error('Failed to update permissions')
+                        return response.json()
+                    }).then(function(data) {
+                        wrapper.dataset.currentPermissions = data.permissions
+                        wrapper.querySelectorAll('.snapshot-permissions-menu-item').forEach(function(button) {
+                            button.classList.toggle('is-active', button.dataset.permissions === data.permissions)
+                        })
+                        const icon = wrapper.querySelector('.snapshot-permissions-icon')
+                        if (icon) {
+                            icon.textContent = data.icon
+                            icon.style.color = data.fg
+                            icon.style.background = data.bg
+                        }
+                        const toggle = wrapper.querySelector('.snapshot-permissions-button')
+                        if (toggle) {
+                            toggle.className = 'snapshot-permissions-button snapshot-permissions-' + data.permissions
+                            toggle.title = data.label
+                            toggle.setAttribute('aria-label', 'Change snapshot permissions: ' + data.label)
+                            toggle.setAttribute('aria-expanded', 'false')
+                        }
+                        wrapper.classList.remove('is-open')
+                        const menu = wrapper.querySelector('.snapshot-permissions-menu')
+                        if (menu) menu.hidden = true
+                    }).catch(function(error) {
+                        window.alert(error.message)
+                    }).finally(function() {
+                        wrapper.dataset.saving = ''
+                    })
+                })
+            }
+            function setupChangelistFormHandlers() {
+                const form = document.querySelector('#changelist-form')
+                if (form && !form.dataset.archiveboxActionsReady) {
+                    form.dataset.archiveboxActionsReady = '1'
+                    form.addEventListener('change', updateTagWidgetVisibility)
+                }
             }
             function fixInlineAddRow() {
                 $('#id_snapshottag-MAX_NUM_FORMS').val('1000')
@@ -1858,28 +2153,30 @@
                 })
                 return false
             }
-            if ($) {
-                $(document).ready(function() {
+            window.archiveboxInitAdminChangelist = function() {
+                if (window.jQuery) {
                     fix_actions()
-                    updateTagWidgetVisibility()
-                    const form = document.querySelector('#changelist-form')
-                    if (form) {
-                        form.addEventListener('change', updateTagWidgetVisibility)
-                    }
                     fixInlineAddRow()
                     setupSnapshotGridListToggle()
+                }
+                updateTagWidgetVisibility()
+                setupActionSummary()
+                setupSearchModeSelect()
+                setupEmbeddedChangelistSearch()
+                setupChangelistFormHandlers()
+                selectSnapshotIfHotlinked()
+            }
+            if ($) {
+                $(document).ready(function() {
+                    window.archiveboxInitAdminChangelist()
+                    setupSnapshotPermissionsQuickEdit()
                     setTimeOffset()
-                    selectSnapshotIfHotlinked()
                 })
             } else {
                 document.addEventListener('DOMContentLoaded', function() {
-                    updateTagWidgetVisibility()
-                    const form = document.querySelector('#changelist-form')
-                    if (form) {
-                        form.addEventListener('change', updateTagWidgetVisibility)
-                    }
+                    window.archiveboxInitAdminChangelist()
+                    setupSnapshotPermissionsQuickEdit()
                     setTimeOffset()
-                    selectSnapshotIfHotlinked()
                 })
             }
         </script>

archivebox/templates/admin/change_list.html

@@ -46,7 +46,7 @@
 <script src="{% static 'admin/js/filters.js' %}" defer></script>
 {% endblock %}
 
-{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-list{% endblock %}
+{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-list{% if embedded_changelist %} embedded-change-list{% endif %}{% endblock %}
 
 {% if not is_popup %}
 {% block breadcrumbs %}
@@ -63,11 +63,13 @@
 {% block content %}
   <div id="content-main">
     {% block object-tools %}
+      {% if not embedded_changelist %}
         <ul class="object-tools">
           {% block object-tools-items %}
             {% change_list_object_tools %}
           {% endblock %}
         </ul>
+      {% endif %}
     {% endblock %}
     {% if cl.formset and cl.formset.errors %}
         <p class="errornote">
@@ -75,101 +77,6 @@
         </p>
         {{ cl.formset.non_form_errors }}
     {% endif %}
-    {% if cl.model_admin.show_search_mode_selector %}
-      {% with current_search_mode=cl.params.search_mode|default:cl.model_admin.get_default_search_mode %}
-        <div class="module{% if cl.has_filters %} filtered{% endif %}{% if current_search_mode == 'contents' %} search-mode-contents{% elif current_search_mode == 'deep' %} search-mode-deep{% endif %}" id="changelist">
-      {% endwith %}
-    {% else %}
-        <div class="module{% if cl.has_filters %} filtered{% endif %}" id="changelist">
-    {% endif %}
-      <div class="changelist-form-container">
-        <div>
-          {% block search %}{% search_form cl %}{% endblock %}
-          {% block date_hierarchy %}{% if cl.date_hierarchy %}{% date_hierarchy cl %}{% endif %}{% endblock %}
-
-          <form id="changelist-form" method="post"{% if cl.formset and cl.formset.is_multipart %} enctype="multipart/form-data"{% endif %} novalidate>{% csrf_token %}
-          {% if cl.formset %}
-            <div>{{ cl.formset.management_form }}</div>
-          {% endif %}
-
-          {% block result_list %}
-            {% if action_form and actions_on_top and cl.show_admin_actions %}{% admin_actions %}{% endif %}
-            {% result_list cl %}
-            {% if action_form and actions_on_bottom and cl.show_admin_actions %}{% admin_actions %}{% endif %}
-          {% endblock %}
-          {% block pagination %}
-          <div class="changelist-footer">
-            {% pagination cl %}
-            {% if cl.formset and cl.result_count %}<input type="submit" name="_save" class="default" value="{% translate 'Save' %}">{% endif %}
-          </div>
-          {% endblock %}
-          </form>
-        </div>
-      </div>
-      {% block filters %}
-        {% if cl.has_filters %}
-        <div id="changelist-filter">
-          <h2>
-            {% translate 'Filter' %}
-            <button
-              type="button"
-              id="changelist-filter-toggle"
-              class="filter-toggle"
-              aria-expanded="true"
-              data-show-label="{% translate 'Filters' %}"
-              data-hide-label="{% translate 'Hide' %}"
-            >
-              {% translate 'Hide' %}
-            </button>
-          </h2>
-          {% if cl.is_facets_optional or cl.has_active_filters %}<div id="changelist-filter-extra-actions">
-            {% if cl.is_facets_optional %}<h3>
-              {% if cl.add_facets %}<a href="{{ cl.remove_facet_link }}" class="hidelink">{% translate "Hide counts" %}</a>
-              {% else %}<a href="{{ cl.add_facet_link }}" class="viewlink">{% translate "Show counts" %}</a>{% endif %}
-            </h3>{% endif %}
-            {% if cl.has_active_filters %}<h3>
-              <a href="{{ cl.clear_all_filters_qs }}">&#10006; {% translate "Clear all filters" %}</a>
-            </h3>{% endif %}
-          </div>{% endif %}
-          {% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %}
-        </div>
-        {% endif %}
-      {% endblock %}
-    </div>
+    {% include "admin/change_list_panel.html" %}
   </div>
-  {% if cl.has_filters %}
-    <script>
-      (function() {
-        var storageKey = 'admin-filters-collapsed';
-        var toggle = document.getElementById('changelist-filter-toggle');
-        var toolbarToggle = document.getElementById('changelist-toolbar-filter-toggle');
-        if (!toggle) return;
-
-        function applyState() {
-          var collapsed = localStorage.getItem(storageKey) === 'true';
-          document.body.classList.toggle('filters-collapsed', collapsed);
-          toggle.textContent = collapsed ? toggle.dataset.showLabel : toggle.dataset.hideLabel;
-          toggle.setAttribute('aria-expanded', collapsed ? 'false' : 'true');
-          if (toolbarToggle) {
-            toolbarToggle.textContent = toggle.dataset.showLabel;
-            toolbarToggle.style.display = collapsed ? 'inline-block' : 'none';
-          }
-        }
-
-        toggle.addEventListener('click', function() {
-          localStorage.setItem(storageKey, document.body.classList.contains('filters-collapsed') ? 'false' : 'true');
-          applyState();
-        });
-
-        if (toolbarToggle) {
-          toolbarToggle.addEventListener('click', function() {
-            localStorage.setItem(storageKey, 'false');
-            applyState();
-          });
-        }
-
-        applyState();
-      })();
-    </script>
-  {% endif %}
 {% endblock %}

archivebox/templates/admin/change_list_panel.html

@@ -0,0 +1,107 @@
+{% load i18n admin_list core_tags %}
+
+{% if cl.model_admin.show_search_mode_selector %}
+  {% with current_search_mode=cl.params.search_mode|default:cl.model_admin.get_default_search_mode %}
+    <div class="module{% if cl.has_filters %} filtered{% endif %}{% if current_search_mode == 'contents' %} search-mode-contents{% elif current_search_mode == 'deep' %} search-mode-deep{% endif %}" id="changelist">
+  {% endwith %}
+{% else %}
+    <div class="module{% if cl.has_filters %} filtered{% endif %}" id="changelist">
+{% endif %}
+  <div class="changelist-form-container">
+    <div>
+      {% search_form cl %}
+      {% if cl.date_hierarchy %}{% date_hierarchy cl %}{% endif %}
+
+      <form id="changelist-form" method="post"{% if changelist_form_action %} action="{{ changelist_form_action }}"{% endif %}{% if cl.formset and cl.formset.is_multipart %} enctype="multipart/form-data"{% endif %} novalidate>{% csrf_token %}
+      {% if cl.formset %}
+        <div>{{ cl.formset.management_form }}</div>
+      {% endif %}
+
+      {% if action_form and actions_on_top and cl.show_admin_actions %}{% admin_actions %}{% endif %}
+      {% if cl.snapshot_is_grid_view %}
+        {% snapshots_grid cl %}
+      {% else %}
+        {% result_list cl %}
+      {% endif %}
+      {% if action_form and actions_on_bottom and cl.show_admin_actions %}{% admin_actions %}{% endif %}
+      <div class="changelist-footer">
+        {% pagination cl %}
+        {% if cl.formset and cl.result_count %}<input type="submit" name="_save" class="default" value="{% translate 'Save' %}">{% endif %}
+      </div>
+      </form>
+    </div>
+  </div>
+  {% if cl.has_filters and not embedded_changelist %}
+  <div id="changelist-filter">
+    <h2>
+      {% translate 'Filter' %}
+      <button
+        type="button"
+        id="changelist-filter-toggle"
+        class="filter-toggle"
+        aria-expanded="true"
+        data-show-label="{% translate 'Filters' %}"
+        data-hide-label="{% translate 'Hide' %}"
+      >
+        {% translate 'Hide' %}
+      </button>
+    </h2>
+    {% if cl.is_facets_optional or cl.has_active_filters %}<div id="changelist-filter-extra-actions">
+      {% if cl.is_facets_optional %}<h3>
+        {% if cl.add_facets %}<a href="{{ cl.remove_facet_link }}" class="hidelink">{% translate "Hide counts" %}</a>
+        {% else %}<a href="{{ cl.add_facet_link }}" class="viewlink">{% translate "Show counts" %}</a>{% endif %}
+      </h3>{% endif %}
+      {% if cl.has_active_filters %}<h3>
+        <a href="{{ cl.clear_all_filters_qs }}">&#10006; {% translate "Clear all filters" %}</a>
+      </h3>{% endif %}
+    </div>{% endif %}
+    {% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %}
+  </div>
+  {% endif %}
+</div>
+
+{% if cl.has_filters and not embedded_changelist %}
+  <script>
+    window.archiveboxInitChangelistFilters = function() {
+      var storageKey = 'admin-filters-collapsed';
+      var toggle = document.getElementById('changelist-filter-toggle');
+      var toolbarToggle = document.getElementById('changelist-toolbar-filter-toggle');
+      if (!toggle) return;
+
+      function applyState() {
+        var storedState = localStorage.getItem(storageKey);
+        var collapsed = storedState === null
+          ? window.matchMedia('(max-width: 1180px)').matches
+          : storedState === 'true';
+        document.body.classList.toggle('filters-collapsed', collapsed);
+        toggle.textContent = collapsed ? toggle.dataset.showLabel : toggle.dataset.hideLabel;
+        toggle.setAttribute('aria-expanded', collapsed ? 'false' : 'true');
+        if (toolbarToggle) {
+          toolbarToggle.textContent = collapsed ? 'Filters ▸' : 'Filters ◂';
+          toolbarToggle.style.display = 'inline-flex';
+          toolbarToggle.setAttribute('aria-expanded', collapsed ? 'false' : 'true');
+        }
+      }
+
+      if (!toggle.dataset.archiveboxFiltersReady) {
+        toggle.dataset.archiveboxFiltersReady = '1';
+        toggle.addEventListener('click', function() {
+          localStorage.setItem(storageKey, document.body.classList.contains('filters-collapsed') ? 'false' : 'true');
+          applyState();
+        });
+      }
+
+      if (toolbarToggle && !toolbarToggle.dataset.archiveboxFiltersReady) {
+        toolbarToggle.dataset.archiveboxFiltersReady = '1';
+        toolbarToggle.addEventListener('click', function() {
+          localStorage.setItem(storageKey, document.body.classList.contains('filters-collapsed') ? 'false' : 'true');
+          applyState();
+        });
+      }
+
+      applyState();
+    };
+    window.archiveboxInitChangelistFilters();
+  </script>
+{% endif %}
+{% include "admin/snapshot_search_stream.html" %}

archivebox/templates/admin/change_list_results.html

@@ -7,6 +7,14 @@
 {% if results %}
 <div class="results">
 <table id="result_list">
+{% if cl.opts.model_name == "snapshot" %}
+<colgroup>
+  <col>
+  <col class="snapshot-permissions-col">
+  <col>
+  <col>
+</colgroup>
+{% endif %}
 <thead>
 <tr>
 {% for header in result_headers %}
@@ -35,4 +43,12 @@
 </tbody>
 </table>
 </div>
+{% elif cl.show_search_index_hint %}
+<div class="results search-empty-state">
+  <p>
+    0 results from deep: {{ cl.search_backend_label }}.
+    If this looks wrong, the search index may need to be updated:
+    <code>archivebox update --index-only</code>
+  </p>
+</div>
 {% endif %}

archivebox/templates/admin/core/archiveresult/change_list.html

@@ -118,8 +118,8 @@
           toggle.textContent = collapsed ? toggle.dataset.showLabel : toggle.dataset.hideLabel;
           toggle.setAttribute('aria-expanded', collapsed ? 'false' : 'true');
           if (toolbarToggle) {
-            toolbarToggle.textContent = toggle.dataset.showLabel;
-            toolbarToggle.style.display = collapsed ? 'inline-block' : 'none';
+            toolbarToggle.textContent = collapsed ? 'Filters ▸' : 'Filters ◂';
+            toolbarToggle.style.display = 'inline-flex';
           }
         }
 

archivebox/templates/admin/crawls/crawl/change_form.html

@@ -0,0 +1,42 @@
+{% extends "admin/change_form.html" %}
+
+{% block object-tools-items %}
+{% if original %}
+  <li class="archivebox-crawl-resume-tool">
+    <span class="crawl-stop-reason-inline">
+      Stop reason:
+      {% if crawl_stop_reason %}
+        <strong>{{ crawl_stop_reason }}</strong>
+      {% else %}
+        <strong>none</strong>
+      {% endif %}
+    </span>
+    {% if original.status != "sealed" and not original.is_paused %}
+      <form method="post" action="{% url 'admin:crawls_crawl_changelist' %}" class="crawl-resume-action-form">
+        {% csrf_token %}
+        <input type="hidden" name="action" value="pause_selected_crawls">
+        <input type="hidden" name="_selected_action" value="{{ original.pk }}">
+        <input type="hidden" name="index" value="0">
+        <button type="submit" class="button crawl-pause-submit">Pause</button>
+      </form>
+    {% endif %}
+    {% if original.status == "sealed" or original.is_paused %}
+      <form method="post" action="{% url 'admin:crawls_crawl_changelist' %}" class="crawl-resume-action-form">
+        {% csrf_token %}
+        <input type="hidden" name="action" value="resume_selected_crawls">
+        <input type="hidden" name="_selected_action" value="{{ original.pk }}">
+        <input type="hidden" name="index" value="0">
+        <button type="submit" class="button default crawl-resume-submit">Resume</button>
+      </form>
+    {% endif %}
+  </li>
+{% endif %}
+{{ block.super }}
+{% endblock %}
+
+{% block content %}
+{{ block.super }}
+{% if crawl_snapshots_changelist %}
+  {{ crawl_snapshots_changelist }}
+{% endif %}
+{% endblock %}

archivebox/templates/admin/crawls/crawl/snapshots_changelist.html

@@ -0,0 +1,9 @@
+{% load i18n %}
+
+<div class="crawl-snapshots-embed snapshot-changelist-widget model-snapshot change-list embedded-change-list">
+  <div class="crawl-snapshots-embed__toolbar">
+    <strong>{% translate "Snapshots in this crawl" %}</strong>
+    <a class="button" href="{{ snapshot_changelist_url }}">{% translate "Open full changelist" %}</a>
+  </div>
+  {% include "admin/change_list_panel.html" with changelist_form_action=snapshot_changelist_url %}
+</div>

archivebox/templates/admin/private_index.html

@@ -1,138 +0,0 @@
-{% extends "admin/base_site.html" %}
-{% load i18n admin_urls static admin_list %}
-{% load core_tags %}
-
-{% block extrastyle %}
-  {{ block.super }}
-  <link rel="stylesheet" type="text/css" href="{% static "admin/css/changelists.css" %}">
-  {% if cl.formset %}
-    <link rel="stylesheet" type="text/css" href="{% static "admin/css/forms.css" %}">
-  {% endif %}
-  {% if cl.formset or action_form %}
-    <script src="{% url 'admin:jsi18n' %}"></script>
-  {% endif %}
-  {{ media.css }}
-  {% if not actions_on_top and not actions_on_bottom %}
-    <style>
-      #changelist table thead th:first-child {width: inherit}
-    </style>
-  {% endif %}
-{% endblock %}
-
-{% block extrahead %}
-{{ block.super }}
-{{ media.js }}
-{% endblock %}
-
-{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-list{% endblock %}
-
-{% if not is_popup %}
-{% block breadcrumbs %}
-<div class="breadcrumbs">
-<a href="{% url 'admin:index' %}">{% translate 'Home' %}</a>
-&rsaquo; <a href="{% url 'admin:app_list' app_label=cl.opts.app_label %}">{{ cl.opts.app_config.verbose_name }}</a>
-&rsaquo; {{ cl.opts.verbose_name_plural|capfirst }}
-</div>
-{% endblock %}
-{% endif %}
-
-{% block coltype %}{% endblock %}
-
-{% block content %}
-  <div id="content-main">
-    {% block object-tools %}
-        <ul class="object-tools">
-          {% block object-tools-items %}
-            {% change_list_object_tools %}
-          {% endblock %}
-        </ul>
-    {% endblock %}
-    {% if cl.formset and cl.formset.errors %}
-        <p class="errornote">
-        {% if cl.formset.total_error_count == 1 %}{% translate "Please correct the error below." %}{% else %}{% translate "Please correct the errors below." %}{% endif %}
-        </p>
-        {{ cl.formset.non_form_errors }}
-    {% endif %}
-    <div class="module{% if cl.has_filters %} filtered{% endif %}" id="changelist">
-      <div class="changelist-form-container">
-        {% block search %}{% search_form cl %}{% endblock %}
-        {% block date_hierarchy %}{% if cl.date_hierarchy %}{% date_hierarchy cl %}{% endif %}{% endblock %}
-
-        <form id="changelist-form" method="post"{% if cl.formset and cl.formset.is_multipart %} enctype="multipart/form-data"{% endif %} novalidate>{% csrf_token %}
-        {% if cl.formset %}
-          <div>{{ cl.formset.management_form }}</div>
-        {% endif %}
-
-        {% block result_list %}
-          {% if action_form and actions_on_top and cl.show_admin_actions %}{% admin_actions %}{% endif %}
-          {% comment %} 
-             Table grid
-            {% result_list cl %} 
-          {% endcomment %}
-          {% snapshots_grid cl %}
-          {% if action_form and actions_on_bottom and cl.show_admin_actions %}{% admin_actions %}{% endif %}
-        {% endblock %}
-        {% block pagination %}{% pagination cl %}{% endblock %}
-        </form>
-      </div>
-      {% block filters %}
-        {% if cl.has_filters %}
-          <div id="changelist-filter">
-            <h2>
-              {% translate 'Filter' %}
-              <button
-                type="button"
-                id="changelist-filter-toggle"
-                class="filter-toggle"
-                aria-expanded="true"
-                data-show-label="{% translate 'Filters' %}"
-                data-hide-label="{% translate 'Hide' %}"
-              >
-                {% translate 'Hide' %}
-              </button>
-            </h2>
-            {% if cl.has_active_filters %}<h3 id="changelist-filter-clear">
-              <a href="{{ cl.clear_all_filters_qs }}">&#10006; {% translate "Clear all filters" %}</a>
-            </h3>{% endif %}
-            {% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %}
-          </div>
-        {% endif %}
-      {% endblock %}
-    </div>
-  </div>
-  {% if cl.has_filters %}
-    <script>
-      (function() {
-        var storageKey = 'admin-filters-collapsed';
-        var toggle = document.getElementById('changelist-filter-toggle');
-        if (!toggle) return;
-        var toolbarToggle = document.getElementById('changelist-toolbar-filter-toggle');
-
-        function applyState() {
-          var collapsed = localStorage.getItem(storageKey) === 'true';
-          document.body.classList.toggle('filters-collapsed', collapsed);
-          toggle.textContent = collapsed ? toggle.dataset.showLabel : toggle.dataset.hideLabel;
-          toggle.setAttribute('aria-expanded', collapsed ? 'false' : 'true');
-          if (toolbarToggle) {
-            toolbarToggle.textContent = toggle.dataset.showLabel;
-            toolbarToggle.style.display = collapsed ? 'inline-block' : 'none';
-          }
-        }
-
-        toggle.addEventListener('click', function() {
-          var collapsed = !document.body.classList.contains('filters-collapsed');
-          localStorage.setItem(storageKey, collapsed ? 'true' : 'false');
-          applyState();
-        });
-        if (toolbarToggle) {
-          toolbarToggle.addEventListener('click', function() {
-            localStorage.setItem(storageKey, 'false');
-            applyState();
-          });
-        }
-
-        applyState();
-      })();
-    </script>
-  {% endif %}
-{% endblock %}