Publish local ArchiveBox changes

archivebox/__main__.py

@@ -17,4 +17,4 @@ ASCII_LOGO_MINI = r"""
 """
 
 if __name__ == "__main__":
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main(args=sys.argv[1:])

archivebox/api/auth.py

@@ -51,6 +51,27 @@ def auth_using_token(token: str | None, request: HttpRequest | None = None) -> U
     return user
 
 
+def token_from_request(request: HttpRequest) -> str:
+    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
+    auth_header = request.headers.get("Authorization", "")
+    if not token and auth_header.lower().startswith("bearer "):
+        token = auth_header.split(None, 1)[1].strip()
+    return token
+
+
+def authenticated_user_from_request(request: HttpRequest) -> User | None:
+    user = request.user
+    if user.is_authenticated and user.is_active:
+        return user
+
+    token = token_from_request(request)
+    token_user = auth_using_token(token=token, request=request) if token else None
+    if token_user and token_user.is_active:
+        request.user = token_user
+        return token_user
+    return None
+
+
 def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
     """Given a username and password, check if they are valid and return the corresponding user"""
     user: User | None = None

archivebox/api/v1_cli.py

@@ -8,8 +8,11 @@ from enum import Enum
 from django.http import HttpRequest
 
 from ninja import Router, Schema
+from ninja.errors import HttpError
+from pydantic import Field
 
 from archivebox.misc.util import ansi_to_html
+from archivebox.core.models import SnapshotQuerySet
 
 
 # from .auth import API_AUTH_METHODS
@@ -21,6 +24,7 @@ router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 # Schemas
 
 JSONType = list[Any] | dict[str, Any] | bool | int | str | None
+FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
 
 
 class CLICommandResponseSchema(Schema):
@@ -32,26 +36,11 @@ class CLICommandResponseSchema(Schema):
     stderr: str
 
 
-class FilterTypeChoices(str, Enum):
-    exact = "exact"
-    substring = "substring"
-    regex = "regex"
-    domain = "domain"
-    tag = "tag"
-    timestamp = "timestamp"
-
-
-class StatusChoices(str, Enum):
-    indexed = "indexed"
-    archived = "archived"
-    unarchived = "unarchived"
-    present = "present"
-    valid = "valid"
-    invalid = "invalid"
-    duplicate = "duplicate"
-    orphaned = "orphaned"
-    corrupted = "corrupted"
-    unrecognized = "unrecognized"
+FilterTypeChoices = Enum(
+    "FilterTypeChoices",
+    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
+    type=str,
+)
 
 
 class AddCommandSchema(Schema):
@@ -71,14 +60,27 @@ class AddCommandSchema(Schema):
     index_only: bool = False
 
 
-class UpdateCommandSchema(Schema):
-    resume: str | None = None
+class SnapshotFilterCommandSchema(Schema):
     after: float | None = 0
-    before: float | None = 999999999999999
+    before: float | None = None
     filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
+    status: str | None = None
+    url__icontains: str | None = None
+    url__istartswith: str | None = None
+    tag: str | None = None
+    crawl_id: str | None = None
+    limit: int | None = None
+    sort: str | None = None
+    search: str | None = None
+
+
+class UpdateCommandSchema(SnapshotFilterCommandSchema):
+    resume: str | None = None
     batch_size: int = 100
     continuous: bool = False
+    index_only: bool = False
+    migrate_only: bool = False
 
 
 class ScheduleCommandSchema(Schema):
@@ -97,24 +99,23 @@ class ScheduleCommandSchema(Schema):
     clear: bool = False
 
 
-class ListCommandSchema(Schema):
-    filter_patterns: list[str] | None = ["https://example.com"]
-    filter_type: str = FilterTypeChoices.substring
-    status: StatusChoices = StatusChoices.indexed
-    after: float | None = 0
-    before: float | None = 999999999999999
-    sort: str = "bookmarked_at"
+class ListCommandSchema(SnapshotFilterCommandSchema):
     as_json: bool = True
     as_html: bool = False
     as_csv: str | None = "timestamp,url"
     with_headers: bool = False
 
 
-class RemoveCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = 999999999999999
+class RemoveCommandSchema(SnapshotFilterCommandSchema):
     filter_type: str = FilterTypeChoices.exact
-    filter_patterns: list[str] | None = ["https://example.com"]
+    timeout: float = 60.0
+
+
+def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
+    kwargs = args.dict()
+    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
+    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
+    return kwargs
 
 
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
@@ -165,24 +166,39 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
 
 @router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
 def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import update
+    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
+    from archivebox.core.snapshot_status import normalize_snapshot_status
 
-    result = update(
-        filter_patterns=args.filter_patterns or [],
-        filter_type=args.filter_type or FilterTypeChoices.substring,
-        after=args.after,
-        before=args.before,
-        resume=args.resume,
-        batch_size=args.batch_size,
-        continuous=args.continuous,
-        stop_daemon_stack=False,
+    try:
+        status = normalize_snapshot_status(args.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+
+    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    update_kwargs["status"] = status
+    update_kwargs["stop_daemon_stack"] = False
+
+    is_filtered_update = any(
+        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
     )
+    matched_snapshot_ids = []
+    if is_filtered_update:
+        matched_snapshot_ids = [
+            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
+        ]
+
+    update(**update_kwargs)
     stdout = request.__dict__.get("stdout")
     stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
-        "result": result,
+        "result": {
+            "matched_count": len(matched_snapshot_ids),
+            "snapshot_ids": matched_snapshot_ids,
+        }
+        if is_filtered_update
+        else None,
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
         "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
     }
@@ -225,29 +241,35 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
 
 @router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
 def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_search import search
+    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    result = search(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
+    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    as_json = search_kwargs.pop("as_json")
+    as_html = search_kwargs.pop("as_html")
+    as_csv = search_kwargs.pop("as_csv")
+    with_headers = search_kwargs.pop("with_headers")
+    try:
+        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
 
     result_format = "txt"
-    if args.as_json:
+    if as_json:
         result_format = "json"
-        result = json.loads(result)
-    elif args.as_html:
+        result = [
+            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
+            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
+        ]
+    elif as_html:
         result_format = "html"
-    elif args.as_csv:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
+    elif as_csv:
         result_format = "csv"
+        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
+        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
+        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
+    else:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
 
     stdout = request.__dict__.get("stdout")
     stderr = request.__dict__.get("stderr")
@@ -264,37 +286,23 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
 @router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
 def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
     from archivebox.cli.archivebox_remove import remove
-    from archivebox.cli.archivebox_search import get_snapshots
     from archivebox.core.models import Snapshot
 
-    filter_patterns = args.filter_patterns or []
-    snapshots_to_remove = get_snapshots(
-        filter_patterns=filter_patterns,
-        filter_type=args.filter_type,
-        after=args.after,
-        before=args.before,
-    )
-    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
+    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
+    timeout_arg = remove_kwargs.pop("timeout")
+    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
+    snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
 
-    remove(
+    result = remove(
         yes=True,  # no way to interactively ask for confirmation via API, so we force yes
         snapshots=snapshots_to_remove,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=filter_patterns,
+        timeout=timeout,
     )
-
-    result = {
-        "removed_count": len(removed_snapshot_ids),
-        "removed_snapshot_ids": removed_snapshot_ids,
-        "remaining_snapshots": Snapshot.objects.count(),
-    }
     stdout = request.__dict__.get("stdout")
     stderr = request.__dict__.get("stderr")
     return {
-        "success": True,
-        "errors": [],
+        "success": bool(result["success"]),
+        "errors": [str(result["error"])] if result["error"] else [],
         "result": result,
         "result_format": "json",
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",

archivebox/api/v1_core.py

@@ -29,7 +29,7 @@ from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot, ArchiveResult, Tag
 from archivebox.core.permissions import public_snapshots_queryset
-from archivebox.api.auth import auth_using_token
+from archivebox.api.auth import authenticated_user_from_request
 from archivebox.config.common import get_config
 from archivebox.core.routes_util import build_web_url
 from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
@@ -49,9 +49,10 @@ from archivebox.core.tag_util import (
     rename_tag as rename_tag_record,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.api.v1_crawls import CrawlSchema
+from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
 from archivebox.search.config import get_search_mode, get_search_mode_backend
 from archivebox.search.query import apply_snapshot_search
+from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
 
 
 router = Router(tags=["Core Models"])
@@ -79,7 +80,7 @@ class CustomPagination(PaginationBase):
     def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
         limit = min(pagination.limit, 500)
         offset = pagination.offset or (pagination.page * limit)
-        total = queryset.count()
+        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
         total_pages = math.ceil(total / limit)
         current_page = math.ceil(offset / (limit + 1))
         items = queryset[offset : offset + limit]
@@ -212,7 +213,10 @@ class ArchiveResultFilterSchema(FilterSchema):
 @paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
     """List all ArchiveResult entries matching these filters."""
-    return filters.filter(ArchiveResult.objects.all()).distinct()
+    queryset = filters.filter(ArchiveResult.objects.all())
+    if filters.search or filters.snapshot_tag:
+        return queryset.distinct()
+    return queryset
 
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
@@ -854,6 +858,7 @@ class SnapshotFilterSchema(FilterSchema):
     modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
     search: str | None = None
     search_mode: str | None = None
+    status: str | None = None
     url: Annotated[str | None, FilterLookup("url")] = None
     tag: Annotated[str | None, FilterLookup("tags__name")] = None
     title: Annotated[str | None, FilterLookup("title__icontains")] = None
@@ -867,13 +872,20 @@ class SnapshotFilterSchema(FilterSchema):
     def filter_search_mode(self, value: str | None) -> Q:
         return Q()
 
+    def filter_status(self, value: str | None) -> Q:
+        return Q()
+
 
 @router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
 @paginate(CustomPagination)
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
     """List all Snapshot entries matching these filters."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = filters.filter(Snapshot.objects.all()).distinct()
+    try:
+        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+    queryset = filters.filter(queryset).distinct()
     query = (filters.search or "").strip()
     if not query:
         return queryset
@@ -916,18 +928,16 @@ def get_snapshots_rss(
 def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
     """Get a specific Snapshot by id."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.all()
-    try:
-        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(_uuid_ref_query("id", snapshot_id))
+    return _get_snapshot_by_ref(snapshot_id)
 
 
 @router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
 def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     tags = normalize_tag_list(data.tags)
-    if data.status is not None and data.status not in Snapshot.StatusChoices.values:
-        raise HttpError(400, f"Invalid status: {data.status}")
+    try:
+        status = normalize_snapshot_status(data.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
     if not data.url.strip():
         raise HttpError(400, "URL is required")
     try:
@@ -938,7 +948,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         raise HttpError(400, "depth must be between 0 and 4")
 
     if data.crawl_id:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), data.crawl_id).get()
+        crawl = get_crawl_by_ref(data.crawl_id)
         crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
         tags = tags or crawl_tags
     else:
@@ -955,7 +965,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         "depth": data.depth,
         "title": data.title,
         "timestamp": str(timezone.now().timestamp()),
-        "status": data.status or Snapshot.StatusChoices.QUEUED,
+        "status": status or Snapshot.StatusChoices.QUEUED,
         "retry_at": timezone.now(),
     }
     snapshot, _ = Snapshot.objects.get_or_create(
@@ -968,10 +978,8 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     if data.title is not None and snapshot.title != data.title:
         snapshot.title = data.title
         update_fields.append("title")
-    if data.status is not None and snapshot.status != data.status:
-        if data.status not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {data.status}")
-        snapshot.status = data.status
+    if status is not None and snapshot.status != status:
+        snapshot.status = status
         update_fields.append("status")
     if update_fields:
         update_fields.append("modified_at")
@@ -992,10 +1000,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
 @router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
 def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
     """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    try:
-        snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        snapshot = filter_queryset_by_uuid_substring(Snapshot.objects.all(), snapshot_id).get()
+    snapshot = _get_snapshot_by_ref(snapshot_id)
 
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
@@ -1018,9 +1023,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
         raise HttpError(400, f"Invalid action: {action}")
 
     if "status" in payload:
-        if payload["status"] not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        snapshot.status = payload["status"]
+        try:
+            snapshot.status = normalize_snapshot_status(payload["status"])
+        except ValueError as err:
+            raise HttpError(400, str(err)) from err
         if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
             snapshot.retry_at = None
         update_fields.append("status")
@@ -1289,16 +1295,7 @@ def _public_tag_listing_enabled() -> bool:
 
 
 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    user = request.user
-    if user.is_authenticated:
-        return True
-
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-
-    if token and auth_using_token(token=token, request=request):
+    if authenticated_user_from_request(request):
         return True
 
     return _public_tag_listing_enabled()

archivebox/api/v1_crawls.py

@@ -24,7 +24,7 @@ from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
 from archivebox.misc.util import filter_queryset_by_uuid_substring
 
-from .auth import API_AUTH_METHODS, auth_using_token
+from .auth import API_AUTH_METHODS, authenticated_user_from_request
 
 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
 
@@ -137,12 +137,16 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
     return crawl
 
 
+def get_crawl_by_ref(crawl_id: str):
+    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+
+
 @router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
 def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
     """Get a specific Crawl by id."""
     setattr(request, "with_snapshots", with_snapshots)
     setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
 
     if crawl and as_rss:
         query = request.GET.copy()
@@ -156,33 +160,18 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
 def crawl_file(request: HttpRequest, crawl_id: str, path: str):
     # Try to resolve the crawl first; if it doesn't exist, return 404.
     try:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+        crawl = get_crawl_by_ref(crawl_id)
     except Crawl.DoesNotExist:
         raise HttpError(404, "Crawl not found")
 
-    # Determine the effective viewer: session user takes precedence, otherwise
-    # fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or
-    # Authorization: Bearer ... (so that programmatic clients still work).
-    user = request.user
-    is_authenticated = bool(user.is_authenticated and user.is_active)
-    if not is_authenticated:
-        token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-        auth_header = request.headers.get("Authorization", "")
-        if not token and auth_header.lower().startswith("bearer "):
-            token = auth_header.split(None, 1)[1].strip()
-        token_user = auth_using_token(token=token, request=request) if token else None
-        if token_user and token_user.is_active:
-            user = token_user
-            is_authenticated = True
-            # Re-bind so is_admin_user() / ownership checks below see the token user.
-            setattr(request, "user", token_user)
+    user = authenticated_user_from_request(request)
 
     # Gate access using the same model as SnapshotView/can_view_snapshot:
     # admins always pass; owners can see their own crawls; otherwise the crawl
     # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
     if not is_admin_user(request):
         permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(is_authenticated and crawl.created_by_id == user.id)
+        is_owner = bool(user and crawl.created_by_id == user.id)
         if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
             raise HttpError(404, "Crawl not found")
 
@@ -217,7 +206,7 @@ def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfol
 @router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
 def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
 
@@ -259,7 +248,7 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
 
 @router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
 def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
     crawl_id_str = str(crawl.id)
     snapshot_count = crawl.snapshot_set.count()
     deleted_count, _ = crawl.delete()

archivebox/api/v1_personas.py

@@ -8,8 +8,10 @@ from uuid import UUID
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.pagination import paginate
 from pydantic import Field
 
+from archivebox.api.v1_core import CustomPagination
 from archivebox.personas.importers import validate_persona_name
 from archivebox.personas.models import Persona
 
@@ -113,6 +115,7 @@ def find_persona(extension_persona_id: str, name: str) -> Persona | None:
 
 
 @router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
+@paginate(CustomPagination)
 def get_personas(request: HttpRequest):
     """List personas available on this ArchiveBox server."""
     return Persona.objects.all().order_by("name")

archivebox/cli/__init__.py

@@ -66,10 +66,6 @@ class ArchiveBoxGroup(click.Group):
         # Introspection commands
         "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
     }
-    legacy_model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
-    }
     all_subcommands = {
         **meta_commands,
         **setup_commands,
@@ -81,34 +77,29 @@ class ArchiveBoxGroup(click.Group):
         "import": "add",
         "archive": "add",
     }
-    legacy_model_subcommands = {
-        "crawl": {"create", "list", "update", "delete"},
-        "snapshot": {"create", "list", "update", "delete"},
-    }
 
     @classmethod
     def get_canonical_name(cls, cmd_name):
         return cls.renamed_commands.get(cmd_name, cmd_name)
 
     @classmethod
-    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
-        if cmd_name not in cls.legacy_model_commands:
-            return False
+    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
+        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)
 
-        try:
-            arg_idx = sys.argv.index(cmd_name)
-        except ValueError:
-            return False
+    @classmethod
+    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
+        if not cls._needs_django_for_lazy_import(cmd_name):
+            return
 
-        remaining_args = sys.argv[arg_idx + 1 :]
-        if not remaining_args:
-            return False
+        from django.apps import apps
 
-        first_arg = remaining_args[0]
-        if first_arg in ("-h", "--help"):
-            return False
+        if apps.ready:
+            return
 
-        return first_arg not in cls.legacy_model_subcommands[cmd_name]
+        from archivebox.config.django import setup_django
+
+        setup_django()
 
     def get_command(self, ctx, cmd_name):
         # handle renamed commands
@@ -120,11 +111,9 @@ class ArchiveBoxGroup(click.Group):
             cmd_name = new_name
             ctx.invoked_subcommand = cmd_name
 
-        if self._should_use_legacy_model_command(cmd_name):
-            return self._lazy_load(self.legacy_model_commands[cmd_name])
-
         # handle lazy loading of commands
         if cmd_name in self.all_subcommands:
+            self._setup_django_for_lazy_import(cmd_name)
             return self._lazy_load(cmd_name)
 
         # fall-back to using click's default command lookup
@@ -199,15 +188,12 @@ def cli(ctx, help=False):
                 raise
 
 
-def main(args=None, prog_name=None, stdin=None):
+def main(args=None, prog_name=None):
     # show `docker run archivebox xyz` in help messages if running in docker
     IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
     IS_TTY = sys.stdin.isatty()
     prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
 
-    # stdin param allows passing input data from caller (used by __main__.py)
-    # currently not used by click-based CLI, but kept for backwards compatibility
-
     previous_unraisablehook = sys.unraisablehook
 
     def ignore_shutdown_unraisable(unraisable):

archivebox/cli/archivebox_add.py

@@ -108,6 +108,7 @@ def add(
 
     # import models once django is set up
     from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.personas.models import Persona
     from archivebox.misc.logging_util import printable_filesize
@@ -124,16 +125,28 @@ def add(
         url_list = [str(url).strip() for url in urls if str(url).strip()]
     if snapshot_ids and len(snapshot_ids) != len(url_list):
         raise ValueError("snapshot_ids length must match urls length")
+    admitted_urls: list[str] = []
+    admitted_snapshot_ids: list[str] | None = [] if snapshot_ids else None
+    for index, url in enumerate(url_list):
+        if Snapshot.is_archivebox_internal_url(url):
+            print(f"[yellow][!] Skipping internal ArchiveBox URL: {url}[/yellow]")
+            continue
+        admitted_urls.append(url)
+        if admitted_snapshot_ids is not None and snapshot_ids:
+            admitted_snapshot_ids.append(snapshot_ids[index])
 
     # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
     sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
     sources_file.parent.mkdir(parents=True, exist_ok=True)
-    if snapshot_ids:
+    if admitted_snapshot_ids is not None:
         sources_file.write_text(
-            "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)),
+            "\n".join(
+                json.dumps({"url": url, "id": admitted_snapshot_ids[index], "tags": tag, "depth": 0})
+                for index, url in enumerate(admitted_urls)
+            ),
         )
     else:
-        sources_file.write_text("\n".join(url_list))
+        sources_file.write_text("\n".join(admitted_urls))
 
     # 2. Create a new Crawl with inline URLs
     cli_args = [*sys.argv]
@@ -143,7 +156,6 @@ def add(
 
     timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
 
-    # Read URLs directly into crawl
     urls_content = sources_file.read_text()
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
@@ -214,8 +226,8 @@ def add(
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
         from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import command_owns_runtime_stack, current_command, standby_until_runtime_stack_needed
-        from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
+        from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed
+        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
 
         command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
         exit_code = 0
@@ -223,15 +235,27 @@ def add(
             try:
                 with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
                     while True:
-                        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
                         exit_code = run_runner_worker(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
                             interactive_interrupts=True,
                         )
+                        crawl.refresh_from_db(fields=["status", "retry_at"])
+                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
+                            break
+                        # A shared supervisord can be stopped by another
+                        # foreground owner (for example `archivebox server`
+                        # shutting down) and stop the one-shot add runner while
+                        # this add's crawl is still runnable. Keep the add
+                        # process alive so it can re-enter the normal
+                        # foreground-runner ownership loop and finish its crawl.
+                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
+                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
+                            continue
                         if exit_code == 0:
                             break
-                        if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                             continue
                         raise SystemExit(exit_code)
             except KeyboardInterrupt:

archivebox/cli/archivebox_config.py

@@ -6,6 +6,7 @@ import sys
 import toml
 import rich_click as click
 from rich import print
+from pathlib import Path
 
 from archivebox.misc.util import docstring, enforce_types
 from archivebox.misc.toml_util import CustomTOMLEncoder
@@ -30,12 +31,17 @@ def config(
     from archivebox.misc.logging_util import printable_config
     from abx_plugins.plugins.base.utils import resolve_alias
     from archivebox.config.collection import write_config_file
+    from archivebox.config import CONSTANTS_CONFIG
     from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
     from archivebox.plugins.discovery import discover_plugin_configs
 
     check_data_folder()
 
     FLAT_CONFIG = get_config().as_dict()
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
+    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
+    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
+    readable_config = {**writable_config, **readonly_config}
     CONFIGS = get_all_configs()
     plugin_schemas = {
         plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
@@ -56,18 +62,23 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
             for config_section in CONFIGS.values():
                 aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}
 
                 for search_key in config_options:
                     # search all aliases in the section
                     for alias_key, key in aliases.items():
-                        if search_key.lower() in alias_key.lower():
+                        if key in readable_config and search_key.lower() in alias_key.lower():
                             matching_config[key] = dict(config_section)[key]
 
                     # search all keys and values in the section
                     for existing_key, value in dict(config_section).items():
+                        if existing_key in readable_config and (
+                            search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
+                        ):
+                            matching_config[existing_key] = value
+                    for existing_key, value in readonly_config.items():
                         if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
                             matching_config[existing_key] = value
 
@@ -79,14 +90,14 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
+            failed_config = [key for key in config_options if key not in readable_config]
             if failed_config:
                 print("\n[red][X] These options failed to get[/red]")
                 print("    {}".format("\n    ".join(config_options)))
                 raise SystemExit(1)
         else:
-            matching_config = FLAT_CONFIG
+            matching_config = readable_config
 
         # Display core config sections
         for config_section in CONFIGS.values():
@@ -100,12 +111,18 @@ def config(
             print(_format_toml(kv_in_section))
             print("[grey53]################################################################[/grey53]")
 
+        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
+        if readonly_keys:
+            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
+            print(_format_toml(readonly_keys))
+            print("[grey53]################################################################[/grey53]")
+
         plugin_keys = {}
 
         # Collect all plugin config keys
         for schema in plugin_schemas.values():
             for key in schema.keys():
-                if key in matching_config:
+                if key in matching_config and key in writable_config:
                     plugin_keys[key] = matching_config[key]
 
         # Display all plugin config in single [PLUGINS] section
@@ -135,7 +152,7 @@ def config(
                     f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
                 )
 
-            if key in FLAT_CONFIG:
+            if key in writable_config:
                 new_config[key] = val.strip()
             else:
                 failed_options.append(line)

archivebox/cli/archivebox_crawl_compat.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_add import add, _collect_input_urls
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
-@click.argument("urls", nargs=-1)
-def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
-    del status, wait
-    add(_collect_input_urls(urls), depth=depth, tag=tag, bg=True)
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_help.py

@@ -13,6 +13,10 @@ from rich.panel import Panel
 
 
 def _command_doc(cmd: str, import_path: str) -> str:
+    def first_doc_line(docstring: str | None) -> str:
+        lines = (docstring or "").splitlines()
+        return lines[0] if lines else ""
+
     modname, _ = import_path.rsplit(".", 1)
     spec = importlib.util.find_spec(modname)
     if spec is None or spec.origin is None:
@@ -26,9 +30,9 @@ def _command_doc(cmd: str, import_path: str) -> str:
     for name in (cmd, "main"):
         for node in tree.body:
             if isinstance(node, ast.FunctionDef) and node.name == name:
-                return (ast.get_docstring(node) or "").splitlines()[0]
+                return first_doc_line(ast.get_docstring(node))
 
-    return (ast.get_docstring(tree) or "").splitlines()[0]
+    return first_doc_line(ast.get_docstring(tree))
 
 
 def help() -> None:

archivebox/cli/archivebox_list.py

@@ -7,50 +7,15 @@ import sys
 
 import rich_click as click
 
-from archivebox.cli.archivebox_snapshot import list_snapshots
+from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options
 
 
 @click.command()
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def main(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-) -> None:
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def main(**kwargs) -> None:
     """List Snapshots."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_remove.py

@@ -23,6 +23,7 @@ from archivebox.misc.logging_util import (
     log_removal_finished,
     TimedProgress,
 )
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 
 @enforce_types
@@ -34,48 +35,54 @@ def remove(
     before: float | None = None,
     yes: bool = False,
     out_dir: Path = CONSTANTS.DATA_DIR,
-) -> QuerySet:
+    timeout: float | None = None,
+    **kwargs,
+) -> dict[str, object]:
     """Remove the specified URLs from the archive"""
 
     setup_django()
     check_data_folder()
+    timeout = float(timeout) if timeout is not None else None
 
-    from archivebox.cli.archivebox_search import get_snapshots
+    from archivebox.core.models import Snapshot
 
+    filter_kwargs = {
+        **kwargs,
+        "filter_patterns": filter_patterns,
+        "filter_type": filter_type,
+        "after": after,
+        "before": before,
+    }
     pattern_list = list(filter_patterns)
 
     log_list_started(pattern_list or None, filter_type)
     timer = TimedProgress(360, prefix="      ")
     try:
-        snapshots = get_snapshots(
-            snapshots=snapshots,
-            filter_patterns=pattern_list or None,
-            filter_type=filter_type,
-            after=after,
-            before=before,
-        )
+        if snapshots is None:
+            snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
+        # Freeze the target set up-front so a concurrent daemon writing new
+        # snapshots can't extend the deletion under us, and so the cursor isn't
+        # held open across the per-row deletes below.
+        snapshot_pks = list(snapshots.values_list("pk", flat=True))
     finally:
         timer.end()
 
-    if not snapshots.exists():
+    if not snapshot_pks:
         log_removal_finished(0, 0)
         raise SystemExit(1)
 
-    log_list_finished(snapshots)
-    log_removal_started(snapshots, yes=yes)
+    if not yes:
+        log_list_finished(snapshots)
+        log_removal_started(snapshots, yes=False)
 
-    from archivebox.core.models import Snapshot
     from archivebox.search.query import flush_search_index
 
-    # Freeze the target set up-front so a concurrent daemon writing new
-    # snapshots can't extend the deletion under us, and so the cursor isn't
-    # held open across the per-row deletes below.
-    snapshot_pks = list(snapshots.values_list("pk", flat=True))
-    to_remove = len(snapshot_pks)
+    started_at = time.monotonic()
+    deadline = started_at + timeout if timeout is not None else None
 
     # Search-index flush touches a separate backend (FTS / sonic), not the
     # main index.sqlite3 writer lock, so it's safe to do once up front.
-    flush_search_index(snapshots=Snapshot.objects.filter(pk__in=snapshot_pks))
+    flush_search_index(snapshots=snapshots)
 
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
@@ -90,40 +97,59 @@ def remove(
     # own retry loop at this outer (non-atomic) level. Each attempt is a
     # fresh atomic; an exception cleanly rolls it back before we sleep.
     retry_interval = 1.0
-    retry_timeout = 60.0
-    for pk in snapshot_pks:
-        deadline = time.monotonic() + retry_timeout
+    deleted_snapshot_pks = []
+    timed_out = False
+    timeout_error = ""
+    for index, pk in enumerate(snapshot_pks):
+        if deadline is not None and time.monotonic() >= deadline:
+            timed_out = True
+            timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
+            break
         while True:
             try:
-                Snapshot.objects.filter(pk=pk).delete()
+                deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
+                if deleted_count:
+                    deleted_snapshot_pks.append(pk)
                 break
             except OperationalError as err:
-                if "database is locked" not in str(err) or time.monotonic() >= deadline:
+                if "database is locked" not in str(err):
                     raise
-                time.sleep(retry_interval)
+                remaining_time = deadline - time.monotonic() if deadline is not None else None
+                if remaining_time is not None and remaining_time <= 0:
+                    timed_out = True
+                    timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
+                    break
+                time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
+        if timed_out:
+            break
 
     all_snapshots = Snapshot.objects.all()
-    log_removal_finished(all_snapshots.count(), to_remove)
+    remaining_count = all_snapshots.count()
+    deleted_snapshot_id_set = set(deleted_snapshot_pks)
+    remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set]
+    log_removal_finished(remaining_count, len(deleted_snapshot_pks))
 
-    return all_snapshots
+    return {
+        "removed_count": len(deleted_snapshot_pks),
+        "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks],
+        "not_removed_count": len(remaining_snapshot_pks),
+        "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks],
+        "success": not timed_out,
+        "error": timeout_error,
+        "timeout": timeout,
+    }
 
 
 @click.command()
 @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
-@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
-    default="exact",
-    help="Type of pattern matching to use when filtering URLs",
-)
-@click.argument("filter_patterns", nargs=-1)
+@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
+@snapshot_filter_options(default_filter_type="exact")
 @docstring(remove.__doc__)
 def main(**kwargs):
     """Remove the specified URLs from the archive"""
-    remove(**kwargs)
+    result = remove(**kwargs)
+    if not result["success"]:
+        raise SystemExit(124)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_run.py

@@ -259,11 +259,24 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
     from archivebox.config import CONSTANTS
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Machine, Process
-    from archivebox.core.takeover_util import enter_single_runner_gate
-    from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls
+    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
+    from archivebox.core.recovery_util import recover_orchestrator_state
+    from archivebox.services.runner import run_pending_crawls
 
     Machine.current()
     current = Process.current()
+    root_command = current.root
+    if daemon and root_command.process_type in (
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
+    ):
+        # Server-owned daemon runners are persistent supervisor workers, but
+        # foreground add/update commands are allowed to borrow runner/sonic
+        # leadership without taking down Daphne. Waiting here keeps the worker
+        # on the normal runner path while preventing a server restart loop from
+        # immediately stealing the single-runner gate back from the newer CLI.
+        standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR)
     if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
         current.mark_exited()
         return 0

archivebox/cli/archivebox_search.py

@@ -3,255 +3,7 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox search"
 
-import sys
-from pathlib import Path
-from typing import TYPE_CHECKING
-from collections.abc import Callable
-
-import rich_click as click
-
-from django.db.models import Q, QuerySet
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
-from archivebox.misc.logging import stderr
-from archivebox.misc.util import enforce_types, docstring
-
-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-
-# Filter types for URL matching
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-STATUS_CHOICES = ["indexed", "archived", "unarchived"]
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        stderr()
-        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
-def _snapshots_to_json(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from archivebox.config import VERSION
-    from archivebox.misc.util import to_json
-
-    config = get_config()
-    main_index_header = (
-        {
-            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-            "schema": "archivebox.index.json",
-            "copyright_info": config.FOOTER_INFO,
-            "meta": {
-                "project": "ArchiveBox",
-                "version": VERSION,
-                "git_sha": VERSION,
-                "website": "https://ArchiveBox.io",
-                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                "source": "https://github.com/ArchiveBox/ArchiveBox",
-                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                "dependencies": {},
-            },
-        }
-        if with_headers
-        else {}
-    )
-
-    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
-    output: dict[str, object] | list[dict[str, object]]
-    if with_headers:
-        output = {
-            **main_index_header,
-            "num_links": len(snapshot_dicts),
-            "updated": datetime.now(tz.utc),
-            "last_run_cmd": sys.argv,
-            "links": snapshot_dicts,
-        }
-    else:
-        output = snapshot_dicts
-
-    return to_json(output, indent=4, sort_keys=True)
-
-
-def _snapshots_to_csv(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    cols: list[str],
-    with_headers: bool,
-) -> str:
-    header = ",".join(cols) if with_headers else ""
-    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
-    return "\n".join((header, *rows))
-
-
-def _snapshots_to_html(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from django.template.loader import render_to_string
-
-    from archivebox.config import VERSION
-    from archivebox.config.version import get_COMMIT_HASH
-
-    config = get_config()
-    template = "static_index.html" if with_headers else "minimal_index.html"
-    snapshot_list = list(snapshots.iterator(chunk_size=500))
-
-    return render_to_string(
-        template,
-        {
-            "version": VERSION,
-            "git_sha": get_COMMIT_HASH() or VERSION,
-            "num_links": str(len(snapshot_list)),
-            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
-            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
-            "links": snapshot_list,
-            "FOOTER_INFO": config.FOOTER_INFO,
-        },
-    )
-
-
-def get_snapshots(
-    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    after: float | None = None,
-    before: float | None = None,
-    out_dir: Path = CONSTANTS.DATA_DIR,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    """Filter and return Snapshots matching the given criteria."""
-    from archivebox.core.models import Snapshot
-
-    if snapshots is not None:
-        result = snapshots
-    else:
-        result = Snapshot.objects.all()
-
-    if after is not None:
-        result = result.filter(timestamp__gte=after)
-    if before is not None:
-        result = result.filter(timestamp__lt=before)
-    if filter_patterns:
-        result = _apply_pattern_filters(result, filter_patterns, filter_type)
-
-    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
-    result = result.select_related("crawl", "crawl__created_by")
-
-    if not result.exists():
-        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
-
-    return result
-
-
-@enforce_types
-def search(
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    status: str = "indexed",
-    before: float | None = None,
-    after: float | None = None,
-    sort: str | None = None,
-    json: bool = False,
-    html: bool = False,
-    csv: str | None = None,
-    with_headers: bool = False,
-):
-    """List, filter, and export information about archive entries"""
-
-    if with_headers and not (json or html or csv):
-        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
-        raise SystemExit(2)
-
-    # Query DB directly - no filesystem scanning
-    snapshots = get_snapshots(
-        filter_patterns=list(filter_patterns) if filter_patterns else None,
-        filter_type=filter_type,
-        before=before,
-        after=after,
-    )
-
-    # Apply status filter
-    if status == "archived":
-        snapshots = snapshots.filter(downloaded_at__isnull=False)
-    elif status == "unarchived":
-        snapshots = snapshots.filter(downloaded_at__isnull=True)
-    # 'indexed' = all snapshots (no filter)
-
-    if sort:
-        snapshots = snapshots.order_by(sort)
-
-    # Export to requested format
-    if json:
-        output = _snapshots_to_json(snapshots, with_headers=with_headers)
-    elif html:
-        output = _snapshots_to_html(snapshots, with_headers=with_headers)
-    elif csv:
-        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
-    else:
-        from archivebox.misc.logging_util import printable_folders
-
-        # Convert to dict for printable_folders
-        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
-        output = printable_folders(folders, with_headers)
-
-    # Structured exports must be written directly to stdout.
-    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
-    sys.stdout.write(output)
-    if not output.endswith("\n"):
-        sys.stdout.write("\n")
-    return output
-
-
-@click.command()
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(["search", *LINK_FILTERS.keys()]),
-    default="substring",
-    help="Pattern matching type for filtering URLs",
-)
-@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
-@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
-@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
-@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
-@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
-@click.help_option("--help", "-h")
-@click.argument("filter_patterns", nargs=-1)
-@docstring(search.__doc__)
-def main(**kwargs):
-    return search(**kwargs)
+from archivebox.cli.archivebox_list import main
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_server.py

@@ -313,6 +313,7 @@ def server(
     from archivebox.core.takeover_util import (
         command_owns_runtime_stack,
         current_command,
+        foreground_runner_owner,
         runtime_stack_owner,
         standby_until_runtime_stack_needed,
     )
@@ -354,7 +355,10 @@ def server(
         ):
             while True:
                 standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
-                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id)
+                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id) or foreground_runner_owner(
+                    data_dir=CONSTANTS.DATA_DIR,
+                    exclude_id=command.id,
+                )
                 takeover_components = active_supervisord_runtime_components(config=config)
                 if older_owner and takeover_components:
                     print(

archivebox/cli/archivebox_snapshot.py

@@ -35,9 +35,9 @@ from collections.abc import Iterable
 
 import rich_click as click
 from rich import print as rprint
-from django.db.models import Case, IntegerField, QuerySet, When
+from django.db.models import QuerySet
 
-from archivebox.cli.cli_util import apply_filters
+SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
 
 
 # =============================================================================
@@ -178,72 +178,63 @@ def create_snapshots(
 # =============================================================================
 
 
+def snapshot_filter_options(*, default_filter_type: str):
+    def decorate(func):
+        for decorator in reversed(
+            (
+                click.option("--status", "-s", help="Filter by status (queued, started, sealed)"),
+                click.option("--url__icontains", help="Filter by URL contains"),
+                click.option("--url__istartswith", help="Filter by URL starts with"),
+                click.option("--tag", "-t", help="Filter by tag name"),
+                click.option("--crawl-id", help="Filter by crawl ID"),
+                click.option("--limit", "-n", type=int, help="Limit number of results"),
+                click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at"),
+                click.option("--search", help="Search mode to use for positional query"),
+                click.option("--before", type=float, help="Only snapshots bookmarked before timestamp"),
+                click.option("--after", type=float, help="Only snapshots bookmarked after timestamp"),
+                click.option(
+                    "--filter-type",
+                    "-f",
+                    type=click.Choice(SNAPSHOT_FILTER_TYPE_CHOICES),
+                    default=default_filter_type,
+                    help="Type of pattern matching to use for positional filters",
+                ),
+                click.argument("filter_patterns", nargs=-1),
+            ),
+        ):
+            func = decorator(func)
+        return func
+
+    return decorate
+
+
+def snapshot_output_options(func):
+    for decorator in reversed(
+        (
+            click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title"),
+            click.option("--json", "as_json", is_flag=True, help="Print output as a JSON array"),
+            click.option("--html", "as_html", is_flag=True, help="Print output as HTML"),
+            click.option("--with-headers", is_flag=True, help="Include column headers in structured output"),
+        ),
+    ):
+        func = decorator(func)
+    return func
+
+
 def build_snapshot_queryset(
-    *,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    query: str | None = None,
-    limit: int | None = None,
+    **kwargs,
 ) -> QuerySet:
     from archivebox.core.models import Snapshot
-    from archivebox.search.query import apply_snapshot_search
 
-    queryset = Snapshot.objects.order_by("-created_at")
-    queryset = apply_filters(
-        queryset,
-        {
-            "status": status,
-            "url__icontains": url__icontains,
-            "url__istartswith": url__istartswith,
-            "crawl_id": crawl_id,
-        },
-    )
-
-    if tag:
-        queryset = queryset.filter(tags__name__iexact=tag)
-
-    query = (query or "").strip()
-    if query:
-        try:
-            queryset = apply_snapshot_search(
-                queryset,
-                query,
-                search_mode=search,
-                ordering=("-created_at",) if not sort else None,
-                max_results=limit,
-                skip_backend_when_metadata_satisfies_limit=True,
-                include_metadata_for_forced_backend=True,
-            )
-        except Exception as err:
-            rprint(
-                f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                file=sys.stderr,
-            )
-            queryset = apply_snapshot_search(queryset, query, search_mode="meta")
-
-    if sort:
-        queryset = queryset.order_by(sort)
-
-    return queryset
+    return Snapshot.objects.order_by("-created_at").search(**kwargs)
 
 
 def list_snapshots(
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
     csv: str | None = None,
+    as_json: bool = False,
+    as_html: bool = False,
     with_headers: bool = False,
-    search: str | None = None,
-    query: str | None = None,
+    **kwargs,
 ) -> int:
     """
     List Snapshots as JSONL with optional filters.
@@ -252,90 +243,71 @@ def list_snapshots(
         0: Success (even if no results)
     """
     from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Snapshot
 
-    if with_headers and not csv:
-        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
+    output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
+    if output_formats > 1:
+        rprint("[red]Choose only one output format: --csv, --json, or --html[/red]", file=sys.stderr)
+        return 2
+    if with_headers and not output_formats:
+        rprint("[red]--with-headers requires --csv, --json, or --html[/red]", file=sys.stderr)
         return 2
 
-    is_tty = sys.stdout.isatty() and not csv
+    is_tty = sys.stdout.isatty() and not output_formats
 
-    queryset = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=query,
-        limit=limit,
-    )
-
-    if not is_tty:
-        if limit:
-            limited_ids = list(queryset.values_list("id", flat=True)[:limit])
-            preserved_order = Case(
-                *(When(id=snapshot_id, then=position) for position, snapshot_id in enumerate(limited_ids)),
-                output_field=IntegerField(),
-            )
-            queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order)
-        queryset = queryset.prefetch_related("tags")
-    elif limit:
-        queryset = queryset[:limit]
+    try:
+        queryset = build_snapshot_queryset(**kwargs)
+    except ValueError as err:
+        rprint(f"[red]{err}[/red]", file=sys.stderr)
+        return 2
 
     count = 0
+    if as_json:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_json(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    if as_html:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_html(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
     if csv:
         cols = [col.strip() for col in csv.split(",") if col.strip()]
         if not cols:
             rprint("[red]No CSV columns provided[/red]", file=sys.stderr)
             return 2
-        rows: list[str] = []
         if with_headers:
-            rows.append(",".join(cols))
-        simple_cols = {
-            "id",
-            "crawl_id",
-            "url",
-            "title",
-            "timestamp",
-            "depth",
-            "status",
-            "fs_version",
-            "bookmarked_at",
-            "created_at",
-            "modified_at",
-            "retry_at",
-            "downloaded_at",
-        }
-        from archivebox.misc.util import to_json
-
-        for snapshot in queryset.iterator(chunk_size=500):
-            if set(cols).issubset(simple_cols):
-                rows.append(
-                    ",".join(to_json(snapshot.serializable_value(col), indent=None) for col in cols),
-                )
-            else:
-                rows.append(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write(",".join(cols))
+            sys.stdout.write("\n")
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=500):
+            sys.stdout.write(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write("\n")
             count += 1
-        output = "\n".join(rows)
-        if output:
-            sys.stdout.write(output)
-            if not output.endswith("\n"):
-                sys.stdout.write("\n")
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 
-    for snapshot in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(snapshot.status, "dim")
-            rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
-        else:
+    if not is_tty:
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=500):
             write_record(snapshot.to_json())
+            count += 1
+        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    for snapshot in queryset.iterator(chunk_size=500):
+        status_color = {
+            "queued": "yellow",
+            "started": "blue",
+            "sealed": "green",
+        }.get(snapshot.status, "dim")
+        rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
         count += 1
 
     rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
@@ -494,46 +466,11 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int):
 
 
 @main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def list_cmd(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-):
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def list_cmd(**kwargs):
     """List Snapshots as JSONL."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 @main.command("update")

archivebox/cli/archivebox_snapshot_compat.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox snapshot"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_snapshot import create_snapshots
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
-@click.argument("urls", nargs=-1)
-def main(tag: str, status: str, depth: int, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox snapshot URL...` entrypoint."""
-    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_update.py

@@ -15,6 +15,7 @@ from pathlib import Path
 import rich_click as click
 
 from archivebox.misc.util import enforce_types, docstring
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -47,51 +48,19 @@ def _get_search_indexing_plugins() -> list[str]:
 
 
 def _build_filtered_snapshots_queryset(
-    *,
-    filter_patterns: Iterable[str],
-    filter_type: str,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    before: float | None = None,
-    after: float | None = None,
-    resume: str | None = None,
+    **kwargs,
 ):
-    from datetime import datetime
+    from archivebox.core.models import Snapshot
     from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    filter_patterns = tuple(filter_patterns)
-    snapshots = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=" ".join(filter_patterns) if search else None,
-    )
-
-    if filter_patterns and not search:
-        snapshots = snapshots.filter_by_patterns(list(filter_patterns), filter_type)
-
-    if before:
-        snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before))
-    if after:
-        snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after))
-    if resume:
-        snapshots = snapshots.filter(timestamp__lte=resume)
-    if not sort:
-        snapshots = snapshots.order_by("-timestamp")
+    limit = kwargs.pop("limit", None)
+    snapshots = build_snapshot_queryset(**kwargs)
+    if kwargs.get("resume"):
+        snapshots = snapshots.filter(timestamp__lte=kwargs["resume"])
     snapshots = snapshots.select_related("crawl")
-    if limit:
-        limited_ids = list(snapshots.values_list("id", flat=True)[:limit])
-        snapshots = snapshots.model.objects.filter(id__in=limited_ids).select_related("crawl")
+    if limit is not None and limit > 0:
+        snapshot_ids = list(snapshots.values_list("id", flat=True)[:limit])
+        snapshots = Snapshot.objects.filter(id__in=snapshot_ids).select_related("crawl")
 
     return snapshots
 
@@ -254,27 +223,29 @@ def update(
     from archivebox.machine.models import Process
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals, raise_if_shutdown_requested
     from archivebox.core.takeover_util import (
-        command_owns_runtime_stack,
+        command_owns_foreground_runner,
         current_command,
         ensure_daemon_stack,
-        standby_until_runtime_stack_needed,
+        standby_until_foreground_runner_needed,
     )
-    from archivebox.workers.supervisord_util import run_runner_worker, stop_existing_supervisord_process, stop_own_supervisord_process
+    from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
 
     command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR)
 
     def wait_for_turn() -> None:
         raise_if_shutdown_requested()
-        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
         raise_if_shutdown_requested()
 
-    def run_scoped_runner(*args: str) -> None:
+    def run_scoped_runner(*args: str, ensure_daemon_reason: str | None = None) -> None:
         while True:
             wait_for_turn()
+            if ensure_daemon_reason:
+                ensure_daemon_stack(reason=ensure_daemon_reason)
             exit_code = run_runner_worker(list(args), name=f"worker_runner_update_{os.getpid()}")
             if exit_code == 0:
                 return
-            if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+            if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                 continue
             raise SystemExit(exit_code)
 
@@ -298,8 +269,6 @@ def update(
 
     try:
         wait_for_turn()
-        if stop_daemon_stack:
-            stop_existing_supervisord_process()
 
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             while True:
@@ -466,7 +435,10 @@ def update(
                         for snapshot_id in sorted(touched_snapshot_ids):
                             run_scoped_runner("--snapshot-id", snapshot_id)
                     else:
-                        run_scoped_runner(*(["--maintenance-only"] if index_only or migrate_only else []))
+                        run_scoped_runner(
+                            *(["--maintenance-only"] if index_only or migrate_only else []),
+                            ensure_daemon_reason="search indexing" if do_index else None,
+                        )
 
                 if not continuous:
                     break
@@ -987,22 +959,11 @@ def print_index_stats(stats: dict[str, Any]) -> None:
 
 @click.command()
 @click.option("--resume", type=str, help="Resume from timestamp")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of snapshots to update")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--search", help="Search mode to use for positional query")
-@click.option("--before", type=float, help="Only snapshots before timestamp")
-@click.option("--after", type=float, help="Only snapshots after timestamp")
-@click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
 @click.option("--batch-size", type=int, default=500, help="Commit every N records")
 @click.option("--continuous", is_flag=True, help="Run continuously as background worker")
 @click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
 @click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
-@click.argument("filter_patterns", nargs=-1)
+@snapshot_filter_options(default_filter_type="exact")
 @docstring(update.__doc__)
 def main(**kwargs):
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@@ -1010,6 +971,8 @@ def main(**kwargs):
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             update(**kwargs)
+    except ValueError as err:
+        raise click.BadParameter(str(err), param_hint="--status") from err
     except KeyboardInterrupt:
         raise SystemExit(130) from None
 

archivebox/core/admin_snapshots.py

@@ -280,7 +280,7 @@ class SnapshotChangeList(SearchResultsChangeList):
         else:
             self.full_result_count = self.model_admin.get_paginator(
                 request,
-                self.model._default_manager.all().order_by(),
+                self.model._default_manager.all().order_by("-id"),
                 self.list_per_page,
             ).count
         self.show_full_result_count = True
@@ -465,7 +465,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         ),
     )
 
-    ordering = ["-timestamp"]
+    ordering = ["-id"]
     actions = [
         "add_tags",
         "remove_tags",

archivebox/core/models.py

@@ -267,6 +267,22 @@ class SnapshotQuerySet(models.QuerySet):
         "tag": lambda pattern: models.Q(tags__name=pattern),
         "timestamp": lambda pattern: models.Q(timestamp=pattern),
     }
+    FILTER_TYPE_CHOICES = tuple(FILTER_TYPES)
+    FILTER_ARG_KEYS = (
+        "after",
+        "before",
+        "filter_type",
+        "filter_patterns",
+        "status",
+        "url__icontains",
+        "url__istartswith",
+        "tag",
+        "crawl_id",
+        "limit",
+        "sort",
+        "search",
+    )
+    SPECIAL_FILTER_ARG_KEYS = frozenset({"filter_patterns", "filter_type", "query", "search", "tag", "before", "after", "limit", "sort"})
 
     def filter_by_patterns(self, patterns: list[str], filter_type: str = "exact") -> "SnapshotQuerySet":
         """Filter snapshots by URL patterns using specified filter type"""
@@ -283,6 +299,62 @@ class SnapshotQuerySet(models.QuerySet):
                 raise SystemExit(2)
         return self.filter(q_filter)
 
+    def search(self, **kwargs) -> "SnapshotQuerySet":
+        from datetime import timezone as dt_timezone
+
+        from archivebox.core.snapshot_status import filter_snapshots_by_status
+        from archivebox.search.query import apply_snapshot_search
+
+        queryset = self
+        filter_patterns = tuple(str(pattern) for pattern in kwargs.get("filter_patterns") or ())
+        filter_type = kwargs.get("filter_type") or "substring"
+        query = kwargs.get("query")
+        if isinstance(query, (list, tuple)):
+            query = " ".join(str(part) for part in query)
+        query = (query or (" ".join(filter_patterns) if kwargs.get("search") else "")).strip()
+
+        field_names = {field.name for field in self.model._meta.get_fields()}
+        field_names.update(field.attname for field in self.model._meta.fields)
+        field_filters = {
+            key: value
+            for key, value in kwargs.items()
+            if value is not None and key not in self.SPECIAL_FILTER_ARG_KEYS and key.split("__", 1)[0] in field_names
+        }
+        status = field_filters.pop("status", None)
+        queryset = filter_snapshots_by_status(queryset, status)
+        if field_filters:
+            queryset = queryset.filter(**field_filters)
+        if kwargs.get("tag"):
+            queryset = queryset.filter(tags__name__iexact=kwargs["tag"])
+        if kwargs.get("before") is not None:
+            queryset = queryset.filter(bookmarked_at__lt=datetime.fromtimestamp(float(kwargs["before"]), tz=dt_timezone.utc))
+        if kwargs.get("after") is not None:
+            queryset = queryset.filter(bookmarked_at__gt=datetime.fromtimestamp(float(kwargs["after"]), tz=dt_timezone.utc))
+
+        if query:
+            queryset = apply_snapshot_search(
+                queryset,
+                query,
+                search_mode=kwargs.get("search"),
+                ordering=("-created_at",) if not kwargs.get("sort") else None,
+                max_results=kwargs.get("limit"),
+                skip_backend_when_metadata_satisfies_limit=True,
+                include_metadata_for_forced_backend=True,
+            )
+        elif filter_patterns:
+            queryset = queryset.filter_by_patterns(list(filter_patterns), filter_type)
+
+        if kwargs.get("sort"):
+            queryset = queryset.order_by(kwargs["sort"])
+        elif not queryset.query.order_by:
+            queryset = queryset.order_by("-created_at")
+
+        limit = kwargs.get("limit")
+        if limit is not None and limit > 0:
+            queryset = queryset[:limit]
+
+        return queryset
+
     # =========================================================================
     # Export Methods
     # =========================================================================
@@ -690,6 +762,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         from archivebox.core.routes_util import (
             get_admin_host,
             get_api_host,
+            get_base_host,
             get_listen_host,
             get_public_host,
             get_web_host,
@@ -704,6 +777,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         protected_roots: set[tuple[str, str | None]] = set()
         for host_value in (
             get_listen_host(config=config),
+            get_base_host(config=config),
             get_admin_host(config=config),
             get_web_host(config=config),
             get_api_host(config=config),
@@ -3173,7 +3247,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         """Convert to CSV string"""
         data = self.to_dict()
         cols = cols or ["timestamp", "is_archived", "url"]
-        return separator.join(to_json(data.get(col, ""), indent=None).ljust(ljust) for col in cols)
+        invalid_cols = [col for col in dict.fromkeys(cols) if col not in data]
+        if invalid_cols:
+            supported_cols = ", ".join(sorted(data))
+            raise ValueError(f"Invalid CSV field(s): {', '.join(invalid_cols)}\nSupported CSV fields: {supported_cols}")
+        return separator.join(to_json(data[col], indent=None).ljust(ljust) for col in cols)
 
     def write_json_details(self, out_dir: Path | str | None = None) -> None:
         """Write JSON index file for this snapshot to its output directory"""

archivebox/core/recovery_util.py

@@ -20,6 +20,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
         "crawls_queued_without_retry_at": 0,
         "snapshots_queued_without_retry_at": 0,
         "archiveresults_backoff": 0,
+        "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0,
         "archiveresults_started_without_running_process": 0,
         "snapshots_started_without_running_results": 0,
         "crawls_started_with_due_snapshots": 0,
@@ -73,6 +74,24 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
         retry_at__isnull=True,
     ).update(retry_at=now, modified_at=now)
     cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
+    # Targeted plugin rows on final/paused Snapshots are scheduled through the
+    # parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a
+    # sealed Snapshot and must not be interpreted as queued work just because
+    # old/synthetic ArchiveResult rows exist. If takeover kills the runner
+    # after it leases the Snapshot but before queued ArchiveResult rows finish,
+    # the rows remain QUEUED while retry_at sits in the future. Recovery runs
+    # only after this runner has won the single-runner gate, so it can safely
+    # unlock those stale plugin leases for immediate processing instead of
+    # waiting out the previous owner's full lock timeout.
+    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED)
+    cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
+        Snapshot.objects.filter(
+            id__in=queued_plugin_results.values("snapshot_id"),
+            status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
+        )
+        .filter(retry_at__gt=now)
+        .update(retry_at=now, modified_at=now)
+    )
     # Impossible state repair: STARTED ArchiveResults without a live Process
     # have no owner left to emit completion. Requeue only the result row; the
     # snapshot/crawl schedulers will pick up normal retry processing.

archivebox/core/settings.py

@@ -77,6 +77,8 @@ INSTALLED_APPS = [
     "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)
 ]
 
+DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST"
+
 
 MIDDLEWARE = [
     "archivebox.core.middleware.TimezoneMiddleware",

archivebox/core/snapshot_status.py

@@ -0,0 +1,25 @@
+__package__ = "archivebox.core"
+
+from django.db.models import QuerySet
+
+
+def snapshot_status_values() -> tuple[str, ...]:
+    from archivebox.core.models import Snapshot
+
+    return tuple(Snapshot.StatusChoices.values)
+
+
+def normalize_snapshot_status(status: str | None) -> str | None:
+    value = str(status or "").strip().lower()
+    if not value:
+        return None
+
+    valid_statuses = snapshot_status_values()
+    if value not in valid_statuses:
+        raise ValueError(f"Invalid snapshot status: {status}. Expected one of: {', '.join(valid_statuses)}")
+    return value
+
+
+def filter_snapshots_by_status(queryset: QuerySet, status: str | None) -> QuerySet:
+    value = normalize_snapshot_status(status)
+    return queryset.filter(status=value) if value else queryset

archivebox/core/takeover_util.py

@@ -18,10 +18,18 @@ def runtime_stack_owner_types():
     from archivebox.machine.models import Process
 
     return (
-        Process.TypeChoices.UPDATE,
         Process.TypeChoices.SERVER,
         Process.TypeChoices.ORCHESTRATOR,
+    )
+
+
+def foreground_runner_owner_types():
+    from archivebox.machine.models import Process
+
+    return (
+        Process.TypeChoices.SERVER,
         Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
     )
 
 
@@ -70,15 +78,13 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
     if exclude_id is not None:
         base_qs = base_qs.exclude(id=exclude_id)
 
-    top_level_types = (
-        Process.TypeChoices.UPDATE,
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ADD,
-    )
     for qs in (
-        base_qs.filter(process_type__in=top_level_types),
+        # Only server parents own HTTP runtime leadership. Foreground add/update
+        # commands can own runner/sonic components, but server startup must never
+        # wait behind them before binding Daphne.
+        base_qs.filter(process_type=Process.TypeChoices.SERVER),
         # A foreground `archivebox run` process is allowed to own the runtime
-        # stack when no server/update/add parent is alive. A runner launched by
+        # stack when no server/add parent is alive. A runner launched by
         # supervisord is only a child worker; after its parent is killed it must
         # not keep stealing leadership from the next foreground command.
         base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
@@ -95,6 +101,30 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
     return bool(owner and owner.id == command.id)
 
 
+def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
+    from archivebox.machine.models import Machine, Process
+
+    machine = Machine.current()
+    qs = Process.objects.filter(
+        machine=machine,
+        status=Process.StatusChoices.RUNNING,
+        pwd=str(data_dir),
+        process_type__in=foreground_runner_owner_types(),
+    )
+    if exclude_id is not None:
+        qs = qs.exclude(id=exclude_id)
+    for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+        if proc.is_running:
+            return proc
+        proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
+    return None
+
+
+def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
+    owner = foreground_runner_owner(data_dir=data_dir)
+    return bool(owner and owner.id == command.id)
+
+
 def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
     try:
         from archivebox.config.common import get_config
@@ -108,9 +138,9 @@ def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
     if not names and owner is not None:
         from archivebox.machine.models import Process
 
-        if owner.process_type in {Process.TypeChoices.SERVER, Process.TypeChoices.ADD}:
+        if owner.process_type == Process.TypeChoices.SERVER:
             names = ["orchestrator", "server"]
-        elif owner.process_type in {Process.TypeChoices.UPDATE, Process.TypeChoices.ORCHESTRATOR}:
+        elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
             names = ["orchestrator"]
 
     return ", ".join(dict.fromkeys(names)) or "runtime stack"
@@ -292,3 +322,26 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
     command.modified_at = timezone.now()
     command.save(update_fields=["modified_at"])
     return {"resumed": announced, "previous_owner_pid": previous_owner_pid}
+
+
+def standby_until_foreground_runner_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
+    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+
+    announced = False
+    previous_owner_pid = None
+    while not command_owns_foreground_runner(command, data_dir=data_dir):
+        reap_foreground_supervisord_process()
+        if not announced:
+            owner = foreground_runner_owner(data_dir=data_dir)
+            owner_pid = owner.pid if owner else "unknown"
+            previous_owner_pid = owner_pid
+            rprint(
+                f"[yellow][*] A newer archivebox process took over the orchestrator, sonic "
+                f"(pid={owner_pid}). Work will continue there, and will resume here if that process exits and work still remains.[/yellow]",
+                file=sys.stderr,
+            )
+            announced = True
+        time.sleep(interval)
+    command.modified_at = timezone.now()
+    command.save(update_fields=["modified_at"])
+    return {"resumed": announced, "previous_owner_pid": previous_owner_pid}

archivebox/crawls/admin.py

@@ -943,7 +943,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
             updated += len(batch)
         return updated
 
-    @action(label="Recrawl", description="Create a new crawl with the same settings")
+    @action(label="Recrawl", description="Create a new crawl with the same settings", methods=("POST",))
     def recrawl(self, request, obj):
         """Duplicate this crawl as a new crawl with the same URLs and settings."""
 

archivebox/machine/admin.py

@@ -548,6 +548,7 @@ class ProcessAdmin(BaseModelAdmin):
         label="Kill",
         description="Kill this process if it is still running",
         attrs={"class": "deletelink"},
+        methods=("POST",),
     )
     def kill_process(self, request, obj):
         self._terminate_processes(request, [obj])

archivebox/services/runner.py

@@ -56,7 +56,6 @@ from abxbus.event_bus import EventBus, get_current_event, in_handler_context
 from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError
 
 from archivebox.config.common import ArchiveBoxBaseConfig, normalize_runtime_config
-from archivebox.core.recovery_util import recover_orchestrator_state
 from archivebox.misc.db import run_db_analyze_batch
 from archivebox.core.shutdown_util import foreground_shutdown_signals, raise_if_shutdown_requested
 from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
@@ -930,15 +929,14 @@ class CrawlRunner:
                             event_handler_slow_timeout=slow_warning_timeout(crawl_setup_phase_timeout),
                         ),
                     )
-                    # Normal crawl shutdown drives cleanup synchronously so
-                    # ProcessKillEvent handlers get their grace period. During
-                    # OS-signal shutdown, asyncio is already cancelling tasks;
-                    # keep the child event attached for any remaining bus tick,
-                    # but do not call now()/wait() because the bus context may
-                    # disappear before delivery and produce noisy shutdown
-                    # exceptions instead of useful cleanup.
-                    if not self._signal_abort_requested:
-                        await _run_event_now(cleanup_event, crawl_setup_phase_timeout)
+                    # Cleanup owns ProcessKillEvent emission for crawl-scoped
+                    # setup hooks. Even during OS-signal shutdown we must drive
+                    # it synchronously before bus teardown; otherwise daemon/bg
+                    # setup hooks can outlive the foreground runner that
+                    # launched them. _run_event_now() is already bounded by the
+                    # crawl setup timeout and cleanup handlers provide their own
+                    # hook-level grace periods.
+                    await _run_event_now(cleanup_event, crawl_setup_phase_timeout)
             finally:
                 cancel_watcher.cancel()
                 await asyncio.gather(cancel_watcher, return_exceptions=True)
@@ -2134,6 +2132,8 @@ def run_pending_crawls(
         if daemon:
             now_monotonic = time.monotonic()
             if now_monotonic - last_recovery_at >= 30.0:
+                from archivebox.core.recovery_util import recover_orchestrator_state
+
                 recover_orchestrator_state()
                 last_recovery_at = now_monotonic
             # SQLite query plans degrade as the snapshot/archiveresult tables grow

archivebox/templates/admin/crawls/crawl/change_form.html

@@ -1,4 +1,5 @@
 {% extends "admin/change_form.html" %}
+{% load add_preserved_filters from admin_urls %}
 
 {% block object-tools-items %}
 {% if original %}
@@ -31,6 +32,12 @@
     {% endif %}
   </li>
 {% endif %}
+{% for tool in objectactions %}
+  <li class="objectaction-item" data-tool-name="{{ tool.name }}">
+    {% url tools_view_name pk=object_id tool=tool.name as action_url %}
+    {% include 'django_object_actions/action_trigger.html' %}
+  </li>
+{% endfor %}
 {{ block.super }}
 {% endblock %}
 

archivebox/templates/static/admin.css

@@ -134,13 +134,23 @@ body.change-list #content .object-tools {
     clear: both;
 }
 
-#content .object-tools a:link, #content .object-tools a:visited {
+#content .object-tools a:link,
+#content .object-tools a:visited,
+#content .object-tools form button {
     border-radius: 0px;
     background-color: #f5dd5d;
     color: #333;
     font-size: 12px;
     font-weight: 800;
 }
+#content .object-tools form {
+    display: inline;
+    margin: 0;
+}
+#content .object-tools form button {
+    border: 0;
+    cursor: pointer;
+}
 
 #content .object-tools a.addlink {
     background-blend-mode: difference;

archivebox/tests/fixtures.py

@@ -1,121 +0,0 @@
-import os
-import subprocess
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from threading import Thread
-
-import pytest
-
-
-@pytest.fixture
-def process(tmp_path):
-    process = subprocess.run(
-        ["archivebox", "init"],
-        capture_output=True,
-        cwd=tmp_path,
-    )
-    return process
-
-
-@pytest.fixture
-def disable_extractors_dict():
-    env = os.environ.copy()
-    env.update(
-        {
-            "SAVE_WGET": "false",
-            "SAVE_SINGLEFILE": "false",
-            "SAVE_READABILITY": "false",
-            "SAVE_MERCURY": "false",
-            "SAVE_HTMLTOTEXT": "false",
-            "SAVE_PDF": "false",
-            "SAVE_SCREENSHOT": "false",
-            "SAVE_DOM": "false",
-            "SAVE_HEADERS": "false",
-            "SAVE_GIT": "false",
-            "SAVE_YTDLP": "false",
-            "SAVE_ARCHIVEDOTORG": "false",
-            "SAVE_TITLE": "false",
-            "SAVE_FAVICON": "false",
-            "PLUGINS": "__archivebox_test_no_plugins__",
-        },
-    )
-    return env
-
-
-@pytest.fixture
-def recursive_test_site():
-    pages = {
-        "/": """
-            <html>
-              <head>
-                <title>Root</title>
-                <link rel="icon" href="/favicon.ico">
-              </head>
-              <body>
-                <a href="/about">About</a>
-                <a href="/blog">Blog</a>
-                <a href="/contact">Contact</a>
-              </body>
-            </html>
-        """.strip().encode("utf-8"),
-        "/about": """
-            <html>
-              <body>
-                <a href="/deep/about">Deep About</a>
-              </body>
-            </html>
-        """.strip().encode("utf-8"),
-        "/blog": """
-            <html>
-              <body>
-                <a href="/deep/blog">Deep Blog</a>
-              </body>
-            </html>
-        """.strip().encode("utf-8"),
-        "/contact": """
-            <html>
-              <body>
-                <a href="/deep/contact">Deep Contact</a>
-              </body>
-            </html>
-        """.strip().encode("utf-8"),
-        "/deep/about": b"<html><body><h1>Deep About</h1></body></html>",
-        "/deep/blog": b"<html><body><h1>Deep Blog</h1></body></html>",
-        "/deep/contact": b"<html><body><h1>Deep Contact</h1></body></html>",
-        "/favicon.ico": b"test-icon",
-    }
-
-    class _RecursiveHandler(BaseHTTPRequestHandler):
-        def do_GET(self):
-            body = pages.get(self.path)
-            if body is None:
-                self.send_response(404)
-                self.end_headers()
-                return
-
-            self.send_response(200)
-            if self.path.endswith(".ico"):
-                self.send_header("Content-Type", "image/x-icon")
-            else:
-                self.send_header("Content-Type", "text/html; charset=utf-8")
-            self.send_header("Content-Length", str(len(body)))
-            self.end_headers()
-            self.wfile.write(body)
-
-        def log_message(self, format, *args):
-            return
-
-    server = ThreadingHTTPServer(("127.0.0.1", 0), _RecursiveHandler)
-    thread = Thread(target=server.serve_forever, daemon=True)
-    thread.start()
-    try:
-        base_url = f"http://127.0.0.1:{server.server_address[1]}"
-        yield {
-            "base_url": base_url,
-            "root_url": f"{base_url}/",
-            "child_urls": [f"{base_url}/about", f"{base_url}/blog", f"{base_url}/contact"],
-            "deep_urls": [f"{base_url}/deep/about", f"{base_url}/deep/blog", f"{base_url}/deep/contact"],
-        }
-    finally:
-        server.shutdown()
-        server.server_close()
-        thread.join(timeout=5)

archivebox/tests/migrations_helpers.py

@@ -8,14 +8,12 @@ This module provides:
 - Helper functions to run archivebox commands and verify results
 """
 
-import os
-import sys
 import json
 import sqlite3
-import subprocess
 from pathlib import Path
 from datetime import datetime, timezone
 
+from archivebox.tests.conftest import cli_env, run_archivebox_cmd
 from archivebox.uuid_compat import uuid7
 
 
@@ -1020,41 +1018,21 @@ def seed_0_8_data(db_path: Path) -> dict[str, list[dict]]:
 # =============================================================================
 
 
-def run_archivebox(data_dir: Path, args: list, timeout: int = 60, env: dict | None = None) -> subprocess.CompletedProcess:
+def run_archivebox_migration_cmd(data_dir: Path, args: list, timeout: int = 60, env: dict | None = None):
     """Run archivebox command in subprocess with given data directory."""
-    base_env = os.environ.copy()
-    base_env["USE_COLOR"] = "False"
-    base_env["SHOW_PROGRESS"] = "False"
-    # Disable ALL extractors for faster tests (can be overridden by env parameter)
-    base_env["PLUGINS"] = "__archivebox_test_no_plugins__"
-    base_env["SAVE_ARCHIVEDOTORG"] = "False"
-    base_env["SAVE_TITLE"] = "False"
-    base_env["SAVE_FAVICON"] = "False"
-    base_env["SAVE_WGET"] = "False"
-    base_env["SAVE_SINGLEFILE"] = "False"
-    base_env["SAVE_SCREENSHOT"] = "False"
-    base_env["SAVE_PDF"] = "False"
-    base_env["SAVE_DOM"] = "False"
-    base_env["SAVE_READABILITY"] = "False"
-    base_env["SAVE_MERCURY"] = "False"
-    base_env["SAVE_GIT"] = "False"
-    base_env["SAVE_YTDLP"] = "False"
-    base_env["SAVE_HEADERS"] = "False"
-    base_env["SAVE_HTMLTOTEXT"] = "False"
-
-    # Override with any custom env vars
+    base_env = cli_env(
+        disable_extractors=True,
+        PLUGINS="__archivebox_test_no_plugins__",
+    )
     if env:
         base_env.update(env)
 
-    cmd = [sys.executable, "-m", "archivebox"] + args
-
-    return subprocess.run(
-        cmd,
-        capture_output=True,
-        text=True,
+    return run_archivebox_cmd(
+        args,
         env=base_env,
-        cwd=str(data_dir),
+        cwd=data_dir,
         timeout=timeout,
+        replace_env=True,
     )
 
 

archivebox/tests/test_api_archiveresult.py

@@ -0,0 +1 @@
+# Tests moved to test_api_v1_core_archiveresults.py and test_api_v1_core_archiveresult_archiveresult_id.py.

archivebox/tests/test_api_cli.py

@@ -1,155 +1 @@
-import os
-import time
-
-import pytest
-import requests
-
-from archivebox.core.models import Snapshot
-from archivebox.crawls.models import Crawl
-from archivebox.tests.test_orm_helpers import use_archivebox_db
-from .conftest import (
-    build_test_env,
-    create_admin_and_token,
-    get_free_port,
-    init_archive,
-    start_server,
-    stop_server,
-    wait_for_http,
-)
-
-pytestmark = pytest.mark.django_db(transaction=True)
-
-
-@pytest.mark.timeout(180)
-def test_cli_api_add_search_update_remove_over_server(tmp_path):
-    os.chdir(tmp_path)
-    init_archive(tmp_path)
-
-    port = get_free_port()
-    env = build_test_env(port, PUBLIC_INDEX="True")
-    api_token = create_admin_and_token(tmp_path)
-    api_headers = {
-        "Host": f"api.archivebox.localhost:{port}",
-        "X-ArchiveBox-API-Key": api_token,
-    }
-    target_url = "https://example.com/"
-
-    try:
-        start_server(tmp_path, env=env, port=port)
-        wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs")
-
-        add_response = requests.post(
-            f"http://127.0.0.1:{port}/api/v1/cli/add",
-            headers=api_headers,
-            json={
-                "urls": [target_url],
-                "tag": "api-cli",
-                "depth": 0,
-                "parser": "url_list",
-                "plugins": "wget",
-                "update": True,
-                "overwrite": False,
-                "index_only": True,
-            },
-            timeout=10,
-        )
-        assert add_response.status_code == 200, add_response.text
-        add_payload = add_response.json()
-        assert add_payload["success"] is True
-        assert add_payload["result_format"] == "json"
-        assert add_payload["result"]["num_snapshots"] == 0
-        crawl_id = add_payload["result"]["crawl_id"]
-        assert add_payload["result"]["snapshot_ids"] == []
-        stop_server(tmp_path)
-        from archivebox.services.runner import run_crawl
-
-        with use_archivebox_db(tmp_path):
-            run_crawl(crawl_id, show_progress=False)
-        start_server(tmp_path, env=env, port=port)
-        wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs")
-
-        deadline = time.time() + 180
-        snapshot_id = None
-        while time.time() < deadline:
-            with use_archivebox_db(tmp_path):
-                snapshot = Snapshot.objects.filter(crawl_id=crawl_id, url=target_url).first()
-                if snapshot is not None:
-                    snapshot_id = str(snapshot.id)
-                    break
-            time.sleep(1)
-        assert snapshot_id is not None
-
-        search_response = requests.post(
-            f"http://127.0.0.1:{port}/api/v1/cli/search",
-            headers=api_headers,
-            json={
-                "filter_patterns": [target_url],
-                "filter_type": "exact",
-                "status": "indexed",
-                "sort": "bookmarked_at",
-                "as_json": True,
-                "as_html": False,
-                "as_csv": "",
-                "with_headers": False,
-            },
-            timeout=10,
-        )
-        assert search_response.status_code == 200, search_response.text
-        search_payload = search_response.json()
-        assert search_payload["success"] is True
-        assert search_payload["result_format"] == "json"
-        assert any(item["url"] == target_url for item in search_payload["result"])
-
-        update_response = requests.post(
-            f"http://127.0.0.1:{port}/api/v1/cli/update",
-            headers=api_headers,
-            json={
-                "resume": None,
-                "after": 0,
-                "before": 4102444800,
-                "filter_type": "exact",
-                "filter_patterns": [target_url],
-                "batch_size": 1,
-                "continuous": False,
-            },
-            timeout=20,
-        )
-        assert update_response.status_code == 200, update_response.text
-        assert update_response.json()["success"] is True
-        stop_server(tmp_path)
-        start_server(tmp_path, env=env, port=port)
-        wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs")
-
-        with use_archivebox_db(tmp_path):
-            crawl_obj = Crawl.objects.filter(pk=crawl_id).first()
-            crawl = (crawl_obj.max_depth, crawl_obj.tags_str, crawl_obj.config) if crawl_obj else None
-
-        assert crawl is not None
-        assert crawl[0] == 0
-        assert crawl[1] == "api-cli"
-        assert crawl[2]["INDEX_ONLY"] is True
-
-        remove_response = requests.post(
-            f"http://127.0.0.1:{port}/api/v1/cli/remove",
-            headers=api_headers,
-            json={
-                "delete": True,
-                "after": 0,
-                "before": 4102444800,
-                "filter_type": "exact",
-                "filter_patterns": [target_url],
-            },
-            timeout=20,
-        )
-        assert remove_response.status_code == 200, remove_response.text
-        remove_payload = remove_response.json()
-        assert remove_payload["success"] is True
-        assert remove_payload["result"]["removed_count"] == 1
-        assert snapshot_id in remove_payload["result"]["removed_snapshot_ids"]
-
-        with use_archivebox_db(tmp_path):
-            snapshot_count = Snapshot.objects.filter(pk=snapshot_id).count()
-
-        assert snapshot_count == 0
-    finally:
-        stop_server(tmp_path)
+# CLI endpoint tests moved to test_api_v1_cli_add.py and test_api_v1_cli_update.py.

archivebox/tests/test_api_cli_schedule.py

@@ -1,83 +1 @@
-import os
-from io import StringIO
-
-import pytest
-import requests
-from django.contrib.auth import get_user_model
-from django.test import RequestFactory
-
-from archivebox.api.v1_cli import ScheduleCommandSchema, cli_schedule
-from archivebox.crawls.models import CrawlSchedule
-from .conftest import (
-    build_test_env,
-    create_admin_and_token,
-    get_free_port,
-    init_archive,
-    start_server,
-    stop_server,
-    wait_for_http,
-)
-
-User = get_user_model()
-
-
-@pytest.mark.django_db
-def test_schedule_api_creates_schedule_via_view_request():
-    user = User.objects.create_user(
-        username="api-user",
-        password="testpass123",
-        email="api@example.com",
-    )
-    request = RequestFactory().post("/api/v1/cli/schedule")
-    request.user = user
-    setattr(request, "stdout", StringIO())
-    setattr(request, "stderr", StringIO())
-    args = ScheduleCommandSchema(
-        every="daily",
-        import_path="https://example.com/feed.xml",
-        quiet=True,
-    )
-
-    response = cli_schedule(request, args)
-
-    assert response["success"] is True
-    assert response["result_format"] == "json"
-    assert CrawlSchedule.objects.count() == 1
-    assert len(response["result"]["created_schedule_ids"]) == 1
-
-
-@pytest.mark.django_db(transaction=True)
-@pytest.mark.timeout(180)
-def test_api_v1_cli_schedule_creates_schedule_over_server(tmp_path, recursive_test_site):
-    os.chdir(tmp_path)
-    init_archive(tmp_path)
-
-    port = get_free_port()
-    env = build_test_env(port)
-    api_token = create_admin_and_token(tmp_path)
-
-    try:
-        start_server(tmp_path, env=env, port=port)
-        wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs")
-
-        response = requests.post(
-            f"http://127.0.0.1:{port}/api/v1/cli/schedule",
-            headers={
-                "Host": f"api.archivebox.localhost:{port}",
-                "X-ArchiveBox-API-Key": api_token,
-            },
-            json={
-                "every": "daily",
-                "import_path": recursive_test_site["root_url"],
-                "quiet": True,
-            },
-            timeout=10,
-        )
-
-        assert response.status_code == 200, response.text
-        payload = response.json()
-        assert payload["success"] is True
-        assert payload["result_format"] == "json"
-        assert len(payload["result"]["created_schedule_ids"]) == 1
-    finally:
-        stop_server(tmp_path)
+# CLI schedule endpoint tests moved to test_api_v1_cli_schedule.py.

archivebox/tests/test_api_core.py

@@ -1,374 +0,0 @@
-import os
-from datetime import timedelta
-
-import pytest
-import requests
-from django.core.files.uploadedfile import SimpleUploadedFile
-from django.contrib.auth import get_user_model
-from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart
-from django.utils import timezone
-
-from archivebox.core.models import ArchiveResult, Snapshot, Tag
-from archivebox.crawls.models import Crawl
-from archivebox.tests.test_orm_helpers import use_archivebox_db
-from .conftest import (
-    build_test_env,
-    create_admin_and_token,
-    get_free_port,
-    init_archive,
-    start_server,
-    stop_server,
-    wait_for_http,
-)
-
-pytestmark = pytest.mark.django_db(transaction=True)
-
-
-def test_archiveresult_upload_api_queues_snapshot_maintenance_without_finalizing(client):
-    from archivebox.api.auth import get_or_create_api_token
-
-    user = get_user_model().objects.create_superuser(
-        username="uploadapiadmin",
-        email="uploadapiadmin@example.com",
-        password="testpass123",
-    )
-    api_token = get_or_create_api_token(user)
-    assert api_token is not None
-
-    crawl = Crawl.objects.create(
-        urls="https://example.com",
-        created_by=user,
-        status=Crawl.StatusChoices.STARTED,
-        retry_at=timezone.now(),
-    )
-    active_retry_at = timezone.now() + timedelta(minutes=5)
-    active_snapshot = Snapshot.objects.create(
-        url="https://example.com/active",
-        crawl=crawl,
-        status=Snapshot.StatusChoices.STARTED,
-        retry_at=active_retry_at,
-    )
-    sealed_snapshot = Snapshot.objects.create(
-        url="https://example.com/sealed",
-        crawl=crawl,
-        status=Snapshot.StatusChoices.SEALED,
-        retry_at=None,
-    )
-
-    active_response = client.post(
-        "/api/v1/core/archiveresults",
-        {
-            "snapshot_id": str(active_snapshot.id),
-            "plugin": "chrome_extension_dom",
-            "hook_name": "on_Snapshot__archivebox_browser_extension_upload",
-            "status": ArchiveResult.StatusChoices.SUCCEEDED,
-            "output_str": "uploaded active snapshot output",
-        },
-        HTTP_HOST="api.archivebox.localhost:8000",
-        HTTP_X_ARCHIVEBOX_API_KEY=api_token.token,
-    )
-    assert active_response.status_code == 200, active_response.content
-    active_snapshot.refresh_from_db()
-    assert active_snapshot.status == Snapshot.StatusChoices.STARTED
-    assert active_snapshot.retry_at == active_retry_at
-    assert active_snapshot.downloaded_at is not None
-
-    sealed_response = client.post(
-        "/api/v1/core/archiveresults",
-        {
-            "snapshot_id": str(sealed_snapshot.id),
-            "plugin": "chrome_extension_mhtml",
-            "hook_name": "on_Snapshot__archivebox_browser_extension_upload",
-            "status": ArchiveResult.StatusChoices.SUCCEEDED,
-            "output_str": "uploaded sealed snapshot output",
-        },
-        HTTP_HOST="api.archivebox.localhost:8000",
-        HTTP_X_ARCHIVEBOX_API_KEY=api_token.token,
-    )
-    assert sealed_response.status_code == 200, sealed_response.content
-    sealed_snapshot.refresh_from_db()
-    assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
-    assert sealed_snapshot.retry_at is not None
-    assert sealed_snapshot.downloaded_at is not None
-
-
-def test_archiveresult_patch_upload_finalizes_queued_result(client):
-    from archivebox.api.auth import get_or_create_api_token
-
-    user = get_user_model().objects.create_superuser(
-        username="patchuploadapiadmin",
-        email="patchuploadapiadmin@example.com",
-        password="testpass123",
-    )
-    api_token = get_or_create_api_token(user)
-    assert api_token is not None
-
-    crawl = Crawl.objects.create(
-        urls="https://example.com",
-        created_by=user,
-        status=Crawl.StatusChoices.SEALED,
-        retry_at=None,
-    )
-    snapshot = Snapshot.objects.create(
-        url="https://example.com/upload-patch",
-        crawl=crawl,
-        status=Snapshot.StatusChoices.SEALED,
-        retry_at=None,
-    )
-    result = ArchiveResult.objects.create(
-        snapshot=snapshot,
-        plugin="dom",
-        hook_name="on_Snapshot__archivebox_browser_extension_upload",
-        status=ArchiveResult.StatusChoices.QUEUED,
-    )
-
-    response = client.generic(
-        "PATCH",
-        f"/api/v1/core/archiveresult/{result.id}",
-        encode_multipart(
-            BOUNDARY,
-            {
-                "files": SimpleUploadedFile("output.html", b"<html>uploaded</html>", content_type="text/html"),
-                "output_paths": "output.html",
-                "output_str": "output.html",
-            },
-        ),
-        content_type=MULTIPART_CONTENT,
-        HTTP_HOST="api.archivebox.localhost:8000",
-        HTTP_X_ARCHIVEBOX_API_KEY=api_token.token,
-    )
-    assert response.status_code == 200, response.content
-
-    result.refresh_from_db()
-    snapshot.refresh_from_db()
-    assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
-    assert result.output_str == "output.html"
-    assert snapshot.status == Snapshot.StatusChoices.SEALED
-    assert snapshot.retry_at is not None
-
-
-def test_crawl_cancel_api_defers_cleanup_to_runner(client):
-    from archivebox.api.auth import get_or_create_api_token
-    from archivebox.services.runner import run_due_crawl
-
-    user = get_user_model().objects.create_superuser(
-        username="cancelapiadmin",
-        email="cancelapiadmin@example.com",
-        password="testpass123",
-    )
-    api_token = get_or_create_api_token(user)
-    assert api_token is not None
-
-    crawl = Crawl.objects.create(
-        urls="https://example.com",
-        created_by=user,
-        status=Crawl.StatusChoices.STARTED,
-        retry_at=timezone.now() + timedelta(minutes=5),
-    )
-    child = Snapshot.objects.create(
-        url="https://example.com/cancel-child",
-        crawl=crawl,
-        status=Snapshot.StatusChoices.STARTED,
-        retry_at=timezone.now() + timedelta(minutes=5),
-    )
-    crawl.output_dir.mkdir(parents=True, exist_ok=True)
-    pid_file = crawl.output_dir / "cleanup-test.pid"
-    pid_file.write_text("12345")
-
-    response = client.patch(
-        f"/api/v1/crawls/crawl/{crawl.id}",
-        {"action": "cancel"},
-        content_type="application/json",
-        HTTP_HOST="api.archivebox.localhost:8000",
-        HTTP_X_ARCHIVEBOX_API_KEY=api_token.token,
-    )
-    assert response.status_code == 200, response.content
-
-    crawl.refresh_from_db()
-    child.refresh_from_db()
-    assert crawl.status == Crawl.StatusChoices.SEALED
-    assert crawl.retry_at is not None
-    assert crawl.retry_at <= timezone.now()
-    assert child.status == Snapshot.StatusChoices.STARTED
-    assert child.retry_at is not None
-    assert child.retry_at <= timezone.now()
-    assert pid_file.exists()
-
-    assert run_due_crawl(crawl, lock_seconds=60) is True
-    crawl.refresh_from_db()
-    assert crawl.retry_at is None
-    assert not pid_file.exists()
-
-
-@pytest.mark.timeout(180)
-def test_core_api_crud_uses_token_auth_and_persists_side_effects_over_server(tmp_path, recursive_test_site):
-    os.chdir(tmp_path)
-    init_archive(tmp_path)
-
-    port = get_free_port()
-    env = build_test_env(port, PUBLIC_INDEX="True")
-    api_token = create_admin_and_token(tmp_path)
-    api_headers = {
-        "Host": f"api.archivebox.localhost:{port}",
-        "X-ArchiveBox-API-Key": api_token,
-    }
-
-    try:
-        start_server(tmp_path, env=env, port=port)
-        docs = wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs")
-        assert docs.status_code == 200
-        openapi = wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/openapi.json")
-        assert openapi.status_code == 200
-        paths = openapi.json()["paths"]
-        assert "/api/v1/core/snapshots" in paths
-        assert "/api/v1/crawls/crawls" in paths
-
-        unauth = requests.get(
-            f"http://127.0.0.1:{port}/api/v1/crawls/crawls",
-            headers={"Host": f"api.archivebox.localhost:{port}"},
-            timeout=10,
-        )
-        assert unauth.status_code in (401, 403)
-        bad_auth = requests.get(
-            f"http://127.0.0.1:{port}/api/v1/crawls/crawls",
-            headers={"Host": f"api.archivebox.localhost:{port}", "X-ArchiveBox-API-Key": "bad-token"},
-            timeout=10,
-        )
-        assert bad_auth.status_code in (401, 403)
-
-        crawl_response = requests.post(
-            f"http://127.0.0.1:{port}/api/v1/crawls/crawls",
-            headers=api_headers,
-            json={
-                "urls": [recursive_test_site["root_url"]],
-                "max_depth": 2,
-                "tags": ["api-depth-two"],
-                "label": "api crawl",
-                "notes": "created through REST API",
-                "config": {
-                    "PLUGINS": "wget,parse_html_urls",
-                    "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
-                    "CRAWL_MAX_URLS": 7,
-                    "CRAWL_MAX_SIZE": "0",
-                    "SNAPSHOT_MAX_SIZE": "0",
-                },
-            },
-            timeout=10,
-        )
-        assert crawl_response.status_code == 200, crawl_response.text
-        crawl_payload = crawl_response.json()
-        crawl_id = crawl_payload["id"]
-        assert crawl_payload["max_depth"] == 2
-        assert crawl_payload["tags_str"] == "api-depth-two"
-        assert crawl_payload["config"]["PLUGINS"] == "wget,parse_html_urls"
-        assert crawl_payload["config"]["CRAWL_MAX_URLS"] == 7
-
-        snapshot_response = requests.post(
-            f"http://127.0.0.1:{port}/api/v1/core/snapshots",
-            headers=api_headers,
-            json={
-                "url": recursive_test_site["child_urls"][0],
-                "crawl_id": crawl_id,
-                "depth": 1,
-                "title": "API child snapshot",
-                "tags": ["api-child"],
-                "status": "queued",
-            },
-            timeout=10,
-        )
-        assert snapshot_response.status_code == 200, snapshot_response.text
-        snapshot_payload = snapshot_response.json()
-        snapshot_id = snapshot_payload["id"]
-        assert snapshot_payload["url"] == recursive_test_site["child_urls"][0]
-        assert snapshot_payload["tags"] == ["api-child"]
-
-        patch_snapshot = requests.patch(
-            f"http://127.0.0.1:{port}/api/v1/core/snapshot/{snapshot_id}",
-            headers=api_headers,
-            json={"status": "sealed", "tags": ["api-child", "api-patched"]},
-            timeout=10,
-        )
-        assert patch_snapshot.status_code == 200, patch_snapshot.text
-        assert patch_snapshot.json()["status"] == "sealed"
-        assert set(patch_snapshot.json()["tags"]) == {"api-child", "api-patched"}
-
-        tag_create = requests.post(
-            f"http://127.0.0.1:{port}/api/v1/core/tags/create/",
-            headers=api_headers,
-            json={"name": "api-extra"},
-            timeout=10,
-        )
-        assert tag_create.status_code == 200, tag_create.text
-        tag_id = tag_create.json()["tag_id"]
-
-        add_tag = requests.post(
-            f"http://127.0.0.1:{port}/api/v1/core/tags/add-to-snapshot/",
-            headers=api_headers,
-            json={"snapshot_id": snapshot_id, "tag_id": tag_id},
-            timeout=10,
-        )
-        assert add_tag.status_code == 200, add_tag.text
-        remove_tag = requests.post(
-            f"http://127.0.0.1:{port}/api/v1/core/tags/remove-from-snapshot/",
-            headers=api_headers,
-            json={"snapshot_id": snapshot_id, "tag_name": "api-extra"},
-            timeout=10,
-        )
-        assert remove_tag.status_code == 200, remove_tag.text
-
-        crawl_patch = requests.patch(
-            f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}",
-            headers=api_headers,
-            json={"status": "sealed", "tags": ["api-sealed"]},
-            timeout=10,
-        )
-        assert crawl_patch.status_code == 200, crawl_patch.text
-        assert crawl_patch.json()["status"] == "sealed"
-        assert crawl_patch.json()["tags_str"] == "api-sealed"
-
-        snapshots_list = requests.get(
-            f"http://127.0.0.1:{port}/api/v1/core/snapshots?tag=api-patched&with_archiveresults=true",
-            headers=api_headers,
-            timeout=10,
-        )
-        assert snapshots_list.status_code == 200, snapshots_list.text
-        snapshot_items = snapshots_list.json()["items"]
-        assert len(snapshot_items) == 1
-        assert snapshot_items[0]["id"] == snapshot_id
-        assert snapshot_items[0]["archiveresults"] == []
-
-        bearer_response = requests.get(
-            f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}",
-            headers={"Host": f"api.archivebox.localhost:{port}", "Authorization": f"Bearer {api_token}"},
-            timeout=10,
-        )
-        assert bearer_response.status_code == 200, bearer_response.text
-        query_response = requests.get(
-            f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}?api_key={api_token}",
-            headers={"Host": f"api.archivebox.localhost:{port}"},
-            timeout=10,
-        )
-        assert query_response.status_code == 200, query_response.text
-
-        delete_snapshot = requests.delete(
-            f"http://127.0.0.1:{port}/api/v1/core/snapshot/{snapshot_id}",
-            headers=api_headers,
-            timeout=10,
-        )
-        assert delete_snapshot.status_code == 200, delete_snapshot.text
-        assert delete_snapshot.json()["success"] is True
-
-        delete_crawl = requests.delete(
-            f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}",
-            headers=api_headers,
-            timeout=10,
-        )
-        assert delete_crawl.status_code == 200, delete_crawl.text
-        assert delete_crawl.json()["success"] is True
-
-        with use_archivebox_db(tmp_path):
-            assert Crawl.objects.filter(pk=crawl_id).count() == 0
-            assert Snapshot.objects.filter(pk=snapshot_id).count() == 0
-            assert Tag.objects.filter(name="api-extra").count() == 1
-    finally:
-        stop_server(tmp_path)

archivebox/tests/test_api_crawl.py

@@ -0,0 +1 @@
+# Tests moved to test_api_v1_crawls_crawl_crawl_id.py.

archivebox/tests/test_api_crud.py

@@ -0,0 +1 @@
+# Tests moved to test_api_v1_workflow_core_token_auth_side_effects.py and exact endpoint files under test_api_v1_core_*.

archivebox/tests/test_api_delete_paths.py

@@ -1,88 +1 @@
-import json
-from pathlib import Path
-
-import pytest
-from django.contrib.auth import get_user_model
-
-from archivebox.api.auth import get_or_create_api_token
-from archivebox.cli.archivebox_add import add
-from archivebox.cli.archivebox_run import run_runner
-from archivebox.core.models import Snapshot
-from archivebox.crawls.models import Crawl
-
-
-pytestmark = pytest.mark.django_db(transaction=True)
-
-API_HOST = "api.archivebox.localhost:8000"
-
-
-def _api_headers(token):
-    return {
-        "HTTP_HOST": API_HOST,
-        "HTTP_X_ARCHIVEBOX_API_KEY": token.token,
-    }
-
-
-def _admin_token():
-    user = get_user_model().objects.create_superuser(
-        username="deletepathadmin",
-        email="deletepathadmin@test.com",
-        password="testpassword",
-    )
-    token = get_or_create_api_token(user)
-    assert token is not None
-    return token
-
-
-def test_rest_snapshot_delete_removes_output_dir(client):
-    token = _admin_token()
-    url = "https://example.com/delete-path-snapshot"
-
-    response = client.post(
-        "/api/v1/core/snapshots",
-        data=json.dumps({"url": url, "depth": 0, "status": Snapshot.StatusChoices.QUEUED}),
-        content_type="application/json",
-        **_api_headers(token),
-    )
-    assert response.status_code == 200, response.content.decode()
-
-    snapshot = Snapshot.objects.get(url=url)
-    snapshot_dir = Path(snapshot.output_dir)
-    snapshot_dir.mkdir(parents=True, exist_ok=True)
-    (snapshot_dir / "delete-path-test.txt").write_text("snapshot output")
-    assert snapshot_dir.exists()
-
-    response = client.delete(f"/api/v1/core/snapshot/{snapshot.id}", **_api_headers(token))
-    assert response.status_code == 200, response.content.decode()
-    assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
-    assert not snapshot_dir.exists()
-
-
-def test_rest_crawl_delete_removes_crawl_and_snapshot_output_dirs(client):
-    token = _admin_token()
-    url = "https://example.com/delete-path-crawl"
-
-    crawl, _snapshots = add(
-        urls=[url],
-        depth=0,
-        max_urls=1,
-        plugins="__archivebox_test_no_plugins__",
-        bg=True,
-    )
-    assert run_runner(daemon=False, crawl_id=str(crawl.id)) == 0
-    snapshot = Snapshot.objects.get(crawl=crawl, url=url)
-    crawl_dir = Path(crawl.output_dir)
-    snapshot_dir = Path(snapshot.output_dir)
-    crawl_dir.mkdir(parents=True, exist_ok=True)
-    snapshot_dir.mkdir(parents=True, exist_ok=True)
-    (crawl_dir / "delete-path-crawl.txt").write_text("crawl output")
-    (snapshot_dir / "delete-path-snapshot.txt").write_text("snapshot output")
-    assert crawl_dir.exists()
-    assert snapshot_dir.exists()
-
-    response = client.delete(f"/api/v1/crawls/crawl/{crawl.id}", **_api_headers(token))
-    assert response.status_code == 200, response.content.decode()
-    assert not Crawl.objects.filter(pk=crawl.pk).exists()
-    assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
-    assert not crawl_dir.exists()
-    assert not snapshot_dir.exists()
+# Tests moved to test_api_v1_core_snapshot_snapshot_id.py and test_api_v1_crawls_crawl_crawl_id.py.

archivebox/tests/test_api_personas.py

@@ -0,0 +1 @@
+# Tests moved to test_api_v1_personas_sync.py and test_api_v1_personas_personas.py.

archivebox/tests/test_api_remove.py

@@ -0,0 +1 @@
+# CLI remove endpoint tests moved to test_api_v1_cli_remove.py.

archivebox/tests/test_api_rss.py

@@ -1,166 +1 @@
-from datetime import datetime
-from typing import cast
-
-import pytest
-from django.contrib.auth import get_user_model
-from django.contrib.auth.models import UserManager
-from django.utils import timezone
-
-
-pytestmark = pytest.mark.django_db
-
-
-User = get_user_model()
-ADMIN_HOST = "admin.archivebox.localhost:8000"
-
-
-@pytest.fixture
-def admin_user(db):
-    return cast(UserManager, User.objects).create_superuser(
-        username="rssadmin",
-        email="rssadmin@test.com",
-        password="testpassword",
-    )
-
-
-@pytest.fixture
-def other_user(db):
-    return cast(UserManager, User.objects).create_user(
-        username="rssother",
-        email="rssother@test.com",
-        password="testpassword",
-    )
-
-
-@pytest.fixture
-def api_token(admin_user):
-    from archivebox.api.auth import get_or_create_api_token
-
-    token = get_or_create_api_token(admin_user)
-    assert token is not None
-    return token.token
-
-
-def make_snapshot(*, user, url: str, title: str, bookmarked_at: datetime):
-    from archivebox.core.models import Snapshot
-    from archivebox.crawls.models import Crawl
-
-    crawl = Crawl.objects.create(urls=url, created_by=user)
-    snapshot = Snapshot.objects.create(
-        url=url,
-        title=title,
-        crawl=crawl,
-        bookmarked_at=bookmarked_at,
-    )
-    return crawl, snapshot
-
-
-def test_snapshots_rss_filters_by_user_and_orders_newest_first(client, api_token, admin_user, other_user):
-    from archivebox.core.models import Tag
-
-    older_at = timezone.make_aware(datetime(2026, 5, 22, 8, 0, 0))
-    newer_at = timezone.make_aware(datetime(2026, 5, 23, 8, 0, 0))
-    _crawl, older_snapshot = make_snapshot(
-        user=admin_user,
-        url="https://example.com/rss-older",
-        title="Older & Escaped",
-        bookmarked_at=older_at,
-    )
-    make_snapshot(
-        user=admin_user,
-        url="https://example.com/rss-newer",
-        title="Newer Snapshot",
-        bookmarked_at=newer_at,
-    )
-    make_snapshot(
-        user=other_user,
-        url="https://example.com/rss-other-user",
-        title="Other User",
-        bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 9, 0, 0)),
-    )
-    older_snapshot.tags.add(Tag.objects.create(name="rss-tag", created_by=admin_user))
-
-    response = client.get(
-        "/api/v1/core/snapshots.rss",
-        {"created_by": admin_user.username, "limit": 50, "api_key": api_token},
-        HTTP_HOST=ADMIN_HOST,
-    )
-
-    assert response.status_code == 200
-    assert response["Content-Type"].startswith("application/rss+xml")
-    body = response.content.decode()
-    assert '<rss version="2.0"' in body
-    assert api_token not in body
-    assert 'href="http://admin.archivebox.localhost:8000/api/v1/core/snapshots.rss?created_by=rssadmin&amp;limit=50"' in body
-    assert "Newer Snapshot" in body
-    assert "Older &amp; Escaped" in body
-    assert "Tags: rss-tag" not in body
-    assert "<category>rss-tag</category>" in body
-    assert "rss-other-user" not in body
-    assert body.index("rss-newer") < body.index("rss-older")
-
-
-def test_snapshots_rss_supports_before_yyyymmdd_and_limit(client, api_token, admin_user):
-    make_snapshot(
-        user=admin_user,
-        url="https://example.com/rss-before-too-new",
-        title="Too New",
-        bookmarked_at=timezone.make_aware(datetime(2026, 5, 24, 8, 0, 0)),
-    )
-    make_snapshot(
-        user=admin_user,
-        url="https://example.com/rss-before-keep-one",
-        title="Keep One",
-        bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 12, 0, 0)),
-    )
-    make_snapshot(
-        user=admin_user,
-        url="https://example.com/rss-before-keep-two",
-        title="Keep Two",
-        bookmarked_at=timezone.make_aware(datetime(2026, 5, 22, 12, 0, 0)),
-    )
-
-    response = client.get(
-        "/api/v1/core/snapshots.rss",
-        {"created_by": str(admin_user.pk), "before": "20260523", "limit": 1, "api_key": api_token},
-        HTTP_HOST=ADMIN_HOST,
-    )
-
-    assert response.status_code == 200
-    body = response.content.decode()
-    assert "rss-before-too-new" not in body
-    assert "rss-before-keep-one" in body
-    assert "rss-before-keep-two" not in body
-
-
-def test_crawl_as_rss_redirects_to_canonical_snapshots_feed(client, api_token, admin_user, other_user):
-    crawl, _snapshot = make_snapshot(
-        user=admin_user,
-        url="https://example.com/rss-crawl-feed",
-        title="Crawl Feed Snapshot",
-        bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 8, 0, 0)),
-    )
-    make_snapshot(
-        user=other_user,
-        url="https://example.com/rss-crawl-other",
-        title="Other Crawl Snapshot",
-        bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 9, 0, 0)),
-    )
-
-    response = client.get(
-        f"/api/v1/crawls/crawl/{crawl.id}",
-        {"as_rss": "true", "limit": 50, "api_key": api_token},
-        HTTP_HOST=ADMIN_HOST,
-        follow=True,
-    )
-
-    assert response.status_code == 200
-    assert response.redirect_chain
-    redirect_url = response.redirect_chain[0][0]
-    assert redirect_url.startswith("/api/v1/core/snapshots.rss?")
-    assert f"crawl_id={crawl.id}" in redirect_url
-    assert "as_rss" not in redirect_url
-    assert response["Content-Type"].startswith("application/rss+xml")
-    body = response.content.decode()
-    assert "rss-crawl-feed" in body
-    assert "rss-crawl-other" not in body
+# Tests moved to test_api_v1_core_snapshots_rss.py and test_api_v1_crawls_crawl_crawl_id.py.

archivebox/tests/test_api_search.py

@@ -0,0 +1 @@
+# Tests moved to test_api_v1_core_snapshots.py.

archivebox/tests/test_api_v1_auth_check_api_token.py

@@ -0,0 +1,19 @@
+import pytest
+
+from archivebox.tests.conftest import API_TEST_HOST, api_client_request
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_token):
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/auth/check_api_token",
+        payload={"token": api_token.token},
+        headers={"HTTP_HOST": API_TEST_HOST},
+    )
+
+    assert response.status_code == 200, response.content
+    assert response.json()["success"] is True

archivebox/tests/test_api_v1_auth_get_api_token.py

@@ -0,0 +1,22 @@
+import pytest
+
+from archivebox.tests.conftest import API_TEST_HOST, api_client_request
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user):
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/auth/get_api_token",
+        payload={
+            "username": api_admin_user.username,
+            "password": "testpass123",
+        },
+        headers={"HTTP_HOST": API_TEST_HOST},
+    )
+
+    assert response.status_code == 200, response.content
+    assert response.json()["success"] is True

archivebox/tests/test_api_v1_cli_add.py

@@ -0,0 +1,29 @@
+import pytest
+
+from .conftest import (
+    api_client_request,
+    init_archive,
+)
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_headers):
+    init_archive(tmp_path)
+
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/cli/add",
+        payload={
+            "urls": ["https://example.com/api-cli-add-basic"],
+            "depth": 0,
+            "parser": "url_list",
+            "plugins": "__archivebox_test_no_plugins__",
+            "index_only": True,
+        },
+        headers=api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+    assert response.json()["success"] is True

archivebox/tests/test_api_v1_cli_remove.py

@@ -0,0 +1,209 @@
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import pytest
+from django.utils import timezone
+
+from archivebox.core.models import Snapshot, Tag
+from archivebox.crawls.models import Crawl
+from archivebox.tests.conftest import api_client_request, init_archive
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def _crawl(user, urls: str, label: str) -> Crawl:
+    return Crawl.objects.create(
+        urls=urls,
+        created_by=user,
+        status=Crawl.StatusChoices.SEALED,
+        retry_at=None,
+        tags_str=label,
+    )
+
+
+def _snapshot(crawl: Crawl, url: str, *, status: str = Snapshot.StatusChoices.SEALED, tag: str = "", bookmarked_at=None) -> Snapshot:
+    snapshot = Snapshot.objects.create(
+        url=url,
+        crawl=crawl,
+        status=status,
+        retry_at=None,
+        bookmarked_at=bookmarked_at or timezone.now(),
+    )
+    if tag:
+        tag_obj, _ = Tag.objects.get_or_create(name=tag)
+        snapshot.tags.add(tag_obj)
+    return snapshot
+
+
+def _touch_output(snapshot: Snapshot) -> Path:
+    output_dir = Path(snapshot.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "api-remove-test.txt").write_text(str(snapshot.id))
+    return output_dir
+
+
+def _bulk_timeout_snapshots(crawl: Crawl, *, count: int = 30000) -> tuple[list[Snapshot], dict[str, Path]]:
+    base = timezone.make_aware(datetime(2026, 2, 1, 12, 0, 0))
+    snapshots = [
+        Snapshot(
+            url=f"https://example.com/remove-timeout-{idx}",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=None,
+            timestamp=f"88{idx:030d}",
+            bookmarked_at=base + timedelta(seconds=idx),
+            created_at=base + timedelta(seconds=idx),
+        )
+        for idx in range(count)
+    ]
+    Snapshot.objects.bulk_create(snapshots, batch_size=1000)
+
+    sample = [*snapshots[-200:], *snapshots[:200]]
+    return snapshots, {str(snapshot.id): _touch_output(snapshot) for snapshot in sample}
+
+
+def _post_remove(client, api_headers, body: dict):
+    return api_client_request(
+        client,
+        "post",
+        "/api/v1/cli/remove",
+        payload=body,
+        headers=api_headers,
+    )
+
+
+def test_cli_remove_api_removes_rows_and_respects_snapshot_filters(client, tmp_path, api_admin_user, api_headers):
+    init_archive(tmp_path)
+    base = timezone.make_aware(datetime(2026, 1, 1, 12, 0, 0))
+
+    crawl_a = _crawl(api_admin_user, "https://alpha.example.com/articles/needle", "crawl-a")
+    crawl_b = _crawl(api_admin_user, "https://beta.example.org/posts/needle", "crawl-b")
+    exact = _snapshot(crawl_a, "https://alpha.example.com/articles/exact", tag="api-keep", bookmarked_at=base)
+    keep = _snapshot(crawl_a, "https://alpha.example.com/articles/needle", tag="api-keep", bookmarked_at=base + timedelta(hours=1))
+    wrong_status = _snapshot(
+        crawl_a,
+        "https://alpha.example.com/articles/needle-queued",
+        status=Snapshot.StatusChoices.QUEUED,
+        tag="api-keep",
+        bookmarked_at=base + timedelta(hours=2),
+    )
+    other_crawl = _snapshot(crawl_b, "https://beta.example.org/posts/needle", tag="api-other", bookmarked_at=base + timedelta(hours=3))
+    exact_dir = _touch_output(exact)
+    keep_dir = _touch_output(keep)
+    wrong_status_dir = _touch_output(wrong_status)
+    other_crawl_dir = _touch_output(other_crawl)
+
+    exact_response = _post_remove(
+        client,
+        api_headers,
+        {
+            "filter_type": "exact",
+            "filter_patterns": [exact.url],
+            "timeout": 60,
+        },
+    )
+    assert exact_response.status_code == 200, exact_response.content
+    exact_payload = exact_response.json()
+    assert exact_payload["success"] is True
+    assert exact_payload["result"]["removed_count"] == 1
+    assert exact_payload["result"]["removed_snapshot_ids"] == [str(exact.id)]
+    assert exact_payload["result"]["not_removed_count"] == 0
+    assert not Snapshot.objects.filter(pk=exact.pk).exists()
+    assert not exact_dir.exists()
+
+    filtered_response = _post_remove(
+        client,
+        api_headers,
+        {
+            "filter_type": "substring",
+            "filter_patterns": ["needle"],
+            "status": Snapshot.StatusChoices.SEALED,
+            "tag": "api-keep",
+            "url__istartswith": "https://alpha.example.com",
+            "crawl_id": str(crawl_a.id),
+            "after": (base + timedelta(minutes=30)).timestamp(),
+            "before": (base + timedelta(hours=2)).timestamp(),
+            "timeout": 60,
+        },
+    )
+    assert filtered_response.status_code == 200, filtered_response.content
+    filtered_payload = filtered_response.json()
+    assert filtered_payload["success"] is True
+    assert filtered_payload["result"]["removed_count"] == 1
+    assert filtered_payload["result"]["removed_snapshot_ids"] == [str(keep.id)]
+    assert filtered_payload["result"]["not_removed_count"] == 0
+    assert not Snapshot.objects.filter(pk=keep.pk).exists()
+    assert not keep_dir.exists()
+    assert Snapshot.objects.filter(pk=wrong_status.pk).exists()
+    assert Snapshot.objects.filter(pk=other_crawl.pk).exists()
+    assert wrong_status_dir.exists()
+    assert other_crawl_dir.exists()
+
+
+def test_cli_remove_api_reports_timeout_and_clamps_timeout_to_sixty_seconds(client, tmp_path, api_admin_user, api_headers):
+    init_archive(tmp_path)
+    crawl = _crawl(api_admin_user, "https://example.com/remove-timeout-0", "timeout")
+    snapshots, output_dirs_by_id = _bulk_timeout_snapshots(crawl)
+
+    timeout_response = _post_remove(
+        client,
+        api_headers,
+        {
+            "filter_type": "substring",
+            "filter_patterns": ["remove-timeout-"],
+            "timeout": 3,
+        },
+    )
+    assert timeout_response.status_code == 200, timeout_response.content
+    timeout_payload = timeout_response.json()
+    assert timeout_payload["success"] is False
+    assert timeout_payload["errors"]
+    assert set(timeout_payload["result"]) == {
+        "removed_count",
+        "removed_snapshot_ids",
+        "not_removed_count",
+        "not_removed_snapshot_ids",
+        "success",
+        "error",
+        "timeout",
+    }
+    assert timeout_payload["result"]["success"] is False
+    assert timeout_payload["result"]["timeout"] == 3.0
+    assert timeout_payload["result"]["error"]
+    assert timeout_payload["result"]["removed_count"] == len(timeout_payload["result"]["removed_snapshot_ids"])
+    assert timeout_payload["result"]["not_removed_count"] == len(timeout_payload["result"]["not_removed_snapshot_ids"])
+    assert timeout_payload["result"]["removed_count"] > 0
+    assert timeout_payload["result"]["not_removed_count"] > 0
+    assert timeout_payload["result"]["removed_count"] + timeout_payload["result"]["not_removed_count"] == len(snapshots)
+
+    removed_ids = set(timeout_payload["result"]["removed_snapshot_ids"])
+    not_removed_ids = set(timeout_payload["result"]["not_removed_snapshot_ids"])
+    assert Snapshot.objects.filter(url__icontains="remove-timeout-").count() == len(not_removed_ids)
+    assert removed_ids & set(output_dirs_by_id)
+    assert not_removed_ids & set(output_dirs_by_id)
+    for snapshot_id in removed_ids & set(output_dirs_by_id):
+        assert not Snapshot.objects.filter(pk=snapshot_id).exists()
+        assert not output_dirs_by_id[snapshot_id].exists()
+    for snapshot_id in not_removed_ids & set(output_dirs_by_id):
+        assert Snapshot.objects.filter(pk=snapshot_id).exists()
+        assert output_dirs_by_id[snapshot_id].exists()
+
+    clamp_snapshot = _snapshot(crawl, "https://example.com/remove-timeout-clamp")
+    clamp_dir = _touch_output(clamp_snapshot)
+    clamp_response = _post_remove(
+        client,
+        api_headers,
+        {
+            "filter_type": "exact",
+            "filter_patterns": [clamp_snapshot.url],
+            "timeout": 999,
+        },
+    )
+    assert clamp_response.status_code == 200, clamp_response.content
+    clamp_payload = clamp_response.json()
+    assert clamp_payload["success"] is True
+    assert clamp_payload["result"]["timeout"] == 60.0
+    assert clamp_payload["result"]["removed_snapshot_ids"] == [str(clamp_snapshot.id)]
+    assert not Snapshot.objects.filter(pk=clamp_snapshot.pk).exists()
+    assert not clamp_dir.exists()

archivebox/tests/test_api_v1_cli_schedule.py

@@ -0,0 +1,71 @@
+from io import StringIO
+
+import pytest
+import requests
+from django.test import RequestFactory
+
+from archivebox.api.v1_cli import ScheduleCommandSchema, cli_schedule
+from archivebox.crawls.models import CrawlSchedule
+from .conftest import (
+    api_auth_headers,
+    cli_env,
+    create_admin_and_token,
+    get_free_port,
+    init_archive,
+    start_archivebox_server,
+    stop_server,
+    wait_for_http,
+)
+
+
+@pytest.mark.django_db
+def test_schedule_api_creates_schedule_via_view_request(api_admin_user):
+    request = RequestFactory().post("/api/v1/cli/schedule")
+    request.user = api_admin_user
+    setattr(request, "stdout", StringIO())
+    setattr(request, "stderr", StringIO())
+    args = ScheduleCommandSchema(
+        every="daily",
+        import_path="https://example.com/feed.xml",
+        quiet=True,
+    )
+
+    response = cli_schedule(request, args)
+
+    assert response["success"] is True
+    assert response["result_format"] == "json"
+    assert CrawlSchedule.objects.count() == 1
+    assert len(response["result"]["created_schedule_ids"]) == 1
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.timeout(180)
+def test_api_v1_cli_schedule_creates_schedule_over_server(tmp_path, recursive_test_site):
+    init_archive(tmp_path)
+
+    port = get_free_port()
+    env = cli_env(port=port, server=True)
+    api_token = create_admin_and_token(tmp_path)
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs")
+
+        response = requests.post(
+            f"http://127.0.0.1:{port}/api/v1/cli/schedule",
+            headers=api_auth_headers(api_token, port=port),
+            json={
+                "every": "daily",
+                "import_path": recursive_test_site["root_url"],
+                "quiet": True,
+            },
+            timeout=10,
+        )
+
+        assert response.status_code == 200, response.text
+        payload = response.json()
+        assert payload["success"] is True
+        assert payload["result_format"] == "json"
+        assert len(payload["result"]["created_schedule_ids"]) == 1
+    finally:
+        stop_server(tmp_path)

archivebox/tests/test_api_v1_cli_search.py

@@ -0,0 +1,31 @@
+import pytest
+
+from archivebox.core.models import Snapshot
+from archivebox.crawls.models import Crawl
+from .conftest import api_client_request
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/api-cli-search-basic", created_by=api_admin_user)
+    Snapshot.objects.create(url="https://example.com/api-cli-search-basic", crawl=crawl)
+
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/cli/search",
+        payload={
+            "filter_patterns": ["https://example.com/api-cli-search-basic"],
+            "filter_type": "exact",
+            "as_json": True,
+            "as_html": False,
+            "as_csv": "",
+            "with_headers": False,
+        },
+        headers=api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+    assert response.json()["success"] is True

archivebox/tests/test_api_v1_cli_update.py

@@ -0,0 +1,137 @@
+import json
+
+import pytest
+
+from .conftest import (
+    api_client_request,
+    cli_env,
+    create_admin_and_token,
+    get_free_port,
+    init_archive,
+    live_api_request,
+    parse_jsonl_output,
+    run_archivebox_cmd,
+    start_archivebox_server,
+    stop_server,
+    wait_for_live_api,
+)
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_cli_update_api_accepts_empty_json_without_traceback(client, tmp_path, api_headers):
+    init_archive(tmp_path)
+
+    try:
+        response = api_client_request(
+            client,
+            "post",
+            "/api/v1/cli/update",
+            payload={},
+            headers=api_headers,
+        )
+    finally:
+        stop_server(tmp_path)
+
+    assert response.status_code == 200, response.content
+    payload = response.json()
+    assert payload["success"] is True
+    assert "Traceback" not in response.content.decode()
+
+
+@pytest.mark.timeout(180)
+def test_cli_update_api_supports_all_snapshot_list_filters_with_real_rows(tmp_path):
+    env = cli_env(disable_extractors=True)
+    init_archive(tmp_path)
+
+    records = [
+        {
+            "type": "Snapshot",
+            "url": "https://alpha.example.com/articles/needle",
+            "title": "Needle Alpha",
+            "tags": "api-keep",
+            "timestamp": "1700000000",
+            "bookmarked_at": "2023-11-14T22:13:20+00:00",
+        },
+        {
+            "type": "Snapshot",
+            "url": "https://beta.example.org/posts/haystack",
+            "title": "Haystack Beta",
+            "tags": "api-other",
+            "timestamp": "1710000000",
+            "bookmarked_at": "2024-03-09T16:00:00+00:00",
+        },
+        {
+            "type": "Snapshot",
+            "url": "https://docs.archivebox.io/manual",
+            "title": "Manual Gamma",
+            "tags": "api-docs",
+            "timestamp": "1720000000",
+            "bookmarked_at": "2024-07-03T09:46:40+00:00",
+        },
+    ]
+    stdin = "\n".join(json.dumps(record) for record in records) + "\n"
+    run_archivebox_cmd(["snapshot", "create"], cwd=tmp_path, stdin=stdin, env=env, check=True)
+
+    port = get_free_port()
+    env = {
+        **cli_env(port=port, server=True, PUBLIC_INDEX="True"),
+        **env,
+    }
+    api_token = create_admin_and_token(tmp_path)
+
+    def assert_update_filter(label, body, expected_records):
+        response = live_api_request(
+            port,
+            "post",
+            "/api/v1/cli/update",
+            api_token=api_token,
+            json={**body, "batch_size": 100},
+            timeout=30,
+        )
+        assert response.status_code == 200, f"{label}: {response.text}"
+        assert "Traceback" not in response.text
+        payload = response.json()
+        assert payload["success"] is True, label
+        expected_ids = {record["id"] for record in expected_records}
+        assert set(payload["result"]["snapshot_ids"]) == expected_ids, label
+        assert payload["result"]["matched_count"] == len(expected_ids), label
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
+        list_result = run_archivebox_cmd(["snapshot", "list", "--sort", "timestamp"], cwd=tmp_path, env=env, check=True)
+        snapshots = {record["url"]: record for record in parse_jsonl_output(list_result.stdout) if record.get("type") == "Snapshot"}
+        alpha = snapshots["https://alpha.example.com/articles/needle"]
+        beta = snapshots["https://beta.example.org/posts/haystack"]
+        gamma = snapshots["https://docs.archivebox.io/manual"]
+        status_result = run_archivebox_cmd(
+            ["snapshot", "list", "--status", alpha["status"]],
+            cwd=tmp_path,
+            env=env,
+            check=True,
+        )
+        status_records = [record for record in parse_jsonl_output(status_result.stdout) if record.get("type") == "Snapshot"]
+
+        cases = [
+            ("status", {"status": alpha["status"]}, status_records),
+            ("filter_type exact", {"filter_type": "exact", "filter_patterns": [alpha["url"]]}, [alpha]),
+            ("filter_type substring", {"filter_type": "substring", "filter_patterns": ["needle"]}, [alpha]),
+            ("filter_type regex", {"filter_type": "regex", "filter_patterns": [r"alpha\.example\.com/.+needle"]}, [alpha]),
+            ("filter_type domain", {"filter_type": "domain", "filter_patterns": ["alpha.example.com"]}, [alpha]),
+            ("filter_type tag", {"filter_type": "tag", "filter_patterns": ["api-keep"]}, [alpha]),
+            ("filter_type timestamp", {"filter_type": "timestamp", "filter_patterns": [alpha["timestamp"]]}, [alpha]),
+            ("url__icontains", {"url__icontains": "needle"}, [alpha]),
+            ("url__istartswith", {"url__istartswith": "https://alpha.example.com"}, [alpha]),
+            ("tag", {"tag": "api-keep"}, [alpha]),
+            ("crawl_id", {"crawl_id": alpha["crawl_id"]}, [alpha]),
+            ("limit and sort", {"limit": 1, "sort": "timestamp"}, [alpha]),
+            ("search", {"search": "meta", "filter_patterns": ["Needle Alpha"]}, [alpha]),
+            ("before", {"before": 1715000000}, [alpha, beta]),
+            ("after", {"after": 1715000000}, [gamma]),
+            ("resume", {"resume": beta["timestamp"]}, [alpha, beta]),
+        ]
+        for label, body, expected_records in cases:
+            assert_update_filter(label, body, expected_records)
+    finally:
+        stop_server(tmp_path)

archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py

@@ -0,0 +1,160 @@
+import time
+
+import pytest
+
+from archivebox.core.models import Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.tests.test_orm_helpers import use_archivebox_db
+from .conftest import (
+    cli_env,
+    create_admin_and_token,
+    get_free_port,
+    init_archive,
+    live_api_request,
+    start_archivebox_server,
+    stop_server,
+    wait_for_live_api,
+)
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+@pytest.mark.timeout(180)
+def test_cli_api_add_search_update_remove_over_server(tmp_path):
+    init_archive(tmp_path)
+
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PUBLIC_INDEX="True")
+    api_token = create_admin_and_token(tmp_path)
+    target_url = "https://example.com/"
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
+
+        add_response = live_api_request(
+            port,
+            "post",
+            "/api/v1/cli/add",
+            api_token=api_token,
+            json={
+                "urls": [target_url],
+                "tag": "api-cli",
+                "depth": 0,
+                "parser": "url_list",
+                "plugins": "wget",
+                "update": True,
+                "overwrite": False,
+                "index_only": True,
+            },
+            timeout=10,
+        )
+        assert add_response.status_code == 200, add_response.text
+        add_payload = add_response.json()
+        assert add_payload["success"] is True
+        assert add_payload["result_format"] == "json"
+        assert add_payload["result"]["num_snapshots"] == 0
+        crawl_id = add_payload["result"]["crawl_id"]
+        assert add_payload["result"]["snapshot_ids"] == []
+        stop_server(tmp_path)
+        from archivebox.services.runner import run_crawl
+
+        with use_archivebox_db(tmp_path):
+            run_crawl(crawl_id, show_progress=False)
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
+
+        deadline = time.time() + 180
+        snapshot_id = None
+        snapshot_status = None
+        while time.time() < deadline:
+            with use_archivebox_db(tmp_path):
+                snapshot = Snapshot.objects.filter(crawl_id=crawl_id, url=target_url).first()
+                if snapshot is not None:
+                    snapshot_id = str(snapshot.id)
+                    snapshot_status = snapshot.status
+                    break
+            time.sleep(1)
+        assert snapshot_id is not None
+        assert snapshot_status is not None
+
+        search_response = live_api_request(
+            port,
+            "post",
+            "/api/v1/cli/search",
+            api_token=api_token,
+            json={
+                "filter_patterns": [target_url],
+                "filter_type": "exact",
+                "status": snapshot_status,
+                "sort": "bookmarked_at",
+                "as_json": True,
+                "as_html": False,
+                "as_csv": "",
+                "with_headers": False,
+            },
+            timeout=10,
+        )
+        assert search_response.status_code == 200, search_response.text
+        search_payload = search_response.json()
+        assert search_payload["success"] is True
+        assert search_payload["result_format"] == "json"
+        assert any(item["url"] == target_url for item in search_payload["result"])
+
+        update_response = live_api_request(
+            port,
+            "post",
+            "/api/v1/cli/update",
+            api_token=api_token,
+            json={
+                "resume": None,
+                "after": 0,
+                "before": 4102444800,
+                "filter_type": "exact",
+                "filter_patterns": [target_url],
+                "batch_size": 1,
+                "continuous": False,
+            },
+            timeout=20,
+        )
+        assert update_response.status_code == 200, update_response.text
+        assert update_response.json()["success"] is True
+        stop_server(tmp_path)
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
+
+        with use_archivebox_db(tmp_path):
+            crawl_obj = Crawl.objects.filter(pk=crawl_id).first()
+            crawl = (crawl_obj.max_depth, crawl_obj.tags_str, crawl_obj.config) if crawl_obj else None
+
+        assert crawl is not None
+        assert crawl[0] == 0
+        assert crawl[1] == "api-cli"
+        assert crawl[2]["INDEX_ONLY"] is True
+
+        remove_response = live_api_request(
+            port,
+            "post",
+            "/api/v1/cli/remove",
+            api_token=api_token,
+            json={
+                "delete": True,
+                "after": 0,
+                "before": 4102444800,
+                "filter_type": "exact",
+                "filter_patterns": [target_url],
+            },
+            timeout=20,
+        )
+        assert remove_response.status_code == 200, remove_response.text
+        remove_payload = remove_response.json()
+        assert remove_payload["success"] is True
+        assert remove_payload["result"]["removed_count"] == 1
+        assert snapshot_id in remove_payload["result"]["removed_snapshot_ids"]
+
+        with use_archivebox_db(tmp_path):
+            snapshot_count = Snapshot.objects.filter(pk=snapshot_id).count()
+
+        assert snapshot_count == 0
+    finally:
+        stop_server(tmp_path)

archivebox/tests/test_api_v1_core_any_id.py

@@ -0,0 +1,16 @@
+import pytest
+
+from archivebox.core.models import Snapshot
+from archivebox.crawls.models import Crawl
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/any", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/any", crawl=crawl)
+
+    response = client.get(f"/api/v1/core/any/{snapshot.id}", follow=True, **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py

@@ -0,0 +1,69 @@
+import pytest
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart
+
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.crawls.models import Crawl
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/archiveresult-detail", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/archiveresult-detail", crawl=crawl)
+    result = ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="api-basic",
+        hook_name="on_Snapshot__api_basic",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_str="ok",
+    )
+
+    response = client.get(f"/api/v1/core/archiveresult/{result.id}", **api_headers)
+
+    assert response.status_code == 200, response.content
+
+
+def test_archiveresult_patch_upload_finalizes_queued_result(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        created_by=api_admin_user,
+        status=Crawl.StatusChoices.SEALED,
+        retry_at=None,
+    )
+    snapshot = Snapshot.objects.create(
+        url="https://example.com/upload-patch",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.SEALED,
+        retry_at=None,
+    )
+    result = ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="dom",
+        hook_name="on_Snapshot__archivebox_browser_extension_upload",
+        status=ArchiveResult.StatusChoices.QUEUED,
+    )
+
+    response = client.generic(
+        "PATCH",
+        f"/api/v1/core/archiveresult/{result.id}",
+        encode_multipart(
+            BOUNDARY,
+            {
+                "files": SimpleUploadedFile("output.html", b"<html>uploaded</html>", content_type="text/html"),
+                "output_paths": "output.html",
+                "output_str": "output.html",
+            },
+        ),
+        content_type=MULTIPART_CONTENT,
+        **api_headers,
+    )
+    assert response.status_code == 200, response.content
+
+    result.refresh_from_db()
+    snapshot.refresh_from_db()
+    assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
+    assert result.output_str == "output.html"
+    assert snapshot.status == Snapshot.StatusChoices.SEALED
+    assert snapshot.retry_at is not None

archivebox/tests/test_api_v1_core_archiveresults.py

@@ -0,0 +1,170 @@
+from datetime import timedelta
+
+import pytest
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
+from django.utils import timezone
+
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.tests.conftest import api_client_request
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_archiveresult_upload_api_queues_snapshot_maintenance_without_finalizing(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        created_by=api_admin_user,
+        status=Crawl.StatusChoices.STARTED,
+        retry_at=timezone.now(),
+    )
+    active_retry_at = timezone.now() + timedelta(minutes=5)
+    active_snapshot = Snapshot.objects.create(
+        url="https://example.com/active",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.STARTED,
+        retry_at=active_retry_at,
+    )
+    sealed_snapshot = Snapshot.objects.create(
+        url="https://example.com/sealed",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.SEALED,
+        retry_at=None,
+    )
+
+    active_response = client.post(
+        "/api/v1/core/archiveresults",
+        {
+            "snapshot_id": str(active_snapshot.id),
+            "plugin": "chrome_extension_dom",
+            "hook_name": "on_Snapshot__archivebox_browser_extension_upload",
+            "status": ArchiveResult.StatusChoices.SUCCEEDED,
+            "output_str": "uploaded active snapshot output",
+        },
+        **api_headers,
+    )
+    assert active_response.status_code == 200, active_response.content
+    active_snapshot.refresh_from_db()
+    assert active_snapshot.status == Snapshot.StatusChoices.STARTED
+    assert active_snapshot.retry_at == active_retry_at
+    assert active_snapshot.downloaded_at is not None
+
+    sealed_response = client.post(
+        "/api/v1/core/archiveresults",
+        {
+            "snapshot_id": str(sealed_snapshot.id),
+            "plugin": "chrome_extension_mhtml",
+            "hook_name": "on_Snapshot__archivebox_browser_extension_upload",
+            "status": ArchiveResult.StatusChoices.SUCCEEDED,
+            "output_str": "uploaded sealed snapshot output",
+        },
+        **api_headers,
+    )
+    assert sealed_response.status_code == 200, sealed_response.content
+    sealed_snapshot.refresh_from_db()
+    assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
+    assert sealed_snapshot.retry_at is not None
+    assert sealed_snapshot.downloaded_at is not None
+
+
+def test_archiveresults_api_limit_uses_exact_count_without_full_row_distinct(client, api_headers):
+    snapshot_response = api_client_request(
+        client,
+        "post",
+        "/api/v1/core/snapshots",
+        payload={
+            "url": "https://example.com/archive-result-pagination",
+            "title": "ArchiveResult pagination",
+            "status": Snapshot.StatusChoices.QUEUED,
+        },
+        headers=api_headers,
+    )
+    assert snapshot_response.status_code == 200, snapshot_response.content
+    snapshot_id = snapshot_response.json()["id"]
+
+    for plugin_name in ("dom", "screenshot"):
+        result_response = client.post(
+            "/api/v1/core/archiveresults",
+            {
+                "snapshot_id": snapshot_id,
+                "plugin": plugin_name,
+                "hook_name": f"on_Snapshot__test_{plugin_name}",
+                "status": ArchiveResult.StatusChoices.SUCCEEDED,
+                "output_str": f"{plugin_name} output",
+            },
+            **api_headers,
+        )
+        assert result_response.status_code == 200, result_response.content
+
+    total_archiveresults = ArchiveResult.objects.count()
+    with CaptureQueriesContext(connection) as captured_queries:
+        response = client.get(
+            "/api/v1/core/archiveresults?limit=1",
+            **api_headers,
+        )
+
+    assert response.status_code == 200, response.content
+    payload = response.json()
+    assert payload["count"] == total_archiveresults
+    assert payload["total_items"] == total_archiveresults
+    assert payload["limit"] == 1
+    assert payload["num_items"] == 1
+
+    count_queries = [
+        query["sql"] for query in captured_queries if "COUNT" in query["sql"].upper() and '"core_archiveresult"' in query["sql"]
+    ]
+    assert count_queries
+    assert not any("SELECT DISTINCT" in query.upper() for query in count_queries), count_queries
+
+
+def test_archiveresults_api_join_filters_count_distinct_primary_keys(client, api_headers):
+    snapshot_response = api_client_request(
+        client,
+        "post",
+        "/api/v1/core/snapshots",
+        payload={
+            "url": "https://example.com/archive-result-tag-pagination",
+            "title": "ArchiveResult tag pagination",
+            "tags": ["api-tag-pagination-one", "api-tag-pagination-two"],
+            "status": Snapshot.StatusChoices.QUEUED,
+        },
+        headers=api_headers,
+    )
+    assert snapshot_response.status_code == 200, snapshot_response.content
+    snapshot_id = snapshot_response.json()["id"]
+
+    result_response = client.post(
+        "/api/v1/core/archiveresults",
+        {
+            "snapshot_id": snapshot_id,
+            "plugin": "dom",
+            "hook_name": "on_Snapshot__test_tag_pagination",
+            "status": ArchiveResult.StatusChoices.SUCCEEDED,
+            "output_str": "tag pagination output",
+        },
+        **api_headers,
+    )
+    assert result_response.status_code == 200, result_response.content
+
+    with CaptureQueriesContext(connection) as captured_queries:
+        response = client.get(
+            "/api/v1/core/archiveresults?search=api-tag-pagination&limit=1",
+            **api_headers,
+        )
+
+    assert response.status_code == 200, response.content
+    payload = response.json()
+    assert payload["count"] == 1
+    assert payload["total_items"] == 1
+    assert payload["num_items"] == 1
+    assert [item["id"] for item in payload["items"]] == [result_response.json()["id"]]
+
+    count_queries = [
+        query["sql"] for query in captured_queries if "COUNT" in query["sql"].upper() and '"core_archiveresult"' in query["sql"]
+    ]
+    assert count_queries
+    assert any("SELECT DISTINCT" in query.upper() for query in count_queries), count_queries
+    assert not any('"core_archiveresult"."output_files" AS' in query for query in count_queries), count_queries
+    assert not any('"core_archiveresult"."notes" AS' in query for query in count_queries), count_queries

archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py

@@ -0,0 +1,497 @@
+import json
+import time
+from pathlib import Path
+
+import pytest
+from django.utils import timezone
+
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.tests.conftest import run_archivebox_cmd
+from archivebox.tests.test_orm_helpers import use_archivebox_db
+from archivebox.workers.models import RETRY_AT_MAX
+
+from .conftest import (
+    api_client_request,
+    cli_env,
+    create_admin_and_token,
+    get_crawl_runtime_state,
+    get_free_port,
+    init_archive,
+    live_api_request,
+    start_archivebox_server,
+    stop_server,
+    wait_for_live_api,
+    wait_for_snapshot_capture,
+)
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def _seed_archiveresult(
+    snapshot: Snapshot,
+    *,
+    plugin: str,
+    hook_name: str,
+    status: str,
+    output_text: str = "",
+    output_path: str | None = None,
+) -> ArchiveResult:
+    output_files = {}
+    output_size = 0
+    output_mimetypes = ""
+    if output_path is not None:
+        output_bytes = output_text.encode()
+        absolute_path = Path(snapshot.output_dir) / output_path
+        absolute_path.parent.mkdir(parents=True, exist_ok=True)
+        absolute_path.write_bytes(output_bytes)
+        output_size = len(output_bytes)
+        output_mimetypes = "text/plain"
+        output_files[output_path] = {
+            "extension": Path(output_path).suffix.lstrip("."),
+            "mimetype": "text/plain",
+            "size": output_size,
+        }
+
+    now = timezone.now()
+    return ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin=plugin,
+        hook_name=hook_name,
+        status=status,
+        output_str=output_path or output_text,
+        output_files=output_files,
+        output_size=output_size,
+        output_mimetypes=output_mimetypes,
+        start_ts=now if status != ArchiveResult.StatusChoices.QUEUED else None,
+        end_ts=now if status in ArchiveResult.FINAL_STATES else None,
+    )
+
+
+def _snapshot_hook_name(plugin_name: str) -> str:
+    from abx_dl.models import discover_plugins
+
+    plugin = discover_plugins().get(plugin_name)
+    assert plugin is not None, f"missing test plugin {plugin_name}"
+    hooks = plugin.filter_hooks("Snapshot")
+    assert hooks, f"missing Snapshot hooks for {plugin_name}"
+    return hooks[0].name
+
+
+def _snapshot_state(cwd: Path, url: str) -> dict[str, object]:
+    with use_archivebox_db(cwd):
+        snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").get(url=url)
+        snapshot_dir = Path(snapshot.output_dir)
+        crawl_dir = Path(snapshot.crawl.output_dir)
+        crawl_link = crawl_dir / "snapshots" / Snapshot.extract_domain_from_url(snapshot.url) / str(snapshot.id)
+        results = list(
+            ArchiveResult.objects.filter(snapshot=snapshot)
+            .order_by("plugin", "hook_name")
+            .values("plugin", "hook_name", "status", "output_files", "output_size"),
+        )
+        return {
+            "id": str(snapshot.id),
+            "crawl_id": str(snapshot.crawl_id),
+            "status": snapshot.status,
+            "retry_at": snapshot.retry_at,
+            "downloaded_at": snapshot.downloaded_at,
+            "output_size": snapshot.output_size,
+            "snapshot_dir": snapshot_dir,
+            "crawl_dir": crawl_dir,
+            "crawl_link": crawl_link,
+            "results": results,
+        }
+
+
+def _paused_snapshot_state(cwd: Path, snapshot_id: str) -> dict[str, object]:
+    with use_archivebox_db(cwd):
+        snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+        succeeded_results = ArchiveResult.objects.filter(snapshot=snapshot, status=ArchiveResult.StatusChoices.SUCCEEDED).count()
+        return {
+            "status": snapshot.status,
+            "retry_at": snapshot.retry_at,
+            "crawl_status": snapshot.crawl.status,
+            "succeeded_results": succeeded_results,
+            "snapshot_dir": Path(snapshot.output_dir),
+        }
+
+
+def _wait_for_paused_scheduler_marker(cwd: Path, snapshot_id: str, timeout: int = 60) -> dict[str, object]:
+    deadline = time.time() + timeout
+    last_state: dict[str, object] = {}
+    while time.time() < deadline:
+        last_state = _paused_snapshot_state(cwd, snapshot_id)
+        if last_state["status"] == Snapshot.StatusChoices.PAUSED and last_state["retry_at"] == RETRY_AT_MAX:
+            return last_state
+        if last_state["status"] == Snapshot.StatusChoices.SEALED:
+            return last_state
+        time.sleep(1)
+    raise AssertionError(f"paused snapshot did not settle back to retry_at=MAX: {last_state}")
+
+
+def _wait_for_crawl_snapshot_rows(cwd: Path, crawl_id: str, timeout: int = 45) -> dict[str, object]:
+    deadline = time.time() + timeout
+    latest_state: dict[str, object] | None = None
+    while time.time() < deadline:
+        latest_state = get_crawl_runtime_state(cwd, crawl_id)
+        if latest_state["snapshots"]:
+            return latest_state
+        time.sleep(0.2)
+    raise AssertionError(f"timed out waiting for snapshot rows for crawl {crawl_id}: {latest_state}")
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/snapshot-detail", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/snapshot-detail", crawl=crawl)
+
+    response = client.get(f"/api/v1/core/snapshot/{snapshot.id}", **api_headers)
+
+    assert response.status_code == 200, response.content
+
+
+def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_finished_rows(
+    tmp_path,
+    client,
+    recursive_test_site,
+):
+    init_archive(tmp_path)
+    api_token = create_admin_and_token(tmp_path)
+
+    with use_archivebox_db(tmp_path):
+        create_response = api_client_request(
+            client,
+            "post",
+            "/api/v1/core/snapshots",
+            api_token=api_token,
+            payload={
+                "url": recursive_test_site["root_url"],
+                "depth": 0,
+                "title": "Snapshot pause target",
+                "tags": ["snapshot-pause-e2e"],
+                "status": "queued",
+            },
+        )
+        assert create_response.status_code == 200, create_response.content.decode()
+        snapshot_id = json.loads(create_response.content.decode())["id"]
+        snapshot = Snapshot.objects.get(id=snapshot_id)
+
+        queued_result = _seed_archiveresult(
+            snapshot,
+            plugin="manualqueue",
+            hook_name="on_Snapshot__manual_queue",
+            status=ArchiveResult.StatusChoices.QUEUED,
+        )
+        started_result = _seed_archiveresult(
+            snapshot,
+            plugin="manualstart",
+            hook_name="on_Snapshot__manual_start",
+            status=ArchiveResult.StatusChoices.STARTED,
+        )
+        succeeded_result = _seed_archiveresult(
+            snapshot,
+            plugin="manualdone",
+            hook_name="on_Snapshot__manual_done",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+            output_text="finished result should stay finished",
+            output_path="manualdone/final.txt",
+        )
+        failed_result = _seed_archiveresult(
+            snapshot,
+            plugin="manualfail",
+            hook_name="on_Snapshot__manual_fail",
+            status=ArchiveResult.StatusChoices.FAILED,
+            output_text="failed result should stay failed",
+        )
+
+        invalid_response = api_client_request(
+            client,
+            "patch",
+            f"/api/v1/core/snapshot/{snapshot_id}",
+            api_token=api_token,
+            payload={"action": "hold"},
+        )
+        assert invalid_response.status_code == 400
+        snapshot = Snapshot.objects.get(id=snapshot_id)
+        assert snapshot.status == Snapshot.StatusChoices.QUEUED
+
+        pause_response = api_client_request(
+            client,
+            "patch",
+            f"/api/v1/core/snapshot/{snapshot_id}",
+            api_token=api_token,
+            payload={"action": "pause"},
+        )
+        assert pause_response.status_code == 200, pause_response.content.decode()
+        assert json.loads(pause_response.content.decode())["status"] == Snapshot.StatusChoices.PAUSED
+
+        snapshot.refresh_from_db()
+        crawl = Crawl.objects.get(id=snapshot.crawl_id)
+        assert snapshot.status == Snapshot.StatusChoices.PAUSED
+        assert snapshot.retry_at == RETRY_AT_MAX
+        assert crawl.status == Crawl.StatusChoices.QUEUED
+
+        active_rows = {
+            row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id])
+        }
+        assert active_rows == {
+            "manualqueue": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
+            "manualstart": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
+        }
+
+        finished_rows = {
+            row.plugin: (row.status, row.retry_at, row.output_size)
+            for row in ArchiveResult.objects.filter(id__in=[succeeded_result.id, failed_result.id])
+        }
+        assert finished_rows["manualdone"][0] == ArchiveResult.StatusChoices.SUCCEEDED
+        assert finished_rows["manualdone"][1] is None
+        assert finished_rows["manualdone"][2] == len("finished result should stay finished")
+        assert finished_rows["manualfail"] == (ArchiveResult.StatusChoices.FAILED, None, 0)
+
+        succeeded_row = ArchiveResult.objects.get(id=succeeded_result.id)
+        output_path = Path(snapshot.output_dir) / next(iter(succeeded_row.output_files))
+        assert output_path.read_text() == "finished result should stay finished"
+
+        resume_response = api_client_request(
+            client,
+            "patch",
+            f"/api/v1/core/snapshot/{snapshot_id}",
+            api_token=api_token,
+            payload={"action": "resume"},
+        )
+        assert resume_response.status_code == 200, resume_response.content.decode()
+        assert json.loads(resume_response.content.decode())["status"] == Snapshot.StatusChoices.QUEUED
+
+        snapshot.refresh_from_db()
+        crawl.refresh_from_db()
+        assert snapshot.status == Snapshot.StatusChoices.QUEUED
+        assert snapshot.retry_at is not None
+        assert snapshot.retry_at != RETRY_AT_MAX
+        assert crawl.status == Crawl.StatusChoices.QUEUED
+        assert crawl.retry_at is not None
+        assert crawl.retry_at != RETRY_AT_MAX
+
+        resumed_rows = {
+            row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id])
+        }
+        assert resumed_rows["manualqueue"][0] == ArchiveResult.StatusChoices.QUEUED
+        assert resumed_rows["manualqueue"][1] is not None
+        assert resumed_rows["manualqueue"][1] != RETRY_AT_MAX
+        assert resumed_rows["manualstart"][0] == ArchiveResult.StatusChoices.QUEUED
+        assert resumed_rows["manualstart"][1] is not None
+        assert resumed_rows["manualstart"][1] != RETRY_AT_MAX
+
+        assert ArchiveResult.objects.get(id=succeeded_result.id).status == ArchiveResult.StatusChoices.SUCCEEDED
+        assert ArchiveResult.objects.get(id=failed_result.id).status == ArchiveResult.StatusChoices.FAILED
+        assert output_path.read_text() == "finished result should stay finished"
+
+
+def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_paused(
+    tmp_path,
+    client,
+    recursive_test_site,
+):
+    init_archive(tmp_path)
+    api_token = create_admin_and_token(tmp_path)
+
+    with use_archivebox_db(tmp_path):
+        snapshot_response = api_client_request(
+            client,
+            "post",
+            "/api/v1/core/snapshots",
+            api_token=api_token,
+            payload={
+                "url": recursive_test_site["root_url"],
+                "depth": 0,
+                "title": "Paused targeted retry",
+                "tags": ["targeted-extract-pause"],
+                "status": "queued",
+            },
+        )
+        assert snapshot_response.status_code == 200, snapshot_response.content.decode()
+        snapshot_id = json.loads(snapshot_response.content.decode())["id"]
+        snapshot = Snapshot.objects.get(id=snapshot_id)
+
+        wget_result = _seed_archiveresult(
+            snapshot,
+            plugin="wget",
+            hook_name=_snapshot_hook_name("wget"),
+            status=ArchiveResult.StatusChoices.FAILED,
+            output_text="initial failure before targeted retry",
+        )
+        unrelated_result = _seed_archiveresult(
+            snapshot,
+            plugin="manualqueue",
+            hook_name="on_Snapshot__manual_queue",
+            status=ArchiveResult.StatusChoices.QUEUED,
+        )
+        finished_result = _seed_archiveresult(
+            snapshot,
+            plugin="manualdone",
+            hook_name="on_Snapshot__manual_done",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+            output_text="finished row must survive targeted retry",
+            output_path="manualdone/targeted.txt",
+        )
+
+        pause_response = api_client_request(
+            client,
+            "patch",
+            f"/api/v1/core/snapshot/{snapshot_id}",
+            api_token=api_token,
+            payload={"action": "pause"},
+        )
+        assert pause_response.status_code == 200, pause_response.content.decode()
+        assert json.loads(pause_response.content.decode())["status"] == Snapshot.StatusChoices.PAUSED
+
+        snapshot = Snapshot.objects.get(id=snapshot_id)
+        assert snapshot.status == Snapshot.StatusChoices.PAUSED
+        assert snapshot.retry_at == RETRY_AT_MAX
+        assert ArchiveResult.objects.get(id=wget_result.id).status == ArchiveResult.StatusChoices.FAILED
+        assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.PAUSED
+        finished_row = ArchiveResult.objects.get(id=finished_result.id)
+        finished_output_path = Path(snapshot.output_dir) / next(iter(finished_row.output_files))
+        assert finished_output_path.read_text() == "finished row must survive targeted retry"
+
+    env = cli_env(
+        port=get_free_port(),
+        PLUGINS="wget",
+        SAVE_WGET="True",
+        WGET_WARC_ENABLED="False",
+        URL_ALLOWLIST=r"127\.0\.0\.1[:/].*",
+    )
+    extract = run_archivebox_cmd(
+        ["extract", str(wget_result.id)],
+        cwd=tmp_path,
+        env=env,
+        timeout=150,
+    )
+    assert extract.returncode == 0, f"STDOUT:\n{extract.stdout}\nSTDERR:\n{extract.stderr}"
+
+    with use_archivebox_db(tmp_path):
+        snapshot = Snapshot.objects.get(id=snapshot_id)
+        assert snapshot.status == Snapshot.StatusChoices.PAUSED
+        assert snapshot.retry_at == RETRY_AT_MAX
+
+        retried_wget = ArchiveResult.objects.get(id=wget_result.id)
+        assert retried_wget.status == ArchiveResult.StatusChoices.SUCCEEDED
+        assert retried_wget.output_size > 0
+        assert retried_wget.output_files
+
+        unrelated = ArchiveResult.objects.get(id=unrelated_result.id)
+        assert unrelated.status == ArchiveResult.StatusChoices.PAUSED
+        assert unrelated.retry_at == RETRY_AT_MAX
+
+        finished = ArchiveResult.objects.get(id=finished_result.id)
+        assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED
+        assert finished.retry_at is None
+        assert finished_output_path.read_text() == "finished row must survive targeted retry"
+
+
+@pytest.mark.timeout(240)
+def test_paused_snapshot_survives_server_restart_and_resumes_via_api(tmp_path, recursive_test_site):
+    init_archive(tmp_path)
+
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True")
+    api_token = create_admin_and_token(tmp_path)
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
+
+        crawl_response = live_api_request(
+            port,
+            "post",
+            "/api/v1/crawls/crawls",
+            api_token=api_token,
+            json={
+                "urls": [recursive_test_site["root_url"]],
+                "max_depth": 0,
+                "tags": ["snapshot-pause-restart-e2e"],
+                "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
+            },
+            timeout=10,
+        )
+        assert crawl_response.status_code == 200, crawl_response.text
+        crawl_id = crawl_response.json()["id"]
+        crawl_state = _wait_for_crawl_snapshot_rows(tmp_path, crawl_id)
+        snapshot_id = crawl_state["snapshots"][0]["id"]
+
+        pause_response = live_api_request(
+            port,
+            "patch",
+            f"/api/v1/crawls/crawl/{crawl_id}",
+            api_token=api_token,
+            json={"action": "pause"},
+            timeout=10,
+        )
+        assert pause_response.status_code == 200, pause_response.text
+
+        current_state = _paused_snapshot_state(tmp_path, snapshot_id)
+        if current_state["status"] == Snapshot.StatusChoices.SEALED:
+            assert current_state["succeeded_results"] > 0
+            return
+
+        paused_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id)
+        if paused_state["status"] == Snapshot.StatusChoices.SEALED:
+            assert paused_state["succeeded_results"] > 0
+            return
+        assert paused_state["succeeded_results"] == 0
+        assert not list((paused_state["snapshot_dir"] / "wget").rglob("*.html"))
+
+        stop_server(tmp_path)
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
+
+        restarted_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id)
+        assert restarted_state["status"] == Snapshot.StatusChoices.PAUSED
+        assert restarted_state["succeeded_results"] == 0
+
+        resume_response = live_api_request(
+            port,
+            "patch",
+            f"/api/v1/core/snapshot/{snapshot_id}",
+            api_token=api_token,
+            json={"action": "resume"},
+            timeout=10,
+        )
+        assert resume_response.status_code == 200, resume_response.text
+        assert resume_response.json()["status"] == Snapshot.StatusChoices.QUEUED
+
+        captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180)
+        assert "Root" in captured_text
+        assert "About" in captured_text
+
+        final_state = _snapshot_state(tmp_path, recursive_test_site["root_url"])
+        assert final_state["status"] == Snapshot.StatusChoices.SEALED
+        assert final_state["downloaded_at"] is not None
+        assert any(
+            result["plugin"] == "wget" and result["status"] == ArchiveResult.StatusChoices.SUCCEEDED for result in final_state["results"]
+        )
+    finally:
+        stop_server(tmp_path)
+
+
+def test_rest_snapshot_delete_removes_output_dir(client, api_headers):
+    url = "https://example.com/delete-path-snapshot"
+
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/core/snapshots",
+        payload={"url": url, "depth": 0, "status": Snapshot.StatusChoices.QUEUED},
+        headers=api_headers,
+    )
+    assert response.status_code == 200, response.content.decode()
+
+    snapshot = Snapshot.objects.get(url=url)
+    snapshot_dir = Path(snapshot.output_dir)
+    snapshot_dir.mkdir(parents=True, exist_ok=True)
+    (snapshot_dir / "delete-path-test.txt").write_text("snapshot output")
+    assert snapshot_dir.exists()
+
+    response = client.delete(f"/api/v1/core/snapshot/{snapshot.id}", **api_headers)
+    assert response.status_code == 200, response.content.decode()
+    assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
+    assert not snapshot_dir.exists()

archivebox/tests/test_api_v1_core_snapshots.py

@@ -0,0 +1,46 @@
+import pytest
+
+from archivebox.core.models import Snapshot
+from archivebox.crawls.models import Crawl
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_snapshots_api_filters_status_column_and_rejects_legacy_status(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        created_by=api_admin_user,
+        status=Crawl.StatusChoices.SEALED,
+        retry_at=None,
+    )
+    Snapshot.objects.create(
+        url="https://example.com/api-status-queued",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.QUEUED,
+    )
+    sealed_snapshot = Snapshot.objects.create(
+        url="https://example.com/api-status-sealed",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.SEALED,
+        retry_at=None,
+    )
+
+    response = client.get(
+        "/api/v1/core/snapshots",
+        {"status": "sealed"},
+        **api_headers,
+    )
+    assert response.status_code == 200, response.content
+    payload = response.json()
+    items = payload["items"] if isinstance(payload, dict) and "items" in payload else payload
+    assert [item["id"] for item in items] == [str(sealed_snapshot.id)]
+    assert [item["status"] for item in items] == ["sealed"]
+
+    legacy_response = client.get(
+        "/api/v1/core/snapshots",
+        {"status": "unarchived"},
+        **api_headers,
+    )
+    assert legacy_response.status_code == 400
+    assert "Invalid snapshot status" in legacy_response.content.decode()

archivebox/tests/test_api_v1_core_snapshots_rss.py

@@ -0,0 +1,115 @@
+from datetime import datetime
+from typing import cast
+
+import pytest
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import UserManager
+from django.utils import timezone
+
+
+pytestmark = pytest.mark.django_db
+
+
+User = get_user_model()
+ADMIN_HOST = "admin.archivebox.localhost:8000"
+
+
+@pytest.fixture
+def other_user(db):
+    return cast(UserManager, User.objects).create_user(
+        username="rssother",
+        email="rssother@test.com",
+        password="testpassword",
+    )
+
+
+def make_snapshot(*, user, url: str, title: str, bookmarked_at: datetime):
+    from archivebox.core.models import Snapshot
+    from archivebox.crawls.models import Crawl
+
+    crawl = Crawl.objects.create(urls=url, created_by=user)
+    snapshot = Snapshot.objects.create(
+        url=url,
+        title=title,
+        crawl=crawl,
+        bookmarked_at=bookmarked_at,
+    )
+    return crawl, snapshot
+
+
+def test_snapshots_rss_filters_by_user_and_orders_newest_first(client, api_token, api_admin_user, other_user):
+    from archivebox.core.models import Tag
+
+    older_at = timezone.make_aware(datetime(2026, 5, 22, 8, 0, 0))
+    newer_at = timezone.make_aware(datetime(2026, 5, 23, 8, 0, 0))
+    _crawl, older_snapshot = make_snapshot(
+        user=api_admin_user,
+        url="https://example.com/rss-older",
+        title="Older & Escaped",
+        bookmarked_at=older_at,
+    )
+    make_snapshot(
+        user=api_admin_user,
+        url="https://example.com/rss-newer",
+        title="Newer Snapshot",
+        bookmarked_at=newer_at,
+    )
+    make_snapshot(
+        user=other_user,
+        url="https://example.com/rss-other-user",
+        title="Other User",
+        bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 9, 0, 0)),
+    )
+    older_snapshot.tags.add(Tag.objects.create(name="rss-tag", created_by=api_admin_user))
+
+    response = client.get(
+        "/api/v1/core/snapshots.rss",
+        {"created_by": api_admin_user.username, "limit": 50, "api_key": api_token.token},
+        HTTP_HOST=ADMIN_HOST,
+    )
+
+    assert response.status_code == 200
+    assert response["Content-Type"].startswith("application/rss+xml")
+    body = response.content.decode()
+    assert '<rss version="2.0"' in body
+    assert api_token.token not in body
+    assert f"created_by={api_admin_user.username}&amp;limit=50" in body
+    assert "Newer Snapshot" in body
+    assert "Older &amp; Escaped" in body
+    assert "Tags: rss-tag" not in body
+    assert "<category>rss-tag</category>" in body
+    assert "rss-other-user" not in body
+    assert body.index("rss-newer") < body.index("rss-older")
+
+
+def test_snapshots_rss_supports_before_yyyymmdd_and_limit(client, api_token, api_admin_user):
+    make_snapshot(
+        user=api_admin_user,
+        url="https://example.com/rss-before-too-new",
+        title="Too New",
+        bookmarked_at=timezone.make_aware(datetime(2026, 5, 24, 8, 0, 0)),
+    )
+    make_snapshot(
+        user=api_admin_user,
+        url="https://example.com/rss-before-keep-one",
+        title="Keep One",
+        bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 12, 0, 0)),
+    )
+    make_snapshot(
+        user=api_admin_user,
+        url="https://example.com/rss-before-keep-two",
+        title="Keep Two",
+        bookmarked_at=timezone.make_aware(datetime(2026, 5, 22, 12, 0, 0)),
+    )
+
+    response = client.get(
+        "/api/v1/core/snapshots.rss",
+        {"created_by": str(api_admin_user.pk), "before": "20260523", "limit": 1, "api_key": api_token.token},
+        HTTP_HOST=ADMIN_HOST,
+    )
+
+    assert response.status_code == 200
+    body = response.content.decode()
+    assert "rss-before-too-new" not in body
+    assert "rss-before-keep-one" in body
+    assert "rss-before-keep-two" not in body

archivebox/tests/test_api_v1_core_tag_tag_id.py

@@ -0,0 +1,15 @@
+import pytest
+
+from archivebox.core.models import Tag
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+@pytest.mark.parametrize("request_method", ("get", "delete"))
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers, request_method):
+    tag = Tag.objects.create(name="api-basic-tag", created_by=api_admin_user)
+
+    response = getattr(client, request_method)(f"/api/v1/core/tag/{tag.id}", **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_core_tag_tag_id_rename.py

@@ -0,0 +1,38 @@
+import pytest
+
+from archivebox.core.models import Tag
+from archivebox.tests.conftest import ADMIN_TEST_HOST, api_client_request
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    tag = Tag.objects.create(name="api-basic-tag", created_by=api_admin_user)
+
+    response = api_client_request(
+        client,
+        "post",
+        f"/api/v1/core/tag/{tag.id}/rename",
+        payload={"name": "api-basic-renamed"},
+        headers=api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+
+
+def test_tag_rename_api_updates_name(client, api_token, tagged_data):
+    tag, _ = tagged_data
+
+    response = api_client_request(
+        client,
+        "post",
+        f"/api/v1/core/tag/{tag.id}/rename?api_key={api_token.token}",
+        payload={"name": "Alpha Archive"},
+        headers={"HTTP_HOST": ADMIN_TEST_HOST},
+    )
+
+    assert response.status_code == 200
+
+    tag.refresh_from_db()
+    assert tag.name == "Alpha Archive"

archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py

@@ -0,0 +1,36 @@
+import pytest
+
+from archivebox.core.models import Snapshot, Tag
+from archivebox.crawls.models import Crawl
+from archivebox.tests.conftest import ADMIN_TEST_HOST
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    tag = Tag.objects.create(name="api-basic-tag", created_by=api_admin_user)
+    crawl = Crawl.objects.create(urls="https://example.com/tag-jsonl-export", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/tag-jsonl-export", crawl=crawl)
+    snapshot.tags.add(tag)
+
+    response = client.get(f"/api/v1/core/tag/{tag.id}/snapshots.jsonl", **api_headers)
+
+    assert response.status_code == 200, response.content
+
+
+def test_tag_snapshots_export_returns_jsonl(client, api_token, tagged_data):
+    tag, _ = tagged_data
+
+    response = client.get(
+        f"/api/v1/core/tag/{tag.id}/snapshots.jsonl",
+        {"api_key": api_token.token},
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+
+    assert response.status_code == 200
+    assert response["Content-Type"].startswith("application/x-ndjson")
+    assert f"tag-{tag.slug}-snapshots.jsonl" in response["Content-Disposition"]
+    body = response.content.decode()
+    assert '"type": "Snapshot"' in body
+    assert '"tags": "Alpha Research"' in body

archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py

@@ -0,0 +1,35 @@
+import pytest
+
+from archivebox.core.models import Snapshot, Tag
+from archivebox.crawls.models import Crawl
+from archivebox.tests.conftest import ADMIN_TEST_HOST
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    tag = Tag.objects.create(name="api-basic-tag", created_by=api_admin_user)
+    crawl = Crawl.objects.create(urls="https://example.com/tag-url-export", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/tag-url-export", crawl=crawl)
+    snapshot.tags.add(tag)
+
+    response = client.get(f"/api/v1/core/tag/{tag.id}/urls.txt", **api_headers)
+
+    assert response.status_code == 200, response.content
+
+
+def test_tag_urls_export_returns_plain_text_urls(client, api_token, tagged_data):
+    tag, snapshots = tagged_data
+
+    response = client.get(
+        f"/api/v1/core/tag/{tag.id}/urls.txt",
+        {"api_key": api_token.token},
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+
+    assert response.status_code == 200
+    assert response["Content-Type"].startswith("text/plain")
+    assert f"tag-{tag.slug}-urls.txt" in response["Content-Disposition"]
+    exported_urls = set(filter(None, response.content.decode().splitlines()))
+    assert exported_urls == {snapshot.url for snapshot in snapshots}

archivebox/tests/test_api_v1_core_tags.py

@@ -0,0 +1,14 @@
+import pytest
+
+from archivebox.core.models import Tag
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    Tag.objects.create(name="api-basic-tag", created_by=api_admin_user)
+
+    response = client.get("/api/v1/core/tags", **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py

@@ -0,0 +1,25 @@
+import pytest
+
+from archivebox.core.models import Snapshot, Tag
+from archivebox.crawls.models import Crawl
+from archivebox.tests.conftest import api_client_request
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/tag-add", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/tag-add", crawl=crawl)
+    tag = Tag.objects.create(name="api-basic-add-tag", created_by=api_admin_user)
+
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/core/tags/add-to-snapshot/",
+        payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
+        headers=api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+    assert response.json()["success"] is True

archivebox/tests/test_api_v1_core_tags_autocomplete.py

@@ -0,0 +1,18 @@
+import pytest
+
+from archivebox.core.models import Tag
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_token):
+    Tag.objects.create(name="api-basic-tag", created_by=api_admin_user)
+
+    response = client.get(
+        "/api/v1/core/tags/autocomplete/",
+        {"q": "api-basic", "api_key": api_token.token},
+        HTTP_HOST="api.archivebox.localhost:8000",
+    )
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_core_tags_create.py

@@ -0,0 +1,19 @@
+import pytest
+
+from archivebox.tests.conftest import api_client_request
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, api_headers):
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/core/tags/create/",
+        payload={"name": "api-basic-created-tag"},
+        headers=api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+    assert response.json()["success"] is True

archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py

@@ -0,0 +1,26 @@
+import pytest
+
+from archivebox.core.models import Snapshot, Tag
+from archivebox.crawls.models import Crawl
+from archivebox.tests.conftest import api_client_request
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/tag-remove", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/tag-remove", crawl=crawl)
+    tag = Tag.objects.create(name="api-basic-remove-tag", created_by=api_admin_user)
+    snapshot.tags.add(tag)
+
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/core/tags/remove-from-snapshot/",
+        payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
+        headers=api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+    assert response.json()["success"] is True

archivebox/tests/test_api_v1_core_tags_search.py

@@ -0,0 +1,83 @@
+import pytest
+from django.contrib.auth import get_user_model
+from django.utils import timezone
+
+from archivebox.core.models import Snapshot, Tag
+from archivebox.tests.conftest import ADMIN_TEST_HOST
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    Tag.objects.create(name="api-basic-tag", created_by=api_admin_user)
+
+    response = client.get("/api/v1/core/tags/search/", {"q": "api-basic"}, **api_headers)
+
+    assert response.status_code == 200, response.content
+
+
+def test_tag_search_api_returns_card_payload(client, api_token, tagged_data):
+    tag, snapshots = tagged_data
+
+    response = client.get(
+        "/api/v1/core/tags/search/",
+        {"q": "Alpha", "api_key": api_token.token},
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["sort"] == "created_desc"
+    assert payload["created_by"] == ""
+    assert payload["year"] == ""
+    assert payload["has_snapshots"] == "all"
+    assert payload["tags"][0]["id"] == tag.id
+    assert payload["tags"][0]["name"] == "Alpha Research"
+    assert payload["tags"][0]["num_snapshots"] == 2
+    assert payload["tags"][0]["snapshots"] == []
+    assert payload["tags"][0]["export_jsonl_url"].endswith(f"/api/v1/core/tag/{tag.id}/snapshots.jsonl")
+    assert payload["tags"][0]["filter_url"].endswith(f"/admin/core/snapshot/?tags__id__exact={tag.id}")
+    assert {snap.url for snap in snapshots} == {"https://example.com/one", "https://example.com/two"}
+
+
+def test_tag_search_api_respects_sort_and_filters(client, api_token, admin_user, crawl, tagged_data):
+    from datetime import datetime
+
+    other_user = get_user_model().objects.create_user(
+        username="tagother",
+        email="tagother@test.com",
+        password="unused",
+    )
+    tag_with_snapshots = tagged_data[0]
+    empty_tag = Tag.objects.create(name="Zulu Empty", created_by=other_user)
+    alpha_tag = Tag.objects.create(name="Alpha Empty", created_by=other_user)
+    Snapshot.objects.create(
+        url="https://example.com/three",
+        title="Example Three",
+        crawl=crawl,
+    ).tags.add(alpha_tag)
+
+    Tag.objects.filter(pk=empty_tag.pk).update(created_at=timezone.make_aware(datetime(2024, 1, 1, 12, 0, 0)))
+    Tag.objects.filter(pk=alpha_tag.pk).update(created_at=timezone.make_aware(datetime(2025, 1, 1, 12, 0, 0)))
+    Tag.objects.filter(pk=tag_with_snapshots.pk).update(created_at=timezone.make_aware(datetime(2026, 1, 1, 12, 0, 0)))
+
+    response = client.get(
+        "/api/v1/core/tags/search/",
+        {
+            "sort": "name_desc",
+            "created_by": str(other_user.pk),
+            "year": "2024",
+            "has_snapshots": "no",
+            "api_key": api_token.token,
+        },
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["sort"] == "name_desc"
+    assert payload["created_by"] == str(other_user.pk)
+    assert payload["year"] == "2024"
+    assert payload["has_snapshots"] == "no"
+    assert [tag["name"] for tag in payload["tags"]] == ["Zulu Empty"]

archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py

@@ -0,0 +1,614 @@
+import json
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import cast
+
+import pytest
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import UserManager
+from django.utils import timezone
+
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.tests.test_orm_helpers import use_archivebox_db
+from archivebox.workers.models import RETRY_AT_MAX
+
+from .conftest import (
+    api_client_request,
+    cli_env,
+    create_admin_and_token,
+    get_crawl_runtime_state,
+    get_free_port,
+    init_archive,
+    live_api_request,
+    run_archivebox_cmd,
+    start_archivebox_server,
+    stop_server,
+    wait_for_live_api,
+    wait_for_snapshot_capture,
+)
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+User = get_user_model()
+ADMIN_HOST = "admin.archivebox.localhost:8000"
+
+
+@pytest.fixture
+def other_user(db):
+    return cast(UserManager, User.objects).create_user(
+        username="rssother",
+        email="rssother@test.com",
+        password="testpassword",
+    )
+
+
+def _seed_archiveresult(
+    snapshot: Snapshot,
+    *,
+    plugin: str,
+    hook_name: str,
+    status: str,
+    output_text: str = "",
+    output_path: str | None = None,
+) -> ArchiveResult:
+    output_files = {}
+    output_size = 0
+    output_mimetypes = ""
+    if output_path is not None:
+        output_bytes = output_text.encode()
+        absolute_path = Path(snapshot.output_dir) / output_path
+        absolute_path.parent.mkdir(parents=True, exist_ok=True)
+        absolute_path.write_bytes(output_bytes)
+        output_size = len(output_bytes)
+        output_mimetypes = "text/plain"
+        output_files[output_path] = {
+            "extension": Path(output_path).suffix.lstrip("."),
+            "mimetype": "text/plain",
+            "size": output_size,
+        }
+
+    now = timezone.now()
+    return ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin=plugin,
+        hook_name=hook_name,
+        status=status,
+        output_str=output_path or output_text,
+        output_files=output_files,
+        output_size=output_size,
+        output_mimetypes=output_mimetypes,
+        start_ts=now if status != ArchiveResult.StatusChoices.QUEUED else None,
+        end_ts=now if status in ArchiveResult.FINAL_STATES else None,
+    )
+
+
+def wait_for_crawl_snapshot_rows(cwd, crawl_id, timeout=45):
+    deadline = time.time() + timeout
+    latest_state = None
+    while time.time() < deadline:
+        latest_state = get_crawl_runtime_state(cwd, crawl_id)
+        if latest_state["snapshots"]:
+            return latest_state
+        time.sleep(0.2)
+    raise AssertionError(f"timed out waiting for runner to create snapshots for crawl {crawl_id}: {latest_state}")
+
+
+def wait_for_crawl_child_snapshots_paused_or_sealed(cwd, crawl_id, timeout=45):
+    deadline = time.time() + timeout
+    latest_state = None
+    while time.time() < deadline:
+        latest_state = get_crawl_runtime_state(cwd, crawl_id)
+        snapshots = latest_state["snapshots"]
+        if snapshots and all(snapshot["status"] in {"paused", "sealed"} for snapshot in snapshots):
+            return latest_state
+        time.sleep(0.2)
+    raise AssertionError(f"timed out waiting for runner to pause or seal snapshots for crawl {crawl_id}: {latest_state}")
+
+
+def make_snapshot(*, user, url: str, title: str, bookmarked_at: datetime):
+    crawl = Crawl.objects.create(urls=url, created_by=user)
+    snapshot = Snapshot.objects.create(
+        url=url,
+        title=title,
+        crawl=crawl,
+        bookmarked_at=bookmarked_at,
+    )
+    return crawl, snapshot
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/crawl-detail", created_by=api_admin_user)
+
+    response = client.get(f"/api/v1/crawls/crawl/{crawl.id}", **api_headers)
+
+    assert response.status_code == 200, response.content
+
+
+def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone(
+    tmp_path,
+    client,
+    recursive_test_site,
+):
+    init_archive(tmp_path)
+    api_token = create_admin_and_token(tmp_path)
+
+    with use_archivebox_db(tmp_path):
+        crawl_response = api_client_request(
+            client,
+            "post",
+            "/api/v1/crawls/crawls",
+            api_token=api_token,
+            payload={
+                "urls": [recursive_test_site["root_url"]],
+                "max_depth": 0,
+                "tags": ["crawl-archiveresult-pause"],
+                "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
+            },
+        )
+        assert crawl_response.status_code == 200, crawl_response.content.decode()
+        crawl_id = json.loads(crawl_response.content.decode())["id"]
+        from archivebox.services.runner import run_due_snapshot
+
+        active_response = api_client_request(
+            client,
+            "post",
+            "/api/v1/core/snapshots",
+            api_token=api_token,
+            payload={
+                "url": recursive_test_site["root_url"],
+                "crawl_id": crawl_id,
+                "depth": 0,
+                "title": "Active child",
+                "status": "queued",
+            },
+        )
+        assert active_response.status_code == 200, active_response.content.decode()
+        active_snapshot = Snapshot.objects.get(id=json.loads(active_response.content.decode())["id"])
+
+        sealed_response = api_client_request(
+            client,
+            "post",
+            "/api/v1/core/snapshots",
+            api_token=api_token,
+            payload={
+                "url": recursive_test_site["child_urls"][0],
+                "crawl_id": crawl_id,
+                "depth": 0,
+                "title": "Already sealed child",
+                "status": "queued",
+            },
+        )
+        assert sealed_response.status_code == 200, sealed_response.content.decode()
+        sealed_snapshot_id = json.loads(sealed_response.content.decode())["id"]
+        sealed_snapshot = Snapshot.objects.get(id=sealed_snapshot_id)
+        sealed_done = _seed_archiveresult(
+            sealed_snapshot,
+            plugin="sealedone",
+            hook_name="on_Snapshot__sealed_done",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+            output_text="sealed snapshot result remains finished",
+            output_path="sealedone/final.txt",
+        )
+        sealed_snapshot.sm.seal()
+        sealed_snapshot.refresh_from_db()
+        assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
+        assert sealed_snapshot.retry_at is None
+
+        active_queued = _seed_archiveresult(
+            active_snapshot,
+            plugin="manualqueue",
+            hook_name="on_Snapshot__manual_queue",
+            status=ArchiveResult.StatusChoices.QUEUED,
+        )
+        active_started = _seed_archiveresult(
+            active_snapshot,
+            plugin="manualstart",
+            hook_name="on_Snapshot__manual_start",
+            status=ArchiveResult.StatusChoices.STARTED,
+        )
+        active_done = _seed_archiveresult(
+            active_snapshot,
+            plugin="manualdone",
+            hook_name="on_Snapshot__manual_done",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+            output_text="parent cascade should not rewrite finished rows",
+            output_path="manualdone/cascade.txt",
+        )
+        pause_response = api_client_request(
+            client,
+            "patch",
+            f"/api/v1/crawls/crawl/{crawl_id}",
+            api_token=api_token,
+            payload={"action": "pause"},
+        )
+        assert pause_response.status_code == 200, pause_response.content.decode()
+        assert json.loads(pause_response.content.decode())["status"] == Crawl.StatusChoices.PAUSED
+
+        active_snapshot.refresh_from_db()
+        sealed_snapshot.refresh_from_db()
+        crawl = Crawl.objects.get(id=crawl_id)
+        assert crawl.status == Crawl.StatusChoices.PAUSED
+        assert crawl.retry_at == RETRY_AT_MAX
+        assert active_snapshot.status == Snapshot.StatusChoices.QUEUED
+        assert active_snapshot.retry_at is not None
+        assert active_snapshot.retry_at <= timezone.now()
+        assert ArchiveResult.objects.get(id=active_queued.id).status == ArchiveResult.StatusChoices.QUEUED
+        assert ArchiveResult.objects.get(id=active_started.id).status == ArchiveResult.StatusChoices.STARTED
+
+        assert run_due_snapshot(active_snapshot, lock_seconds=60) is True
+        active_snapshot.refresh_from_db()
+        sealed_snapshot.refresh_from_db()
+        assert active_snapshot.status == Snapshot.StatusChoices.PAUSED
+        assert active_snapshot.retry_at == RETRY_AT_MAX
+        assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
+        assert sealed_snapshot.retry_at is None
+
+        paused_rows = {
+            row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id])
+        }
+        assert paused_rows == {
+            "manualqueue": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
+            "manualstart": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
+        }
+
+        active_done_row = ArchiveResult.objects.get(id=active_done.id)
+        sealed_done_row = ArchiveResult.objects.get(id=sealed_done.id)
+        active_done_path = Path(active_snapshot.output_dir) / next(iter(active_done_row.output_files))
+        sealed_done_path = Path(sealed_snapshot.output_dir) / next(iter(sealed_done_row.output_files))
+        assert active_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED
+        assert active_done_row.retry_at is None
+        assert active_done_path.read_text() == "parent cascade should not rewrite finished rows"
+        assert sealed_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED
+        assert sealed_done_row.retry_at is None
+        assert sealed_done_path.read_text() == "sealed snapshot result remains finished"
+
+        resume_response = api_client_request(
+            client,
+            "patch",
+            f"/api/v1/crawls/crawl/{crawl_id}",
+            api_token=api_token,
+            payload={"action": "resume"},
+        )
+        assert resume_response.status_code == 200, resume_response.content.decode()
+        assert json.loads(resume_response.content.decode())["status"] == Crawl.StatusChoices.QUEUED
+
+        active_snapshot.refresh_from_db()
+        sealed_snapshot.refresh_from_db()
+        crawl.refresh_from_db()
+        assert crawl.status == Crawl.StatusChoices.QUEUED
+        assert crawl.retry_at is not None
+        assert crawl.retry_at != RETRY_AT_MAX
+        assert active_snapshot.status == Snapshot.StatusChoices.QUEUED
+        assert active_snapshot.retry_at is not None
+        assert active_snapshot.retry_at != RETRY_AT_MAX
+        assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
+        assert sealed_snapshot.retry_at is None
+
+        resumed_rows = {
+            row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id])
+        }
+        assert resumed_rows["manualqueue"][0] == ArchiveResult.StatusChoices.QUEUED
+        assert resumed_rows["manualqueue"][1] is not None
+        assert resumed_rows["manualqueue"][1] != RETRY_AT_MAX
+        assert resumed_rows["manualstart"][0] == ArchiveResult.StatusChoices.QUEUED
+        assert resumed_rows["manualstart"][1] is not None
+        assert resumed_rows["manualstart"][1] != RETRY_AT_MAX
+        assert ArchiveResult.objects.get(id=active_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
+        assert ArchiveResult.objects.get(id=sealed_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
+        assert active_done_path.read_text() == "parent cascade should not rewrite finished rows"
+        assert sealed_done_path.read_text() == "sealed snapshot result remains finished"
+
+
+@pytest.mark.timeout(240)
+def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resume(tmp_path, recursive_test_site):
+    init_archive(tmp_path)
+
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True")
+    api_token = create_admin_and_token(tmp_path)
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
+
+        crawl_response = live_api_request(
+            port,
+            "post",
+            "/api/v1/crawls/crawls",
+            api_token=api_token,
+            json={
+                "urls": [recursive_test_site["root_url"]],
+                "max_depth": 0,
+                "tags": ["pause-resume-e2e"],
+                "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
+            },
+            timeout=10,
+        )
+        assert crawl_response.status_code == 200, crawl_response.text
+        crawl_id = crawl_response.json()["id"]
+        wait_for_crawl_snapshot_rows(tmp_path, crawl_id)
+
+        pause_response = live_api_request(
+            port,
+            "patch",
+            f"/api/v1/crawls/crawl/{crawl_id}",
+            api_token=api_token,
+            json={"action": "pause"},
+            timeout=10,
+        )
+        assert pause_response.status_code == 200, pause_response.text
+        assert pause_response.json()["status"] == "paused"
+
+        paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id)
+        assert paused_state["crawl_status"] == "paused"
+        assert paused_state["crawl_retry_at"] == paused_state["retry_at_max"]
+        assert len(paused_state["snapshots"]) == 1
+        snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed"
+        if snapshot_finished_before_pause:
+            assert any(result["status"] == "succeeded" for result in paused_state["results"])
+        else:
+            assert paused_state["snapshots"][0]["status"] == "paused"
+            assert paused_state["snapshots"][0]["retry_at"] == paused_state["retry_at_max"]
+
+        stop_server(tmp_path)
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
+
+        restarted_state = get_crawl_runtime_state(tmp_path, crawl_id)
+        assert restarted_state["crawl_status"] == "paused"
+        assert restarted_state["crawl_retry_at"] == restarted_state["retry_at_max"]
+        if snapshot_finished_before_pause:
+            assert restarted_state["snapshots"][0]["status"] == "sealed"
+            assert any(result["status"] == "succeeded" for result in restarted_state["results"])
+            return
+        assert restarted_state["snapshots"][0]["status"] == "paused"
+        assert restarted_state["snapshots"][0]["retry_at"] == restarted_state["retry_at_max"]
+        assert not any(result["status"] == "succeeded" for result in restarted_state["results"])
+
+        resume_response = live_api_request(
+            port,
+            "patch",
+            f"/api/v1/crawls/crawl/{crawl_id}",
+            api_token=api_token,
+            json={"action": "resume"},
+            timeout=10,
+        )
+        assert resume_response.status_code == 200, resume_response.text
+        assert resume_response.json()["status"] == "queued"
+
+        captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180)
+        assert "Root" in captured_text
+        assert "About" in captured_text
+
+        final_state = get_crawl_runtime_state(tmp_path, crawl_id)
+        assert final_state["snapshots"][0]["status"] == "sealed"
+        wget_results = [result for result in final_state["results"] if result["plugin"] == "wget"]
+        assert wget_results
+        assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results)
+    finally:
+        stop_server(tmp_path)
+
+
+@pytest.mark.timeout(180)
+def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(tmp_path, recursive_test_site):
+    init_archive(tmp_path)
+
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True")
+    api_token = create_admin_and_token(tmp_path)
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
+
+        crawl_response = live_api_request(
+            port,
+            "post",
+            "/api/v1/crawls/crawls",
+            api_token=api_token,
+            json={
+                "urls": [recursive_test_site["root_url"]],
+                "max_depth": 0,
+                "tags": ["paused-index-e2e"],
+                "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
+            },
+            timeout=10,
+        )
+        assert crawl_response.status_code == 200, crawl_response.text
+        crawl_id = crawl_response.json()["id"]
+        wait_for_crawl_snapshot_rows(tmp_path, crawl_id)
+
+        pause_response = live_api_request(
+            port,
+            "patch",
+            f"/api/v1/crawls/crawl/{crawl_id}",
+            api_token=api_token,
+            json={"action": "pause"},
+            timeout=10,
+        )
+        assert pause_response.status_code == 200, pause_response.text
+        assert pause_response.json()["status"] == "paused"
+        paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id)
+        snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed"
+    finally:
+        stop_server(tmp_path)
+
+    if snapshot_finished_before_pause:
+        indexed_state = get_crawl_runtime_state(tmp_path, crawl_id)
+        assert indexed_state["crawl_status"] == "paused"
+        assert indexed_state["snapshots"][0]["status"] == "sealed"
+        return
+
+    update_env = cli_env(
+        port=port,
+        PLUGINS="search_backend_sqlite",
+        SEARCH_BACKEND_ENGINE="sqlite",
+    )
+    update_process = run_archivebox_cmd(
+        [
+            "update",
+            "--index-only",
+            "--crawl-id",
+            crawl_id,
+            "--limit",
+            "1",
+            "--batch-size",
+            "1",
+        ],
+        cwd=tmp_path,
+        env=update_env,
+        timeout=120,
+    )
+    assert update_process.returncode == 0, update_process.stderr
+
+    indexed_state = get_crawl_runtime_state(tmp_path, crawl_id)
+    assert indexed_state["crawl_status"] == "paused"
+    assert indexed_state["crawl_retry_at"] == indexed_state["retry_at_max"]
+    assert indexed_state["snapshots"][0]["status"] == "paused"
+    assert indexed_state["snapshots"][0]["retry_at"] == indexed_state["retry_at_max"]
+    search_results = [result for result in indexed_state["results"] if result["plugin"] == "search_backend_sqlite"]
+    assert search_results
+    assert all(result["status"] not in {"queued", "started", "paused"} for result in search_results)
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        wait_for_live_api(port)
+
+        still_paused_state = get_crawl_runtime_state(tmp_path, crawl_id)
+        assert still_paused_state["crawl_status"] == "paused"
+        assert still_paused_state["snapshots"][0]["status"] == "paused"
+        assert not any(result["plugin"] == "wget" and result["status"] == "succeeded" for result in still_paused_state["results"])
+
+        resume_response = live_api_request(
+            port,
+            "patch",
+            f"/api/v1/crawls/crawl/{crawl_id}",
+            api_token=api_token,
+            json={"action": "resume"},
+            timeout=10,
+        )
+        assert resume_response.status_code == 200, resume_response.text
+        assert resume_response.json()["status"] == "queued"
+
+        captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180)
+        assert "Root" in captured_text
+        assert "About" in captured_text
+
+        resumed_state = get_crawl_runtime_state(tmp_path, crawl_id)
+        assert resumed_state["snapshots"][0]["status"] == "sealed"
+        wget_results = [result for result in resumed_state["results"] if result["plugin"] == "wget"]
+        assert wget_results
+        assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results)
+    finally:
+        stop_server(tmp_path)
+
+
+def test_crawl_cancel_api_defers_cleanup_to_runner(client, api_admin_user, api_headers):
+    from archivebox.services.runner import run_due_crawl
+
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        created_by=api_admin_user,
+        status=Crawl.StatusChoices.STARTED,
+        retry_at=timezone.now() + timedelta(minutes=5),
+    )
+    child = Snapshot.objects.create(
+        url="https://example.com/cancel-child",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.STARTED,
+        retry_at=timezone.now() + timedelta(minutes=5),
+    )
+    crawl.output_dir.mkdir(parents=True, exist_ok=True)
+    pid_file = crawl.output_dir / "cleanup-test.pid"
+    pid_file.write_text("12345")
+
+    response = api_client_request(
+        client,
+        "patch",
+        f"/api/v1/crawls/crawl/{crawl.id}",
+        payload={"action": "cancel"},
+        headers=api_headers,
+    )
+    assert response.status_code == 200, response.content
+
+    crawl.refresh_from_db()
+    child.refresh_from_db()
+    assert crawl.status == Crawl.StatusChoices.SEALED
+    assert crawl.retry_at is not None
+    assert crawl.retry_at <= timezone.now()
+    assert child.status == Snapshot.StatusChoices.STARTED
+    assert child.retry_at is not None
+    assert child.retry_at <= timezone.now()
+    assert pid_file.exists()
+
+    assert run_due_crawl(crawl, lock_seconds=60) is True
+    crawl.refresh_from_db()
+    assert crawl.retry_at is None
+    assert not pid_file.exists()
+
+
+def test_rest_crawl_delete_removes_crawl_and_snapshot_output_dirs(client, api_admin_user, api_headers):
+    url = "https://example.com/delete-path-crawl"
+
+    crawl = Crawl.objects.create(
+        urls=url,
+        max_depth=0,
+        created_by=api_admin_user,
+        status=Crawl.StatusChoices.SEALED,
+    )
+    snapshot = Snapshot.objects.create(
+        crawl=crawl,
+        url=url,
+        depth=0,
+        status=Snapshot.StatusChoices.SEALED,
+    )
+    crawl_dir = Path(crawl.output_dir)
+    snapshot_dir = Path(snapshot.output_dir)
+    crawl_dir.mkdir(parents=True, exist_ok=True)
+    snapshot_dir.mkdir(parents=True, exist_ok=True)
+    (crawl_dir / "delete-path-crawl.txt").write_text("crawl output")
+    (snapshot_dir / "delete-path-snapshot.txt").write_text("snapshot output")
+    assert crawl_dir.exists()
+    assert snapshot_dir.exists()
+
+    response = client.delete(f"/api/v1/crawls/crawl/{crawl.id}", **api_headers)
+    assert response.status_code == 200, response.content.decode()
+    assert not Crawl.objects.filter(pk=crawl.pk).exists()
+    assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
+    assert not crawl_dir.exists()
+    assert not snapshot_dir.exists()
+
+
+def test_crawl_as_rss_redirects_to_canonical_snapshots_feed(client, api_token, api_admin_user, other_user):
+    crawl, _snapshot = make_snapshot(
+        user=api_admin_user,
+        url="https://example.com/rss-crawl-feed",
+        title="Crawl Feed Snapshot",
+        bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 8, 0, 0)),
+    )
+    make_snapshot(
+        user=other_user,
+        url="https://example.com/rss-crawl-other",
+        title="Other Crawl Snapshot",
+        bookmarked_at=timezone.make_aware(datetime(2026, 5, 23, 9, 0, 0)),
+    )
+
+    response = client.get(
+        f"/api/v1/crawls/crawl/{crawl.id}",
+        {"as_rss": "true", "limit": 50, "api_key": api_token.token},
+        HTTP_HOST=ADMIN_HOST,
+        follow=True,
+    )
+
+    assert response.status_code == 200
+    assert response.redirect_chain
+    redirect_url = response.redirect_chain[0][0]
+    assert redirect_url.startswith("/api/v1/core/snapshots.rss?")
+    assert f"crawl_id={crawl.id}" in redirect_url
+    assert "as_rss" not in redirect_url
+    assert response["Content-Type"].startswith("application/rss+xml")
+    body = response.content.decode()
+    assert "rss-crawl-feed" in body
+    assert "rss-crawl-other" not in body

archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py

@@ -0,0 +1,16 @@
+import pytest
+
+from archivebox.crawls.models import Crawl
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/crawl-file-root", created_by=api_admin_user)
+    crawl.output_dir.mkdir(parents=True, exist_ok=True)
+    (crawl.output_dir / "basic.txt").write_text("ok")
+
+    response = client.get(f"/api/v1/crawls/crawl/{crawl.id}/files/basic.txt", **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py

@@ -0,0 +1,17 @@
+import pytest
+
+from archivebox.crawls.models import Crawl
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/crawl-file-nested", created_by=api_admin_user)
+    nested_dir = crawl.output_dir / "folder"
+    nested_dir.mkdir(parents=True, exist_ok=True)
+    (nested_dir / "basic.txt").write_text("ok")
+
+    response = client.get(f"/api/v1/crawls/crawl/{crawl.id}/files/folder/basic.txt", **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py

@@ -0,0 +1,17 @@
+import pytest
+
+from archivebox.crawls.models import Crawl
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com/crawl-file-deep", created_by=api_admin_user)
+    nested_dir = crawl.output_dir / "folder" / "subfolder"
+    nested_dir.mkdir(parents=True, exist_ok=True)
+    (nested_dir / "basic.txt").write_text("ok")
+
+    response = client.get(f"/api/v1/crawls/crawl/{crawl.id}/files/folder/subfolder/basic.txt", **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_crawls_crawls.py

@@ -0,0 +1,38 @@
+from django.test import RequestFactory
+
+import pytest
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_create_crawl_api_queues_crawl_without_spawning_runner():
+    from django.contrib.auth import get_user_model
+
+    from archivebox.api.v1_crawls import CrawlCreateSchema, create_crawl
+
+    user = get_user_model().objects.create_superuser(
+        username="runner-api-admin",
+        email="runner-api-admin@example.com",
+        password="testpassword",
+    )
+    request = RequestFactory().post("/api/v1/crawls")
+    request.user = user
+
+    crawl = create_crawl(
+        request,
+        CrawlCreateSchema(
+            urls=["https://example.com"],
+            max_depth=0,
+            tags=[],
+            tags_str="",
+            label="",
+            notes="",
+            config={},
+        ),
+    )
+
+    assert str(crawl.id)
+    assert crawl.status == "queued"
+    assert crawl.retry_at is not None
+    assert crawl.snapshot_set.count() == 0

archivebox/tests/test_api_v1_machine_binaries.py

@@ -0,0 +1,22 @@
+import pytest
+
+from archivebox.machine.models import Binary, Machine
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_headers):
+    machine = Machine.current(refresh=True)
+    Binary.objects.create(
+        machine=machine,
+        name="api-basic-bin",
+        binprovider="env",
+        abspath="/usr/bin/env",
+        version="1.0",
+        status=Binary.StatusChoices.INSTALLED,
+    )
+
+    response = client.get("/api/v1/machine/binaries", **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_machine_binary_binary_id.py

@@ -0,0 +1,22 @@
+import pytest
+
+from archivebox.machine.models import Binary, Machine
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_headers):
+    machine = Machine.current(refresh=True)
+    binary = Binary.objects.create(
+        machine=machine,
+        name="api-basic-bin",
+        binprovider="env",
+        abspath="/usr/bin/env",
+        version="1.0",
+        status=Binary.StatusChoices.INSTALLED,
+    )
+
+    response = client.get(f"/api/v1/machine/binary/{binary.id}", **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_machine_binary_by_name_name.py

@@ -0,0 +1,22 @@
+import pytest
+
+from archivebox.machine.models import Binary, Machine
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_headers):
+    machine = Machine.current(refresh=True)
+    Binary.objects.create(
+        machine=machine,
+        name="api-basic-bin",
+        binprovider="env",
+        abspath="/usr/bin/env",
+        version="1.0",
+        status=Binary.StatusChoices.INSTALLED,
+    )
+
+    response = client.get("/api/v1/machine/binary/by-name/api-basic-bin", **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_machine_machine_current.py

@@ -0,0 +1,10 @@
+import pytest
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_headers):
+    response = client.get("/api/v1/machine/machine/current", **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_machine_machine_machine_id.py

@@ -0,0 +1,14 @@
+import pytest
+
+from archivebox.machine.models import Machine
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_headers):
+    machine = Machine.current(refresh=True)
+
+    response = client.get(f"/api/v1/machine/machine/{machine.id}", **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_machine_machines.py

@@ -0,0 +1,14 @@
+import pytest
+
+from archivebox.machine.models import Machine
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, tmp_path, api_headers):
+    Machine.current(refresh=True)
+
+    response = client.get("/api/v1/machine/machines", **api_headers)
+
+    assert response.status_code == 200, response.content

archivebox/tests/test_api_v1_personas_personas.py

@@ -0,0 +1,57 @@
+import pytest
+
+from archivebox.personas.models import Persona
+from archivebox.tests.conftest import api_client_request
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_personas_api_returns_paginated_envelope(client, api_headers):
+    for name in ("api-persona-alpha", "api-persona-bravo", "api-persona-charlie"):
+        create_response = api_client_request(
+            client,
+            "post",
+            "/api/v1/personas/sync",
+            payload={
+                "extension_persona_id": f"extension-{name}",
+                "name": name,
+                "settings": {},
+                "cookies_txt": "",
+                "auth_json": {},
+            },
+            headers=api_headers,
+        )
+        assert create_response.status_code == 200, create_response.content
+        assert create_response.json()["created"] is True
+
+    total_personas = Persona.objects.count()
+    response = client.get(
+        "/api/v1/personas/personas?limit=1",
+        **api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+    payload = response.json()
+    assert isinstance(payload, dict)
+    assert set(payload) >= {"items", "count", "total_items", "total_pages", "page", "limit", "offset", "num_items"}
+    assert payload["count"] == total_personas
+    assert payload["total_items"] == total_personas
+    assert payload["limit"] == 1
+    assert payload["offset"] == 0
+    assert payload["num_items"] == 1
+    assert len(payload["items"]) == 1
+
+    limit_two_response = client.get(
+        "/api/v1/personas/personas?limit=2",
+        **api_headers,
+    )
+    assert limit_two_response.status_code == 200, limit_two_response.content
+    limit_two_payload = limit_two_response.json()
+    assert isinstance(limit_two_payload, dict)
+    assert limit_two_payload["count"] == total_personas
+    assert limit_two_payload["total_items"] == total_personas
+    assert limit_two_payload["limit"] == 2
+    assert limit_two_payload["offset"] == 0
+    assert limit_two_payload["num_items"] == 2
+    assert len(limit_two_payload["items"]) == 2

archivebox/tests/test_api_v1_personas_sync.py

@@ -0,0 +1,25 @@
+import pytest
+
+from archivebox.tests.conftest import api_client_request
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+def test_basic_success_case_request(client, api_headers):
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/personas/sync",
+        payload={
+            "extension_persona_id": "extension-api-persona-basic",
+            "name": "api-persona-basic",
+            "settings": {},
+            "cookies_txt": "",
+            "auth_json": {},
+        },
+        headers=api_headers,
+    )
+
+    assert response.status_code == 200, response.content
+    assert response.json()["success"] is True

archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py

@@ -0,0 +1,207 @@
+import pytest
+import requests
+
+from archivebox.core.models import Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.tests.conftest import (
+    cli_env,
+    create_admin_and_token,
+    get_free_port,
+    init_archive,
+    live_api_request,
+    start_archivebox_server,
+    stop_server,
+    wait_for_live_api,
+)
+from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+@pytest.mark.timeout(180)
+def test_core_api_workflow_uses_token_auth_and_persists_side_effects_over_server(tmp_path, recursive_test_site):
+    init_archive(tmp_path)
+
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PUBLIC_INDEX="True")
+    api_token = create_admin_and_token(tmp_path)
+
+    try:
+        start_archivebox_server(tmp_path, env=env, port=port)
+        docs = wait_for_live_api(port)
+        assert docs.status_code == 200
+        openapi = wait_for_live_api(port, path="/api/v1/openapi.json")
+        assert openapi.status_code == 200
+        paths = openapi.json()["paths"]
+        assert "/api/v1/core/snapshots" in paths
+        assert "/api/v1/crawls/crawls" in paths
+
+        unauth = requests.get(
+            f"http://127.0.0.1:{port}/api/v1/crawls/crawls",
+            headers={"Host": f"api.archivebox.localhost:{port}"},
+            timeout=10,
+        )
+        assert unauth.status_code in (401, 403)
+        bad_auth = requests.get(
+            f"http://127.0.0.1:{port}/api/v1/crawls/crawls",
+            headers={"Host": f"api.archivebox.localhost:{port}", "X-ArchiveBox-API-Key": "bad-token"},
+            timeout=10,
+        )
+        assert bad_auth.status_code in (401, 403)
+
+        crawl_response = live_api_request(
+            port,
+            "post",
+            "/api/v1/crawls/crawls",
+            api_token=api_token,
+            json={
+                "urls": [recursive_test_site["root_url"]],
+                "max_depth": 2,
+                "tags": ["api-depth-two"],
+                "label": "api crawl",
+                "notes": "created through REST API",
+                "config": {
+                    "PLUGINS": "wget,parse_html_urls",
+                    "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
+                    "CRAWL_MAX_URLS": 7,
+                    "CRAWL_MAX_SIZE": "0",
+                    "SNAPSHOT_MAX_SIZE": "0",
+                },
+            },
+            timeout=10,
+        )
+        assert crawl_response.status_code == 200, crawl_response.text
+        crawl_payload = crawl_response.json()
+        crawl_id = crawl_payload["id"]
+        assert crawl_payload["max_depth"] == 2
+        assert crawl_payload["tags_str"] == "api-depth-two"
+        assert crawl_payload["config"]["PLUGINS"] == "wget,parse_html_urls"
+        assert crawl_payload["config"]["CRAWL_MAX_URLS"] == 7
+
+        snapshot_response = live_api_request(
+            port,
+            "post",
+            "/api/v1/core/snapshots",
+            api_token=api_token,
+            json={
+                "url": recursive_test_site["child_urls"][0],
+                "crawl_id": crawl_id,
+                "depth": 1,
+                "title": "API child snapshot",
+                "tags": ["api-child"],
+                "status": "queued",
+            },
+            timeout=10,
+        )
+        assert snapshot_response.status_code == 200, snapshot_response.text
+        snapshot_payload = snapshot_response.json()
+        snapshot_id = snapshot_payload["id"]
+        assert snapshot_payload["url"] == recursive_test_site["child_urls"][0]
+        assert snapshot_payload["tags"] == ["api-child"]
+
+        patch_snapshot = live_api_request(
+            port,
+            "patch",
+            f"/api/v1/core/snapshot/{snapshot_id}",
+            api_token=api_token,
+            json={"status": "sealed", "tags": ["api-child", "api-patched"]},
+            timeout=10,
+        )
+        assert patch_snapshot.status_code == 200, patch_snapshot.text
+        assert patch_snapshot.json()["status"] == "sealed"
+        assert set(patch_snapshot.json()["tags"]) == {"api-child", "api-patched"}
+
+        tag_create = live_api_request(
+            port,
+            "post",
+            "/api/v1/core/tags/create/",
+            api_token=api_token,
+            json={"name": "api-extra"},
+            timeout=10,
+        )
+        assert tag_create.status_code == 200, tag_create.text
+        tag_id = tag_create.json()["tag_id"]
+
+        add_tag = live_api_request(
+            port,
+            "post",
+            "/api/v1/core/tags/add-to-snapshot/",
+            api_token=api_token,
+            json={"snapshot_id": snapshot_id, "tag_id": tag_id},
+            timeout=10,
+        )
+        assert add_tag.status_code == 200, add_tag.text
+        remove_tag = live_api_request(
+            port,
+            "post",
+            "/api/v1/core/tags/remove-from-snapshot/",
+            api_token=api_token,
+            json={"snapshot_id": snapshot_id, "tag_name": "api-extra"},
+            timeout=10,
+        )
+        assert remove_tag.status_code == 200, remove_tag.text
+
+        crawl_patch = live_api_request(
+            port,
+            "patch",
+            f"/api/v1/crawls/crawl/{crawl_id}",
+            api_token=api_token,
+            json={"status": "sealed", "tags": ["api-sealed"]},
+            timeout=10,
+        )
+        assert crawl_patch.status_code == 200, crawl_patch.text
+        assert crawl_patch.json()["status"] == "sealed"
+        assert crawl_patch.json()["tags_str"] == "api-sealed"
+
+        snapshots_list = live_api_request(
+            port,
+            "get",
+            "/api/v1/core/snapshots?tag=api-patched&with_archiveresults=true",
+            api_token=api_token,
+            timeout=10,
+        )
+        assert snapshots_list.status_code == 200, snapshots_list.text
+        snapshot_items = snapshots_list.json()["items"]
+        assert len(snapshot_items) == 1
+        assert snapshot_items[0]["id"] == snapshot_id
+        assert snapshot_items[0]["archiveresults"] == []
+
+        bearer_response = requests.get(
+            f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}",
+            headers={"Host": f"api.archivebox.localhost:{port}", "Authorization": f"Bearer {api_token}"},
+            timeout=10,
+        )
+        assert bearer_response.status_code == 200, bearer_response.text
+        query_response = requests.get(
+            f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}?api_key={api_token}",
+            headers={"Host": f"api.archivebox.localhost:{port}"},
+            timeout=10,
+        )
+        assert query_response.status_code == 200, query_response.text
+
+        delete_snapshot = live_api_request(
+            port,
+            "delete",
+            f"/api/v1/core/snapshot/{snapshot_id}",
+            api_token=api_token,
+            timeout=10,
+        )
+        assert delete_snapshot.status_code == 200, delete_snapshot.text
+        assert delete_snapshot.json()["success"] is True
+
+        delete_crawl = live_api_request(
+            port,
+            "delete",
+            f"/api/v1/crawls/crawl/{crawl_id}",
+            api_token=api_token,
+            timeout=10,
+        )
+        assert delete_crawl.status_code == 200, delete_crawl.text
+        assert delete_crawl.json()["success"] is True
+
+        with use_archivebox_db(tmp_path):
+            assert Crawl.objects.filter(pk=crawl_id).count() == 0
+            assert Snapshot.objects.filter(pk=snapshot_id).count() == 0
+    finally:
+        stop_server(tmp_path)

archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py

@@ -0,0 +1,66 @@
+from django.test import RequestFactory
+
+import pytest
+
+
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
+SENSITIVE_SECRET = "raw-twocaptcha-secret-for-frozen-crawl-test"
+
+
+@pytest.fixture
+def archivebox_db(initialized_archive):
+    from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+    with use_archivebox_db(initialized_archive):
+        yield initialized_archive
+
+
+def _user(username="frozen-config-admin"):
+    from django.contrib.auth import get_user_model
+
+    return get_user_model().objects.create_superuser(
+        username=username,
+        email=f"{username}@example.com",
+        password="testpassword",
+    )
+
+
+def test_api_create_and_cli_add_store_full_frozen_config(archivebox_db):
+    from archivebox.api.v1_crawls import CrawlCreateSchema, CrawlSchema, create_crawl
+    from archivebox.cli.archivebox_add import add
+    from archivebox.config.common import SENSITIVE_CONFIG_VALUE_REDACTED
+
+    user = _user("frozen-config-api-admin")
+    request = RequestFactory().post("/api/v1/crawls")
+    request.user = user
+
+    api_crawl = create_crawl(
+        request,
+        CrawlCreateSchema(
+            urls=["https://example.com/api"],
+            max_depth=0,
+            tags=[],
+            tags_str="",
+            label="API frozen config",
+            notes="",
+            config={"TWOCAPTCHA_API_KEY": SENSITIVE_SECRET, "TIMEOUT": 33, "SECRET_KEY": "must-not-freeze", "PUBLIC_ADD_VIEW": True},
+        ),
+    )
+    assert "CHECK_SSL_VALIDITY" in api_crawl.config
+    assert api_crawl.config["TIMEOUT"] == 33
+    assert api_crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET
+    assert "SECRET_KEY" not in api_crawl.config
+    assert "PUBLIC_ADD_VIEW" not in api_crawl.config
+    assert CrawlSchema.resolve_config(api_crawl)["TWOCAPTCHA_API_KEY"] == SENSITIVE_CONFIG_VALUE_REDACTED
+
+    cli_crawl, _snapshots = add(
+        "https://example.com/cli",
+        bg=True,
+        created_by_id=user.pk,
+        config={"TWOCAPTCHA_API_KEY": SENSITIVE_SECRET, "TIMEOUT": 44},
+    )
+    assert "CHECK_SSL_VALIDITY" in cli_crawl.config
+    assert cli_crawl.config["TIMEOUT"] == 44
+    assert cli_crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET

archivebox/tests/test_archiveresult_pause.py

@@ -1,372 +1,2 @@
-import json
-import os
-import subprocess
-import sys
-from pathlib import Path
-
-import pytest
-from django.utils import timezone
-
-from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.crawls.models import Crawl
-from archivebox.tests.test_orm_helpers import use_archivebox_db
-from archivebox.workers.models import RETRY_AT_MAX
-
-from .conftest import build_test_env, create_admin_and_token, get_free_port, init_archive
-
-pytestmark = pytest.mark.django_db(transaction=True)
-
-API_HOST = "api.archivebox.localhost:8000"
-
-
-def _api_headers(token: str) -> dict[str, str]:
-    return {
-        "HTTP_HOST": API_HOST,
-        "HTTP_X_ARCHIVEBOX_API_KEY": token,
-    }
-
-
-def _json_response(response):
-    return json.loads(response.content.decode())
-
-
-def _post_json(client, path: str, token: str, payload: dict):
-    return client.post(
-        path,
-        data=json.dumps(payload),
-        content_type="application/json",
-        **_api_headers(token),
-    )
-
-
-def _patch_json(client, path: str, token: str, payload: dict):
-    return client.patch(
-        path,
-        data=json.dumps(payload),
-        content_type="application/json",
-        **_api_headers(token),
-    )
-
-
-def _seed_archiveresult(
-    snapshot: Snapshot,
-    *,
-    plugin: str,
-    hook_name: str,
-    status: str,
-    output_text: str = "",
-    output_path: str | None = None,
-) -> ArchiveResult:
-    output_files = {}
-    output_size = 0
-    output_mimetypes = ""
-    if output_path is not None:
-        output_bytes = output_text.encode()
-        absolute_path = Path(snapshot.output_dir) / output_path
-        absolute_path.parent.mkdir(parents=True, exist_ok=True)
-        absolute_path.write_bytes(output_bytes)
-        output_size = len(output_bytes)
-        output_mimetypes = "text/plain"
-        output_files[output_path] = {
-            "extension": Path(output_path).suffix.lstrip("."),
-            "mimetype": "text/plain",
-            "size": output_size,
-        }
-
-    now = timezone.now()
-    return ArchiveResult.objects.create(
-        snapshot=snapshot,
-        plugin=plugin,
-        hook_name=hook_name,
-        status=status,
-        output_str=output_path or output_text,
-        output_files=output_files,
-        output_size=output_size,
-        output_mimetypes=output_mimetypes,
-        start_ts=now if status != ArchiveResult.StatusChoices.QUEUED else None,
-        end_ts=now if status in ArchiveResult.FINAL_STATES else None,
-    )
-
-
-def _snapshot_hook_name(plugin_name: str) -> str:
-    from abx_dl.models import discover_plugins
-
-    plugin = discover_plugins().get(plugin_name)
-    assert plugin is not None, f"missing test plugin {plugin_name}"
-    hooks = plugin.filter_hooks("Snapshot")
-    assert hooks, f"missing Snapshot hooks for {plugin_name}"
-    return hooks[0].name
-
-
-def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone(
-    tmp_path,
-    client,
-    recursive_test_site,
-):
-    os.chdir(tmp_path)
-    init_archive(tmp_path)
-    api_token = create_admin_and_token(tmp_path)
-
-    with use_archivebox_db(tmp_path):
-        crawl_response = _post_json(
-            client,
-            "/api/v1/crawls/crawls",
-            api_token,
-            {
-                "urls": [recursive_test_site["root_url"]],
-                "max_depth": 0,
-                "tags": ["crawl-archiveresult-pause"],
-                "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
-            },
-        )
-        assert crawl_response.status_code == 200, crawl_response.content.decode()
-        crawl_id = _json_response(crawl_response)["id"]
-        from archivebox.services.runner import run_due_snapshot
-
-        active_response = _post_json(
-            client,
-            "/api/v1/core/snapshots",
-            api_token,
-            {
-                "url": recursive_test_site["root_url"],
-                "crawl_id": crawl_id,
-                "depth": 0,
-                "title": "Active child",
-                "status": "queued",
-            },
-        )
-        assert active_response.status_code == 200, active_response.content.decode()
-        active_snapshot = Snapshot.objects.get(id=_json_response(active_response)["id"])
-
-        sealed_response = _post_json(
-            client,
-            "/api/v1/core/snapshots",
-            api_token,
-            {
-                "url": recursive_test_site["child_urls"][0],
-                "crawl_id": crawl_id,
-                "depth": 0,
-                "title": "Already sealed child",
-                "status": "queued",
-            },
-        )
-        assert sealed_response.status_code == 200, sealed_response.content.decode()
-        sealed_snapshot_id = _json_response(sealed_response)["id"]
-        sealed_snapshot = Snapshot.objects.get(id=sealed_snapshot_id)
-        sealed_done = _seed_archiveresult(
-            sealed_snapshot,
-            plugin="sealedone",
-            hook_name="on_Snapshot__sealed_done",
-            status=ArchiveResult.StatusChoices.SUCCEEDED,
-            output_text="sealed snapshot result remains finished",
-            output_path="sealedone/final.txt",
-        )
-        sealed_snapshot.sm.seal()
-        sealed_snapshot.refresh_from_db()
-        assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
-        assert sealed_snapshot.retry_at is None
-
-        active_queued = _seed_archiveresult(
-            active_snapshot,
-            plugin="manualqueue",
-            hook_name="on_Snapshot__manual_queue",
-            status=ArchiveResult.StatusChoices.QUEUED,
-        )
-        active_started = _seed_archiveresult(
-            active_snapshot,
-            plugin="manualstart",
-            hook_name="on_Snapshot__manual_start",
-            status=ArchiveResult.StatusChoices.STARTED,
-        )
-        active_done = _seed_archiveresult(
-            active_snapshot,
-            plugin="manualdone",
-            hook_name="on_Snapshot__manual_done",
-            status=ArchiveResult.StatusChoices.SUCCEEDED,
-            output_text="parent cascade should not rewrite finished rows",
-            output_path="manualdone/cascade.txt",
-        )
-        pause_response = _patch_json(
-            client,
-            f"/api/v1/crawls/crawl/{crawl_id}",
-            api_token,
-            {"action": "pause"},
-        )
-        assert pause_response.status_code == 200, pause_response.content.decode()
-        assert _json_response(pause_response)["status"] == Crawl.StatusChoices.PAUSED
-
-        active_snapshot.refresh_from_db()
-        sealed_snapshot.refresh_from_db()
-        crawl = Crawl.objects.get(id=crawl_id)
-        assert crawl.status == Crawl.StatusChoices.PAUSED
-        assert crawl.retry_at == RETRY_AT_MAX
-        assert active_snapshot.status == Snapshot.StatusChoices.QUEUED
-        assert active_snapshot.retry_at is not None
-        assert active_snapshot.retry_at <= timezone.now()
-        assert ArchiveResult.objects.get(id=active_queued.id).status == ArchiveResult.StatusChoices.QUEUED
-        assert ArchiveResult.objects.get(id=active_started.id).status == ArchiveResult.StatusChoices.STARTED
-
-        assert run_due_snapshot(active_snapshot, lock_seconds=60) is True
-        active_snapshot.refresh_from_db()
-        sealed_snapshot.refresh_from_db()
-        assert active_snapshot.status == Snapshot.StatusChoices.PAUSED
-        assert active_snapshot.retry_at == RETRY_AT_MAX
-        assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
-        assert sealed_snapshot.retry_at is None
-
-        paused_rows = {
-            row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id])
-        }
-        assert paused_rows == {
-            "manualqueue": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
-            "manualstart": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
-        }
-
-        active_done_row = ArchiveResult.objects.get(id=active_done.id)
-        sealed_done_row = ArchiveResult.objects.get(id=sealed_done.id)
-        active_done_path = Path(active_snapshot.output_dir) / next(iter(active_done_row.output_files))
-        sealed_done_path = Path(sealed_snapshot.output_dir) / next(iter(sealed_done_row.output_files))
-        assert active_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED
-        assert active_done_row.retry_at is None
-        assert active_done_path.read_text() == "parent cascade should not rewrite finished rows"
-        assert sealed_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED
-        assert sealed_done_row.retry_at is None
-        assert sealed_done_path.read_text() == "sealed snapshot result remains finished"
-
-        resume_response = _patch_json(
-            client,
-            f"/api/v1/crawls/crawl/{crawl_id}",
-            api_token,
-            {"action": "resume"},
-        )
-        assert resume_response.status_code == 200, resume_response.content.decode()
-        assert _json_response(resume_response)["status"] == Crawl.StatusChoices.QUEUED
-
-        active_snapshot.refresh_from_db()
-        sealed_snapshot.refresh_from_db()
-        crawl.refresh_from_db()
-        assert crawl.status == Crawl.StatusChoices.QUEUED
-        assert crawl.retry_at is not None
-        assert crawl.retry_at != RETRY_AT_MAX
-        assert active_snapshot.status == Snapshot.StatusChoices.QUEUED
-        assert active_snapshot.retry_at is not None
-        assert active_snapshot.retry_at != RETRY_AT_MAX
-        assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
-        assert sealed_snapshot.retry_at is None
-
-        resumed_rows = {
-            row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id])
-        }
-        assert resumed_rows["manualqueue"][0] == ArchiveResult.StatusChoices.QUEUED
-        assert resumed_rows["manualqueue"][1] is not None
-        assert resumed_rows["manualqueue"][1] != RETRY_AT_MAX
-        assert resumed_rows["manualstart"][0] == ArchiveResult.StatusChoices.QUEUED
-        assert resumed_rows["manualstart"][1] is not None
-        assert resumed_rows["manualstart"][1] != RETRY_AT_MAX
-        assert ArchiveResult.objects.get(id=active_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
-        assert ArchiveResult.objects.get(id=sealed_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
-        assert active_done_path.read_text() == "parent cascade should not rewrite finished rows"
-        assert sealed_done_path.read_text() == "sealed snapshot result remains finished"
-
-
-def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_paused(
-    tmp_path,
-    client,
-    recursive_test_site,
-):
-    os.chdir(tmp_path)
-    init_archive(tmp_path)
-    api_token = create_admin_and_token(tmp_path)
-
-    with use_archivebox_db(tmp_path):
-        snapshot_response = _post_json(
-            client,
-            "/api/v1/core/snapshots",
-            api_token,
-            {
-                "url": recursive_test_site["root_url"],
-                "depth": 0,
-                "title": "Paused targeted retry",
-                "tags": ["targeted-extract-pause"],
-                "status": "queued",
-            },
-        )
-        assert snapshot_response.status_code == 200, snapshot_response.content.decode()
-        snapshot_id = _json_response(snapshot_response)["id"]
-        snapshot = Snapshot.objects.get(id=snapshot_id)
-
-        wget_result = _seed_archiveresult(
-            snapshot,
-            plugin="wget",
-            hook_name=_snapshot_hook_name("wget"),
-            status=ArchiveResult.StatusChoices.FAILED,
-            output_text="initial failure before targeted retry",
-        )
-        unrelated_result = _seed_archiveresult(
-            snapshot,
-            plugin="manualqueue",
-            hook_name="on_Snapshot__manual_queue",
-            status=ArchiveResult.StatusChoices.QUEUED,
-        )
-        finished_result = _seed_archiveresult(
-            snapshot,
-            plugin="manualdone",
-            hook_name="on_Snapshot__manual_done",
-            status=ArchiveResult.StatusChoices.SUCCEEDED,
-            output_text="finished row must survive targeted retry",
-            output_path="manualdone/targeted.txt",
-        )
-
-        pause_response = _patch_json(
-            client,
-            f"/api/v1/core/snapshot/{snapshot_id}",
-            api_token,
-            {"action": "pause"},
-        )
-        assert pause_response.status_code == 200, pause_response.content.decode()
-        assert _json_response(pause_response)["status"] == Snapshot.StatusChoices.PAUSED
-
-        snapshot = Snapshot.objects.get(id=snapshot_id)
-        assert snapshot.status == Snapshot.StatusChoices.PAUSED
-        assert snapshot.retry_at == RETRY_AT_MAX
-        assert ArchiveResult.objects.get(id=wget_result.id).status == ArchiveResult.StatusChoices.FAILED
-        assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.PAUSED
-        finished_row = ArchiveResult.objects.get(id=finished_result.id)
-        finished_output_path = Path(snapshot.output_dir) / next(iter(finished_row.output_files))
-        assert finished_output_path.read_text() == "finished row must survive targeted retry"
-
-    env = build_test_env(
-        get_free_port(),
-        PLUGINS="wget",
-        SAVE_WGET="True",
-        WGET_WARC_ENABLED="False",
-        URL_ALLOWLIST=r"127\.0\.0\.1[:/].*",
-    )
-    extract = subprocess.run(
-        [sys.executable, "-m", "archivebox", "extract", str(wget_result.id)],
-        cwd=tmp_path,
-        capture_output=True,
-        text=True,
-        env=env,
-        timeout=150,
-    )
-    assert extract.returncode == 0, f"STDOUT:\n{extract.stdout}\nSTDERR:\n{extract.stderr}"
-
-    with use_archivebox_db(tmp_path):
-        snapshot = Snapshot.objects.get(id=snapshot_id)
-        assert snapshot.status == Snapshot.StatusChoices.PAUSED
-        assert snapshot.retry_at == RETRY_AT_MAX
-
-        retried_wget = ArchiveResult.objects.get(id=wget_result.id)
-        assert retried_wget.status == ArchiveResult.StatusChoices.SUCCEEDED
-        assert retried_wget.output_size > 0
-        assert retried_wget.output_files
-
-        unrelated = ArchiveResult.objects.get(id=unrelated_result.id)
-        assert unrelated.status == ArchiveResult.StatusChoices.PAUSED
-        assert unrelated.retry_at == RETRY_AT_MAX
-
-        finished = ArchiveResult.objects.get(id=finished_result.id)
-        assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED
-        assert finished.retry_at is None
-        assert finished_output_path.read_text() == "finished row must survive targeted retry"
+# test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone moved to test_api_v1_crawls_crawl_crawl_id.py.
+# test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_paused moved to test_api_v1_core_snapshot_snapshot_id.py.

archivebox/tests/test_auth_ldap.py

@@ -132,27 +132,33 @@ class TestArchiveBoxWithLDAP:
 
     def test_archivebox_init_without_ldap(self, tmp_path):
         """Test that archivebox init works without LDAP enabled."""
-        _, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["init"],
-            data_dir=tmp_path,
+            cwd=tmp_path,
             timeout=45,
             env={"LDAP_ENABLED": "False"},
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should succeed
         assert code == 0, f"archivebox init failed: {stderr}"
 
     def test_archivebox_version_with_ldap_config(self, tmp_path):
         """Test that archivebox version works with LDAP config set."""
-        _, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["version"],
-            data_dir=tmp_path,
+            cwd=tmp_path,
             timeout=10,
             env={
                 "LDAP_ENABLED": "False",
                 "LDAP_SERVER_URI": "ldap://ldap-test.localhost:389",
             },
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should succeed
         assert code == 0, f"archivebox version failed: {stderr}"
@@ -163,15 +169,18 @@ class TestLDAPConfigValidationInArchiveBox:
 
     def test_archivebox_init_with_incomplete_ldap_config(self, tmp_path):
         """Test that archivebox init fails with helpful error when LDAP config is incomplete."""
-        _, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["init"],
-            data_dir=tmp_path,
+            cwd=tmp_path,
             timeout=45,
             env={
                 "LDAP_ENABLED": "True",
                 # Missing: LDAP_SERVER_URI, LDAP_BIND_DN, etc.
             },
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should fail with validation error
         assert code != 0, "Should fail with incomplete LDAP config"

archivebox/tests/test_binary_service.py

@@ -24,10 +24,6 @@ def _link_real_binary(bin_dir: Path, name: str, *, source: str | None = None) ->
     return link
 
 
-def _binary_request(name: str, *, binproviders: str = "env") -> str:
-    return json.dumps({"type": "BinaryRequest", "name": name, "binproviders": binproviders}) + "\n"
-
-
 def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]:
     return {
         "LIB_DIR": str(data_dir / "lib"),
@@ -100,13 +96,16 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     _link_real_binary(bootstrap_bin_dir, "uv")
     _link_real_binary(provider_bin_dir, name, source="rg")
 
-    stdout, stderr, returncode = run_archivebox_cmd(
+    _cmd_result = run_archivebox_cmd(
         ["run"],
-        data_dir=initialized_archive,
-        stdin=_binary_request(name),
+        cwd=initialized_archive,
+        stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n",
         timeout=120,
         env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        default_cli_env=True,
+        disable_extractors=True,
     )
+    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert returncode == 0, stderr
     output_records = parse_jsonl_output(stdout)
@@ -135,12 +134,15 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert binary_processes[-1].exit_code == 0
     assert any(f"--name={name}" in arg for arg in binary_processes[-1].cmd)
 
-    version_stdout, version_stderr, version_code = run_archivebox_cmd(
+    _cmd_result = run_archivebox_cmd(
         ["version"],
-        data_dir=initialized_archive,
+        cwd=initialized_archive,
         timeout=60,
         env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        default_cli_env=True,
+        disable_extractors=True,
     )
+    version_stdout, version_stderr, version_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
     assert version_code == 0, version_stderr
     assert name in version_stdout
     assert binary.version in version_stdout
@@ -149,12 +151,15 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     (initialized_archive / "lib" / "bin" / name).unlink(missing_ok=True)
     _link_real_binary(bootstrap_bin_dir, name, source="rg")
 
-    rerun_stdout, rerun_stderr, rerun_code = run_archivebox_cmd(
+    _cmd_result = run_archivebox_cmd(
         ["run", f"--binary-id={first_binary_id}"],
-        data_dir=initialized_archive,
+        cwd=initialized_archive,
         timeout=120,
         env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        default_cli_env=True,
+        disable_extractors=True,
     )
+    rerun_stdout, rerun_stderr, rerun_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert rerun_code == 0, rerun_stdout + rerun_stderr
     with use_archivebox_db(initialized_archive):
@@ -174,13 +179,16 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
     provider_bin_dir = initialized_archive / "lib" / "env" / "bin"
     _link_real_binary(bootstrap_bin_dir, "uv")
 
-    stdout, stderr, returncode = run_archivebox_cmd(
+    _cmd_result = run_archivebox_cmd(
         ["run"],
-        data_dir=initialized_archive,
-        stdin=_binary_request(name),
+        cwd=initialized_archive,
+        stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n",
         timeout=120,
         env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        default_cli_env=True,
+        disable_extractors=True,
     )
+    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert returncode == 0, stderr
     assert any(record["type"] == "BinaryRequest" and record["name"] == name for record in parse_jsonl_output(stdout))
@@ -201,12 +209,15 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
 
     _link_real_binary(provider_bin_dir, name, source="rg")
 
-    recover_stdout, recover_stderr, recover_code = run_archivebox_cmd(
+    _cmd_result = run_archivebox_cmd(
         ["run", f"--binary-id={queued_id}"],
-        data_dir=initialized_archive,
+        cwd=initialized_archive,
         timeout=120,
         env=_runtime_env(initialized_archive, bootstrap_bin_dir),
+        default_cli_env=True,
+        disable_extractors=True,
     )
+    recover_stdout, recover_stderr, recover_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert recover_code == 0, recover_stdout + recover_stderr
     with use_archivebox_db(initialized_archive):

archivebox/tests/test_cli_add.py

@@ -5,64 +5,49 @@ Verify add creates snapshots in DB, crawls, source files, and archive directorie
 """
 
 import os
-import subprocess
-from pathlib import Path
+import json
 
 import pytest
 
-from archivebox.core.models import Snapshot
+from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.crawls.models import Crawl
-from archivebox.tests.conftest import run_queued_crawls
+from archivebox.machine.models import Process
+from archivebox.tests.conftest import _find_system_browser, find_snapshot_dir, run_archivebox_cmd, run_queued_crawls, cli_env
+
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def _find_snapshot_dir(data_dir: Path, snapshot_id: str) -> Path | None:
-    candidates = {snapshot_id}
-    if len(snapshot_id) == 32:
-        candidates.add(f"{snapshot_id[:8]}-{snapshot_id[8:12]}-{snapshot_id[12:16]}-{snapshot_id[16:20]}-{snapshot_id[20:]}")
-    elif len(snapshot_id) == 36 and "-" in snapshot_id:
-        candidates.add(snapshot_id.replace("-", ""))
-
-    for needle in candidates:
-        for path in data_dir.rglob(needle):
-            if path.is_dir():
-                return path
-    return None
-
-
-def test_add_single_url_creates_snapshot_in_db(tmp_path, process, disable_extractors_dict):
+def test_add_single_url_creates_snapshot_in_db(initialized_archive):
     """Test that adding a single URL queues a crawl whose runner creates the snapshot."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    result = run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
     )
 
     assert result.returncode == 0
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         snapshots = list(Snapshot.objects.values_list("url", flat=True))
 
     assert len(snapshots) == 1
     assert snapshots[0] == "https://example.com"
 
 
-def test_add_bg_queues_crawl_without_creating_snapshots(tmp_path, process, disable_extractors_dict):
+def test_add_bg_queues_crawl_without_creating_snapshots(initialized_archive):
     """Background add should leave root snapshot creation to the runner."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "add", "--bg", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    result = run_archivebox_cmd(
+        ["add", "--bg", "--depth=0", "https://example.com"],
+        env=env,
     )
 
     assert result.returncode == 0
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
         snapshot_count = Snapshot.objects.count()
 
@@ -71,25 +56,23 @@ def test_add_bg_queues_crawl_without_creating_snapshots(tmp_path, process, disab
     assert snapshot_count == 0
 
 
-def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(tmp_path, process, disable_extractors_dict):
+def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initialized_archive):
     """Index-only add only creates the crawl; rejected URLs are sealed by the runner."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
+    env = cli_env(disable_extractors=True)
+    result = run_archivebox_cmd(
         [
-            "archivebox",
             "add",
             "--index-only",
             "--depth=0",
             "--url-denylist=example.com",
             "https://example.com",
         ],
-        capture_output=True,
-        env=disable_extractors_dict,
+        env=env,
     )
 
     assert result.returncode == 0
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
         snapshot_count = Snapshot.objects.count()
 
@@ -97,9 +80,9 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(tmp_p
     assert crawl.retry_at is None
     assert snapshot_count == 0
 
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
         snapshot_count = Snapshot.objects.count()
 
@@ -108,31 +91,55 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(tmp_p
     assert snapshot_count == 0
 
 
-def test_add_creates_crawl_record(tmp_path, process, disable_extractors_dict):
-    """Test that add command creates a Crawl record in the database."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive):
+    """Index-only add must apply the same internal URL guard as snapshot creation."""
+    env = cli_env(disable_extractors=True)
+    internal_urls = [
+        "http://archivebox.localhost:9292/admin/",
+        "http://web.archivebox.localhost:9292/",
+        "http://api.archivebox.localhost:9292/api/v1/docs",
+        "http://snap-2fb8e923c58c.archivebox.localhost:9292/index.html",
+    ]
+    result = run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", *internal_urls],
+        env={**env, "BASE_URL": "http://archivebox.localhost:9292"},
     )
 
-    with use_archivebox_db(tmp_path):
+    assert result.returncode == 0
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.get()
+        snapshot_count = Snapshot.objects.count()
+
+    assert crawl.get_urls_list() == []
+    assert crawl.status == Crawl.StatusChoices.QUEUED
+    assert crawl.retry_at is None
+    assert snapshot_count == 0
+
+
+def test_add_creates_crawl_record(initialized_archive):
+    """Test that add command creates a Crawl record in the database."""
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
+    )
+
+    with use_archivebox_db(initialized_archive):
         crawl_count = Crawl.objects.count()
 
     assert crawl_count == 1
 
 
-def test_add_creates_source_file(tmp_path, process, disable_extractors_dict):
+def test_add_creates_source_file(initialized_archive):
     """Test that add creates a source file with the URL."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
     )
 
-    sources_dir = tmp_path / "sources"
+    sources_dir = initialized_archive / "sources"
     assert sources_dir.exists()
 
     source_files = list(sources_dir.glob("*cli_add.txt"))
@@ -142,19 +149,18 @@ def test_add_creates_source_file(tmp_path, process, disable_extractors_dict):
     assert "https://example.com" in source_content
 
 
-def test_add_multiple_urls_single_command(tmp_path, process, disable_extractors_dict):
+def test_add_multiple_urls_single_command(initialized_archive):
     """Test adding multiple URLs in a single command."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com", "https://example.org"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    result = run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com", "https://example.org"],
+        env=env,
     )
 
     assert result.returncode == 0
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         snapshot_count = Snapshot.objects.count()
         urls = list(Snapshot.objects.order_by("url").values_list("url", flat=True))
 
@@ -163,28 +169,27 @@ def test_add_multiple_urls_single_command(tmp_path, process, disable_extractors_
     assert urls[1] == "https://example.org"
 
 
-def test_add_from_file(tmp_path, process, disable_extractors_dict):
+def test_add_from_file(initialized_archive):
     """Test adding URLs from a file.
 
     The add command should treat a file argument as URL input and create snapshots
     for each URL it contains.
     """
-    os.chdir(tmp_path)
 
+    env = cli_env(disable_extractors=True)
     # Create a file with URLs
-    urls_file = tmp_path / "urls.txt"
+    urls_file = initialized_archive / "urls.txt"
     urls_file.write_text("https://example.com\nhttps://example.org\n")
 
-    result = subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", str(urls_file)],
-        capture_output=True,
-        env=disable_extractors_dict,
+    result = run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", str(urls_file)],
+        env=env,
     )
 
     assert result.returncode == 0
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         crawl_count = Crawl.objects.count()
         snapshot_count = Snapshot.objects.count()
 
@@ -193,91 +198,85 @@ def test_add_from_file(tmp_path, process, disable_extractors_dict):
     assert snapshot_count == 2
 
 
-def test_add_with_depth_0_flag(tmp_path, process, disable_extractors_dict):
+def test_add_with_depth_0_flag(initialized_archive):
     """Test that --depth=0 flag is accepted and works."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    result = run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
     )
 
     assert result.returncode == 0
-    assert "unrecognized arguments: --depth" not in result.stderr.decode("utf-8")
+    assert "unrecognized arguments: --depth" not in result.stderr
 
 
-def test_add_with_depth_1_flag(tmp_path, process, disable_extractors_dict):
+def test_add_with_depth_1_flag(initialized_archive):
     """Test that --depth=1 flag is accepted."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=1", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    result = run_archivebox_cmd(
+        ["add", "--index-only", "--depth=1", "https://example.com"],
+        env=env,
     )
 
     assert result.returncode == 0
-    assert "unrecognized arguments: --depth" not in result.stderr.decode("utf-8")
+    assert "unrecognized arguments: --depth" not in result.stderr
 
 
-def test_add_rejects_invalid_depth_values(tmp_path, process, disable_extractors_dict):
+def test_add_rejects_invalid_depth_values(initialized_archive):
     """Test that add rejects depth values outside the supported range."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     for depth in ("5", "-1"):
-        result = subprocess.run(
-            ["archivebox", "add", "--index-only", f"--depth={depth}", "https://example.com"],
-            capture_output=True,
-            env=disable_extractors_dict,
+        result = run_archivebox_cmd(
+            ["add", "--index-only", f"--depth={depth}", "https://example.com"],
+            env=env,
         )
-        stderr = result.stderr.decode("utf-8").lower()
+        stderr = result.stderr.lower()
         assert result.returncode != 0
         assert "invalid" in stderr or "not one of" in stderr
 
 
-def test_add_with_tags(tmp_path, process, disable_extractors_dict):
+def test_add_with_tags(initialized_archive):
     """Test adding URL with tags stores tags_str in crawl.
 
     With --index-only, Tag objects are not created until archiving happens.
     Tags are stored as a string in the Crawl.tags_str field.
     """
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "--tag=test,example", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "--tag=test,example", "https://example.com"],
+        env=env,
     )
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         tags_str = Crawl.objects.values_list("tags_str", flat=True).get()
 
     # Tags are stored as a comma-separated string in crawl
     assert "test" in tags_str or "example" in tags_str
 
 
-def test_add_records_selected_persona_on_crawl(tmp_path, process, disable_extractors_dict):
+def test_add_records_selected_persona_on_crawl(initialized_archive):
     """Test add persists the selected persona so browser config derives from it later."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "--persona=Default", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    result = run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "--persona=Default", "https://example.com"],
+        env=env,
     )
 
     assert result.returncode == 0
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
 
     assert crawl.persona_id
     assert "ACTIVE_PERSONA" not in crawl.config
-    assert (tmp_path / "personas" / "Default" / "chrome_profile").is_dir()
+    assert (initialized_archive / "personas" / "Default" / "chrome_profile").is_dir()
 
 
-def test_add_records_url_filter_overrides_on_crawl(tmp_path, process, disable_extractors_dict):
-    os.chdir(tmp_path)
-    result = subprocess.run(
+def test_add_records_url_filter_overrides_on_crawl(initialized_archive):
+    env = cli_env(disable_extractors=True)
+    result = run_archivebox_cmd(
         [
-            "archivebox",
             "add",
             "--index-only",
             "--depth=0",
@@ -285,45 +284,42 @@ def test_add_records_url_filter_overrides_on_crawl(tmp_path, process, disable_ex
             "--domain-denylist=static.example.com",
             "https://example.com",
         ],
-        capture_output=True,
-        env=disable_extractors_dict,
+        env=env,
     )
 
     assert result.returncode == 0
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
 
     assert crawl.config["URL_ALLOWLIST"] == "example.com,*.example.com"
     assert crawl.config["URL_DENYLIST"] == "static.example.com"
-    assert not (tmp_path / "personas" / "Default" / "chrome_extensions").exists()
+    assert not (initialized_archive / "personas" / "Default" / "chrome_extensions").exists()
 
 
-def test_add_duplicate_url_creates_separate_crawls(tmp_path, process, disable_extractors_dict):
+def test_add_duplicate_url_creates_separate_crawls(initialized_archive):
     """Test that adding the same URL twice creates separate crawls and snapshots.
 
     Each 'add' command creates a new Crawl. Multiple crawls can archive the same URL.
     This allows re-archiving URLs at different times.
     """
-    os.chdir(tmp_path)
 
+    env = cli_env(disable_extractors=True)
     # Add URL first time
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
     # Add same URL second time with --update to opt out of ONLY_NEW.
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--update", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "--update", "--depth=0", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         snapshot_count = Snapshot.objects.filter(url="https://example.com").count()
         crawl_count = Crawl.objects.count()
 
@@ -332,54 +328,48 @@ def test_add_duplicate_url_creates_separate_crawls(tmp_path, process, disable_ex
     assert snapshot_count == 2
 
 
-def test_add_with_overwrite_flag(tmp_path, process, disable_extractors_dict):
+def test_add_with_overwrite_flag(initialized_archive):
     """Test that --overwrite flag forces re-archiving."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     # Add URL first time
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
     )
 
     # Add with overwrite
-    result = subprocess.run(
-        ["archivebox", "add", "--index-only", "--overwrite", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    result = run_archivebox_cmd(
+        ["add", "--index-only", "--overwrite", "https://example.com"],
+        env=env,
     )
 
     assert result.returncode == 0
-    assert "unrecognized arguments: --overwrite" not in result.stderr.decode("utf-8")
+    assert "unrecognized arguments: --overwrite" not in result.stderr
 
 
-def test_add_creates_snapshot_output_directory(tmp_path, process, disable_extractors_dict):
+def test_add_creates_snapshot_output_directory(initialized_archive):
     """Test that add creates the current snapshot output directory on disk."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         snapshot_id = str(Snapshot.objects.values_list("id", flat=True).get())
 
-    snapshot_dir = _find_snapshot_dir(tmp_path, snapshot_id)
+    snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
     assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
     assert snapshot_dir.is_dir()
 
 
-def test_add_help_shows_depth_and_tag_options(tmp_path, process):
+def test_add_help_shows_depth_and_tag_options(initialized_archive):
     """Test that add --help documents the main filter and crawl options."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "add", "--help"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["add", "--help"],
     )
 
     assert result.returncode == 0
@@ -391,11 +381,10 @@ def test_add_help_shows_depth_and_tag_options(tmp_path, process):
     assert "--tag" in result.stdout
 
 
-def test_add_records_max_url_and_size_limits_on_crawl(tmp_path, process, disable_extractors_dict):
-    os.chdir(tmp_path)
-    result = subprocess.run(
+def test_add_records_max_url_and_size_limits_on_crawl(initialized_archive):
+    env = cli_env(disable_extractors=True)
+    result = run_archivebox_cmd(
         [
-            "archivebox",
             "add",
             "--index-only",
             "--depth=1",
@@ -405,14 +394,13 @@ def test_add_records_max_url_and_size_limits_on_crawl(tmp_path, process, disable
             "--snapshot-max-size=5mb",
             "https://example.com",
         ],
-        capture_output=True,
-        env=disable_extractors_dict,
+        env=env,
     )
 
     assert result.returncode == 0
 
     columns = {field.name for field in Crawl._meta.local_fields}
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         config = Crawl.objects.values_list("config", flat=True).get() or {}
 
     assert {"max_urls", "crawl_max_size", "crawl_timeout", "snapshot_max_size"}.isdisjoint(columns)
@@ -422,14 +410,11 @@ def test_add_records_max_url_and_size_limits_on_crawl(tmp_path, process, disable
     assert config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024
 
 
-def test_add_without_args_shows_usage(tmp_path, process):
+def test_add_without_args_shows_usage(initialized_archive):
     """Test that add without URLs fails with a usage hint instead of crashing."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "add"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["add"],
     )
 
     combined = result.stdout + result.stderr
@@ -437,19 +422,18 @@ def test_add_without_args_shows_usage(tmp_path, process):
     assert "usage" in combined.lower() or "url" in combined.lower()
 
 
-def test_add_index_only_queues_crawl_without_starting_runner(tmp_path, process, disable_extractors_dict):
+def test_add_index_only_queues_crawl_without_starting_runner(initialized_archive):
     """Test that --index-only creates only a queued crawl and returns fast."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    result = run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
         timeout=30,  # Should be fast
     )
 
     assert result.returncode == 0
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.get()
         snapshot_count = Snapshot.objects.count()
 
@@ -458,35 +442,260 @@ def test_add_index_only_queues_crawl_without_starting_runner(tmp_path, process,
     assert snapshot_count == 0
 
 
-def test_add_links_snapshot_to_crawl(tmp_path, process, disable_extractors_dict):
+def test_add_links_snapshot_to_crawl(initialized_archive):
     """Test that add links the snapshot to the crawl via crawl_id."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         crawl_id = Crawl.objects.values_list("id", flat=True).get()
         snapshot_crawl = Snapshot.objects.values_list("crawl_id", flat=True).get()
 
     assert snapshot_crawl == crawl_id
 
 
-def test_add_sets_snapshot_timestamp(tmp_path, process, disable_extractors_dict):
+def test_add_sets_snapshot_timestamp(initialized_archive):
     """Test that add sets a timestamp on the snapshot."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         timestamp = Snapshot.objects.values_list("timestamp", flat=True).get()
 
     assert timestamp is not None
     assert len(str(timestamp)) > 0
+
+
+@pytest.mark.timeout(180)
+def test_cli_add_real_urls_with_options_writes_inspectable_outputs(initialized_archive):
+
+    wget_urls = [
+        "https://example.com",
+        "https://pirate.github.io/stress-tests/challenge.html",
+    ]
+    chrome_url = "https://example.com/?archivebox-chrome-flow=1"
+    env = os.environ.copy()
+    env.pop("CHROME_BINARY", None)
+    env.update(
+        {
+            "USE_COLOR": "false",
+            "SHOW_PROGRESS": "false",
+            "TIMEOUT": "60",
+            "SAVE_WGET": "true",
+            "SAVE_HEADERS": "false",
+            "SAVE_TITLE": "false",
+            "SAVE_READABILITY": "false",
+            "SAVE_SINGLEFILE": "false",
+            "SAVE_MERCURY": "false",
+            "SAVE_SCREENSHOT": "false",
+            "SAVE_PDF": "false",
+            "SAVE_DOM": "false",
+            "SAVE_ARCHIVEDOTORG": "false",
+            "SAVE_GIT": "false",
+            "SAVE_YTDLP": "false",
+            "SAVE_FAVICON": "false",
+        },
+    )
+    _cmd_result = run_archivebox_cmd(
+        [
+            "add",
+            "--depth=0",
+            "--max-urls=2",
+            "--crawl-max-size=10mb",
+            "--tag=real-flow,challenge",
+            "--parser=url_list",
+            "--plugins=wget",
+            *wget_urls,
+        ],
+        cwd=initialized_archive,
+        env=env,
+        timeout=180,
+    )
+    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    assert returncode == 0, stderr or stdout
+
+    chrome_env = env | {
+        "SAVE_WGET": "false",
+        "SAVE_HEADERS": "true",
+        "SAVE_TITLE": "true",
+        "CHROME_HEADLESS": "true",
+        "CHROME_SANDBOX": "false",
+        "CHROME_ISOLATION": "snapshot",
+    }
+    system_browser = _find_system_browser()
+    if system_browser:
+        chrome_env["CHROME_BINARY"] = str(system_browser)
+    else:
+        _cmd_result = run_archivebox_cmd(
+            ["install", "chrome"],
+            cwd=initialized_archive,
+            env=chrome_env,
+            timeout=600,
+        )
+        install_stdout, install_stderr, install_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+        assert install_returncode == 0, install_stderr or install_stdout
+    _cmd_result = run_archivebox_cmd(
+        [
+            "add",
+            "--depth=0",
+            "--max-urls=1",
+            "--crawl-max-size=10mb",
+            "--tag=chrome-flow",
+            "--parser=url_list",
+            "--plugins=chrome,wget,headers,title",
+            chrome_url,
+        ],
+        cwd=initialized_archive,
+        env=chrome_env,
+        timeout=180,
+    )
+    chrome_stdout, chrome_stderr, chrome_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    assert chrome_returncode == 0, chrome_stderr or chrome_stdout
+
+    _cmd_result = run_archivebox_cmd(
+        ["list", "--tag=real-flow"],
+        cwd=initialized_archive,
+        env=env,
+        timeout=60,
+    )
+    list_stdout, list_stderr, list_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    assert list_returncode == 0, list_stderr or list_stdout
+    listed = [json.loads(line) for line in list_stdout.splitlines() if line.strip()]
+    assert {item["url"] for item in listed} >= set(wget_urls)
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first()
+        real_flow_crawl = Crawl.objects.filter(tags_str="real-flow,challenge").values_list("max_depth", "tags_str", "config").first()
+        snapshots = list(Snapshot.objects.order_by("url").values_list("id", "url", "depth", "status", "title"))
+        archive_results = list(
+            ArchiveResult.objects.select_related("snapshot")
+            .order_by("snapshot__url", "plugin")
+            .values_list("snapshot__url", "plugin", "status", "output_files", "output_size", "output_str"),
+        )
+        processes = list(Process.objects.filter(process_type="hook").values_list("process_type", "status", "exit_code", "pwd", "cmd"))
+
+    assert real_flow_crawl is not None
+    assert real_flow_crawl[0] == 0
+    assert real_flow_crawl[1] == "real-flow,challenge"
+    real_flow_config = real_flow_crawl[2] or {}
+    assert real_flow_config["CRAWL_MAX_URLS"] == 2
+    assert real_flow_config["CRAWL_MAX_SIZE"] == 10 * 1024 * 1024
+    assert real_flow_config.get("SNAPSHOT_MAX_SIZE", 0) == 0
+    assert "wget" in real_flow_config["PLUGINS"]
+    assert crawl is not None
+    assert crawl[1] == "chrome-flow"
+    assert "wget,headers,title" in json.dumps(crawl[2] or {})
+
+    snapshot_urls = {url for _id, url, _depth, _status, _title in snapshots}
+    assert snapshot_urls >= {*wget_urls, chrome_url}
+    assert all(depth == 0 for _id, _url, depth, _status, _title in snapshots)
+
+    by_url_plugin = {(url, plugin): status for url, plugin, status, _files, _size, _output in archive_results}
+    assert by_url_plugin[("https://example.com", "wget")] == "succeeded"
+    assert by_url_plugin[("https://pirate.github.io/stress-tests/challenge.html", "wget")] == "succeeded"
+    assert (chrome_url, "headers") in by_url_plugin
+    assert (chrome_url, "title") in by_url_plugin
+    failed_results = [(url, plugin, output) for url, plugin, status, _files, _size, output in archive_results if status == "failed"]
+    assert len(failed_results) <= 2, failed_results
+
+    snapshot_root = initialized_archive / "archive/users/system/snapshots"
+    html_outputs = [path for path in snapshot_root.rglob("wget/**/*.html") if path.is_file()]
+    header_outputs = [path for path in snapshot_root.rglob("headers/**/headers.json") if path.is_file() and path.stat().st_size > 0]
+    title_outputs = [path for path in snapshot_root.rglob("title/title.txt") if path.is_file() and path.stat().st_size > 0]
+    index_outputs = [path for path in snapshot_root.rglob("index.jsonl") if path.is_file()]
+    assert html_outputs
+    if by_url_plugin[(chrome_url, "headers")] == "succeeded":
+        assert header_outputs
+    if by_url_plugin[(chrome_url, "title")] == "succeeded":
+        assert title_outputs
+        assert any("Example Domain" in path.read_text(errors="ignore") for path in title_outputs)
+    assert len(index_outputs) >= len(wget_urls) + 1
+
+    combined_html = "\n".join(path.read_text(errors="ignore") for path in html_outputs)
+    assert "Example Domain" in combined_html
+    assert "Browser Agent Challenge for AI Browser Drivers" in combined_html
+
+    assert processes
+    assert any("wget" in (pwd or "") or "wget" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes)
+    assert any("headers" in (pwd or "") or "headers" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes)
+
+
+@pytest.mark.timeout(180)
+def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive):
+
+    env = os.environ.copy()
+    env.update(
+        {
+            "USE_COLOR": "false",
+            "SHOW_PROGRESS": "false",
+            "TIMEOUT": "60",
+            "SAVE_WGET": "true",
+            "SAVE_HEADERS": "false",
+            "SAVE_TITLE": "false",
+            "SAVE_READABILITY": "false",
+            "SAVE_SINGLEFILE": "false",
+            "SAVE_MERCURY": "false",
+            "SAVE_SCREENSHOT": "false",
+            "SAVE_PDF": "false",
+            "SAVE_DOM": "false",
+            "SAVE_ARCHIVEDOTORG": "false",
+            "SAVE_GIT": "false",
+            "SAVE_YTDLP": "false",
+            "SAVE_FAVICON": "false",
+            "PARSE_HTML_URLS_ENABLED": "true",
+            "PARSE_DOM_OUTLINKS_ENABLED": "false",
+        },
+    )
+
+    _cmd_result = run_archivebox_cmd(
+        [
+            "add",
+            "--depth=2",
+            "--max-urls=2",
+            "--crawl-max-size=50mb",
+            "--tag=recursive-flow",
+            "--parser=url_list",
+            "--plugins=wget,parse_html_urls",
+            "https://example.com",
+        ],
+        cwd=initialized_archive,
+        env=env,
+        timeout=180,
+    )
+    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    assert returncode == 0, stderr or stdout
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first()
+        snapshots = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "status"))
+        archive_results = list(
+            ArchiveResult.objects.select_related("snapshot")
+            .order_by("snapshot__depth", "snapshot__url", "plugin")
+            .values_list("snapshot__url", "plugin", "status", "output_files"),
+        )
+
+    assert crawl[0] == 2
+    assert crawl[1] == "recursive-flow"
+    crawl_config = crawl[2] or {}
+    assert crawl_config["CRAWL_MAX_URLS"] == 2
+    assert crawl_config["CRAWL_MAX_SIZE"] == 50 * 1024 * 1024
+    assert crawl_config.get("SNAPSHOT_MAX_SIZE", 0) == 0
+    assert ("https://example.com", 0, "sealed") in snapshots
+    assert any(url == "https://iana.org/domains/example" and depth == 1 and status == "sealed" for url, depth, status in snapshots)
+
+    by_url_plugin = {(url, plugin): status for url, plugin, status, _files in archive_results}
+    assert by_url_plugin[("https://example.com", "wget")] == "succeeded"
+    assert by_url_plugin[("https://example.com", "parse_html_urls")] == "succeeded"
+    assert by_url_plugin[("https://iana.org/domains/example", "wget")] == "succeeded"
+
+    urls_outputs = list((initialized_archive / "archive/users/system/snapshots").rglob("parse_html_urls/urls.jsonl"))
+    assert urls_outputs
+    assert any("https://iana.org/domains/example" in path.read_text() for path in urls_outputs)

archivebox/tests/test_cli_archiveresult.py

@@ -32,15 +32,24 @@ class TestArchiveResultCreate:
         url = create_test_url()
 
         # Create a snapshot first
-        stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(
+            ["snapshot", "create", url],
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
+        )
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         snapshot = parse_jsonl_output(stdout1)[0]
 
         # Pipe snapshot to archiveresult create
-        stdout2, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "create", "--plugin=title"],
             stdin=json.dumps(snapshot),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0, f"Command failed: {stderr}"
 
@@ -57,14 +66,23 @@ class TestArchiveResultCreate:
     def test_create_with_specific_plugin(self, initialized_archive):
         """Create archive result for specific plugin."""
         url = create_test_url()
-        stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(
+            ["snapshot", "create", url],
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
+        )
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         snapshot = parse_jsonl_output(stdout1)[0]
 
-        stdout2, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "create", "--plugin=screenshot"],
             stdin=json.dumps(snapshot),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout2, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout2)
@@ -77,21 +95,28 @@ class TestArchiveResultCreate:
         url = create_test_url()
 
         # Create crawl and snapshot
-        stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         crawl = parse_jsonl_output(stdout1)[0]
 
-        stdout2, _, _ = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["snapshot", "create"],
             stdin=json.dumps(crawl),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Now pipe all to archiveresult create
-        stdout3, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "create", "--plugin=title"],
             stdin=stdout2,
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout3, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout3)
@@ -105,11 +130,14 @@ class TestArchiveResultCreate:
         """Only pass-through records but no new snapshots returns success."""
         crawl_record = {"type": "Crawl", "id": "fake-id", "urls": "https://example.com"}
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "create"],
             stdin=json.dumps(crawl_record),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         assert "Passed through" in stderr
@@ -120,10 +148,13 @@ class TestArchiveResultList:
 
     def test_list_empty(self, initialized_archive):
         """List with no archive results returns empty."""
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "list"],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         assert "Listed 0 archive results" in stderr
@@ -132,36 +163,53 @@ class TestArchiveResultList:
         """Filter archive results by status."""
         # Create snapshot and materialize an archive result via the runner
         url = create_test_url()
-        stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(
+            ["snapshot", "create", url],
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
+        )
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         snapshot = parse_jsonl_output(stdout1)[0]
-        stdout2, _, _ = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "create", "--plugin=favicon"],
             stdin=json.dumps(snapshot),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         run_archivebox_cmd(
             ["run"],
             stdin=stdout2,
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
             timeout=120,
             env=PROJECTOR_TEST_ENV,
+            default_cli_env=True,
+            disable_extractors=True,
         )
-        created = parse_jsonl_output(
-            run_archivebox_cmd(
-                ["archiveresult", "list", "--plugin=favicon"],
-                data_dir=initialized_archive,
-            )[0],
-        )[0]
+        _cmd_result = run_archivebox_cmd(
+            ["archiveresult", "list", "--plugin=favicon"],
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
+        )
+        created = parse_jsonl_output(_cmd_result.stdout)[0]
         run_archivebox_cmd(
             ["archiveresult", "update", "--status=queued"],
             stdin=json.dumps(created),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "list", "--status=queued"],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout)
@@ -171,25 +219,39 @@ class TestArchiveResultList:
     def test_list_filter_by_plugin(self, initialized_archive):
         """Filter archive results by plugin."""
         url = create_test_url()
-        stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(
+            ["snapshot", "create", url],
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
+        )
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         snapshot = parse_jsonl_output(stdout1)[0]
-        stdout2, _, _ = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "create", "--plugin=favicon"],
             stdin=json.dumps(snapshot),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         run_archivebox_cmd(
             ["run"],
             stdin=stdout2,
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
             timeout=120,
             env=PROJECTOR_TEST_ENV,
+            default_cli_env=True,
+            disable_extractors=True,
         )
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "list", "--plugin=favicon"],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout)
@@ -201,25 +263,39 @@ class TestArchiveResultList:
         # Create multiple archive results
         for _ in range(3):
             url = create_test_url()
-            stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+            _cmd_result = run_archivebox_cmd(
+                ["snapshot", "create", url],
+                cwd=initialized_archive,
+                default_cli_env=True,
+                disable_extractors=True,
+            )
+            stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
             snapshot = parse_jsonl_output(stdout1)[0]
-            stdout2, _, _ = run_archivebox_cmd(
+            _cmd_result = run_archivebox_cmd(
                 ["archiveresult", "create", "--plugin=favicon"],
                 stdin=json.dumps(snapshot),
-                data_dir=initialized_archive,
+                cwd=initialized_archive,
+                default_cli_env=True,
+                disable_extractors=True,
             )
+            stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
             run_archivebox_cmd(
                 ["run"],
                 stdin=stdout2,
-                data_dir=initialized_archive,
+                cwd=initialized_archive,
                 timeout=120,
                 env=PROJECTOR_TEST_ENV,
+                default_cli_env=True,
+                disable_extractors=True,
             )
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "list", "--limit=2"],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout)
@@ -232,32 +308,50 @@ class TestArchiveResultUpdate:
     def test_update_status(self, initialized_archive):
         """Update archive result status."""
         url = create_test_url()
-        stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(
+            ["snapshot", "create", url],
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
+        )
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         snapshot = parse_jsonl_output(stdout1)[0]
 
-        stdout2, _, _ = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "create", "--plugin=favicon"],
             stdin=json.dumps(snapshot),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
-        stdout_run, _, _ = run_archivebox_cmd(
+        stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+        _cmd_result = run_archivebox_cmd(
             ["run"],
             stdin=stdout2,
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
             timeout=120,
             env=PROJECTOR_TEST_ENV,
+            default_cli_env=True,
+            disable_extractors=True,
         )
-        stdout_list, _, _ = run_archivebox_cmd(
+        _stdout_run, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "list", "--plugin=favicon"],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout_list, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         ar = parse_jsonl_output(stdout_list)[0]
 
-        stdout3, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "update", "--status=failed"],
             stdin=json.dumps(ar),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout3, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         assert "Updated 1 archive results" in stderr
@@ -272,32 +366,50 @@ class TestArchiveResultDelete:
     def test_delete_requires_yes(self, initialized_archive):
         """Delete requires --yes flag."""
         url = create_test_url()
-        stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(
+            ["snapshot", "create", url],
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
+        )
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         snapshot = parse_jsonl_output(stdout1)[0]
 
-        stdout2, _, _ = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "create", "--plugin=favicon"],
             stdin=json.dumps(snapshot),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
-        stdout_run, _, _ = run_archivebox_cmd(
+        stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+        _cmd_result = run_archivebox_cmd(
             ["run"],
             stdin=stdout2,
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
             timeout=120,
             env=PROJECTOR_TEST_ENV,
+            default_cli_env=True,
+            disable_extractors=True,
         )
-        stdout_list, _, _ = run_archivebox_cmd(
+        _stdout_run, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "list", "--plugin=favicon"],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout_list, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         ar = parse_jsonl_output(stdout_list)[0]
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "delete"],
             stdin=json.dumps(ar),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 1
         assert "--yes" in stderr
@@ -305,32 +417,50 @@ class TestArchiveResultDelete:
     def test_delete_with_yes(self, initialized_archive):
         """Delete with --yes flag works."""
         url = create_test_url()
-        stdout1, _, _ = run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(
+            ["snapshot", "create", url],
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
+        )
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         snapshot = parse_jsonl_output(stdout1)[0]
 
-        stdout2, _, _ = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "create", "--plugin=favicon"],
             stdin=json.dumps(snapshot),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
-        stdout_run, _, _ = run_archivebox_cmd(
+        stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+        _cmd_result = run_archivebox_cmd(
             ["run"],
             stdin=stdout2,
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
             timeout=120,
             env=PROJECTOR_TEST_ENV,
+            default_cli_env=True,
+            disable_extractors=True,
         )
-        stdout_list, _, _ = run_archivebox_cmd(
+        _stdout_run, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "list", "--plugin=favicon"],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout_list, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         ar = parse_jsonl_output(stdout_list)[0]
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["archiveresult", "delete", "--yes"],
             stdin=json.dumps(ar),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         assert "Deleted 1 archive results" in stderr

archivebox/tests/test_cli_binary.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+"""
+Tests for archivebox binary command.
+
+TODO: expand beyond command discovery into create/list/update/delete behavior.
+"""
+
+from archivebox.tests.conftest import run_archivebox_cmd
+
+
+def test_binary_help_runs_successfully(tmp_path):
+    """The binary command should be registered and expose help."""
+
+    result = run_archivebox_cmd(["binary", "--help"])
+
+    assert result.returncode == 0
+    assert "binary" in result.stdout.lower()
+    assert "list" in result.stdout

archivebox/tests/test_cli_config.py

@@ -4,14 +4,12 @@ Comprehensive tests for archivebox config command.
 Verify config reads/writes ArchiveBox.conf file correctly.
 """
 
-import os
-import subprocess
+from archivebox.tests.conftest import run_archivebox_cmd
 
 
-def test_config_displays_all_config(tmp_path, process):
+def test_config_displays_all_config(initialized_archive):
     """Test that config without args displays all configuration."""
-    os.chdir(tmp_path)
-    result = subprocess.run(["archivebox", "config"], capture_output=True, text=True)
+    result = run_archivebox_cmd(["config"])
 
     assert result.returncode == 0
     output = result.stdout
@@ -21,180 +19,232 @@ def test_config_displays_all_config(tmp_path, process):
     assert "TIMEOUT" in output or "OUTPUT_PERMISSIONS" in output
 
 
-def test_config_get_specific_key(tmp_path, process):
+def test_config_shows_derived_collection_paths_but_not_runtime_dirs(initialized_archive):
+    """The CLI should expose collection paths, not per-crawl/per-snapshot runtime dirs."""
+    run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
+
+    result = run_archivebox_cmd(["config"], cwd=initialized_archive)
+
+    assert result.returncode == 0, result.stderr
+    unwrapped_output = result.stdout.replace("\n", "")
+    assert "DATA_DIR" in result.stdout
+    assert result.stdout.count("\nDATA_DIR =") == 1
+    assert str(initialized_archive) in unwrapped_output
+    assert "PERSONAS_DIR" in result.stdout
+    assert result.stdout.count("\nPERSONAS_DIR =") == 1
+    assert "SNAP_DIR" not in result.stdout
+    assert "CRAWL_DIR" not in result.stdout
+
+
+def test_config_get_derived_path_but_rejects_runtime_dir(initialized_archive):
+    run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
+
+    data_dir = run_archivebox_cmd(["config", "--get", "DATA_DIR"], cwd=initialized_archive)
+    snap_dir = run_archivebox_cmd(["config", "--get", "SNAP_DIR"], cwd=initialized_archive)
+
+    assert data_dir.returncode == 0, data_dir.stderr
+    assert "DATA_DIR" in data_dir.stdout
+    assert str(initialized_archive) in data_dir.stdout.replace("\n", "")
+    assert snap_dir.returncode != 0
+    assert "SNAP_DIR =" not in snap_dir.stdout
+
+
+def test_config_set_rejects_readonly_and_runtime_dirs(initialized_archive):
+    run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
+
+    data_dir = run_archivebox_cmd(
+        ["config", "--set", f"DATA_DIR={initialized_archive / 'other'}"],
+        cwd=initialized_archive,
+    )
+    crawl_dir = run_archivebox_cmd(
+        ["config", "--set", f"CRAWL_DIR={initialized_archive / 'crawl'}"],
+        cwd=initialized_archive,
+    )
+
+    assert data_dir.returncode != 0
+    assert crawl_dir.returncode != 0
+    content = (initialized_archive / "ArchiveBox.conf").read_text()
+    assert "DATA_DIR" not in content
+    assert "CRAWL_DIR" not in content
+
+
+def test_config_get_specific_key(initialized_archive):
     """Test that config --get KEY retrieves specific value."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "config", "--get", "TIMEOUT"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["config", "--get", "TIMEOUT"],
     )
 
     assert result.returncode == 0
     assert "TIMEOUT" in result.stdout
 
 
-def test_config_set_writes_to_file(tmp_path, process):
+def test_config_set_writes_to_file(initialized_archive):
     """Test that config --set KEY=VALUE writes to ArchiveBox.conf."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "config", "--set", "TIMEOUT=120"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["config", "--set", "TIMEOUT=120"],
     )
 
     assert result.returncode == 0
 
     # Verify config file was updated
-    config_file = tmp_path / "ArchiveBox.conf"
+    config_file = initialized_archive / "ArchiveBox.conf"
     assert config_file.exists()
 
     content = config_file.read_text()
     assert "TIMEOUT" in content or "120" in content
 
 
-def test_config_set_and_get_roundtrip(tmp_path, process):
+def test_config_set_and_get_roundtrip(initialized_archive):
     """Test that set value can be retrieved with get."""
-    os.chdir(tmp_path)
 
     # Set a unique value
-    subprocess.run(
-        ["archivebox", "config", "--set", "TIMEOUT=987"],
-        capture_output=True,
-        text=True,
+    run_archivebox_cmd(
+        ["config", "--set", "TIMEOUT=987"],
     )
 
     # Get the value back
-    result = subprocess.run(
-        ["archivebox", "config", "--get", "TIMEOUT"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["config", "--get", "TIMEOUT"],
     )
 
     assert "987" in result.stdout
 
 
-def test_config_set_multiple_values(tmp_path, process):
+def test_config_set_multiple_values(initialized_archive):
     """Test setting multiple config values at once."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "config", "--set", "TIMEOUT=111", "YTDLP_TIMEOUT=222"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["config", "--set", "TIMEOUT=111", "YTDLP_TIMEOUT=222"],
     )
 
     assert result.returncode == 0
 
     # Verify both were written
-    config_file = tmp_path / "ArchiveBox.conf"
+    config_file = initialized_archive / "ArchiveBox.conf"
     content = config_file.read_text()
     assert "111" in content
     assert "222" in content
 
 
-def test_config_set_invalid_key_fails(tmp_path, process):
+def test_config_set_invalid_key_fails(initialized_archive):
     """Test that setting invalid config key fails."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "config", "--set", "TOTALLY_INVALID_KEY_XYZ=value"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["config", "--set", "TOTALLY_INVALID_KEY_XYZ=value"],
     )
 
     assert result.returncode != 0
 
 
-def test_config_set_requires_equals_sign(tmp_path, process):
+def test_config_set_requires_equals_sign(initialized_archive):
     """Test that set requires KEY=VALUE format."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "config", "--set", "TIMEOUT"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["config", "--set", "TIMEOUT"],
     )
 
     assert result.returncode != 0
 
 
-def test_config_search_finds_keys(tmp_path, process):
+def test_config_search_finds_keys(initialized_archive):
     """Test that config --search finds matching keys."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "config", "--search", "TIMEOUT"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["config", "--search", "TIMEOUT"],
     )
 
     # Should find timeout-related config
     assert "TIMEOUT" in result.stdout
 
 
-def test_config_preserves_existing_values(tmp_path, process):
+def test_config_preserves_existing_values(initialized_archive):
     """Test that setting new values preserves existing ones."""
-    os.chdir(tmp_path)
 
     # Set first value
-    subprocess.run(
-        ["archivebox", "config", "--set", "TIMEOUT=100"],
-        capture_output=True,
+    run_archivebox_cmd(
+        ["config", "--set", "TIMEOUT=100"],
     )
 
     # Set second value
-    subprocess.run(
-        ["archivebox", "config", "--set", "YTDLP_TIMEOUT=200"],
-        capture_output=True,
+    run_archivebox_cmd(
+        ["config", "--set", "YTDLP_TIMEOUT=200"],
     )
 
     # Verify both are in config file
-    config_file = tmp_path / "ArchiveBox.conf"
+    config_file = initialized_archive / "ArchiveBox.conf"
     content = config_file.read_text()
     assert "TIMEOUT" in content
     assert "YTDLP_TIMEOUT" in content
 
 
-def test_config_file_is_valid_toml(tmp_path, process):
+def test_config_file_is_valid_toml(initialized_archive):
     """Test that config file remains valid TOML after set."""
-    os.chdir(tmp_path)
 
-    subprocess.run(
-        ["archivebox", "config", "--set", "TIMEOUT=150"],
-        capture_output=True,
+    run_archivebox_cmd(
+        ["config", "--set", "TIMEOUT=150"],
     )
 
-    config_file = tmp_path / "ArchiveBox.conf"
+    config_file = initialized_archive / "ArchiveBox.conf"
     content = config_file.read_text()
 
     # Basic TOML validation - should have sections and key=value pairs
     assert "[" in content or "=" in content
 
 
-def test_config_updates_existing_value(tmp_path, process):
+def test_config_updates_existing_value(initialized_archive):
     """Test that setting same key twice updates the value."""
-    os.chdir(tmp_path)
 
     # Set initial value
-    subprocess.run(
-        ["archivebox", "config", "--set", "TIMEOUT=100"],
-        capture_output=True,
+    run_archivebox_cmd(
+        ["config", "--set", "TIMEOUT=100"],
     )
 
     # Update to new value
-    subprocess.run(
-        ["archivebox", "config", "--set", "TIMEOUT=200"],
-        capture_output=True,
+    run_archivebox_cmd(
+        ["config", "--set", "TIMEOUT=200"],
     )
 
     # Get current value
-    result = subprocess.run(
-        ["archivebox", "config", "--get", "TIMEOUT"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["config", "--get", "TIMEOUT"],
     )
 
     # Should show updated value
     assert "200" in result.stdout
+
+
+def test_config_ignores_legacy_unknown_keys(tmp_path, initialized_archive):
+    """Old ArchiveBox.conf keys should not prevent startup during upgrades."""
+    (tmp_path / "ArchiveBox.conf").write_text(
+        """
+[ARCHIVING_CONFIG]
+MAX_MEDIA_SIZE = "750m"
+
+[SEARCH_BACKEND_CONFIG]
+SEARCH_BACKEND_HOST_NAME = "sonic"
+SEARCH_BACKEND_PASSWORD = "SecretPassword"
+""",
+    )
+
+    result = run_archivebox_cmd(
+        ["version"],
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert "Extra inputs are not permitted" not in result.stderr
+
+
+class TestConfigCLI:
+    """Test the CLI interface for config command."""
+
+    def test_cli_help(self, tmp_path, initialized_archive):
+        """Test that --help works for config command."""
+
+        result = run_archivebox_cmd(
+            ["config", "--help"],
+        )
+
+        assert result.returncode == 0
+        assert "--get" in result.stdout
+        assert "--set" in result.stdout

archivebox/tests/test_cli_crawl.py

@@ -10,11 +10,20 @@ Tests cover:
 
 import json
 
+import pytest
+
+from archivebox.core.models import Snapshot
+from archivebox.crawls.models import Crawl
 from archivebox.tests.conftest import (
-    run_archivebox_cmd,
-    parse_jsonl_output,
+    cli_env,
     create_test_url,
+    parse_jsonl_output,
+    run_archivebox_cmd,
+    run_queued_crawls,
 )
+from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+pytestmark = pytest.mark.django_db(transaction=True)
 
 
 class TestCrawlCreate:
@@ -24,10 +33,13 @@ class TestCrawlCreate:
         """Create crawl from URL arguments."""
         url = create_test_url()
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "create", url],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0, f"Command failed: {stderr}"
         assert "Created crawl" in stderr
@@ -43,11 +55,14 @@ class TestCrawlCreate:
         urls = [create_test_url() for _ in range(3)]
         stdin = "\n".join(urls)
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "create"],
             stdin=stdin,
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0, f"Command failed: {stderr}"
 
@@ -63,10 +78,13 @@ class TestCrawlCreate:
         """Create crawl with --depth flag."""
         url = create_test_url()
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "create", "--depth=2", url],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout)
@@ -76,10 +94,13 @@ class TestCrawlCreate:
         """Create crawl with --tag flag."""
         url = create_test_url()
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "create", "--tag=test-tag", url],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout)
@@ -91,11 +112,14 @@ class TestCrawlCreate:
         url = create_test_url()
         stdin = json.dumps(tag_record) + "\n" + json.dumps({"url": url})
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "create"],
             stdin=stdin,
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout)
@@ -109,15 +133,19 @@ class TestCrawlCreate:
         """Existing Crawl records (with id) are passed through."""
         # First create a crawl
         url = create_test_url()
-        stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         crawl = parse_jsonl_output(stdout1)[0]
 
         # Now pipe it back - should pass through
-        stdout2, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "create"],
             stdin=json.dumps(crawl),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout2, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout2)
@@ -130,10 +158,13 @@ class TestCrawlList:
 
     def test_list_empty(self, initialized_archive):
         """List with no crawls returns empty."""
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "list"],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         assert "Listed 0 crawls" in stderr
@@ -141,12 +172,15 @@ class TestCrawlList:
     def test_list_returns_created(self, initialized_archive):
         """List returns previously created crawls."""
         url = create_test_url()
-        run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive)
+        run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "list"],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout)
@@ -156,12 +190,15 @@ class TestCrawlList:
     def test_list_filter_by_status(self, initialized_archive):
         """Filter crawls by status."""
         url = create_test_url()
-        run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive)
+        run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "list", "--status=queued"],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout)
@@ -172,12 +209,20 @@ class TestCrawlList:
         """Limit number of results."""
         # Create multiple crawls
         for _ in range(3):
-            run_archivebox_cmd(["crawl", "create", create_test_url()], data_dir=initialized_archive)
+            run_archivebox_cmd(
+                ["crawl", "create", create_test_url()],
+                cwd=initialized_archive,
+                default_cli_env=True,
+                disable_extractors=True,
+            )
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "list", "--limit=2"],
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         records = parse_jsonl_output(stdout)
@@ -191,15 +236,19 @@ class TestCrawlUpdate:
         """Update crawl status."""
         # Create a crawl
         url = create_test_url()
-        stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         crawl = parse_jsonl_output(stdout1)[0]
 
         # Update it
-        stdout2, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "update", "--status=started"],
             stdin=json.dumps(crawl),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        stdout2, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         assert "Updated 1 crawls" in stderr
@@ -214,14 +263,18 @@ class TestCrawlDelete:
     def test_delete_requires_yes(self, initialized_archive):
         """Delete requires --yes flag."""
         url = create_test_url()
-        stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         crawl = parse_jsonl_output(stdout1)[0]
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "delete"],
             stdin=json.dumps(crawl),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 1
         assert "--yes" in stderr
@@ -229,14 +282,18 @@ class TestCrawlDelete:
     def test_delete_with_yes(self, initialized_archive):
         """Delete with --yes flag works."""
         url = create_test_url()
-        stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         crawl = parse_jsonl_output(stdout1)[0]
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "delete", "--yes"],
             stdin=json.dumps(crawl),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         assert "Deleted 1 crawls" in stderr
@@ -244,15 +301,172 @@ class TestCrawlDelete:
     def test_delete_dry_run(self, initialized_archive):
         """Dry run shows what would be deleted."""
         url = create_test_url()
-        stdout1, _, _ = run_archivebox_cmd(["crawl", "create", url], data_dir=initialized_archive)
+        _cmd_result = run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+        stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
         crawl = parse_jsonl_output(stdout1)[0]
 
-        stdout, stderr, code = run_archivebox_cmd(
+        _cmd_result = run_archivebox_cmd(
             ["crawl", "delete", "--dry-run"],
             stdin=json.dumps(crawl),
-            data_dir=initialized_archive,
+            cwd=initialized_archive,
+            default_cli_env=True,
+            disable_extractors=True,
         )
+        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         assert code == 0
         assert "Would delete" in stderr
         assert "dry run" in stderr.lower()
+
+
+def test_crawl_creates_crawl_object(initialized_archive):
+    """Test that crawl command creates a Crawl object."""
+    env = cli_env(disable_extractors=True)
+
+    run_archivebox_cmd(
+        ["crawl", "create", "https://example.com"],
+        cwd=initialized_archive,
+        env=env,
+        check=True,
+    )
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.order_by("-created_at").first()
+
+    assert crawl is not None, "Crawl object should be created"
+
+
+def test_crawl_depth_sets_max_depth_in_crawl(initialized_archive):
+    """Test that --depth option sets max_depth in the Crawl object."""
+    env = cli_env(disable_extractors=True)
+
+    run_archivebox_cmd(
+        ["crawl", "create", "--depth=2", "https://example.com"],
+        cwd=initialized_archive,
+        env=env,
+        check=True,
+    )
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.order_by("-created_at").first()
+
+    assert crawl is not None
+    assert crawl.max_depth == 2, "Crawl max_depth should match --depth=2"
+
+
+def test_crawl_creates_snapshot_for_url(initialized_archive):
+    """Test that crawl creates a Snapshot for the input URL."""
+    env = cli_env(disable_extractors=True)
+
+    run_archivebox_cmd(
+        ["crawl", "create", "https://example.com"],
+        cwd=initialized_archive,
+        env=env,
+        check=True,
+    )
+    run_queued_crawls(initialized_archive, env)
+
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.filter(url="https://example.com").first()
+
+    assert snapshot is not None, "Snapshot should be created for input URL"
+
+
+def test_crawl_links_snapshot_to_crawl(initialized_archive):
+    """Test that Snapshot is linked to Crawl via crawl_id."""
+    env = cli_env(disable_extractors=True)
+
+    run_archivebox_cmd(
+        ["crawl", "create", "https://example.com"],
+        cwd=initialized_archive,
+        env=env,
+        check=True,
+    )
+    run_queued_crawls(initialized_archive, env)
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.order_by("-created_at").first()
+        assert crawl is not None
+        snapshot = Snapshot.objects.filter(url="https://example.com").first()
+
+    assert snapshot is not None
+    assert snapshot.crawl_id == crawl.id, "Snapshot should be linked to Crawl"
+
+
+def test_crawl_multiple_urls_creates_multiple_snapshots(initialized_archive):
+    """Test that crawling multiple URLs creates multiple snapshots."""
+    env = cli_env(disable_extractors=True)
+
+    run_archivebox_cmd(
+        [
+            "crawl",
+            "create",
+            "https://example.com",
+            "https://iana.org",
+        ],
+        cwd=initialized_archive,
+        env=env,
+        check=True,
+    )
+    run_queued_crawls(initialized_archive, env)
+
+    with use_archivebox_db(initialized_archive):
+        urls = list(Snapshot.objects.order_by("url").values_list("url", flat=True))
+
+    assert "https://example.com" in urls
+    assert "https://iana.org" in urls
+
+
+def test_crawl_from_file_creates_snapshot(initialized_archive):
+    """Test that crawl can create snapshots from a file of URLs."""
+    env = cli_env(disable_extractors=True)
+
+    # Write URLs to a file
+    urls_file = initialized_archive / "urls.txt"
+    urls_file.write_text("https://example.com\n")
+
+    run_archivebox_cmd(
+        ["crawl", "create", str(urls_file)],
+        cwd=initialized_archive,
+        env=env,
+        check=True,
+    )
+    run_queued_crawls(initialized_archive, env)
+
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.first()
+
+    # Should create at least one snapshot (the source file or the URL)
+    assert snapshot is not None, "Should create at least one snapshot"
+
+
+def test_crawl_persists_input_urls_on_crawl(initialized_archive):
+    """Test that crawl input URLs are stored on the Crawl record."""
+    env = cli_env(disable_extractors=True)
+
+    run_archivebox_cmd(
+        ["crawl", "create", "https://example.com"],
+        cwd=initialized_archive,
+        env=env,
+        check=True,
+    )
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.order_by("-created_at").first()
+
+    assert crawl is not None, "Crawl should be created for crawl input"
+    assert "https://example.com" in crawl.urls, "Crawl should persist input URLs"
+
+
+class TestCrawlCLI:
+    """Test the CLI interface for crawl command."""
+
+    def test_cli_help(self, tmp_path, initialized_archive):
+        """Test that --help works for crawl command."""
+
+        result = run_archivebox_cmd(
+            ["crawl", "--help"],
+        )
+
+        assert result.returncode == 0
+        assert "create" in result.stdout

archivebox/tests/test_cli_extract.py

@@ -4,35 +4,31 @@ Tests for archivebox extract command.
 Verify extract re-runs extractors on existing snapshots.
 """
 
-import os
-import subprocess
-
 import pytest
 
 from archivebox.core.models import Snapshot
-from archivebox.tests.conftest import run_queued_crawls
+from archivebox.tests.conftest import run_queued_crawls, run_archivebox_cmd, cli_env
+
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def test_extract_runs_on_existing_snapshots(tmp_path, process, disable_extractors_dict):
+def test_extract_runs_on_existing_snapshots(initialized_archive):
     """Test that extract command runs on existing snapshots."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     # Add a snapshot first
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
     # Run extract
-    result = subprocess.run(
-        ["archivebox", "extract"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    result = run_archivebox_cmd(
+        ["extract"],
+        env=env,
         timeout=30,
     )
 
@@ -40,30 +36,28 @@ def test_extract_runs_on_existing_snapshots(tmp_path, process, disable_extractor
     assert result.returncode in [0, 1]
 
 
-def test_extract_preserves_snapshot_count(tmp_path, process, disable_extractors_dict):
+def test_extract_preserves_snapshot_count(initialized_archive):
     """Test that extract doesn't change snapshot count."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     # Add snapshot
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         count_before = Snapshot.objects.count()
 
     # Run extract
-    subprocess.run(
-        ["archivebox", "extract", "--overwrite"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["extract", "--overwrite"],
+        env=env,
         timeout=30,
     )
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         count_after = Snapshot.objects.count()
 
     assert count_after == count_before

archivebox/tests/test_cli_extract_input.py

@@ -1,221 +1,210 @@
 """Tests for archivebox extract input handling and pipelines."""
 
-import os
 import subprocess
 import json
 
 import pytest
 
 from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.tests.conftest import run_queued_crawls
+from archivebox.tests.conftest import run_archivebox_cmd, run_queued_crawls, cli_env
+
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def _snapshot_id(data_dir):
-    with use_archivebox_db(data_dir):
-        return Snapshot.objects.values_list("id", flat=True).first()
-
-
-def test_extract_runs_on_snapshot_id(tmp_path, process, disable_extractors_dict):
+def test_extract_runs_on_snapshot_id(initialized_archive):
     """Test that extract command accepts a snapshot ID."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     # First create a snapshot
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    snapshot_id = _snapshot_id(tmp_path)
+    with use_archivebox_db(initialized_archive):
+        snapshot_id = Snapshot.objects.values_list("id", flat=True).first()
 
     # Run extract on the snapshot
-    result = subprocess.run(
-        ["archivebox", "extract", "--no-wait", str(snapshot_id)],
-        capture_output=True,
-        text=True,
-        env=disable_extractors_dict,
+    result = run_archivebox_cmd(
+        ["extract", "--no-wait", str(snapshot_id)],
+        env=env,
     )
 
     # Should not error about invalid snapshot ID
     assert "not found" not in result.stderr.lower()
 
 
-def test_extract_with_enabled_extractor_creates_archiveresult(tmp_path, process, disable_extractors_dict):
+def test_extract_with_enabled_extractor_creates_archiveresult(initialized_archive):
     """Test that extract creates ArchiveResult when extractor is enabled."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     # First create a snapshot
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    snapshot_id = _snapshot_id(tmp_path)
+    with use_archivebox_db(initialized_archive):
+        snapshot_id = Snapshot.objects.values_list("id", flat=True).first()
 
     # Run extract with title extractor enabled
-    env = disable_extractors_dict.copy()
+    env = env.copy()
     env["SAVE_TITLE"] = "true"
 
-    subprocess.run(
-        ["archivebox", "extract", "--no-wait", str(snapshot_id)],
-        capture_output=True,
-        text=True,
+    run_archivebox_cmd(
+        ["extract", "--no-wait", str(snapshot_id)],
         env=env,
     )
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         count = ArchiveResult.objects.filter(snapshot_id=snapshot_id).count()
 
     # May or may not have results depending on timing
     assert count >= 0
 
 
-def test_extract_plugin_option_accepted(tmp_path, process, disable_extractors_dict):
+def test_extract_plugin_option_accepted(initialized_archive):
     """Test that --plugin option is accepted."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     # First create a snapshot
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    snapshot_id = _snapshot_id(tmp_path)
+    with use_archivebox_db(initialized_archive):
+        snapshot_id = Snapshot.objects.values_list("id", flat=True).first()
 
-    result = subprocess.run(
-        ["archivebox", "extract", "--plugin=title", "--no-wait", str(snapshot_id)],
-        capture_output=True,
-        text=True,
-        env=disable_extractors_dict,
+    result = run_archivebox_cmd(
+        ["extract", "--plugin=title", "--no-wait", str(snapshot_id)],
+        env=env,
     )
 
     assert "unrecognized arguments: --plugin" not in result.stderr
 
 
-def test_extract_stdin_snapshot_id(tmp_path, process, disable_extractors_dict):
+def test_extract_stdin_snapshot_id(initialized_archive):
     """Test that extract reads snapshot IDs from stdin."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     # First create a snapshot
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    snapshot_id = _snapshot_id(tmp_path)
+    with use_archivebox_db(initialized_archive):
+        snapshot_id = Snapshot.objects.values_list("id", flat=True).first()
 
-    result = subprocess.run(
-        ["archivebox", "extract", "--no-wait"],
+    result = run_archivebox_cmd(
+        ["extract", "--no-wait"],
         input=f"{snapshot_id}\n",
-        capture_output=True,
-        text=True,
-        env=disable_extractors_dict,
+        env=env,
     )
 
     # Should not show "not found" error
     assert "not found" not in result.stderr.lower() or result.returncode == 0
 
 
-def test_extract_stdin_jsonl_input(tmp_path, process, disable_extractors_dict):
+def test_extract_stdin_jsonl_input(initialized_archive):
     """Test that extract reads JSONL records from stdin."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     # First create a snapshot
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    snapshot_id = _snapshot_id(tmp_path)
+    with use_archivebox_db(initialized_archive):
+        snapshot_id = Snapshot.objects.values_list("id", flat=True).first()
 
     jsonl_input = json.dumps({"type": "Snapshot", "id": str(snapshot_id)}) + "\n"
 
-    result = subprocess.run(
-        ["archivebox", "extract", "--no-wait"],
+    result = run_archivebox_cmd(
+        ["extract", "--no-wait"],
         input=jsonl_input,
-        capture_output=True,
-        text=True,
-        env=disable_extractors_dict,
+        env=env,
     )
 
     # Should not show "not found" error
     assert "not found" not in result.stderr.lower() or result.returncode == 0
 
 
-def test_extract_pipeline_from_snapshot(tmp_path, process, disable_extractors_dict):
+def test_extract_pipeline_from_snapshot(initialized_archive):
     """Test piping snapshot output to extract."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     # Create snapshot and pipe to extract
-    snapshot_proc = subprocess.Popen(
-        ["archivebox", "snapshot", "https://example.com"],
+    snapshot_proc = run_archivebox_cmd(
+        ["snapshot", "https://example.com"],
+        cwd=initialized_archive,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
-        env=disable_extractors_dict,
+        env=env,
+        wait=False,
     )
 
-    subprocess.run(
-        ["archivebox", "extract", "--no-wait"],
+    extract_proc = run_archivebox_cmd(
+        ["extract", "--no-wait"],
+        cwd=initialized_archive,
         stdin=snapshot_proc.stdout,
-        capture_output=True,
-        text=True,
-        env=disable_extractors_dict,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=env,
+        wait=False,
     )
+    if snapshot_proc.stdout is not None:
+        snapshot_proc.stdout.close()
 
-    snapshot_proc.wait()
+    extract_stdout, extract_stderr = extract_proc.communicate(timeout=60)
+    snapshot_stdout, snapshot_stderr = snapshot_proc.communicate(timeout=60)
+    assert snapshot_proc.returncode == 0, (snapshot_stdout or "") + (snapshot_stderr or "")
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         snapshot = Snapshot.objects.filter(url="https://example.com").first()
 
     assert snapshot is not None, "Snapshot should be created by pipeline"
 
 
-def test_extract_multiple_snapshots(tmp_path, process, disable_extractors_dict):
+def test_extract_multiple_snapshots(initialized_archive):
     """Test extracting from multiple snapshots."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     # Create multiple snapshots one at a time to avoid deduplication issues
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "https://example.com"],
+        env=env,
     )
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "https://iana.org"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "https://iana.org"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         snapshot_ids = list(Snapshot.objects.values_list("id", flat=True))
 
     assert len(snapshot_ids) >= 2, "Should have at least 2 snapshots"
 
     # Extract from all snapshots
     ids_input = "\n".join(str(snapshot_id) for snapshot_id in snapshot_ids) + "\n"
-    result = subprocess.run(
-        ["archivebox", "extract", "--no-wait"],
+    result = run_archivebox_cmd(
+        ["extract", "--no-wait"],
         input=ids_input,
-        capture_output=True,
-        text=True,
-        env=disable_extractors_dict,
+        env=env,
     )
     assert result.returncode == 0, result.stderr
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         count = Snapshot.objects.count()
 
     assert count >= 2, "Both snapshots should still exist after extraction"
@@ -224,29 +213,23 @@ def test_extract_multiple_snapshots(tmp_path, process, disable_extractors_dict):
 class TestExtractCLI:
     """Test the CLI interface for extract command."""
 
-    def test_cli_help(self, tmp_path, process):
+    def test_cli_help(self, initialized_archive):
         """Test that --help works for extract command."""
-        os.chdir(tmp_path)
 
-        result = subprocess.run(
-            ["archivebox", "extract", "--help"],
-            capture_output=True,
-            text=True,
+        result = run_archivebox_cmd(
+            ["extract", "--help"],
         )
 
         assert result.returncode == 0
         assert "--plugin" in result.stdout or "-p" in result.stdout
         assert "--wait" in result.stdout or "--no-wait" in result.stdout
 
-    def test_cli_no_snapshots_shows_warning(self, tmp_path, process):
+    def test_cli_no_snapshots_shows_warning(self, initialized_archive):
         """Test that running without snapshots shows a warning."""
-        os.chdir(tmp_path)
 
-        result = subprocess.run(
-            ["archivebox", "extract", "--no-wait"],
+        result = run_archivebox_cmd(
+            ["extract", "--no-wait"],
             input="",
-            capture_output=True,
-            text=True,
         )
 
         # Should show warning about no snapshots or exit normally (empty input)

archivebox/tests/test_cli_help.py

@@ -4,14 +4,12 @@ Tests for archivebox help command.
 Verify command runs successfully and produces output.
 """
 
-import os
-import subprocess
+from archivebox.tests.conftest import run_archivebox_cmd
 
 
 def test_help_runs_successfully(tmp_path):
     """Test that help command runs and produces output."""
-    os.chdir(tmp_path)
-    result = subprocess.run(["archivebox", "help"], capture_output=True, text=True)
+    result = run_archivebox_cmd(["help"])
 
     assert result.returncode == 0
     combined = result.stdout + result.stderr
@@ -19,10 +17,9 @@ def test_help_runs_successfully(tmp_path):
     assert "archivebox" in combined.lower()
 
 
-def test_help_in_initialized_dir(tmp_path, process):
+def test_help_in_initialized_dir(initialized_archive):
     """Test help command in initialized data directory."""
-    os.chdir(tmp_path)
-    result = subprocess.run(["archivebox", "help"], capture_output=True, text=True)
+    result = run_archivebox_cmd(["help"])
 
     assert result.returncode == 0
     combined = result.stdout + result.stderr

archivebox/tests/test_cli_init.py

@@ -4,9 +4,6 @@ Comprehensive tests for archivebox init command.
 Verify init creates correct database schema, filesystem structure, and config.
 """
 
-import os
-import subprocess
-
 import pytest
 from django.utils import timezone
 from django.db import connections
@@ -16,7 +13,8 @@ from archivebox.config.common import get_config
 from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.crawls.models import Crawl
 from archivebox.machine.models import Machine
-from archivebox.tests.conftest import run_queued_crawls
+from archivebox.tests.conftest import run_queued_crawls, run_archivebox_cmd, cli_env
+
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
 pytestmark = pytest.mark.django_db(transaction=True)
@@ -27,8 +25,7 @@ DIR_PERMISSIONS = get_config().OUTPUT_PERMISSIONS.replace("6", "7").replace("4",
 
 def test_init_creates_database_file(tmp_path):
     """Test that init creates index.sqlite3 database file."""
-    os.chdir(tmp_path)
-    result = subprocess.run(["archivebox", "init"], capture_output=True)
+    result = run_archivebox_cmd(["init"])
 
     assert result.returncode == 0
     db_path = tmp_path / "index.sqlite3"
@@ -38,8 +35,7 @@ def test_init_creates_database_file(tmp_path):
 
 def test_init_creates_archive_directory(tmp_path):
     """Test that init creates archive directory."""
-    os.chdir(tmp_path)
-    subprocess.run(["archivebox", "init"], capture_output=True)
+    run_archivebox_cmd(["init"])
 
     archive_dir = tmp_path / "archive"
     assert archive_dir.exists()
@@ -48,9 +44,8 @@ def test_init_creates_archive_directory(tmp_path):
 
 def test_init_uses_cwd_archive_and_users_dirs(tmp_path):
     """Test that init creates archive/users storage roots under cwd."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(["archivebox", "init"], capture_output=True)
+    result = run_archivebox_cmd(["init"])
 
     assert result.returncode == 0
     assert (tmp_path / "archive").is_dir()
@@ -59,8 +54,7 @@ def test_init_uses_cwd_archive_and_users_dirs(tmp_path):
 
 def test_init_creates_sources_directory(tmp_path):
     """Test that init creates sources directory."""
-    os.chdir(tmp_path)
-    subprocess.run(["archivebox", "init"], capture_output=True)
+    run_archivebox_cmd(["init"])
 
     sources_dir = tmp_path / "sources"
     assert sources_dir.exists()
@@ -69,8 +63,7 @@ def test_init_creates_sources_directory(tmp_path):
 
 def test_init_creates_logs_directory(tmp_path):
     """Test that init creates logs directory."""
-    os.chdir(tmp_path)
-    subprocess.run(["archivebox", "init"], capture_output=True)
+    run_archivebox_cmd(["init"])
 
     logs_dir = tmp_path / "logs"
     assert logs_dir.exists()
@@ -79,8 +72,7 @@ def test_init_creates_logs_directory(tmp_path):
 
 def test_init_creates_config_file(tmp_path):
     """Test that init creates ArchiveBox.conf config file."""
-    os.chdir(tmp_path)
-    subprocess.run(["archivebox", "init"], capture_output=True)
+    run_archivebox_cmd(["init"])
 
     config_file = tmp_path / "ArchiveBox.conf"
     assert config_file.exists()
@@ -89,8 +81,7 @@ def test_init_creates_config_file(tmp_path):
 
 def test_init_runs_migrations(tmp_path):
     """Test that init runs Django migrations and creates core tables."""
-    os.chdir(tmp_path)
-    subprocess.run(["archivebox", "init"], capture_output=True)
+    run_archivebox_cmd(["init"])
 
     with use_archivebox_db(tmp_path):
         migration_count = MigrationRecorder.Migration.objects.count()
@@ -100,8 +91,7 @@ def test_init_runs_migrations(tmp_path):
 
 def test_init_creates_core_snapshot_table(tmp_path):
     """Test that init creates core_snapshot table."""
-    os.chdir(tmp_path)
-    subprocess.run(["archivebox", "init"], capture_output=True)
+    run_archivebox_cmd(["init"])
 
     assert Snapshot._meta.db_table == "core_snapshot"
     with use_archivebox_db(tmp_path):
@@ -110,8 +100,7 @@ def test_init_creates_core_snapshot_table(tmp_path):
 
 def test_init_creates_crawls_crawl_table(tmp_path):
     """Test that init creates crawls_crawl table."""
-    os.chdir(tmp_path)
-    subprocess.run(["archivebox", "init"], capture_output=True)
+    run_archivebox_cmd(["init"])
 
     assert Crawl._meta.db_table == "crawls_crawl"
     with use_archivebox_db(tmp_path):
@@ -120,8 +109,7 @@ def test_init_creates_crawls_crawl_table(tmp_path):
 
 def test_init_creates_core_archiveresult_table(tmp_path):
     """Test that init creates core_archiveresult table."""
-    os.chdir(tmp_path)
-    subprocess.run(["archivebox", "init"], capture_output=True)
+    run_archivebox_cmd(["init"])
 
     assert ArchiveResult._meta.db_table == "core_archiveresult"
     with use_archivebox_db(tmp_path):
@@ -130,8 +118,7 @@ def test_init_creates_core_archiveresult_table(tmp_path):
 
 def test_init_sets_correct_file_permissions(tmp_path):
     """Test that init sets correct permissions on created files."""
-    os.chdir(tmp_path)
-    subprocess.run(["archivebox", "init"], capture_output=True)
+    run_archivebox_cmd(["init"])
 
     # Check database permissions
     db_path = tmp_path / "index.sqlite3"
@@ -144,15 +131,14 @@ def test_init_sets_correct_file_permissions(tmp_path):
 
 def test_init_is_idempotent(tmp_path):
     """Test that running init multiple times is safe (idempotent)."""
-    os.chdir(tmp_path)
 
     # First init
-    result1 = subprocess.run(["archivebox", "init"], capture_output=True, text=True)
+    result1 = run_archivebox_cmd(["init"])
     assert result1.returncode == 0
     assert "Initializing a new ArchiveBox" in result1.stdout
 
     # Second init should update, not fail
-    result2 = subprocess.run(["archivebox", "init"], capture_output=True, text=True)
+    result2 = run_archivebox_cmd(["init"])
     assert result2.returncode == 0
     assert "updating existing ArchiveBox" in result2.stdout or "up-to-date" in result2.stdout.lower()
 
@@ -164,15 +150,14 @@ def test_init_is_idempotent(tmp_path):
 
 def test_init_refuses_database_migrated_by_newer_code(tmp_path):
     """A downgraded ArchiveBox build must fail before serving a newer DB schema."""
-    os.chdir(tmp_path)
-    result = subprocess.run(["archivebox", "init"], capture_output=True, text=True)
+    result = run_archivebox_cmd(["init"])
     assert result.returncode == 0
 
     with use_archivebox_db(tmp_path):
         MigrationRecorder.Migration.objects.create(app="crawls", name="9999_future_test", applied=timezone.now())
         connections["default"].commit()
 
-    result = subprocess.run(["archivebox", "init"], capture_output=True, text=True)
+    result = run_archivebox_cmd(["init"])
     assert result.returncode == 3
     assert "migrated by a newer version of ArchiveBox" in result.stderr
     assert "crawls.9999_future_test" in result.stderr
@@ -183,8 +168,7 @@ def test_init_recovers_from_pre_squash_dev_history(tmp_path):
     """Pre-squash dev DBs (rows for migrations now absorbed by ``replaces=``)
     must NOT trip the newer-DB guard — every historical squash would otherwise
     brick beta-tester collections that pre-date the squash commit."""
-    os.chdir(tmp_path)
-    result = subprocess.run(["archivebox", "init"], capture_output=True, text=True)
+    result = run_archivebox_cmd(["init"])
     assert result.returncode == 0
 
     # Sampling — one name per affected app, all listed in the ``replaces=``
@@ -205,43 +189,41 @@ def test_init_recovers_from_pre_squash_dev_history(tmp_path):
             MigrationRecorder.Migration.objects.create(app=app, name=name, applied=timezone.now())
         connections["default"].commit()
 
-    result = subprocess.run(["archivebox", "init"], capture_output=True, text=True)
+    result = run_archivebox_cmd(["init"])
     assert result.returncode == 0, f"init refused to recover pre-squash dev DB.\nstdout={result.stdout}\nstderr={result.stderr}"
     assert "migrated by a newer version of ArchiveBox" not in result.stderr
 
 
-def test_init_with_existing_data_preserves_snapshots(tmp_path, process, disable_extractors_dict):
+def test_init_with_existing_data_preserves_snapshots(initialized_archive):
     """Test that re-running init preserves existing snapshot data."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
 
     # Add a snapshot
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
     # Check snapshot was created
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         count_before = Snapshot.objects.count()
     assert count_before == 1
 
     # Run init again
-    result = subprocess.run(["archivebox", "init"], capture_output=True)
+    result = run_archivebox_cmd(["init"])
     assert result.returncode == 0
 
     # Snapshot should still exist
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         count_after = Snapshot.objects.count()
     assert count_after == count_before
 
 
 def test_init_quick_flag_skips_checks(tmp_path):
     """Test that init --quick runs faster by skipping some checks."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(["archivebox", "init", "--quick"], capture_output=True, text=True)
+    result = run_archivebox_cmd(["init", "--quick"])
 
     assert result.returncode == 0
     # Database should still be created
@@ -251,8 +233,7 @@ def test_init_quick_flag_skips_checks(tmp_path):
 
 def test_init_creates_machine_table(tmp_path):
     """Test that init creates the machine_machine table."""
-    os.chdir(tmp_path)
-    subprocess.run(["archivebox", "init"], capture_output=True)
+    run_archivebox_cmd(["init"])
 
     assert Machine._meta.db_table == "machine_machine"
     with use_archivebox_db(tmp_path):
@@ -261,31 +242,27 @@ def test_init_creates_machine_table(tmp_path):
 
 def test_init_output_shows_collection_info(tmp_path):
     """Test that init output shows helpful collection information."""
-    os.chdir(tmp_path)
-    result = subprocess.run(["archivebox", "init"], capture_output=True, text=True)
+    result = run_archivebox_cmd(["init"])
 
     output = result.stdout
     # Should show some helpful info about the collection
     assert "ArchiveBox" in output or "collection" in output.lower() or "Initializing" in output
 
 
-def test_init_ignores_unrecognized_archive_directories(tmp_path, process, disable_extractors_dict):
+def test_init_ignores_unrecognized_archive_directories(initialized_archive):
     """Test that init upgrades existing dirs without choking on extra folders."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
         check=True,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
-    (tmp_path / "archive" / "some_random_folder").mkdir(parents=True, exist_ok=True)
+    run_queued_crawls(initialized_archive, env)
+    (initialized_archive / "archive" / "some_random_folder").mkdir(parents=True, exist_ok=True)
 
-    result = subprocess.run(
-        ["archivebox", "init"],
-        capture_output=True,
-        text=True,
-        env=disable_extractors_dict,
+    result = run_archivebox_cmd(
+        ["init"],
+        env=env,
     )
 
     assert result.returncode == 0, result.stdout + result.stderr

archivebox/tests/test_cli_install.py

@@ -5,8 +5,8 @@ Verify install detects and records binary dependencies in DB.
 """
 
 import os
-import subprocess
 from pathlib import Path
+from archivebox.tests.conftest import run_archivebox_cmd
 
 import pytest
 
@@ -18,13 +18,10 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def test_install_runs_successfully(tmp_path, process):
+def test_install_runs_successfully(initialized_archive):
     """Test that install command runs without error."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "install", "--dry-run"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["install", "--dry-run"],
         timeout=60,
     )
 
@@ -32,28 +29,23 @@ def test_install_runs_successfully(tmp_path, process):
     assert result.returncode in [0, 1]  # May return 1 if binaries missing
 
 
-def test_install_creates_binary_records_in_db(tmp_path, process):
+def test_install_creates_binary_records_in_db(initialized_archive):
     """Test that install creates Binary records in database."""
-    os.chdir(tmp_path)
 
-    subprocess.run(
-        ["archivebox", "install", "--dry-run"],
-        capture_output=True,
+    run_archivebox_cmd(
+        ["install", "--dry-run"],
         timeout=60,
     )
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         Binary.objects.count()
 
 
-def test_install_dry_run_does_not_install(tmp_path, process):
+def test_install_dry_run_does_not_install(initialized_archive):
     """Test that --dry-run doesn't actually install anything."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "install", "--dry-run"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["install", "--dry-run"],
         timeout=60,
     )
 
@@ -61,14 +53,11 @@ def test_install_dry_run_does_not_install(tmp_path, process):
     assert "dry" in result.stdout.lower() or result.returncode in [0, 1]
 
 
-def test_install_detects_system_binaries(tmp_path, process):
+def test_install_detects_system_binaries(initialized_archive):
     """Test that install detects existing system binaries."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "install", "--dry-run"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["install", "--dry-run"],
         timeout=60,
     )
 
@@ -76,14 +65,11 @@ def test_install_detects_system_binaries(tmp_path, process):
     assert result.returncode in [0, 1]
 
 
-def test_install_shows_binary_status(tmp_path, process):
+def test_install_shows_binary_status(initialized_archive):
     """Test that install shows status of binaries."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "install", "--dry-run"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["install", "--dry-run"],
         timeout=60,
     )
 
@@ -92,13 +78,10 @@ def test_install_shows_binary_status(tmp_path, process):
     assert len(output) > 50
 
 
-def test_install_dry_run_prints_dry_run_message(tmp_path, process):
+def test_install_dry_run_prints_dry_run_message(initialized_archive):
     """Test that install --dry-run clearly reports that no changes will be made."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "install", "--dry-run"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["install", "--dry-run"],
         timeout=60,
     )
 
@@ -108,11 +91,8 @@ def test_install_dry_run_prints_dry_run_message(tmp_path, process):
 
 def test_install_help_lists_dry_run_flag(tmp_path):
     """Test that install --help documents the dry-run option."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "install", "--help"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["install", "--help"],
     )
 
     assert result.returncode == 0
@@ -121,11 +101,8 @@ def test_install_help_lists_dry_run_flag(tmp_path):
 
 def test_install_invalid_option_fails(tmp_path):
     """Test that invalid install options fail cleanly."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "install", "--invalid-option"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["install", "--invalid-option"],
     )
 
     assert result.returncode != 0
@@ -133,11 +110,8 @@ def test_install_invalid_option_fails(tmp_path):
 
 def test_install_from_empty_dir_initializes_collection(tmp_path):
     """Test that install bootstraps an empty dir before performing work."""
-    os.chdir(tmp_path)
-    result = subprocess.run(
-        ["archivebox", "install", "--dry-run"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["install", "--dry-run"],
     )
 
     output = result.stdout + result.stderr
@@ -145,11 +119,10 @@ def test_install_from_empty_dir_initializes_collection(tmp_path):
     assert "Initializing" in output or "Dry run" in output or "init" in output.lower()
 
 
-def test_install_updates_binary_table(tmp_path, process):
+def test_install_updates_binary_table(initialized_archive):
     """Test that install completes and only mutates dependency state."""
-    os.chdir(tmp_path)
     env = os.environ.copy()
-    tmp_short = Path("/tmp") / f"abx-install-{tmp_path.name}"
+    tmp_short = Path("/tmp") / f"abx-install-{initialized_archive.name}"
     tmp_short.mkdir(parents=True, exist_ok=True)
     env.update(
         {
@@ -158,10 +131,8 @@ def test_install_updates_binary_table(tmp_path, process):
         },
     )
 
-    result = subprocess.run(
-        ["archivebox", "install", "git"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["install", "git"],
         timeout=120,
         env=env,
     )
@@ -169,7 +140,7 @@ def test_install_updates_binary_table(tmp_path, process):
     output = result.stdout + result.stderr
     assert result.returncode == 0, output
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         binary_counts = {
             status: Binary.objects.filter(status=status).count() for status in Binary.objects.values_list("status", flat=True).distinct()
         }

archivebox/tests/test_cli_list.py

@@ -5,42 +5,135 @@ Verify list emits snapshot JSONL and applies the documented filters.
 """
 
 import json
-import os
-import subprocess
+import sys
 
 import pytest
+from django.db import connection
+from django.utils import timezone
 
 from archivebox.core.models import Snapshot
-from archivebox.tests.conftest import create_test_url, parse_jsonl_output, run_archivebox_cmd, run_queued_crawls
+from archivebox.tests.conftest import create_test_url, parse_jsonl_output, run_archivebox_cmd, run_queued_crawls, cli_env
+
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def _parse_jsonl(stdout: str) -> list[dict]:
-    return [json.loads(line) for line in stdout.splitlines() if line.strip().startswith("{")]
+class CountingStdout:
+    encoding = "utf-8"
+
+    def __init__(self):
+        self.rows = 0
+        self._pending = ""
+
+    def isatty(self):
+        return False
+
+    def write(self, text):
+        self._pending += text
+        lines = self._pending.split("\n")
+        self._pending = lines.pop()
+        self.rows += sum(1 for line in lines if line.startswith("{"))
+        return len(text)
+
+    def flush(self):
+        return None
 
 
-def test_list_outputs_existing_snapshots_as_jsonl(tmp_path, process, disable_extractors_dict):
+def test_list_limit_zero_streams_one_million_snapshots_without_materializing(admin_user, monkeypatch):
+    """Regression: archivebox list --limit=0 must stream unbounded result sets."""
+    from archivebox.cli.archivebox_snapshot import list_snapshots
+    from archivebox.crawls.models import Crawl
+
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        created_by=admin_user,
+        status=Crawl.StatusChoices.SEALED,
+        retry_at=None,
+    )
+    now = timezone.now().isoformat()
+    with connection.cursor() as cursor:
+        cursor.execute(
+            """
+            WITH RECURSIVE seq(n) AS (
+                SELECT 1
+                UNION ALL
+                SELECT n + 1 FROM seq WHERE n < 1000000
+            )
+            INSERT INTO core_snapshot (
+                id,
+                url,
+                timestamp,
+                title,
+                bookmarked_at,
+                created_at,
+                modified_at,
+                downloaded_at,
+                fs_version,
+                crawl_id,
+                config,
+                current_step,
+                depth,
+                notes,
+                num_uses_failed,
+                num_uses_succeeded,
+                retry_at,
+                status,
+                delete_at,
+                output_size,
+                parent_snapshot_id
+            )
+            SELECT
+                lower(hex(randomblob(16))),
+                'https://example.com/page-' || n,
+                printf('9%031d', n),
+                '',
+                %s,
+                %s,
+                %s,
+                NULL,
+                '0.9.0',
+                %s,
+                '{}',
+                0,
+                0,
+                '',
+                0,
+                0,
+                NULL,
+                'sealed',
+                NULL,
+                0,
+                NULL
+            FROM seq
+            """,
+            [now, now, now, str(crawl.id).replace("-", "")],
+        )
+
+    stdout = CountingStdout()
+    monkeypatch.setattr(sys, "stdout", stdout)
+
+    assert list_snapshots(limit=0) == 0
+    assert stdout.rows == 1000000
+
+
+def test_list_outputs_existing_snapshots_as_jsonl(initialized_archive):
     """Test that list prints one JSON object per stored snapshot."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
     for url in ["https://example.com", "https://iana.org"]:
-        subprocess.run(
-            ["archivebox", "add", "--index-only", "--depth=0", url],
-            capture_output=True,
-            env=disable_extractors_dict,
+        run_archivebox_cmd(
+            ["add", "--index-only", "--depth=0", url],
+            env=env,
             check=True,
         )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    result = subprocess.run(
-        ["archivebox", "list"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["list"],
         timeout=30,
     )
 
-    rows = _parse_jsonl(result.stdout)
+    rows = parse_jsonl_output(result.stdout)
     urls = {row["url"] for row in rows}
 
     assert result.returncode == 0, result.stderr
@@ -48,95 +141,83 @@ def test_list_outputs_existing_snapshots_as_jsonl(tmp_path, process, disable_ext
     assert "https://iana.org" in urls
 
 
-def test_list_filters_by_url_icontains(tmp_path, process, disable_extractors_dict):
+def test_list_filters_by_url_icontains(initialized_archive):
     """Test that list --url__icontains returns only matching snapshots."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
     for url in ["https://example.com", "https://iana.org"]:
-        subprocess.run(
-            ["archivebox", "add", "--index-only", "--depth=0", url],
-            capture_output=True,
-            env=disable_extractors_dict,
+        run_archivebox_cmd(
+            ["add", "--index-only", "--depth=0", url],
+            env=env,
             check=True,
         )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    result = subprocess.run(
-        ["archivebox", "list", "--url__icontains", "example.com"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["list", "--url__icontains", "example.com"],
         timeout=30,
     )
 
-    rows = _parse_jsonl(result.stdout)
+    rows = parse_jsonl_output(result.stdout)
     assert result.returncode == 0, result.stderr
     assert len(rows) == 1
     assert rows[0]["url"] == "https://example.com"
 
 
-def test_list_filters_by_crawl_id_and_limit(tmp_path, process, disable_extractors_dict):
+def test_list_filters_by_crawl_id_and_limit(initialized_archive):
     """Test that crawl-id and limit filters constrain the result set."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
     for url in ["https://example.com", "https://iana.org"]:
-        subprocess.run(
-            ["archivebox", "add", "--index-only", "--depth=0", url],
-            capture_output=True,
-            env=disable_extractors_dict,
+        run_archivebox_cmd(
+            ["add", "--index-only", "--depth=0", url],
+            env=env,
             check=True,
         )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         crawl_id = str(Snapshot.objects.values_list("crawl_id", flat=True).get(url="https://example.com"))
 
-    result = subprocess.run(
-        ["archivebox", "list", "--crawl-id", crawl_id, "--limit", "1"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["list", "--crawl-id", crawl_id, "--limit", "1"],
         timeout=30,
     )
 
-    rows = _parse_jsonl(result.stdout)
+    rows = parse_jsonl_output(result.stdout)
     assert result.returncode == 0, result.stderr
     assert len(rows) == 1
     assert rows[0]["crawl_id"].replace("-", "") == crawl_id.replace("-", "")
     assert rows[0]["url"] == "https://example.com"
 
 
-def test_list_filters_by_status(tmp_path, process, disable_extractors_dict):
+def test_list_filters_by_status(initialized_archive):
     """Test that list can filter using the current snapshot status."""
-    os.chdir(tmp_path)
-    subprocess.run(
-        ["archivebox", "add", "--index-only", "--depth=0", "https://example.com"],
-        capture_output=True,
-        env=disable_extractors_dict,
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(
+        ["add", "--index-only", "--depth=0", "https://example.com"],
+        env=env,
         check=True,
     )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    with use_archivebox_db(tmp_path):
+    with use_archivebox_db(initialized_archive):
         status = Snapshot.objects.values_list("status", flat=True).get()
 
-    result = subprocess.run(
-        ["archivebox", "list", "--status", status],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["list", "--status", status],
         timeout=30,
     )
 
-    rows = _parse_jsonl(result.stdout)
+    rows = parse_jsonl_output(result.stdout)
     assert result.returncode == 0, result.stderr
     assert len(rows) == 1
     assert rows[0]["status"] == status
 
 
-def test_list_help_lists_filter_options(tmp_path, process):
+def test_list_help_lists_filter_options(initialized_archive):
     """Test that list --help documents the supported filter flags."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "list", "--help"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["list", "--help"],
         timeout=30,
     )
 
@@ -145,28 +226,28 @@ def test_list_help_lists_filter_options(tmp_path, process):
     assert "--crawl-id" in result.stdout
     assert "--limit" in result.stdout
     assert "--search" in result.stdout
+    assert "--json" in result.stdout
+    assert "--html" in result.stdout
+    assert "--with-headers" in result.stdout
 
 
-def test_list_allows_sort_with_limit(tmp_path, process, disable_extractors_dict):
+def test_list_allows_sort_with_limit(initialized_archive):
     """Test that list can sort and then apply limit without queryset slicing errors."""
-    os.chdir(tmp_path)
+    env = cli_env(disable_extractors=True)
     for url in ["https://example.com", "https://iana.org", "https://example.net"]:
-        subprocess.run(
-            ["archivebox", "add", "--index-only", "--depth=0", url],
-            capture_output=True,
-            env=disable_extractors_dict,
+        run_archivebox_cmd(
+            ["add", "--index-only", "--depth=0", url],
+            env=env,
             check=True,
         )
-    run_queued_crawls(tmp_path, disable_extractors_dict)
+    run_queued_crawls(initialized_archive, env)
 
-    result = subprocess.run(
-        ["archivebox", "list", "--limit", "2", "--sort", "-created_at"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["list", "--limit", "2", "--sort", "-created_at"],
         timeout=30,
     )
 
-    rows = _parse_jsonl(result.stdout)
+    rows = parse_jsonl_output(result.stdout)
     assert result.returncode == 0, result.stderr
     assert len(rows) == 2
 
@@ -174,12 +255,15 @@ def test_list_allows_sort_with_limit(tmp_path, process, disable_extractors_dict)
 def test_snapshot_list_search_meta(initialized_archive):
     """snapshot list should support metadata search mode."""
     url = create_test_url(domain="meta-search-example.com")
-    run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+    run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    stdout, stderr, code = run_archivebox_cmd(
+    _cmd_result = run_archivebox_cmd(
         ["snapshot", "list", "--search=meta", "meta-search-example.com"],
-        data_dir=initialized_archive,
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
     )
+    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert code == 0, f"Command failed: {stderr}"
     records = parse_jsonl_output(stdout)
@@ -190,12 +274,15 @@ def test_snapshot_list_search_meta(initialized_archive):
 def test_list_search_meta_matches_metadata(initialized_archive):
     """top-level list --search=meta should apply metadata search to the queryset."""
     url = create_test_url(domain="top-level-meta-search-example.com")
-    run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+    run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    stdout, stderr, code = run_archivebox_cmd(
+    _cmd_result = run_archivebox_cmd(
         ["list", "--search=meta", "top-level-meta-search-example.com"],
-        data_dir=initialized_archive,
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
     )
+    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert code == 0, f"Command failed: {stderr}"
     records = parse_jsonl_output(stdout)
@@ -204,62 +291,128 @@ def test_list_search_meta_matches_metadata(initialized_archive):
 
 
 def test_search_command_finds_snapshots(initialized_archive):
-    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+    run_archivebox_cmd(
+        ["snapshot", "create", "https://example.com"],
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
 
-    stdout, stderr, code = run_archivebox_cmd(["search", "example"], data_dir=initialized_archive)
+    _cmd_result = run_archivebox_cmd(["search", "example"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert code == 0, stderr
     assert "example" in stdout
 
 
 def test_search_command_returns_no_results_for_missing_term(initialized_archive):
-    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+    run_archivebox_cmd(
+        ["snapshot", "create", "https://example.com"],
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
 
-    _stdout, _stderr, code = run_archivebox_cmd(["search", "nonexistentterm12345"], data_dir=initialized_archive)
+    _cmd_result = run_archivebox_cmd(
+        ["search", "nonexistentterm12345"],
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    _stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert code in [0, 1]
 
 
 def test_search_command_on_empty_archive(initialized_archive):
-    _stdout, _stderr, code = run_archivebox_cmd(["search", "anything"], data_dir=initialized_archive)
+    _cmd_result = run_archivebox_cmd(["search", "anything"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+    _stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert code in [0, 1]
 
 
-def test_search_command_json_outputs_matching_snapshots(initialized_archive):
-    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+def test_search_command_outputs_matching_snapshots_as_jsonl(initialized_archive):
+    run_archivebox_cmd(
+        ["snapshot", "create", "https://example.com"],
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
 
-    stdout, stderr, code = run_archivebox_cmd(["search", "--json"], data_dir=initialized_archive)
+    _cmd_result = run_archivebox_cmd(["search"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert code == 0, stderr
-    payload = json.loads(stdout)
+    records = parse_jsonl_output(stdout)
+    assert any("example.com" in row.get("url", "") for row in records)
+
+
+def test_search_command_json_outputs_matching_snapshots(initialized_archive):
+    run_archivebox_cmd(
+        ["snapshot", "create", "https://example.com"],
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+
+    result = run_archivebox_cmd(["search", "--json"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+
+    assert result.returncode == 0, result.stderr
+    payload = json.loads(result.stdout)
     assert any("example.com" in row.get("url", "") for row in payload)
 
 
 def test_search_command_json_with_headers_wraps_links_payload(initialized_archive):
-    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+    run_archivebox_cmd(
+        ["snapshot", "create", "https://example.com"],
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
 
-    stdout, stderr, code = run_archivebox_cmd(["search", "--json", "--with-headers"], data_dir=initialized_archive)
+    result = run_archivebox_cmd(
+        ["search", "--json", "--with-headers"],
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
 
-    assert code == 0, stderr
-    payload = json.loads(stdout)
-    links = payload.get("links", payload)
-    assert any("example.com" in row.get("url", "") for row in links)
+    assert result.returncode == 0, result.stderr
+    payload = json.loads(result.stdout)
+    assert "links" in payload
+    assert any("example.com" in row.get("url", "") for row in payload["links"])
 
 
 def test_search_command_html_outputs_markup(initialized_archive):
-    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+    run_archivebox_cmd(
+        ["snapshot", "create", "https://example.com"],
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
 
-    stdout, stderr, code = run_archivebox_cmd(["search", "--html"], data_dir=initialized_archive)
+    result = run_archivebox_cmd(["search", "--html"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    assert code == 0, stderr
-    assert "<" in stdout
+    assert result.returncode == 0, result.stderr
+    assert "<" in result.stdout
+    assert "example.com" in result.stdout
 
 
 def test_search_command_csv_outputs_requested_column(initialized_archive):
-    run_archivebox_cmd(["snapshot", "create", "https://example.com"], data_dir=initialized_archive)
+    run_archivebox_cmd(
+        ["snapshot", "create", "https://example.com"],
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
 
-    stdout, stderr, code = run_archivebox_cmd(["search", "--csv", "url", "--with-headers"], data_dir=initialized_archive)
+    _cmd_result = run_archivebox_cmd(
+        ["search", "--csv", "url", "--with-headers"],
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert code == 0, stderr
     assert "url" in stdout
@@ -267,26 +420,38 @@ def test_search_command_csv_outputs_requested_column(initialized_archive):
 
 
 def test_search_command_with_headers_requires_structured_output_format(initialized_archive):
-    _stdout, stderr, code = run_archivebox_cmd(["search", "--with-headers"], data_dir=initialized_archive)
+    _cmd_result = run_archivebox_cmd(["search", "--with-headers"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+    _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert code != 0
-    assert "requires" in stderr.lower() or "json" in stderr.lower()
+    assert "requires" in stderr.lower()
+    assert "json" in stderr.lower()
 
 
 def test_search_command_sort_option_runs_successfully(initialized_archive):
     for url in ["https://iana.org", "https://example.com"]:
-        run_archivebox_cmd(["snapshot", "create", url], data_dir=initialized_archive)
+        run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    stdout, stderr, code = run_archivebox_cmd(["search", "--csv", "url", "--sort=url"], data_dir=initialized_archive)
+    _cmd_result = run_archivebox_cmd(
+        ["search", "--csv", "url", "--sort=url"],
+        cwd=initialized_archive,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert code == 0, stderr
     assert "example.com" in stdout or "iana.org" in stdout
 
 
 def test_search_command_help_lists_supported_filters(initialized_archive):
-    stdout, _stderr, code = run_archivebox_cmd(["search", "--help"], data_dir=initialized_archive)
+    _cmd_result = run_archivebox_cmd(["search", "--help"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
+    stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert code == 0
-    assert "--filter-type" in stdout or "-f" in stdout
+    assert "--url__icontains" in stdout
+    assert "--crawl-id" in stdout
     assert "--status" in stdout
     assert "--sort" in stdout
+    assert "--json" in stdout
+    assert "--html" in stdout

archivebox/tests/test_cli_machine.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+"""
+Tests for archivebox machine command.
+
+TODO: expand beyond command discovery into list/filter behavior.
+"""
+
+from archivebox.tests.conftest import run_archivebox_cmd
+
+
+def test_machine_help_runs_successfully(tmp_path):
+    """The machine command should be registered and expose help."""
+
+    result = run_archivebox_cmd(["machine", "--help"])
+
+    assert result.returncode == 0
+    assert "machine" in result.stdout.lower()
+    assert "list" in result.stdout

archivebox/tests/test_cli_manage.py

@@ -4,18 +4,14 @@ Tests for archivebox manage command.
 Verify manage command runs Django management commands.
 """
 
-import os
-import subprocess
+from archivebox.tests.conftest import run_archivebox_cmd
 
 
-def test_manage_help_works(tmp_path, process):
+def test_manage_help_works(initialized_archive):
     """Test that manage help command works."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "manage", "help"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["manage", "help"],
         timeout=30,
     )
 
@@ -23,14 +19,11 @@ def test_manage_help_works(tmp_path, process):
     assert len(result.stdout) > 100
 
 
-def test_manage_showmigrations_works(tmp_path, process):
+def test_manage_showmigrations_works(initialized_archive):
     """Test that manage showmigrations works."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "manage", "showmigrations"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["manage", "showmigrations"],
         timeout=30,
     )
 
@@ -39,14 +32,11 @@ def test_manage_showmigrations_works(tmp_path, process):
     assert "core" in result.stdout or "[" in result.stdout
 
 
-def test_manage_dbshell_command_exists(tmp_path, process):
+def test_manage_dbshell_command_exists(initialized_archive):
     """Test that manage dbshell command is recognized."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "manage", "help", "dbshell"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["manage", "help", "dbshell"],
         timeout=30,
     )
 
@@ -55,14 +45,11 @@ def test_manage_dbshell_command_exists(tmp_path, process):
     assert "dbshell" in result.stdout or "database" in result.stdout.lower()
 
 
-def test_manage_check_works(tmp_path, process):
+def test_manage_check_works(initialized_archive):
     """Test that manage check works."""
-    os.chdir(tmp_path)
 
-    result = subprocess.run(
-        ["archivebox", "manage", "check"],
-        capture_output=True,
-        text=True,
+    result = run_archivebox_cmd(
+        ["manage", "check"],
         timeout=30,
     )
 

archivebox/tests/test_cli_mcp.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+"""
+Tests for archivebox mcp command.
+
+TODO: expand beyond command discovery into JSON-RPC stdio behavior.
+"""
+
+from archivebox.tests.conftest import run_archivebox_cmd
+
+
+def test_mcp_help_runs_successfully(tmp_path):
+    """The mcp command should be registered and expose help."""
+
+    result = run_archivebox_cmd(["mcp", "--help"])
+
+    assert result.returncode == 0
+    assert "mcp" in result.stdout.lower()