diff --git a/archivebox/api/auth.py b/archivebox/api/auth.py index 5ef84d8b..68992ca7 100644 --- a/archivebox/api/auth.py +++ b/archivebox/api/auth.py @@ -7,7 +7,7 @@ from django.http import HttpRequest from django.contrib.auth import authenticate from django.contrib.auth.models import User -from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth +from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader from ninja.errors import HttpError @@ -106,33 +106,6 @@ class QueryParamTokenAuth(APIKeyQuery): return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__) -class UsernameAndPasswordAuth(HttpBasicAuth): - """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)""" - - def authenticate(self, request: HttpRequest, username: str, password: str) -> User | None: - return _require_superuser( - auth_using_password(username=username, password=password, request=request), - request, - self.__class__.__name__, - ) - - -class DjangoSessionAuth: - """Allow authenticating with existing Django session cookies (same-origin only).""" - - def __call__(self, request: HttpRequest) -> User | None: - return self.authenticate(request) - - def authenticate(self, request: HttpRequest, **kwargs) -> User | None: - user = getattr(request, "user", None) - if isinstance(user, User) and user.is_authenticated: - setattr(request, "_api_auth_method", self.__class__.__name__) - if not user.is_superuser: - raise HttpError(403, "Valid session but User does not have permission (make sure user.is_superuser=True)") - return user - return None - - ### Enabled Auth Methods API_AUTH_METHODS = [ diff --git a/archivebox/api/v1_cli.py b/archivebox/api/v1_cli.py index a2a37b73..3aa9a117 100644 --- a/archivebox/api/v1_cli.py +++ b/archivebox/api/v1_cli.py @@ -61,6 +61,10 @@ class AddCommandSchema(Schema): snapshot_ids: list[str] | None = None tag: str = "" depth: int = 0 + max_urls: int = 0 + crawl_max_size: int = 0 + crawl_timeout: int = 0 + snapshot_max_size: int = 0 parser: str = "auto" plugins: str = "" update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW) @@ -122,6 +126,10 @@ def cli_add(request: HttpRequest, args: AddCommandSchema): snapshot_ids=args.snapshot_ids, tag=args.tag, depth=args.depth, + max_urls=args.max_urls, + crawl_max_size=args.crawl_max_size, + crawl_timeout=args.crawl_timeout, + snapshot_max_size=args.snapshot_max_size, update=args.update, index_only=args.index_only, overwrite=args.overwrite, diff --git a/archivebox/api/v1_core.py b/archivebox/api/v1_core.py index ec512906..50d35af4 100644 --- a/archivebox/api/v1_core.py +++ b/archivebox/api/v1_core.py @@ -11,8 +11,7 @@ from typing import Union, Any, Annotated from datetime import datetime, time from django.db import transaction -from django.db.models import Model, Q, Sum -from django.db.models.functions import Coalesce +from django.db.models import Model, Q from django.http import HttpRequest, HttpResponse from django.http.multipartparser import MultiPartParser, MultiPartParserError from django.core.exceptions import ValidationError @@ -29,6 +28,7 @@ from ninja.pagination import paginate, PaginationBase from ninja.errors import HttpError from archivebox.core.models import Snapshot, ArchiveResult, Tag +from archivebox.core.permissions import public_snapshots_queryset from archivebox.api.auth import auth_using_token from archivebox.config.common import get_config from archivebox.core.host_utils import build_web_url @@ -671,7 +671,7 @@ class SnapshotSchema(Schema): @staticmethod def resolve_archive_size(obj): - return int(getattr(obj, "output_size_sum", obj.archive_size) or 0) + return int(obj.archive_size or 0) @staticmethod def resolve_output_size(obj): @@ -689,6 +689,7 @@ class SnapshotSchema(Schema): class SnapshotUpdateSchema(Schema): + action: str | None = None status: str | None = None retry_at: datetime | None = None tags: list[str] | None = None @@ -766,7 +767,6 @@ def _filter_snapshots_for_rss( ) .filter(bookmarked_at__lte=before_dt) ) - crawl_id = crawl_id.strip() if crawl_id: queryset = queryset.filter(crawl__id__icontains=crawl_id) @@ -854,7 +854,7 @@ class SnapshotFilterSchema(FilterSchema): def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False): """List all Snapshot entries matching these filters.""" setattr(request, "with_archiveresults", with_archiveresults) - queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)) + queryset = Snapshot.objects.all() return filters.filter(queryset).distinct() @@ -880,7 +880,7 @@ def get_snapshots_rss( def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True): """Get a specific Snapshot by id.""" setattr(request, "with_archiveresults", with_archiveresults) - queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)) + queryset = Snapshot.objects.all() try: return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id)) except Snapshot.DoesNotExist: @@ -963,8 +963,24 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS payload = data.dict(exclude_unset=True) update_fields = ["modified_at"] + action = payload.pop("action", None) tags = payload.pop("tags", None) + if action: + if action == "pause": + snapshot.pause() + setattr(request, "with_archiveresults", False) + return snapshot + if action in ("resume", "unpause"): + snapshot.resume() + setattr(request, "with_archiveresults", False) + return snapshot + if action == "cancel": + snapshot.cancel() + setattr(request, "with_archiveresults", False) + return snapshot + raise HttpError(400, f"Invalid action: {action}") + if "status" in payload: if payload["status"] not in Snapshot.StatusChoices.values: raise HttpError(400, f"Invalid status: {payload['status']}") @@ -980,7 +996,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS if tags is not None: snapshot.save_tags(normalize_tag_list(tags)) - snapshot.save(update_fields=update_fields) + if payload.get("status") == Snapshot.StatusChoices.SEALED: + snapshot.cancel() + else: + snapshot.save(update_fields=update_fields) setattr(request, "with_archiveresults", False) return snapshot @@ -1164,6 +1183,40 @@ class TagSnapshotResponseSchema(Schema): tag_name: str +def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot: + snapshot_ref = str(snapshot_ref or "").strip().lower() + if not snapshot_ref: + raise HttpError(400, "Snapshot id is required") + + snapshot_qs = Snapshot.objects.only("id") + is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref) + if is_full_uuid: + try: + return snapshot_qs.get(pk=snapshot_ref) + except (Snapshot.DoesNotExist, ValueError): + pass + + if len(snapshot_ref) >= 14: + try: + return snapshot_qs.get(timestamp=snapshot_ref) + except Snapshot.DoesNotExist: + pass + except Snapshot.MultipleObjectsReturned: + snapshot = snapshot_qs.filter(timestamp=snapshot_ref).first() + if snapshot is not None: + return snapshot + + try: + return snapshot_qs.get(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref)) + except Snapshot.DoesNotExist: + raise HttpError(404, "Snapshot not found") from None + except Snapshot.MultipleObjectsReturned: + snapshot = snapshot_qs.filter(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref)).first() + if snapshot is None: + raise HttpError(404, "Snapshot not found") + return snapshot + + @router.get("/tags/search/", response=TagSearchResponseSchema, url_name="search_tags") def search_tags( request: HttpRequest, @@ -1195,10 +1248,7 @@ def search_tags( def _public_tag_listing_enabled() -> bool: - config = get_config() - if config.PUBLIC_SNAPSHOTS_LIST is not None: - return config.PUBLIC_SNAPSHOTS_LIST - return config.PUBLIC_INDEX + return get_config().PUBLIC_INDEX def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool: @@ -1223,8 +1273,13 @@ def tags_autocomplete(request: HttpRequest, q: str = ""): if not _request_has_tag_autocomplete_access(request): raise HttpError(401, "Authentication required") - tags = list(get_matching_tags(q, with_snapshot_counts=False)[: 50 if not q else 20]) - add_snapshot_counts(tags) + public_only = not getattr(request.user, "is_authenticated", False) and not getattr(request, "_api_token", None) + queryset = get_matching_tags(q, with_snapshot_counts=False) + public_snapshots = public_snapshots_queryset(Snapshot.objects.all()) + if public_only: + queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct() + tags = list(queryset[: 50 if not q else 20]) + add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None) return { "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags], @@ -1308,19 +1363,7 @@ def tag_snapshots_export(request: HttpRequest, tag_id: int): @router.post("/tags/add-to-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_add_to_snapshot") def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema): """Add a tag to a snapshot. Creates the tag if it doesn't exist.""" - # Get the snapshot - try: - snapshot = Snapshot.objects.get( - Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id), - ) - except Snapshot.DoesNotExist: - raise HttpError(404, "Snapshot not found") - except Snapshot.MultipleObjectsReturned: - snapshot = Snapshot.objects.filter( - Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id), - ).first() - if snapshot is None: - raise HttpError(404, "Snapshot not found") + snapshot = _get_snapshot_for_tag_edit(data.snapshot_id) # Get or create the tag if data.tag_name: @@ -1352,19 +1395,7 @@ def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema): @router.post("/tags/remove-from-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_remove_from_snapshot") def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema): """Remove a tag from a snapshot.""" - # Get the snapshot - try: - snapshot = Snapshot.objects.get( - Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id), - ) - except Snapshot.DoesNotExist: - raise HttpError(404, "Snapshot not found") - except Snapshot.MultipleObjectsReturned: - snapshot = Snapshot.objects.filter( - Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id), - ).first() - if snapshot is None: - raise HttpError(404, "Snapshot not found") + snapshot = _get_snapshot_for_tag_edit(data.snapshot_id) # Get the tag if data.tag_id: diff --git a/archivebox/api/v1_crawls.py b/archivebox/api/v1_crawls.py index d8845999..54e695dc 100644 --- a/archivebox/api/v1_crawls.py +++ b/archivebox/api/v1_crawls.py @@ -14,7 +14,6 @@ from ninja.errors import HttpError from archivebox.core.models import Snapshot from archivebox.crawls.models import Crawl -from archivebox.config.common import get_config from .auth import API_AUTH_METHODS @@ -33,22 +32,15 @@ class CrawlSchema(Schema): status: str retry_at: datetime | None + is_paused: bool urls: str max_depth: int - max_urls: int - crawl_max_size: int - snapshot_max_size: int - crawl_max_concurrent_snapshots: int tags_str: str config: dict # snapshots: List[SnapshotSchema] - @staticmethod - def resolve_crawl_max_concurrent_snapshots(obj): - return int(get_config(crawl=obj).CRAWL_MAX_CONCURRENT_SNAPSHOTS) - @staticmethod def resolve_created_by_id(obj): return str(obj.created_by_id) @@ -68,6 +60,7 @@ class CrawlSchema(Schema): class CrawlUpdateSchema(Schema): + action: str | None = None status: str | None = None retry_at: datetime | None = None tags: list[str] | None = None @@ -77,10 +70,6 @@ class CrawlUpdateSchema(Schema): class CrawlCreateSchema(Schema): urls: list[str] max_depth: int = 0 - max_urls: int = 0 - crawl_max_size: int = 0 - snapshot_max_size: int = 0 - crawl_max_concurrent_snapshots: int | None = None tags: list[str] | None = None tags_str: str = "" label: str = "" @@ -113,25 +102,12 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema): raise HttpError(400, "At least one URL is required") if data.max_depth not in (0, 1, 2, 3, 4): raise HttpError(400, "max_depth must be between 0 and 4") - if data.max_urls < 0: - raise HttpError(400, "max_urls must be >= 0") - if data.crawl_max_size < 0: - raise HttpError(400, "crawl_max_size must be >= 0") - if data.snapshot_max_size < 0: - raise HttpError(400, "snapshot_max_size must be >= 0") - if data.crawl_max_concurrent_snapshots is not None and data.crawl_max_concurrent_snapshots < 1: - raise HttpError(400, "crawl_max_concurrent_snapshots must be >= 1") tags = normalize_tag_list(data.tags, data.tags_str) config = dict(data.config or {}) - if data.crawl_max_concurrent_snapshots is not None: - config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = data.crawl_max_concurrent_snapshots crawl = Crawl.objects.create( urls="\n".join(urls), max_depth=data.max_depth, - max_urls=data.max_urls, - crawl_max_size=data.crawl_max_size, - snapshot_max_size=data.snapshot_max_size, tags_str=",".join(tags), label=data.label, notes=data.notes, @@ -167,6 +143,19 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema): payload = data.dict(exclude_unset=True) update_fields = ["modified_at"] + action = payload.pop("action", None) + if action: + if action == "pause": + crawl.pause() + return crawl + if action in ("resume", "unpause"): + crawl.resume() + return crawl + if action == "cancel": + crawl.cancel() + return crawl + raise HttpError(400, f"Invalid action: {action}") + tags = payload.pop("tags", None) tags_str = payload.pop("tags_str", None) if tags is not None or tags_str is not None: @@ -186,19 +175,7 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema): update_fields.append("retry_at") if payload.get("status") == Crawl.StatusChoices.SEALED: - cancelled_at = timezone.now() - crawl.retry_at = None - if "retry_at" not in update_fields: - update_fields.append("retry_at") - crawl.save(update_fields=update_fields) - Snapshot.objects.filter( - crawl=crawl, - status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED], - ).update( - status=Snapshot.StatusChoices.SEALED, - retry_at=None, - modified_at=cancelled_at, - ) + crawl.cancel() else: crawl.save(update_fields=update_fields) return crawl diff --git a/archivebox/base_models/admin.py b/archivebox/base_models/admin.py index 939f3171..2c38d342 100644 --- a/archivebox/base_models/admin.py +++ b/archivebox/base_models/admin.py @@ -117,7 +117,7 @@ class KeyValueWidget(forms.Widget): config_meta_json = json.dumps(config_options) html = f''' -
+
{datalist_options} @@ -606,7 +606,7 @@ class KeyValueWidget(forms.Widget): var newRow = document.createElement('div'); newRow.className = 'key-value-row'; newRow.style.cssText = 'margin-bottom: 6px;'; - newRow.innerHTML = '
' + + newRow.innerHTML = '
' + '' + ' str: return f'''
-
+
= 0") if crawl_max_size < 0: raise ValueError("crawl_max_size must be >= 0") + if crawl_timeout < 0: + raise ValueError("crawl_timeout must be >= 0") if snapshot_max_size < 0: raise ValueError("snapshot_max_size must be >= 0") if crawl_max_concurrent_snapshots < 1: @@ -158,6 +162,10 @@ def add( and crawl_max_concurrent_snapshots != int(effective_persona_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS) else {} ), + **({"CRAWL_MAX_URLS": max_urls} if max_urls else {}), + **({"CRAWL_MAX_SIZE": crawl_max_size} if crawl_max_size else {}), + **({"CRAWL_TIMEOUT": crawl_timeout} if crawl_timeout else {}), + **({"SNAPSHOT_MAX_SIZE": snapshot_max_size} if snapshot_max_size else {}), **({"PARSER": parser} if parser != "auto" else {}), **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}), **({"URL_DENYLIST": url_denylist} if url_denylist else {}), @@ -166,9 +174,6 @@ def add( crawl = Crawl.objects.create( urls=urls_content, max_depth=depth, - max_urls=max_urls, - crawl_max_size=crawl_max_size, - snapshot_max_size=snapshot_max_size, tags_str=tag, persona_id=persona_obj.id, label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]", @@ -274,6 +279,7 @@ def add( ) @click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)") @click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)") +@click.option("--crawl-timeout", type=int, default=0, help="Maximum total crawl runtime in seconds (0 = unlimited)") @click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)") @click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl") @click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3") @@ -297,6 +303,8 @@ def main(**kwargs): raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.") if int(kwargs.get("max_urls") or 0) < 0: raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls") + if int(kwargs.get("crawl_timeout") or 0) < 0: + raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout") try: kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size")) except ValueError as err: diff --git a/archivebox/cli/archivebox_extract.py b/archivebox/cli/archivebox_extract.py index 054382c1..8f536439 100644 --- a/archivebox/cli/archivebox_extract.py +++ b/archivebox/cli/archivebox_extract.py @@ -32,6 +32,7 @@ __command__ = "archivebox extract" import sys from collections import defaultdict +from itertools import product import rich_click as click @@ -45,7 +46,6 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int: through the shared crawl runner with the corresponding plugin selected. """ from rich import print as rprint - from django.utils import timezone from archivebox.core.models import ArchiveResult from archivebox.services.runner import run_crawl @@ -58,19 +58,29 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int: rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr) try: + was_paused = archiveresult.snapshot.is_paused archiveresult.reset_for_retry() snapshot = archiveresult.snapshot - snapshot.status = snapshot.StatusChoices.QUEUED - snapshot.retry_at = timezone.now() - snapshot.save(update_fields=["status", "retry_at", "modified_at"]) - + if not was_paused: + snapshot.queue_for_extraction() + else: + # A paused snapshot may still accept explicit maintenance for one + # ArchiveResult, but this path must not transition it back to + # queued/startable work. + snapshot.save(update_fields=["retry_at", "modified_at"]) crawl = snapshot.crawl - if crawl.status != crawl.StatusChoices.STARTED: - crawl.status = crawl.StatusChoices.QUEUED - crawl.retry_at = timezone.now() - crawl.save(update_fields=["status", "retry_at", "modified_at"]) + if not crawl.claim_processing_lock(lock_seconds=10): + rprint( + f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]", + file=sys.stderr, + ) + return 1 - run_crawl(str(crawl.id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin]) + try: + run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin]) + finally: + if was_paused: + snapshot.restore_paused_scheduler_marker() archiveresult.refresh_from_db() if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED: @@ -98,6 +108,7 @@ def run_plugins( plugins: str = "", wait: bool = True, emit_results: bool = True, + show_progress: bool = True, ) -> int: """ Run plugins on Snapshots from input. @@ -118,7 +129,9 @@ def run_plugins( TYPE_ARCHIVERESULT, ) from archivebox.core.models import Snapshot + from archivebox.core.models import ArchiveResult from archivebox.services.runner import run_crawl + from abx_dl.models import discover_plugins is_tty = sys.stdout.isatty() @@ -145,7 +158,7 @@ def run_plugins( if record_type == TYPE_SNAPSHOT: snapshot_id = record.get("id") if snapshot_id: - snapshot_ids.add(snapshot_id) + snapshot_ids.add(str(snapshot_id)) elif record.get("url"): # Look up by URL (get most recent if multiple exist) snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first() @@ -157,41 +170,124 @@ def run_plugins( elif record_type == TYPE_ARCHIVERESULT: snapshot_id = record.get("snapshot_id") if snapshot_id: - snapshot_ids.add(snapshot_id) + snapshot_ids.add(str(snapshot_id)) plugin_name = record.get("plugin") if plugin_name and not plugins_list: requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name)) elif "id" in record: # Assume it's a snapshot ID - snapshot_ids.add(record["id"]) + snapshot_ids.add(str(record["id"])) if not snapshot_ids: rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr) return 1 - # Get snapshots and ensure they have pending ArchiveResults - processed_count = 0 - for snapshot_id in snapshot_ids: - try: - snapshot = Snapshot.objects.get(id=snapshot_id) - except Snapshot.DoesNotExist: - rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr) - continue + existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id")) + existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots} + existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots} + missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids) + for snapshot_id in missing_snapshot_ids: + rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr) - requested_plugin_names = set(plugins_list) | requested_plugins_by_snapshot.get(str(snapshot.id), set()) - for plugin_name in requested_plugin_names: - existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first() - if existing_result: - existing_result.reset_for_retry() + # Queue only the target plugin rows. Bulk updates keep large reindex runs + # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start. + requested_pairs: set[tuple[str, str]] = set() + if plugins_list: + requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list)) + else: + requested_pairs.update( + (snapshot_id, plugin_name) + for snapshot_id, plugin_names in requested_plugins_by_snapshot.items() + if snapshot_id in existing_snapshot_ids + for plugin_name in plugin_names + ) + plugins_by_name = discover_plugins() + requested_rows: set[tuple[str, str, str]] = set() + for snapshot_id, plugin_name in requested_pairs: + plugin = plugins_by_name.get(plugin_name) + hooks = plugin.filter_hooks("Snapshot") if plugin is not None else [] + if hooks: + requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks) + else: + requested_rows.add((snapshot_id, plugin_name, "")) - # Reset snapshot status to allow processing - if snapshot.status == Snapshot.StatusChoices.SEALED: - snapshot.status = Snapshot.StatusChoices.STARTED - snapshot.retry_at = timezone.now() - snapshot.save() + reset_fields = { + "status": ArchiveResult.StatusChoices.QUEUED, + "output_str": "", + "output_json": None, + "output_files": {}, + "output_size": 0, + "output_mimetypes": "", + "start_ts": None, + "end_ts": None, + "modified_at": timezone.now(), + } + if plugins_list: + ArchiveResult.objects.filter(snapshot_id__in=existing_snapshot_ids, plugin__in=plugins_list).update(**reset_fields) + elif requested_plugins_by_snapshot: + snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set) + for snapshot_id, plugin_names in requested_plugins_by_snapshot.items(): + if snapshot_id in existing_snapshot_ids: + for plugin_name in plugin_names: + snapshot_ids_by_plugin[plugin_name].add(snapshot_id) + for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items(): + ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(**reset_fields) + existing_rows = set( + ArchiveResult.objects.filter( + snapshot_id__in=existing_snapshot_ids, + plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows}, + ).values_list("snapshot_id", "plugin", "hook_name"), + ) + missing_rows = requested_rows - {(str(snapshot_id), plugin_name, hook_name) for snapshot_id, plugin_name, hook_name in existing_rows} + if missing_rows: + ArchiveResult.objects.bulk_create( + [ + ArchiveResult( + snapshot_id=snapshot_id, + plugin=plugin_name, + hook_name=hook_name, + status=ArchiveResult.StatusChoices.QUEUED, + ) + for snapshot_id, plugin_name, hook_name in sorted(missing_rows) + ], + batch_size=500, + ) - processed_count += 1 + processed_count = len(existing_snapshot_ids) + queue_at = timezone.now() + if existing_snapshot_ids: + if requested_rows: + # Targeted ArchiveResult retries use retry_at as the scheduling + # signal and keep sealed snapshots sealed so extractors are not + # re-run outside the explicitly queued plugin rows. Paused snapshots + # also keep status=paused here: `retry_at` only asks the orchestrator + # to process the queued plugin rows, and run_due_snapshot restores + # retry_at=MAX afterward instead of resuming the snapshot lifecycle. + Snapshot.objects.filter(id__in=existing_snapshot_ids).update( + retry_at=queue_at, + modified_at=queue_at, + ) + else: + # No plugin rows were requested, so this is a full snapshot retry. + Snapshot.objects.filter(id__in=existing_snapshot_ids).update( + status=Snapshot.StatusChoices.QUEUED, + retry_at=queue_at, + current_step=0, + modified_at=queue_at, + ) + if existing_crawl_ids and not requested_rows: + from archivebox.crawls.models import Crawl + + Crawl.objects.filter(id__in=existing_crawl_ids).exclude(status=Crawl.StatusChoices.STARTED).update( + status=Crawl.StatusChoices.QUEUED, + retry_at=queue_at, + modified_at=queue_at, + ) + Crawl.objects.filter(id__in=existing_crawl_ids, status=Crawl.StatusChoices.STARTED).update( + retry_at=queue_at, + modified_at=queue_at, + ) if processed_count == 0: rprint("[red]No snapshots to process[/red]", file=sys.stderr) @@ -203,14 +299,19 @@ def run_plugins( if wait: rprint("[blue]Running plugins...[/blue]", file=sys.stderr) snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set) - for snapshot_id in snapshot_ids: - try: - snapshot = Snapshot.objects.only("id", "crawl_id").get(id=snapshot_id) - except Snapshot.DoesNotExist: - continue - snapshot_ids_by_crawl[str(snapshot.crawl_id)].add(str(snapshot.id)) + for snapshot_id, crawl_id in existing_snapshots: + snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id)) for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items(): + from archivebox.crawls.models import Crawl + + crawl = Crawl.objects.get(id=crawl_id) + if not crawl.claim_processing_lock(lock_seconds=10): + rprint( + f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]", + file=sys.stderr, + ) + return 1 selected_plugins = ( plugins_list or sorted( @@ -222,6 +323,7 @@ def run_plugins( crawl_id, snapshot_ids=sorted(crawl_snapshot_ids), selected_plugins=selected_plugins, + show_progress=show_progress, ) if not emit_results: diff --git a/archivebox/cli/archivebox_persona.py b/archivebox/cli/archivebox_persona.py index d2cb9127..88691790 100644 --- a/archivebox/cli/archivebox_persona.py +++ b/archivebox/cli/archivebox_persona.py @@ -31,12 +31,8 @@ import os import sys import shutil import platform -import subprocess -import tempfile -import json from pathlib import Path from collections.abc import Iterable -from collections import OrderedDict import rich_click as click from rich import print as rprint @@ -214,136 +210,6 @@ CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"} # Cookie Extraction via CDP # ============================================================================= -NETSCAPE_COOKIE_HEADER = [ - "# Netscape HTTP Cookie File", - "# https://curl.se/docs/http-cookies.html", - "# This file was generated by ArchiveBox persona cookie extraction", - "#", - "# Format: domain\\tincludeSubdomains\\tpath\\tsecure\\texpiry\\tname\\tvalue", - "", -] - - -def _parse_netscape_cookies(path: Path) -> "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]": - cookies = OrderedDict() - if not path.exists(): - return cookies - - for line in path.read_text().splitlines(): - if not line or line.startswith("#"): - continue - parts = line.split("\t") - if len(parts) < 7: - continue - domain, include_subdomains, cookie_path, secure, expiry, name, value = parts[:7] - key = (domain, cookie_path, name) - cookies[key] = (domain, include_subdomains, cookie_path, secure, expiry, name, value) - return cookies - - -def _write_netscape_cookies(path: Path, cookies: "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]") -> None: - lines = list(NETSCAPE_COOKIE_HEADER) - for cookie in cookies.values(): - lines.append("\t".join(cookie)) - path.write_text("\n".join(lines) + "\n") - - -def _merge_netscape_cookies(existing_file: Path, new_file: Path) -> None: - existing = _parse_netscape_cookies(existing_file) - new = _parse_netscape_cookies(new_file) - for key, cookie in new.items(): - existing[key] = cookie - _write_netscape_cookies(existing_file, existing) - - -def extract_cookies_via_cdp( - user_data_dir: Path, - output_file: Path, - profile_dir: str | None = None, - chrome_binary: str | None = None, -) -> bool: - """ - Launch Chrome with the given user data dir and extract cookies via CDP. - - Returns True if successful, False otherwise. - """ - from archivebox.config.common import get_config - - # Find the cookie extraction script - chrome_plugin_dir = Path(__file__).parent.parent / "plugins" / "chrome" - extract_script = chrome_plugin_dir / "extract_cookies.js" - - if not extract_script.exists(): - rprint(f"[yellow]Cookie extraction script not found at {extract_script}[/yellow]", file=sys.stderr) - return False - - # Get node modules dir - node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules" - - # Set up environment - env = os.environ.copy() - env["NODE_MODULES_DIR"] = str(node_modules_dir) - env["CHROME_USER_DATA_DIR"] = str(user_data_dir) - env["CHROME_HEADLESS"] = "true" - if chrome_binary: - env["CHROME_BINARY"] = str(chrome_binary) - output_path = output_file - temp_output = None - temp_dir = None - if output_file.exists(): - temp_dir = Path(tempfile.mkdtemp(prefix="ab_cookies_")) - temp_output = temp_dir / "cookies.txt" - output_path = temp_output - if profile_dir: - extra_arg = f"--profile-directory={profile_dir}" - existing_extra = env.get("CHROME_ARGS_EXTRA", "").strip() - args_list = [] - if existing_extra: - if existing_extra.startswith("["): - try: - parsed = json.loads(existing_extra) - if isinstance(parsed, list): - args_list.extend(str(x) for x in parsed) - except Exception: - args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()]) - else: - args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()]) - args_list.append(extra_arg) - env["CHROME_ARGS_EXTRA"] = json.dumps(args_list) - - env["COOKIES_OUTPUT_FILE"] = str(output_path) - - try: - result = subprocess.run( - ["node", str(extract_script)], - env=env, - capture_output=True, - text=True, - timeout=60, - ) - - if result.returncode == 0: - if temp_output and temp_output.exists(): - _merge_netscape_cookies(output_file, temp_output) - return True - else: - rprint(f"[yellow]Cookie extraction failed: {result.stderr}[/yellow]", file=sys.stderr) - return False - - except subprocess.TimeoutExpired: - rprint("[yellow]Cookie extraction timed out[/yellow]", file=sys.stderr) - return False - except FileNotFoundError: - rprint("[yellow]Node.js not found. Cannot extract cookies.[/yellow]", file=sys.stderr) - return False - except Exception as e: - rprint(f"[yellow]Cookie extraction error: {e}[/yellow]", file=sys.stderr) - return False - finally: - if temp_dir and temp_dir.exists(): - shutil.rmtree(temp_dir, ignore_errors=True) - - # ============================================================================= # Validation Helpers # ============================================================================= diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py index 5d0b11ad..2bec20a0 100644 --- a/archivebox/cli/archivebox_run.py +++ b/archivebox/cli/archivebox_run.py @@ -40,6 +40,7 @@ Examples: __package__ = "archivebox.cli" __command__ = "archivebox run" +import os import sys from collections import defaultdict @@ -111,10 +112,10 @@ def process_stdin_records() -> int: crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id}) if crawl: - crawl.retry_at = timezone.now() - if crawl.status not in [Crawl.StatusChoices.SEALED]: - crawl.status = Crawl.StatusChoices.QUEUED - crawl.save() + crawl.update_and_requeue( + status=Crawl.StatusChoices.QUEUED, + retry_at=timezone.now(), + ) full_crawl_ids.add(str(crawl.id)) run_all_plugins_for_crawl.add(str(crawl.id)) output_records.append(crawl.to_json()) @@ -132,15 +133,7 @@ def process_stdin_records() -> int: snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id}) if snapshot: - snapshot.retry_at = timezone.now() - if snapshot.status not in [Snapshot.StatusChoices.SEALED]: - snapshot.status = Snapshot.StatusChoices.QUEUED - snapshot.save() - crawl = snapshot.crawl - crawl.retry_at = timezone.now() - if crawl.status != Crawl.StatusChoices.STARTED: - crawl.status = Crawl.StatusChoices.QUEUED - crawl.save(update_fields=["status", "retry_at", "modified_at"]) + snapshot.queue_for_extraction() crawl_id = str(snapshot.crawl_id) snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id)) run_all_plugins_for_crawl.add(crawl_id) @@ -177,15 +170,7 @@ def process_stdin_records() -> int: snapshot = None if snapshot: - snapshot.retry_at = timezone.now() - if snapshot.status != Snapshot.StatusChoices.STARTED: - snapshot.status = Snapshot.StatusChoices.QUEUED - snapshot.save(update_fields=["status", "retry_at", "modified_at"]) - crawl = snapshot.crawl - crawl.retry_at = timezone.now() - if crawl.status != Crawl.StatusChoices.STARTED: - crawl.status = Crawl.StatusChoices.QUEUED - crawl.save(update_fields=["status", "retry_at", "modified_at"]) + snapshot.queue_for_extraction() crawl_id = str(snapshot.crawl_id) snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id)) if plugin_name: @@ -236,6 +221,13 @@ def process_stdin_records() -> int: targeted_crawl_ids = full_crawl_ids | set(snapshot_ids_by_crawl) if targeted_crawl_ids: for crawl_id in sorted(targeted_crawl_ids): + try: + crawl = Crawl.objects.get(id=crawl_id) + except Crawl.DoesNotExist: + continue + if not crawl.claim_processing_lock(lock_seconds=10): + rprint(f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]", file=sys.stderr) + return 1 run_crawl( crawl_id, snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]), @@ -253,17 +245,20 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int: Returns exit code (0 = success, 1 = error). """ - from django.utils import timezone + from archivebox.config import CONSTANTS from archivebox.machine.models import Machine, Process - from archivebox.services.runner import cleanup_orchestrator_state, run_pending_crawls + from archivebox.services.supervision_service import healthy_orchestrator + from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls - cleanup_orchestrator_state(include_chrome=True) + recover_orchestrator_state(include_chrome=True) Machine.current() + existing = healthy_orchestrator(data_dir=CONSTANTS.DATA_DIR) current = Process.current() - if current.process_type != Process.TypeChoices.ORCHESTRATOR: - current.process_type = Process.TypeChoices.ORCHESTRATOR - current.save(update_fields=["process_type", "modified_at"]) - + existing_pid = existing.get("pid") if isinstance(existing, dict) else getattr(existing, "pid", None) + if existing_pid and existing_pid != os.getpid(): + rprint(f"[green][*] Existing ArchiveBox orchestrator pid={existing_pid} is already running.[/green]", file=sys.stderr) + return 0 + current.mark_running(process_type=Process.TypeChoices.ORCHESTRATOR, pwd=str(CONSTANTS.DATA_DIR), timeout=0) try: run_pending_crawls(daemon=daemon, crawl_id=crawl_id) return 0 @@ -275,9 +270,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int: finally: current.refresh_from_db() if current.status != Process.StatusChoices.EXITED: - current.status = Process.StatusChoices.EXITED - current.ended_at = current.ended_at or timezone.now() - current.save(update_fields=["status", "ended_at", "modified_at"]) + current.mark_exited() @click.command() @@ -328,11 +321,15 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str): def run_snapshot_worker(snapshot_id: str) -> int: from archivebox.core.models import Snapshot - from archivebox.services.runner import run_crawl + from archivebox.services.runner import run_due_snapshot + from django.utils import timezone try: snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id) - run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)]) + if snapshot.retry_at is None: + Snapshot.objects.filter(pk=snapshot.pk).update(retry_at=timezone.now(), modified_at=timezone.now()) + snapshot.refresh_from_db() + run_due_snapshot(snapshot, lock_seconds=60) return 0 except KeyboardInterrupt: return 0 diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index 57abfc49..71c59c5f 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -13,83 +13,6 @@ from archivebox.misc.util import docstring, enforce_types from archivebox.config.common import get_config -def stop_existing_background_runner(*, machine, process_model, supervisor=None, stop_worker_fn=None, log=print) -> int: - """Stop any existing orchestrator process so the server can take ownership.""" - running_runners = list( - process_model.objects.filter( - machine=machine, - status=process_model.StatusChoices.RUNNING, - process_type=process_model.TypeChoices.ORCHESTRATOR, - ).order_by("created_at"), - ) - - if not running_runners: - return 0 - - log("[yellow][*] Stopping existing ArchiveBox background runner...[/yellow]") - - if supervisor is not None and stop_worker_fn is not None: - for worker_name in ("worker_runner", "worker_runner_watch"): - try: - stop_worker_fn(supervisor, worker_name) - except Exception: - pass - - for proc in running_runners: - try: - proc.kill_tree(graceful_timeout=2.0) - except Exception: - try: - proc.terminate(graceful_timeout=2.0) - except Exception: - pass - - return len(running_runners) - - -def _read_supervisor_worker_command(worker_name: str) -> str: - from archivebox.workers.supervisord_util import WORKERS_DIR_NAME, get_sock_file - - worker_conf = get_sock_file().parent / WORKERS_DIR_NAME / f"{worker_name}.conf" - if not worker_conf.exists(): - return "" - - for line in worker_conf.read_text().splitlines(): - if line.startswith("command="): - return line.removeprefix("command=").strip() - return "" - - -def _worker_command_matches_bind(command: str, host: str, port: str) -> bool: - if not command: - return False - return f"{host}:{port}" in command or (f"--bind={host}" in command and f"--port={port}" in command) - - -def stop_existing_server_workers(*, supervisor, stop_worker_fn, host: str, port: str, log=print) -> int: - """Stop existing ArchiveBox web workers if they already own the requested bind.""" - stopped = 0 - - for worker_name in ("worker_runserver", "worker_daphne"): - try: - proc = supervisor.getProcessInfo(worker_name) if supervisor else None - except Exception: - proc = None - if not isinstance(proc, dict) or proc.get("statename") != "RUNNING": - continue - - command = _read_supervisor_worker_command(worker_name) - if not _worker_command_matches_bind(command, host, port): - continue - - if stopped == 0: - log("[yellow][*] Taking over existing ArchiveBox web server on same port...[/yellow]") - stop_worker_fn(supervisor, worker_name) - stopped += 1 - - return stopped - - @enforce_types def server( runserver_args: Iterable[str] | None = None, @@ -144,60 +67,17 @@ def server( pass from archivebox.workers.supervisord_util import ( - get_existing_supervisord_process, - get_worker, - stop_worker, start_server_workers, + stop_existing_supervisord_process, is_port_in_use, ) - from archivebox.machine.models import Machine, Process - - machine = Machine.current() - supervisor = get_existing_supervisord_process() - stop_existing_background_runner( - machine=machine, - process_model=Process, - supervisor=supervisor, - stop_worker_fn=stop_worker, + from archivebox.machine.models import Process + from archivebox.services.supervision_service import ( + command_owns_runtime_stack, + current_command, + standby_until_runtime_stack_needed, ) - if supervisor: - stop_existing_server_workers( - supervisor=supervisor, - stop_worker_fn=stop_worker, - host=host, - port=port, - ) - - # Check if port is already in use - if is_port_in_use(host, int(port)): - print(f"[red][X] Error: Port {port} is already in use[/red]") - print(f" Another process (possibly daphne or runserver) is already listening on {host}:{port}") - print(" Stop the conflicting process or choose a different port") - sys.exit(1) - - supervisor = get_existing_supervisord_process() - if supervisor: - server_worker_name = "worker_runserver" if run_in_debug else "worker_daphne" - server_proc = get_worker(supervisor, server_worker_name) - server_state = server_proc.get("statename") if isinstance(server_proc, dict) else None - if server_state == "RUNNING": - runner_proc = get_worker(supervisor, "worker_runner") - runner_watch_proc = get_worker(supervisor, "worker_runner_watch") - runner_state = runner_proc.get("statename") if isinstance(runner_proc, dict) else None - runner_watch_state = runner_watch_proc.get("statename") if isinstance(runner_watch_proc, dict) else None - print("[red][X] Error: ArchiveBox server is already running[/red]") - print( - f" [green]√[/green] Web server ({server_worker_name}) is RUNNING on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]", - ) - if runner_state == "RUNNING": - print(" [green]√[/green] Background runner (worker_runner) is RUNNING") - if runner_watch_state == "RUNNING": - print(" [green]√[/green] Reload watcher (worker_runner_watch) is RUNNING") - print() - print("[yellow]To stop the existing server, run:[/yellow]") - print(' pkill -f "archivebox server"') - print(" pkill -f supervisord") - sys.exit(1) + from archivebox.core.shutdown_util import foreground_shutdown_signals if run_in_debug: print("[green][+] Starting ArchiveBox webserver in DEBUG mode...[/green]") @@ -211,7 +91,42 @@ def server( ) print(" > Writing ArchiveBox error log to ./logs/errors.log") print() - start_server_workers(host=host, port=port, daemonize=daemonize, debug=run_in_debug, reload=reload, nothreading=nothreading) + bind_url = f"http://{host}:{port}" + command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url) + + try: + with foreground_shutdown_signals(): + while True: + standby_until_runtime_stack_needed(command, data_dir=config.DATA_DIR) + sys.stdout.write(f"[*] ArchiveBox server parent pid={os.getpid()} is now running the orchestrator and server...\n") + sys.stdout.flush() + stop_existing_supervisord_process() + if is_port_in_use(host, int(port)): + print(f"[red][X] Error: Port {port} is already in use[/red]") + print(f" Another process outside this ArchiveBox runtime is listening on {host}:{port}") + sys.exit(1) + + result = start_server_workers( + host=host, + port=port, + daemonize=daemonize, + debug=run_in_debug, + reload=reload, + nothreading=nothreading, + keep_running=lambda: command_owns_runtime_stack(command, data_dir=config.DATA_DIR), + should_stop_supervisord=lambda: command_owns_runtime_stack(command, data_dir=config.DATA_DIR), + ) + if not command_owns_runtime_stack(command, data_dir=config.DATA_DIR): + print("[yellow][*] Another ArchiveBox command took over the runtime stack; standing by.[/yellow]") + continue + if result == "exited": + print("[yellow][*] Runtime stack exited while this parent is still leader; restarting...[/yellow]") + continue + break + except KeyboardInterrupt: + pass + finally: + command.mark_exited() print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]") diff --git a/archivebox/cli/archivebox_snapshot.py b/archivebox/cli/archivebox_snapshot.py index 541319e4..d41b1e2b 100644 --- a/archivebox/cli/archivebox_snapshot.py +++ b/archivebox/cli/archivebox_snapshot.py @@ -35,8 +35,7 @@ from collections.abc import Iterable import rich_click as click from rich import print as rprint -from django.db.models import Case, IntegerField, Q, Sum, When -from django.db.models.functions import Coalesce +from django.db.models import Case, IntegerField, Q, QuerySet, When from archivebox.cli.cli_utils import apply_filters @@ -179,26 +178,17 @@ def create_snapshots( # ============================================================================= -def list_snapshots( +def build_snapshot_queryset( + *, status: str | None = None, url__icontains: str | None = None, url__istartswith: str | None = None, tag: str | None = None, crawl_id: str | None = None, - limit: int | None = None, sort: str | None = None, - csv: str | None = None, - with_headers: bool = False, search: str | None = None, query: str | None = None, -) -> int: - """ - List Snapshots as JSONL with optional filters. - - Exit codes: - 0: Success (even if no results) - """ - from archivebox.misc.jsonl import write_record +) -> QuerySet: from archivebox.core.models import Snapshot from archivebox.search import ( get_default_search_mode, @@ -207,24 +197,17 @@ def list_snapshots( query_search_index, ) - if with_headers and not csv: - rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr) - return 2 - - is_tty = sys.stdout.isatty() and not csv - queryset = Snapshot.objects.order_by("-created_at") + queryset = apply_filters( + queryset, + { + "status": status, + "url__icontains": url__icontains, + "url__istartswith": url__istartswith, + "crawl_id": crawl_id, + }, + ) - # Apply filters - filter_kwargs = { - "status": status, - "url__icontains": url__icontains, - "url__istartswith": url__istartswith, - "crawl_id": crawl_id, - } - queryset = apply_filters(queryset, filter_kwargs) - - # Tag filter requires special handling (M2M) if tag: queryset = queryset.filter(tags__name__iexact=tag) @@ -265,6 +248,48 @@ def list_snapshots( if sort: queryset = queryset.order_by(sort) + return queryset + + +def list_snapshots( + status: str | None = None, + url__icontains: str | None = None, + url__istartswith: str | None = None, + tag: str | None = None, + crawl_id: str | None = None, + limit: int | None = None, + sort: str | None = None, + csv: str | None = None, + with_headers: bool = False, + search: str | None = None, + query: str | None = None, +) -> int: + """ + List Snapshots as JSONL with optional filters. + + Exit codes: + 0: Success (even if no results) + """ + from archivebox.misc.jsonl import write_record + from archivebox.core.models import Snapshot + + if with_headers and not csv: + rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr) + return 2 + + is_tty = sys.stdout.isatty() and not csv + + queryset = build_snapshot_queryset( + status=status, + url__icontains=url__icontains, + url__istartswith=url__istartswith, + tag=tag, + crawl_id=crawl_id, + sort=sort, + search=search, + query=query, + ) + if not is_tty: if limit: limited_ids = list(queryset.values_list("id", flat=True)[:limit]) @@ -273,7 +298,7 @@ def list_snapshots( output_field=IntegerField(), ) queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order) - queryset = queryset.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).prefetch_related("tags") + queryset = queryset.prefetch_related("tags") elif limit: queryset = queryset[:limit] diff --git a/archivebox/cli/archivebox_status.py b/archivebox/cli/archivebox_status.py index 9cbd38ef..781df941 100644 --- a/archivebox/cli/archivebox_status.py +++ b/archivebox/cli/archivebox_status.py @@ -130,7 +130,7 @@ def status(out_dir: Path = DATA_DIR) -> None: print(" [green]archivebox manage createsuperuser[/green]") print() - recent_snapshots = snapshots_qs.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).order_by( + recent_snapshots = snapshots_qs.order_by( "-downloaded_at", "-modified_at", )[:10] @@ -141,7 +141,7 @@ def status(out_dir: Path = DATA_DIR) -> None: ( "[grey53] " f" > {str(snapshot.downloaded_at)[:16]} " - f"[{snapshot.num_outputs} {('X', '√')[snapshot.status == Snapshot.StatusChoices.SEALED]} {printable_filesize(snapshot.output_size_sum or 0)}] " + f"[{snapshot.num_outputs} {('X', '√')[snapshot.status == Snapshot.StatusChoices.SEALED]} {printable_filesize(snapshot.output_size or 0)}] " f'"{snapshot.title}": {snapshot.url}' "[/grey53]" )[: config.TERM_WIDTH], diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index 136a5456..8f357d23 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -3,15 +3,17 @@ __package__ = "archivebox.cli" import os +import asyncio +import shlex import time from typing import TYPE_CHECKING, Any -from collections.abc import Callable, Iterable +from collections.abc import Iterable from pathlib import Path import rich_click as click from django.core.exceptions import ObjectDoesNotExist -from django.db.models import Q, QuerySet +from django.db.models import QuerySet from archivebox.misc.util import enforce_types, docstring @@ -20,33 +22,6 @@ if TYPE_CHECKING: from archivebox.crawls.models import Crawl -LINK_FILTERS: dict[str, Callable[[str], Q]] = { - "exact": lambda pattern: Q(url=pattern), - "substring": lambda pattern: Q(url__icontains=pattern), - "regex": lambda pattern: Q(url__iregex=pattern), - "domain": lambda pattern: ( - Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}") - ), - "tag": lambda pattern: Q(tags__name=pattern), - "timestamp": lambda pattern: Q(timestamp=pattern), -} - - -def _apply_pattern_filters( - snapshots: QuerySet["Snapshot", "Snapshot"], - filter_patterns: list[str], - filter_type: str, -) -> QuerySet["Snapshot", "Snapshot"]: - filter_builder = LINK_FILTERS.get(filter_type) - if filter_builder is None: - raise SystemExit(2) - - query = Q() - for pattern in filter_patterns: - query |= filter_builder(pattern) - return snapshots.filter(query) - - def _get_snapshot_crawl(snapshot: "Snapshot") -> "Crawl | None": try: return snapshot.crawl @@ -73,17 +48,35 @@ def _build_filtered_snapshots_queryset( *, filter_patterns: Iterable[str], filter_type: str, - before: float | None, - after: float | None, + status: str | None = None, + url__icontains: str | None = None, + url__istartswith: str | None = None, + tag: str | None = None, + crawl_id: str | None = None, + limit: int | None = None, + sort: str | None = None, + search: str | None = None, + before: float | None = None, + after: float | None = None, resume: str | None = None, ): - from archivebox.core.models import Snapshot from datetime import datetime + from archivebox.cli.archivebox_snapshot import build_snapshot_queryset - snapshots = Snapshot.objects.all() + filter_patterns = tuple(filter_patterns) + snapshots = build_snapshot_queryset( + status=status, + url__icontains=url__icontains, + url__istartswith=url__istartswith, + tag=tag, + crawl_id=crawl_id, + sort=sort, + search=search, + query=" ".join(filter_patterns) if search else None, + ) - if filter_patterns: - snapshots = _apply_pattern_filters(snapshots, list(filter_patterns), filter_type) + if filter_patterns and not search: + snapshots = snapshots.filter_by_patterns(list(filter_patterns), filter_type) if before: snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before)) @@ -91,8 +84,14 @@ def _build_filtered_snapshots_queryset( snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after)) if resume: snapshots = snapshots.filter(timestamp__lte=resume) + if not sort: + snapshots = snapshots.order_by("-timestamp") + snapshots = snapshots.select_related("crawl") + if limit: + limited_ids = list(snapshots.values_list("id", flat=True)[:limit]) + snapshots = snapshots.model.objects.filter(id__in=limited_ids).select_related("crawl") - return snapshots.select_related("crawl").order_by("-bookmarked_at") + return snapshots def reindex_snapshots( @@ -100,53 +99,64 @@ def reindex_snapshots( *, search_plugins: list[str], batch_size: int, -) -> dict[str, int]: + collect_ids: bool = False, +) -> dict[str, Any]: from archivebox.cli.archivebox_extract import run_plugins - stats = {"processed": 0, "reconciled": 0, "queued": 0, "reindexed": 0} + stats: dict[str, Any] = {"processed": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []} records: list[dict[str, str]] = [] total = snapshots.count() print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}") - for snapshot in snapshots.iterator(chunk_size=batch_size): - stats["processed"] += 1 + def run_batch() -> None: + if not records: + return + batch_records = list(records) + # Index-only backfill intentionally queues only search ArchiveResult + # rows. The extract runner bumps Snapshot.retry_at so the orchestrator + # sees the maintenance work, but it does not change status away from + # PAUSED; run_due_snapshot restores retry_at=MAX after the targeted + # plugin rows finish. + exit_code = run_plugins( + args=(), + records=batch_records, + wait=False, + emit_results=False, + show_progress=False, + ) + if exit_code != 0: + raise SystemExit(exit_code) + print( + f" [{stats['processed']}/{total}] Queued {len(batch_records)} index jobs for orchestrator", + ) + records.clear() - if _get_snapshot_crawl(snapshot) is None: - continue + for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size): + try: + stats["processed"] += 1 - output_dir = Path(snapshot.output_dir) - has_directory = output_dir.exists() and output_dir.is_dir() - if has_directory: - snapshot.reconcile_with_index_json() - stats["reconciled"] += 1 + if _get_snapshot_crawl(snapshot) is None: + continue - for plugin_name in search_plugins: - existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first() - if existing_result: - existing_result.reset_for_retry() - records.append( - { - "type": "ArchiveResult", - "snapshot_id": str(snapshot.id), - "plugin": plugin_name, - }, - ) - stats["queued"] += 1 + if collect_ids: + stats["snapshot_ids"].append(str(snapshot.id)) + for plugin_name in search_plugins: + records.append( + { + "type": "ArchiveResult", + "snapshot_id": str(snapshot.id), + "plugin": plugin_name, + }, + ) + stats["queued"] += 1 + if len(records) >= batch_size: + run_batch() + except KeyboardInterrupt as err: + err.archivebox_resume = snapshot.timestamp + raise - if not records: - return stats - - exit_code = run_plugins( - args=(), - records=records, - wait=True, - emit_results=False, - ) - if exit_code != 0: - raise SystemExit(exit_code) - - stats["reindexed"] = len(records) + run_batch() return stats @@ -154,12 +164,21 @@ def reindex_snapshots( def update( filter_patterns: Iterable[str] = (), filter_type: str = "exact", + status: str | None = None, + url__icontains: str | None = None, + url__istartswith: str | None = None, + tag: str | None = None, + crawl_id: str | None = None, + limit: int | None = None, + sort: str | None = None, + search: str | None = None, before: float | None = None, after: float | None = None, resume: str | None = None, batch_size: int = 100, continuous: bool = False, index_only: bool = False, + migrate_only: bool = False, ) -> None: """ Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving. @@ -174,83 +193,173 @@ def update( """ from rich import print + from archivebox.config import CONSTANTS from archivebox.config.django import setup_django setup_django() + from archivebox.machine.models import Process + from archivebox.services.supervision_service import current_command, ensure_daemon_stack + from archivebox.workers.supervisord_util import stop_existing_supervisord_process - from django.core.management import call_command + command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR) + is_filtered_update = any( + ( + filter_patterns, + status, + url__icontains, + url__istartswith, + tag, + crawl_id, + limit, + sort, + search, + before, + after, + ), + ) + touched_snapshot_ids: set[str] = set() + + from archivebox.misc.checks import check_migrations - # Run migrations first to ensure DB schema is up-to-date - print("[*] Checking for pending migrations...") try: - call_command("migrate", "--no-input", verbosity=0) - except Exception as e: - print(f"[!] Warning: Migration check failed: {e}") + # Run migrations first to ensure DB schema is up-to-date + print("[*] Checking for pending migrations...") + check_migrations(auto_apply=True) + stop_existing_supervisord_process() - while True: - if index_only: - search_plugins = _get_search_indexing_plugins() - if not search_plugins: - print("[*] No search indexing plugins are available, nothing to backfill.") + while True: + do_migrate = migrate_only or not index_only + do_index = index_only or not migrate_only + do_run_until_idle = do_migrate or do_index + + if do_migrate: + if filter_patterns or status or url__icontains or url__istartswith or tag or crawl_id or limit or sort or search or before or after: + print("[*] Processing filtered snapshots from database...") + stats = process_filtered_snapshots( + filter_patterns=filter_patterns, + filter_type=filter_type, + status=status, + url__icontains=url__icontains, + url__istartswith=url__istartswith, + tag=tag, + crawl_id=crawl_id, + limit=limit, + sort=sort, + search=search, + before=before, + after=after, + resume=resume, + batch_size=batch_size, + queue_for_archiving=do_run_until_idle, + ) + print_stats(stats) + touched_snapshot_ids.update(stats.get("snapshot_ids", [])) + else: + stats_combined = {"phase1": {}, "phase2": {}} + + print("[*] Phase 1: Draining old archive/ directories (0.8.x β†’ 0.9.x migration)...") + stats_combined["phase1"] = drain_old_archive_dirs( + resume_from=resume, + batch_size=batch_size, + ) + + print("[*] Phase 2: Processing all database snapshots (most recent first)...") + stats_combined["phase2"] = process_all_db_snapshots(batch_size=batch_size, resume=resume) + print_combined_stats(stats_combined) + + if do_index: + ensure_daemon_stack(reason="search indexing") + search_plugins = _get_search_indexing_plugins() + if not search_plugins: + print("[*] No search indexing plugins are available, nothing to backfill.") + else: + snapshots = _build_filtered_snapshots_queryset( + filter_patterns=filter_patterns, + filter_type=filter_type, + status=status, + url__icontains=url__icontains, + url__istartswith=url__istartswith, + tag=tag, + crawl_id=crawl_id, + limit=limit, + sort=sort, + search=search, + before=before, + after=after, + resume=resume, + ) + stats = reindex_snapshots( + snapshots, + search_plugins=search_plugins, + batch_size=batch_size, + collect_ids=is_filtered_update, + ) + print_index_stats(stats) + touched_snapshot_ids.update(stats.get("snapshot_ids", [])) + + if do_run_until_idle: + print("[*] Phase 3: Running queued/interrupted crawl work until idle...") + from archivebox.cli.archivebox_run import run_runner, run_snapshot_worker + + if is_filtered_update: + if not touched_snapshot_ids: + print("[*] No matching snapshots queued work for the runner.") + for snapshot_id in sorted(touched_snapshot_ids): + exit_code = run_snapshot_worker(snapshot_id) + if exit_code != 0: + raise SystemExit(exit_code) + else: + exit_code = run_runner(daemon=False) + if exit_code != 0: + raise SystemExit(exit_code) + + if not continuous: break - if not (filter_patterns or before or after): - print("[*] Phase 1: Draining old archive/ directories (0.8.x β†’ 0.9.x migration)...") - drain_old_archive_dirs( - resume_from=resume, - batch_size=batch_size, - ) - - snapshots = _build_filtered_snapshots_queryset( - filter_patterns=filter_patterns, - filter_type=filter_type, - before=before, - after=after, - resume=resume, - ) - stats = reindex_snapshots( - snapshots, - search_plugins=search_plugins, - batch_size=batch_size, - ) - print_index_stats(stats) - elif filter_patterns or before or after: - # Filtered mode: query DB only - print("[*] Processing filtered snapshots from database...") - stats = process_filtered_snapshots( - filter_patterns=filter_patterns, - filter_type=filter_type, - before=before, - after=after, - resume=resume, - batch_size=batch_size, - ) - print_stats(stats) - else: - # Full mode: drain old dirs + process DB - stats_combined = {"phase1": {}, "phase2": {}} - - print("[*] Phase 1: Draining old archive/ directories (0.8.x β†’ 0.9.x migration)...") - stats_combined["phase1"] = drain_old_archive_dirs( - resume_from=resume, - batch_size=batch_size, - ) - - print("[*] Phase 2: Processing all database snapshots (most recent first)...") - stats_combined["phase2"] = process_all_db_snapshots(batch_size=batch_size, resume=resume) - - # Phase 3: Deduplication (disabled for now) - # print('[*] Phase 3: Deduplicating...') - # stats_combined['deduplicated'] = Snapshot.find_and_merge_duplicates() - - print_combined_stats(stats_combined) - - if not continuous: - break - - print("[yellow]Sleeping 60s before next pass...[/yellow]") - time.sleep(60) - resume = None + print("[yellow]Sleeping 60s before next pass...[/yellow]") + time.sleep(60) + resume = None + except (KeyboardInterrupt, asyncio.CancelledError) as err: + exact_resume = getattr(err, "archivebox_resume", None) + resume_cmd = ["archivebox", "update"] + if migrate_only: + resume_cmd.append("--migrate-only") + if index_only: + resume_cmd.append("--index-only") + if batch_size != 100: + resume_cmd.extend(["--batch-size", str(batch_size)]) + if exact_resume or resume: + resume_cmd.extend(["--resume", str(exact_resume or resume)]) + if before is not None: + resume_cmd.extend(["--before", str(before)]) + if after is not None: + resume_cmd.extend(["--after", str(after)]) + if filter_type != "exact": + resume_cmd.extend(["--filter-type", filter_type]) + if status: + resume_cmd.extend(["--status", status]) + if url__icontains: + resume_cmd.extend(["--url__icontains", url__icontains]) + if url__istartswith: + resume_cmd.extend(["--url__istartswith", url__istartswith]) + if tag: + resume_cmd.extend(["--tag", tag]) + if crawl_id: + resume_cmd.extend(["--crawl-id", crawl_id]) + if limit: + resume_cmd.extend(["--limit", str(limit)]) + if sort: + resume_cmd.extend(["--sort", sort]) + if search: + resume_cmd.extend(["--search", search]) + resume_cmd.extend(str(pattern) for pattern in filter_patterns) + print("\n[red][X] archivebox update interrupted.[/red]") + print("[yellow]Hint: resume this idempotent update with:[/yellow]") + print(f" [green]{' '.join(shlex.quote(part) for part in resume_cmd)}[/green]") + raise SystemExit(130) + finally: + command.mark_exited() + stop_existing_supervisord_process() def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100) -> dict[str, int]: @@ -269,11 +378,9 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 from archivebox.core.models import Snapshot from archivebox.config.common import get_config from archivebox.crawls.models import Crawl - from django.db import transaction from django.utils import timezone - stats = {"processed": 0, "migrated": 0, "skipped": 0, "invalid": 0} - crawl_output_dirs: dict[str, Path] = {} + stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0} crawl_url_lines: dict[str, list[str]] = {} crawl_url_sets: dict[str, set[str]] = {} dirty_crawl_ids: set[str] = set() @@ -283,18 +390,27 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 if not archive_dir.exists(): return stats - for crawl in Crawl.objects.filter(label__startswith="[migration] orphaned").iterator(): - url_entries = crawl._iter_url_lines() - existing_urls = {url for _raw_line, url in url_entries if url} - lines = (crawl.urls or "").splitlines() - changed = False - for url in crawl.snapshot_set.order_by("timestamp").values_list("url", flat=True): - if url not in existing_urls: - lines.append(url) - existing_urls.add(url) - changed = True - if changed: - Crawl.objects.filter(pk=crawl.pk).update(urls="\n".join(lines), modified_at=timezone.now()) + last_crawl_id = None + while True: + crawl_qs = Crawl.objects.filter(label__startswith="[migration] orphaned").order_by("id") + if last_crawl_id is not None: + crawl_qs = crawl_qs.filter(id__gt=last_crawl_id) + crawl_batch = list(crawl_qs[:batch_size]) + if not crawl_batch: + break + for crawl in crawl_batch: + last_crawl_id = crawl.id + url_entries = crawl._iter_url_lines() + existing_urls = {url for _raw_line, url in url_entries if url} + lines = (crawl.urls or "").splitlines() + changed = False + for url in crawl.snapshot_set.order_by("timestamp").values_list("url", flat=True): + if url not in existing_urls: + lines.append(url) + existing_urls.add(url) + changed = True + if changed: + Crawl.objects.filter(pk=crawl.pk).update(urls="\n".join(lines), modified_at=timezone.now()) # Scan for real directories only (skip symlinks - they're already migrated) all_entries = list(os.scandir(archive_dir)) @@ -329,29 +445,17 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 continue try: + snapshot.status = Snapshot.StatusChoices.SEALED + snapshot.retry_at = timezone.now() Snapshot.objects.bulk_create([snapshot]) - snapshot.migrate_filesystem_to_current_version(source_dir=entry_path, config=runtime_config) Snapshot.objects.filter(pk=snapshot.pk).update( - fs_version=snapshot.fs_version, + status=Snapshot.StatusChoices.SEALED, + retry_at=snapshot.retry_at, ) - migration_cleanup = getattr(snapshot, "_pending_fs_migration_cleanup", None) - new_dir = None - if migration_cleanup: - old_dir, new_dir = migration_cleanup - transaction.on_commit( - lambda old_dir=old_dir, new_dir=new_dir, snapshot=snapshot: snapshot._cleanup_old_migration_dir(old_dir, new_dir), - ) - delattr(snapshot, "_pending_fs_migration_cleanup") crawl = _get_snapshot_crawl(snapshot) - crawl_dir = None if crawl is not None: crawl_cache_key = str(crawl.id) - crawl_dir = crawl_output_dirs.get(crawl_cache_key) - if crawl_dir is None: - crawl_dir = Path(crawl.output_dir) - crawl_output_dirs[crawl_cache_key] = crawl_dir - existing_urls = crawl_url_sets.get(crawl_cache_key) if existing_urls is None: url_entries = crawl._iter_url_lines() @@ -363,9 +467,8 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 existing_urls.add(snapshot.url) dirty_crawl_ids.add(crawl_cache_key) - snapshot.ensure_crawl_symlink(crawl_dir=crawl_dir, snapshot_dir=new_dir) - stats["migrated"] += 1 - print(f" [{stats['processed']}] Imported orphaned snapshot: {entry_path.name}") + stats["queued"] += 1 + print(f" [{stats['processed']}] Imported orphaned snapshot and queued migration: {entry_path.name}") except Exception as e: stats["skipped"] += 1 print(f" [{stats['processed']}] Skipped (error: {e}): {entry_path.name}") @@ -377,7 +480,9 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 if not has_valid_crawl: # Create a new crawl (created_by will default to system user) crawl = Crawl.objects.create(urls=snapshot.url) - # Use queryset update to avoid triggering save() hooks + # Use queryset update to avoid save() hooks and keep the SQLite + # write to one statement while the migration loop does filesystem + # work outside any transaction. from archivebox.core.models import Snapshot as SnapshotModel SnapshotModel.objects.filter(pk=snapshot.pk).update(crawl=crawl) @@ -386,32 +491,13 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 # Check if needs migration (0.8.x β†’ 0.9.x) try: - old_version = snapshot.fs_version - snapshot.migrate_filesystem_to_current_version(source_dir=entry_path, config=runtime_config) - if snapshot.fs_version != old_version or getattr(snapshot, "_pending_fs_migration_cleanup", None): + if snapshot.fs_migration_needed: Snapshot.objects.filter(pk=snapshot.pk).update( - fs_version=snapshot.fs_version, + retry_at=timezone.now(), + modified_at=timezone.now(), ) - migration_cleanup = getattr(snapshot, "_pending_fs_migration_cleanup", None) - new_dir = None - if migration_cleanup: - old_dir, new_dir = migration_cleanup - transaction.on_commit( - lambda old_dir=old_dir, new_dir=new_dir, snapshot=snapshot: snapshot._cleanup_old_migration_dir(old_dir, new_dir), - ) - delattr(snapshot, "_pending_fs_migration_cleanup") - crawl_dir = None - if snapshot.crawl_id: - crawl_cache_key = str(snapshot.crawl_id) - crawl_dir = crawl_output_dirs.get(crawl_cache_key) - if crawl_dir is None: - crawl = _get_snapshot_crawl(snapshot) - if crawl is not None: - crawl_dir = Path(crawl.output_dir) - crawl_output_dirs[crawl_cache_key] = crawl_dir - snapshot.ensure_crawl_symlink(crawl_dir=crawl_dir, snapshot_dir=new_dir) - stats["migrated"] += 1 - print(f" [{stats['processed']}] Migrated: {entry_path.name}") + stats["queued"] += 1 + print(f" [{stats['processed']}] Queued filesystem migration: {entry_path.name}") else: stats["skipped"] += 1 except Exception as e: @@ -425,7 +511,6 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 modified_at=timezone.now(), ) dirty_crawl_ids.clear() - transaction.commit() for crawl_id in tuple(dirty_crawl_ids): Crawl.objects.filter(pk=crawl_id).update( @@ -433,7 +518,6 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 modified_at=timezone.now(), ) dirty_crawl_ids.clear() - transaction.commit() return stats @@ -448,14 +532,21 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) - No orphan detection needed - we trust 1:1 mapping between DB and filesystem after Phase 1 has drained all old archive/ directories. """ - from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.config.common import get_config + import uuid + from archivebox.core.models import Snapshot from archivebox.crawls.models import Crawl - from django.db import transaction from django.utils import timezone - stats = {"processed": 0, "reconciled": 0, "sealed": 0, "crawls_sealed": 0} - runtime_config = get_config() + stats = { + "processed": 0, + "scanned_dirs": 0, + "updated_json": 0, + "updated_db": 0, + "queued": 0, + "sealed": 0, + "crawls_sealed": 0, + } + current_fs_version = Snapshot._fs_current_version() queryset = Snapshot.objects.all() if resume: @@ -463,89 +554,83 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) - total = queryset.count() print(f"[*] Processing {total} snapshots from database (most recent first)...") - # Process from most recent to least recent - for snapshot in queryset.select_related("crawl__created_by").order_by("-bookmarked_at").iterator(chunk_size=batch_size): - stats["processed"] += 1 + def update_in_batches(rows, *, label: str, **updates) -> int: + updated = 0 + checked = 0 + while True: + ids = list(rows.order_by("-timestamp").values_list("id", flat=True)[:batch_size]) + if not ids: + if updated: + print(f" [{label}] complete: {updated} rows updated") + return updated + checked += len(ids) + print(f" [{label}] updating next {len(ids)} rows (seen {checked})...") + # Each batch is one short UPDATE and is intentionally idempotent. + # If the command is interrupted, these rows no longer match on the + # next run and remaining rows continue from DB state. + updated += Snapshot.objects.filter(id__in=ids).update(**updates) + print(f" [{label}] updated {updated} rows so far") - # Skip snapshots with missing crawl references (orphaned by migration errors) - if _get_snapshot_crawl(snapshot) is None: - continue + now = timezone.now() + updated_rows = update_in_batches( + queryset.exclude( + status__in=[ + Snapshot.StatusChoices.QUEUED, + Snapshot.StatusChoices.STARTED, + Snapshot.StatusChoices.PAUSED, + Snapshot.StatusChoices.SEALED, + ], + ), + label="snapshot status normalization", + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + modified_at=now, + ) + stats["sealed"] += updated_rows + stats["updated_db"] += updated_rows + fs_version_rows = queryset.exclude(fs_version=current_fs_version) + stale_batch: list[tuple[uuid.UUID, uuid.UUID | None, str]] = [] + + def queue_stale_fs_batch() -> None: + if not stale_batch: + return + now = timezone.now() + snapshot_ids = [snapshot_id for snapshot_id, _crawl_id, _timestamp in stale_batch] + # Do not bump fs_version here. The orchestrator calls Snapshot.save(), + # which performs the idempotent filesystem migration and commits the new + # fs_version in the same serialized worker path as normal crawls. + updated = Snapshot.objects.filter(id__in=snapshot_ids).update( + retry_at=now, + modified_at=now, + ) + stats["processed"] += len(stale_batch) + stats["updated_db"] += updated + stats["queued"] += updated + print(f" [{stats['processed']}/{total}] Queued {updated} filesystem migrations for orchestrator...") + stale_batch.clear() + + for snapshot in fs_version_rows.only("id", "crawl_id", "timestamp").order_by("-timestamp").paged_iterator(chunk_size=batch_size): try: - # Check if snapshot has a directory on disk - from pathlib import Path + stale_batch.append((snapshot.id, snapshot.crawl_id, snapshot.timestamp)) + if len(stale_batch) >= batch_size: + queue_stale_fs_batch() + except KeyboardInterrupt as err: + err.archivebox_resume = snapshot.timestamp + raise + queue_stale_fs_batch() - output_dir = Path(snapshot.get_storage_path_for_version(snapshot.fs_version, config=runtime_config)) - has_directory = output_dir.exists() and output_dir.is_dir() - current_fs_version = Snapshot._fs_current_version() - update_values = { - "status": Snapshot.StatusChoices.SEALED, - "retry_at": None, - } - - # Only reconcile if directory exists (don't create empty directories for orphans) - if has_directory: - old_title = snapshot.title - snapshot.reconcile_with_index_json(output_dir=output_dir, update_existing_archive_results=False) - metadata_updates = [] - for archiveresult in ArchiveResult.objects.filter(snapshot=snapshot).only( - "id", - "snapshot_id", - "plugin", - "output_str", - "output_files", - "output_size", - "output_mimetypes", - "modified_at", - ): - if archiveresult.update_output_metadata_from_filesystem(snapshot_dir=output_dir, save=False): - metadata_updates.append(archiveresult) - if metadata_updates: - ArchiveResult.objects.bulk_update( - metadata_updates, - ["output_files", "output_size", "output_mimetypes", "modified_at"], - batch_size=batch_size, - ) - if snapshot.title != old_title: - update_values["title"] = snapshot.title - update_values["modified_at"] = timezone.now() - - # Clean up invalid field values from old migrations - if not isinstance(snapshot.current_step, int): - update_values["current_step"] = 0 - - if snapshot.fs_migration_needed: - legacy_dir = snapshot.get_storage_path_for_version("0.8.0", config=runtime_config) - current_dir = snapshot.get_storage_path_for_version(current_fs_version, config=runtime_config) - if legacy_dir.exists() or current_dir.exists(): - snapshot.migrate_filesystem_to_current_version(config=runtime_config) - update_values["fs_version"] = snapshot.fs_version - Snapshot.objects.filter(pk=snapshot.pk).update(**update_values) - else: - update_values["fs_version"] = current_fs_version - Snapshot.objects.filter(pk=snapshot.pk).update(**update_values) - else: - Snapshot.objects.filter(pk=snapshot.pk).update(**update_values) - - stats["reconciled"] += 1 if has_directory else 0 - stats["sealed"] += 1 - except Exception as e: - # Skip snapshots that can't be processed (e.g., missing crawl) - print(f" [!] Skipping snapshot {snapshot.id}: {e}") - continue - - if stats["processed"] % batch_size == 0: - transaction.commit() - print(f" [{stats['processed']}/{total}] Processed...") - - transaction.commit() now = timezone.now() stats["crawls_sealed"] = ( Crawl.objects.filter( status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED], ) .exclude( - snapshot_set__status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED], + snapshot_set__status__in=[ + Snapshot.StatusChoices.QUEUED, + Snapshot.StatusChoices.STARTED, + Snapshot.StatusChoices.PAUSED, + ], ) .update( status=Crawl.StatusChoices.SEALED, @@ -553,26 +638,44 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) - modified_at=now, ) ) + stats["updated_db"] += stats["crawls_sealed"] return stats def process_filtered_snapshots( filter_patterns: Iterable[str], filter_type: str, + status: str | None, + url__icontains: str | None, + url__istartswith: str | None, + tag: str | None, + crawl_id: str | None, + limit: int | None, + sort: str | None, + search: str | None, before: float | None, after: float | None, resume: str | None, batch_size: int, -) -> dict[str, int]: + queue_for_archiving: bool = True, +) -> dict[str, Any]: """Process snapshots matching filters (DB query only).""" - from django.db import transaction + from archivebox.core.models import Snapshot from django.utils import timezone - stats = {"processed": 0, "reconciled": 0, "queued": 0} + stats: dict[str, Any] = {"processed": 0, "updated_json": 0, "updated_db": 0, "queued": 0, "snapshot_ids": []} snapshots = _build_filtered_snapshots_queryset( filter_patterns=filter_patterns, filter_type=filter_type, + status=status, + url__icontains=url__icontains, + url__istartswith=url__istartswith, + tag=tag, + crawl_id=crawl_id, + limit=limit, + sort=sort, + search=search, before=before, after=after, resume=resume, @@ -581,7 +684,7 @@ def process_filtered_snapshots( total = snapshots.count() print(f"[*] Found {total} matching snapshots") - for snapshot in snapshots.select_related("crawl").iterator(chunk_size=batch_size): + for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size): stats["processed"] += 1 # Skip snapshots with missing crawl references @@ -589,30 +692,40 @@ def process_filtered_snapshots( continue try: - # Reconcile index.json with DB - snapshot.reconcile_with_index_json() - - # Clean up invalid field values from old migrations + stats["snapshot_ids"].append(str(snapshot.id)) + update_values = {} if not isinstance(snapshot.current_step, int): - snapshot.current_step = 0 + update_values["current_step"] = 0 + if queue_for_archiving: + update_values.update( + { + "status": Snapshot.StatusChoices.QUEUED, + "retry_at": timezone.now(), + "modified_at": timezone.now(), + }, + ) + if update_values: + # update() is intentionally used instead of save(); save() + # runs output-dir hooks, which must not happen while SQLite + # is holding the write lock for this state change. Index-only + # maintenance goes through reindex_snapshots/run_plugins instead + # so paused snapshots keep status=paused while only their + # targeted search ArchiveResult rows run. + Snapshot.objects.filter(pk=snapshot.pk).update(**update_values) + stats["updated_db"] += 1 - # Queue for archiving - snapshot.status = Snapshot.StatusChoices.QUEUED - snapshot.retry_at = timezone.now() - snapshot.save() - - stats["reconciled"] += 1 - stats["queued"] += 1 + stats["queued"] += 1 if queue_for_archiving else 0 + except KeyboardInterrupt as err: + err.archivebox_resume = snapshot.timestamp + raise except Exception as e: # Skip snapshots that can't be processed print(f" [!] Skipping snapshot {snapshot.id}: {e}") continue if stats["processed"] % batch_size == 0: - transaction.commit() print(f" [{stats['processed']}/{total}] Processed...") - transaction.commit() return stats @@ -622,9 +735,10 @@ def print_stats(stats: dict): print(f""" [green]Update Complete[/green] - Processed: {stats["processed"]} - Reconciled: {stats["reconciled"]} - Queued: {stats["queued"]} + Scanned rows: {stats["processed"]} + Updated JSON: {stats.get("updated_json", 0)} + Updated DB rows: {stats.get("updated_db", 0)} + Queued snapshots: {stats["queued"]} """) @@ -639,16 +753,17 @@ def print_combined_stats(stats_combined: dict): [green]Archive Update Complete[/green] Phase 1 (Drain Old Dirs): - Checked: {s1.get("processed", 0)} - Migrated: {s1.get("migrated", 0)} - Skipped: {s1.get("skipped", 0)} - Invalid: {s1.get("invalid", 0)} + Scanned dirs: {s1.get("processed", 0)} + Moved files: {s1.get("migrated", 0)} + Skipped dirs: {s1.get("skipped", 0)} + Invalid dirs: {s1.get("invalid", 0)} Phase 2 (Process DB): - Processed: {s2.get("processed", 0)} - Reconciled: {s2.get("reconciled", 0)} - Sealed: {s2.get("sealed", 0)} - Crawls: {s2.get("crawls_sealed", 0)} sealed + Scanned dirs: {s2.get("scanned_dirs", 0)} + Updated JSON: {s2.get("updated_json", 0)} + Updated DB rows: {s2.get("updated_db", 0)} + Sealed snapshots: {s2.get("sealed", 0)} + Sealed crawls: {s2.get("crawls_sealed", 0)} """) @@ -657,21 +772,28 @@ def print_index_stats(stats: dict[str, Any]) -> None: print(f""" [green]Search Reindex Complete[/green] - Processed: {stats["processed"]} - Reconciled: {stats["reconciled"]} - Queued: {stats["queued"]} - Reindexed: {stats["reindexed"]} + Scanned rows: {stats["processed"]} + Queued index jobs: {stats["queued"]} """) @click.command() @click.option("--resume", type=str, help="Resume from timestamp") +@click.option("--status", "-s", help="Filter by status (queued, started, sealed)") +@click.option("--url__icontains", help="Filter by URL contains") +@click.option("--url__istartswith", help="Filter by URL starts with") +@click.option("--tag", "-t", help="Filter by tag name") +@click.option("--crawl-id", help="Filter by crawl ID") +@click.option("--limit", "-n", type=int, help="Limit number of snapshots to update") +@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at") +@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for positional query") @click.option("--before", type=float, help="Only snapshots before timestamp") @click.option("--after", type=float, help="Only snapshots after timestamp") -@click.option("--filter-type", "-t", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact") +@click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact") @click.option("--batch-size", type=int, default=100, help="Commit every N snapshots") @click.option("--continuous", is_flag=True, help="Run continuously as background worker") @click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content") +@click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state") @click.argument("filter_patterns", nargs=-1) @docstring(update.__doc__) def main(**kwargs): diff --git a/archivebox/config/common.py b/archivebox/config/common.py index c986efc7..958f1700 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -139,8 +139,6 @@ class ServerConfig(BaseConfigSet): # CUSTOM_TEMPLATES_DIR: Path = Field(default=None) # this is now a constant PUBLIC_INDEX: bool = Field(default=True) - PUBLIC_SNAPSHOTS: bool = Field(default=True) - PUBLIC_SNAPSHOTS_LIST: bool | None = Field(default=None) PUBLIC_ADD_VIEW: bool = Field(default=False) ADMIN_USERNAME: str | None = Field(default=None) @@ -254,6 +252,7 @@ class ArchivingConfig(BaseConfigSet): MAX_DEPTH: int = Field(default=0) CRAWL_MAX_URLS: int = Field(default=0) CRAWL_MAX_SIZE: int = Field(default=0) + CRAWL_TIMEOUT: int = Field(default=0, description="Maximum total crawl runtime in seconds (0 = unlimited).") CRAWL_MAX_CONCURRENT_SNAPSHOTS: int = Field( default=4, description="Maximum number of snapshots to archive concurrently within one crawl.", @@ -274,6 +273,10 @@ class ArchivingConfig(BaseConfigSet): SAVE_DENYLIST: dict[str, list[str]] = Field(default={}) DEFAULT_PERSONA: str = Field(default="Default") + PERMISSIONS: str = Field( + default="public", + description="Snapshot visibility: public lists and serves content, unlisted serves direct links only, private requires admin login.", + ) DELETE_AFTER: str = Field( default="0", description=( @@ -310,6 +313,14 @@ class ArchivingConfig(BaseConfigSet): return "0" return str(value).strip() or "0" + @field_validator("PERMISSIONS", mode="before") + @classmethod + def validate_permissions(cls, value): + normalized = str(value or "public").strip().lower() + if normalized not in {"public", "unlisted", "private"}: + raise ValueError("PERMISSIONS must be one of: public, unlisted, private.") + return normalized + @property def URL_ALLOWLIST_PTN(self) -> re.Pattern | None: return re.compile(self.URL_ALLOWLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS) if self.URL_ALLOWLIST else None @@ -561,20 +572,7 @@ def get_config( machine = None if persona is None and crawl is not None: - from archivebox.personas.models import Persona - - persona_id = crawl.persona_id - if persona_id: - persona = Persona.objects.filter(id=persona_id).first() - if persona is None: - raise Persona.DoesNotExist(f"Crawl {crawl.id} references missing Persona {persona_id}") - - if persona is None: - crawl_config = crawl.config or {} - default_persona_name = str(crawl_config.get("DEFAULT_PERSONA") or "").strip() - if default_persona_name: - persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default") - persona.ensure_dirs() + persona = crawl.resolve_persona() config_data: ConfigPayload = dict(defaults or {}) config_data.update(ArchiveBoxConfig().model_dump(mode="json")) diff --git a/archivebox/config/django.py b/archivebox/config/django.py index 530c3afa..cab18036 100644 --- a/archivebox/config/django.py +++ b/archivebox/config/django.py @@ -28,14 +28,6 @@ STDERR = Console(stderr=True) logging.CONSOLE = CONSOLE -def setup_django_minimal(): - # sys.path.append(str(CONSTANTS.PACKAGE_DIR)) - # os.environ.setdefault('ARCHIVEBOX_DATA_DIR', str(CONSTANTS.DATA_DIR)) - # os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') - # django.setup() - raise Exception("dont use this anymore") - - DJANGO_SET_UP = False diff --git a/archivebox/core/actors.py b/archivebox/core/actors.py deleted file mode 100644 index e69de29b..00000000 diff --git a/archivebox/core/admin_archiveresults.py b/archivebox/core/admin_archiveresults.py index 529f108b..1970fdd4 100644 --- a/archivebox/core/admin_archiveresults.py +++ b/archivebox/core/admin_archiveresults.py @@ -105,6 +105,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None): "failed": ("#991b1b", "#fee2e2"), # red "queued": ("#6b7280", "#f3f4f6"), # gray "started": ("#92400e", "#fef3c7"), # amber + "paused": ("#1d4ed8", "#dbeafe"), # blue "backoff": ("#92400e", "#fef3c7"), "skipped": ("#475569", "#f1f5f9"), "noresults": ("#475569", "#f1f5f9"), @@ -152,7 +153,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None): ''' # Truncate output for display - full_output = result.output_str or "-" + full_output = result.output_str_for_display() or "-" output_display = full_output[:60] if len(full_output) > 60: output_display += "..." @@ -718,12 +719,12 @@ class ArchiveResultAdmin(BaseModelAdmin): return format_html( '↗️
{}
', build_snapshot_url(snapshot_id, output_path, request=request, config=config), - result.output_str, + result.output_str_for_display(), ) @admin.display(description="Output", ordering="output_str") def output_str_display(self, result): - output_text = str(result.output_str or "").strip() + output_text = str(result.output_str_for_display() or "").strip() if not output_text: return "-" @@ -787,7 +788,7 @@ class ArchiveResultAdmin(BaseModelAdmin): snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1] output_html = format_html( '
{}

', - result.output_str, + result.output_str_for_display(), ) snapshot_id = str(result.snapshot_id) request = getattr(self, "request", None) diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index 59e57f1f..41a9ed36 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -1,15 +1,24 @@ __package__ = "archivebox.core" -from functools import lru_cache +import asyncio import json +import threading +from copy import copy +from functools import lru_cache +from queue import Full, Queue +from types import SimpleNamespace +from urllib.parse import urlsplit +from uuid import UUID from django.contrib import admin, messages -from django.urls import path +from django.urls import path, reverse from django.shortcuts import get_object_or_404, redirect -from django.utils.html import format_html +from django.core.cache import cache +from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed, QueryDict, StreamingHttpResponse +from django.utils import timezone +from django.utils.html import format_html, format_html_join from django.utils.safestring import mark_safe -from django.db.models import Q, Sum, Count, Prefetch -from django.db.models.functions import Coalesce +from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch from django import forms from django.template import Template, RequestContext from django.contrib.admin.helpers import ActionForm @@ -18,8 +27,9 @@ from archivebox.config.common import get_config from archivebox.misc.util import htmldecode, urldecode from archivebox.misc.paginators import AcceleratedPaginator from archivebox.misc.logging_util import printable_filesize -from archivebox.search.admin import SearchResultsAdminMixin +from archivebox.search.admin import SEARCH_RESULT_CACHE_TTL, SearchResultsAdminMixin, SearchResultsChangeList, get_admin_search_cache_key from archivebox.core.host_utils import build_snapshot_url, build_web_url +from archivebox.core.tag_utils import get_or_create_tag from archivebox.hooks import discover_hooks, get_plugin_icon, get_plugin_name, get_plugins from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin @@ -27,13 +37,27 @@ from archivebox.workers.tasks import bg_archive_snapshots, bg_add from archivebox.core.models import Tag, Snapshot, ArchiveResult from archivebox.core.admin_archiveresults import render_archiveresults_list +from archivebox.core.permissions import ( + PERMISSIONS_CHOICES, + PERMISSIONS_PRIVATE, + PERMISSIONS_PUBLIC, + PERMISSIONS_UNLISTED, + get_snapshot_permissions, +) from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget from archivebox.crawls.models import Crawl +from archivebox.personas.models import Persona # GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False} GLOBAL_CONTEXT = {} +SNAPSHOT_PERMISSION_META = { + PERMISSIONS_PUBLIC: ("πŸ‘₯", "Public", "#047857", "#d1fae5"), + PERMISSIONS_UNLISTED: ("πŸ”—", "Unlisted", "#1d4ed8", "#dbeafe"), + PERMISSIONS_PRIVATE: ("πŸ”’", "Private", "#991b1b", "#fee2e2"), +} + @lru_cache(maxsize=1) def _plugin_sort_order() -> dict[str, int]: @@ -51,22 +75,12 @@ class SnapshotActionForm(ActionForm): ) def clean_tags(self): - """Parse comma-separated tag names into Tag objects.""" + """Parse comma-separated tag names without touching the DB.""" tags_str = self.cleaned_data.get("tags", "") if not tags_str: return [] - tag_names = [name.strip() for name in tags_str.split(",") if name.strip()] - tags = [] - for name in tag_names: - tag, _ = Tag.objects.get_or_create( - name__iexact=name, - defaults={"name": name}, - ) - # Use the existing tag if found by case-insensitive match - tag = Tag.objects.filter(name__iexact=name).first() or tag - tags.append(tag) - return tags + return [name.strip() for name in tags_str.split(",") if name.strip()] # TODO: allow selecting actions for specific extractor plugins? is this useful? # plugin = forms.ChoiceField( @@ -95,6 +109,268 @@ class TagNameListFilter(admin.SimpleListFilter): return queryset +class SnapshotPermissionsListFilter(admin.SimpleListFilter): + title = "permission" + parameter_name = "permissions" + + def lookups(self, request, model_admin): + return PERMISSIONS_CHOICES + + def queryset(self, request, queryset): + value = self.value() + if value: + global_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower() + has_overrides = ( + Snapshot.objects.filter(permissions__gt="").exists() + or Crawl.objects.filter(permissions__gt="").exists() + or Persona.objects.filter(permissions__gt="").exists() + ) + if not has_overrides: + return queryset if value == global_permissions else queryset.none() + + persona_query = Q(crawl__persona_id__in=self.persona_ids_for_value()) + if global_permissions == value: + valid_persona_ids = Persona.objects.values_list("id", flat=True) + persona_query |= Q(crawl__persona_id__isnull=True) | ~Q(crawl__persona_id__in=valid_persona_ids) + return queryset.filter( + Q(permissions=value) + | (Q(permissions__isnull=True) & Q(crawl__permissions=value)) + | (Q(permissions__isnull=True) & Q(crawl__permissions__isnull=True) & persona_query) + ) + return queryset + + def persona_ids_for_value(self): + global_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower() + query = Q(permissions=self.value()) + if global_permissions == self.value(): + query |= Q(permissions__isnull=True) + return Persona.objects.filter(query).values_list("id", flat=True) + + +class SnapshotStatusListFilter(admin.SimpleListFilter): + title = "snapshot status" + parameter_name = "snapshot_status" + + def lookups(self, request, model_admin): + return Snapshot.StatusChoices.choices + + def queryset(self, request, queryset): + value = self.value() + if value in Snapshot.StatusChoices.values: + return queryset.filter(status=value) + return queryset + + +class SnapshotDepthListFilter(admin.SimpleListFilter): + title = "depth" + parameter_name = "depth_bucket" + + def lookups(self, request, model_admin): + return ( + ("0", "0 root"), + ("1", "1"), + ("2", "2"), + ("3plus", "3+"), + ) + + def queryset(self, request, queryset): + value = self.value() + if value == "0": + return queryset.filter(depth=0) + if value == "1": + return queryset.filter(depth=1) + if value == "2": + return queryset.filter(depth=2) + if value == "3plus": + return queryset.filter(depth__gte=3) + return queryset + + +class SnapshotRelationListFilter(admin.SimpleListFilter): + title = "crawl position" + parameter_name = "position" + + def lookups(self, request, model_admin): + return ( + ("root", "Root URL"), + ("discovered", "Discovered URL"), + ("has_children", "Has discovered URLs"), + ("no_children", "No discovered URLs"), + ) + + def queryset(self, request, queryset): + value = self.value() + if value == "root": + return queryset.filter(parent_snapshot__isnull=True) + if value == "discovered": + return queryset.filter(parent_snapshot__isnull=False) + if value in {"has_children", "no_children"}: + child_snapshots = Snapshot.objects.filter(parent_snapshot_id=OuterRef("pk")) + queryset = queryset.annotate(has_child_snapshots=Exists(child_snapshots)) + return queryset.filter(has_child_snapshots=value == "has_children") + return queryset + + +class SnapshotArchiveStateListFilter(admin.SimpleListFilter): + title = "archive state" + parameter_name = "archive_state" + + def lookups(self, request, model_admin): + return ( + ("downloaded", "Downloaded"), + ("not_downloaded", "Not downloaded"), + ("has_output", "Has saved files"), + ("empty_output", "No saved files"), + ("has_title", "Has title"), + ("missing_title", "Missing title"), + ) + + def queryset(self, request, queryset): + value = self.value() + if value == "downloaded": + return queryset.filter(downloaded_at__isnull=False) + if value == "not_downloaded": + return queryset.filter(downloaded_at__isnull=True) + if value == "has_output": + return queryset.filter(output_size__gt=0) + if value == "empty_output": + return queryset.filter(output_size=0) + if value == "has_title": + return queryset.exclude(Q(title__isnull=True) | Q(title="")) + if value == "missing_title": + return queryset.filter(Q(title__isnull=True) | Q(title="")) + return queryset + + +class SnapshotSizeListFilter(admin.SimpleListFilter): + title = "size" + parameter_name = "size" + + def lookups(self, request, model_admin): + return ( + ("1gb", ">1GB"), + ("500mb", ">500MB"), + ("250mb", ">250MB"), + ("100mb", ">100MB"), + ("50mb", ">50MB"), + ("25mb", ">25MB"), + ) + + def queryset(self, request, queryset): + value = self.value() + thresholds = { + "1gb": 1024 * 1024 * 1024, + "500mb": 500 * 1024 * 1024, + "250mb": 250 * 1024 * 1024, + "100mb": 100 * 1024 * 1024, + "50mb": 50 * 1024 * 1024, + "25mb": 25 * 1024 * 1024, + } + if value in thresholds: + return queryset.filter(output_size__gt=thresholds[value]) + return queryset + + +class SnapshotRetryListFilter(admin.SimpleListFilter): + title = "retry" + parameter_name = "retry" + + def lookups(self, request, model_admin): + return ( + ("due", "Due now"), + ("future", "Scheduled later"), + ("none", "No retry time"), + ) + + def queryset(self, request, queryset): + value = self.value() + if value == "due": + return queryset.filter(retry_at__isnull=False, retry_at__lte=timezone.now()) + if value == "future": + return queryset.filter(retry_at__gt=timezone.now()) + if value == "none": + return queryset.filter(retry_at__isnull=True) + return queryset + + +class SnapshotResultHealthListFilter(admin.SimpleListFilter): + title = "ArchiveResult status" + parameter_name = "archiveresult_status" + + def lookups(self, request, model_admin): + return ( + ("none", "No ArchiveResults"), + ("has_results", "Has ArchiveResults"), + ("succeeded", ">50% succeeded"), + ("failed", ">50% failed"), + ("running", ">50% running"), + ("pending", ">50% queued"), + ("backoff", ">50% waiting to retry"), + ("noresults", ">50% noresults"), + ) + + def queryset(self, request, queryset): + value = self.value() + if value: + results = ArchiveResult.objects.filter(snapshot_id=OuterRef("pk")) + if value == "none": + return queryset.annotate(has_results=Exists(results)).filter(has_results=False) + if value == "has_results": + return queryset.annotate(has_results=Exists(results)).filter(has_results=True) + status_by_value = { + "succeeded": ArchiveResult.StatusChoices.SUCCEEDED, + "failed": ArchiveResult.StatusChoices.FAILED, + "running": ArchiveResult.StatusChoices.STARTED, + "pending": ArchiveResult.StatusChoices.QUEUED, + "backoff": ArchiveResult.StatusChoices.BACKOFF, + "noresults": ArchiveResult.StatusChoices.NORESULTS, + } + if value in status_by_value: + queryset = queryset.annotate( + total_results=Count("archiveresult"), + matching_results=Count( + "archiveresult", + filter=Q(archiveresult__status=status_by_value[value]), + ), + ) + return queryset.filter(matching_results__gt=F("total_results") / 2) + return queryset + + +class SnapshotChangeList(SearchResultsChangeList): + def __init__(self, request, *args, **kwargs): + super().__init__(request, *args, **kwargs) + resolver_name = getattr(getattr(request, "resolver_match", None), "url_name", "") + self.embedded_changelist = request.GET.get("_embedded") == "crawl" + self.snapshot_is_grid_view = not self.embedded_changelist and (resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")) + + def get_results(self, request): + super().get_results(request) + if request.GET.get("_embedded") == "crawl": + self.full_result_count = self.result_count + else: + self.full_result_count = self.model_admin.get_paginator(request, self.model._default_manager.all().order_by(), self.list_per_page).count + self.show_full_result_count = True + + snapshot_ids = [obj.pk for obj in self.result_list] + if snapshot_ids: + results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids} + seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids} + rows = ( + ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0) + .order_by("snapshot_id", "plugin") + .values_list("snapshot_id", "plugin", "status", "output_size") + ) + for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000): + if plugin in seen_plugins[snapshot_id]: + continue + seen_plugins[snapshot_id].add(plugin) + results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size)) + + for obj in self.result_list: + obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk] + + class SnapshotAdminForm(forms.ModelForm): """Custom form for Snapshot admin with tag editor widget.""" @@ -104,6 +380,12 @@ class SnapshotAdminForm(forms.ModelForm): widget=TagEditorWidget(), help_text="Type tag names and press Enter or Space to add. Click Γ— to remove.", ) + permissions_config = forms.ChoiceField( + label="Permissions", + choices=PERMISSIONS_CHOICES, + required=True, + help_text="Per-snapshot visibility. Matching the crawl/persona default clears the per-snapshot override.", + ) class Meta: model = Snapshot @@ -116,9 +398,18 @@ class SnapshotAdminForm(forms.ModelForm): self.initial["tags_editor"] = ",".join( sorted(tag.name for tag in self.instance.tags.all()), ) + self.initial["permissions_config"] = get_snapshot_permissions(self.instance) def save(self, commit=True): instance = super().save(commit=False) + permissions = self.cleaned_data["permissions_config"] + inherited_permissions = str(get_config(crawl=instance.crawl, resolve_plugins=False).PERMISSIONS).strip().lower() + config = dict(instance.config or {}) + if permissions == inherited_permissions: + config.pop("PERMISSIONS", None) + else: + config["PERMISSIONS"] = permissions + instance.config = config # Handle tags_editor field if commit: @@ -150,7 +441,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): form = SnapshotAdminForm raw_id_fields = ("crawl", "parent_snapshot") list_select_related = () - list_display = ("created_at", "preview_icon", "title_str", "tags_inline", "status_with_progress", "files", "size_with_stats") + list_display = ("permissions_badge", "created_at", "preview_icon", "title_str", "tags_inline", "status_with_progress", "files", "size_with_stats") + list_display_links = ("created_at",) sort_fields = ("title_str", "created_at", "status", "crawl") readonly_fields = ( "admin_actions", @@ -165,7 +457,20 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): "archiveresults_list", ) search_fields = ("id", "url", "timestamp", "title", "tags__name") - list_filter = ("created_at", "downloaded_at", "archiveresult__status", "crawl__created_by", TagNameListFilter) + list_filter = ( + SnapshotPermissionsListFilter, + SnapshotStatusListFilter, + SnapshotResultHealthListFilter, + SnapshotDepthListFilter, + SnapshotRelationListFilter, + SnapshotArchiveStateListFilter, + SnapshotSizeListFilter, + SnapshotRetryListFilter, + "created_at", + "downloaded_at", + "crawl__created_by", + TagNameListFilter, + ) fieldsets = ( ( @@ -192,7 +497,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): ( "Tags", { - "fields": ("tags_editor",), + "fields": ("tags_editor", "permissions_config"), "classes": ("card",), }, ), @@ -244,7 +549,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): ordering = ["-timestamp"] actions = ["add_tags", "remove_tags", "resnapshot_snapshot", "update_snapshots", "overwrite_snapshots", "delete_snapshots"] inlines = [] # Removed TagInline, using TagEditorWidget instead - list_per_page = 40 + list_per_page = 50 action_form = SnapshotActionForm paginator = AcceleratedPaginator @@ -252,6 +557,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): save_on_top = True show_full_result_count = False + def get_changelist(self, request, **kwargs): + return SnapshotChangeList + + def get_ordering(self, request): + if request.GET.get("o"): + return [] + return super().get_ordering(request) + def change_view(self, request, object_id, form_url="", extra_context=None): request.archivebox_config = getattr(request, "archivebox_config", None) or get_config() extra_context = extra_context or {} @@ -262,8 +575,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): self.request = request request.archivebox_config = getattr(request, "archivebox_config", None) or get_config() saved_list_per_page = self.list_per_page - self.list_per_page = min(max(5, request.archivebox_config.SNAPSHOTS_PER_PAGE), 25) + embedded_changelist = request.GET.get("_embedded") == "crawl" + if embedded_changelist: + try: + requested_per_page = int(request.GET.get("per_page", "200")) + except ValueError: + requested_per_page = 200 + self.list_per_page = min(max(200, requested_per_page), 500) + else: + self.list_per_page = min(max(50, request.archivebox_config.SNAPSHOTS_PER_PAGE), 500) extra_context = extra_context or {} + extra_context["embedded_changelist"] = embedded_changelist extra_context["CONFIG"] = request.archivebox_config try: try: @@ -281,6 +603,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): actions.pop("delete_selected", None) return actions + def lookup_allowed(self, lookup, value, request=None): + if lookup in {"crawl__id__exact", "crawl_id__exact", "crawl_id"}: + return True + return super().lookup_allowed(lookup, value, request=request) + def get_snapshot_view_url(self, obj: Snapshot) -> str: request = getattr(self, "request", None) return build_snapshot_url(str(obj.id), request=request, config=getattr(request, "archivebox_config", None)) @@ -296,10 +623,185 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): urls = super().get_urls() custom_urls = [ path("grid/", self.admin_site.admin_view(self.grid_view), name="grid"), + path("search-stream/", self.admin_site.admin_view(self.search_stream_view), name="core_snapshot_search_stream"), path("/redo-failed/", self.admin_site.admin_view(self.redo_failed_view), name="core_snapshot_redo_failed"), + path("/set-permissions/", self.admin_site.admin_view(self.set_permissions_view), name="core_snapshot_set_permissions"), ] return custom_urls + urls + def search_stream_view(self, request): + from archivebox.search import iter_query_search_ids + + query = (request.GET.get("q") or "").strip() + from archivebox.search import get_search_mode, get_search_mode_base + + search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None)) + if not query: + return StreamingHttpResponse((), content_type="text/plain") + + search_url = request.GET.get("search_url") or request.get_full_path() + target_url = urlsplit(search_url) + target_get = QueryDict(target_url.query, mutable=True) + for key in ("q", "search_mode", "p", "search_url"): + target_get.pop(key, None) + + filter_request = copy(request) + filter_request.path = target_url.path or request.path + filter_request.path_info = target_url.path or request.path_info + filter_request.GET = target_get + filter_request.archivebox_config = getattr(request, "archivebox_config", None) + + # Build the same filtered base queryset the changelist uses, but with + # the search params stripped. The stream then intersects each wave with + # this queryset before writing IDs into the short-lived cache. + current_request = getattr(self, "request", None) + try: + base_queryset = self.get_changelist_instance(filter_request).queryset + finally: + self.request = current_request + + async def snapshot_ids(): + seen = set() + ids = [] + last_sent = 0 + stream_batch_size = 100 + stream_padding = " " * 4096 + cache_key = get_admin_search_cache_key(request, search_url) + cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL) + yield f"0{stream_padding}\n" + queue = Queue(maxsize=8) + stop_event = threading.Event() + + def emit(item): + while not stop_event.is_set(): + try: + queue.put(item, timeout=0.1) + return + except Full: + continue + + def run_search(): + nonlocal last_sent + iterator = None + try: + search_mode_base = get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None)) + iterator = ( + self.iter_meta_search_ids(query, base_queryset) + if search_mode_base == "meta" + else self.iter_backend_search_ids( + iter_query_search_ids(query, search_mode=search_mode, config=getattr(request, "archivebox_config", None)), + base_queryset, + ) + ) + for snapshot_id in iterator: + if stop_event.is_set(): + break + snapshot_id = str(snapshot_id).strip().lower() + if len(snapshot_id.replace("-", "")) != 32 or snapshot_id in seen: + continue + seen.add(snapshot_id) + ids.append(snapshot_id) + if len(ids) - last_sent >= stream_batch_size: + cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL) + last_sent = len(ids) + emit(f"{last_sent}{stream_padding}\n") + if not stop_event.is_set() and len(ids) != last_sent: + cache.set(cache_key, {"ids": ids, "done": False}, SEARCH_RESULT_CACHE_TTL) + emit(f"{len(ids)}{stream_padding}\n") + except BaseException as err: + emit(err) + finally: + if iterator is not None: + try: + iterator.close() + except AttributeError: + pass + cache.set(cache_key, {"ids": ids, "done": True}, SEARCH_RESULT_CACHE_TTL) + emit(None) + + threading.Thread(target=run_search, name="admin-snapshot-search-stream", daemon=True).start() + try: + while True: + item = await asyncio.to_thread(queue.get) + if item is None: + break + if isinstance(item, BaseException): + raise item + yield item + finally: + stop_event.set() + + response = StreamingHttpResponse(snapshot_ids(), content_type="text/plain") + response["X-Accel-Buffering"] = "no" + return response + + def iter_meta_search_ids(self, query, queryset): + seen = set() + try: + snapshot_id = UUID(query) + except ValueError: + snapshot_id = None + if snapshot_id: + for pk in queryset.filter(pk=snapshot_id).values_list("pk", flat=True): + seen.add(pk) + yield pk + + for wave in ( + Q(timestamp__startswith=query) | Q(url__istartswith=query) | Q(title__istartswith=query), + Q(url__icontains=query), + Q(title__icontains=query), + Q(tags__name__icontains=query), + ): + for pk in queryset.filter(wave).values_list("pk", flat=True).distinct().iterator(chunk_size=500): + if pk in seen: + continue + seen.add(pk) + yield pk + + def iter_backend_search_ids(self, iterator, queryset): + batch = [] + seen = set() + + def flush_batch(): + valid = {str(pk) for pk in queryset.filter(pk__in=batch).values_list("pk", flat=True)} + for snapshot_id in batch: + if snapshot_id in valid and snapshot_id not in seen: + seen.add(snapshot_id) + yield snapshot_id + + for snapshot_id in iterator: + snapshot_id = str(snapshot_id).strip().lower() + if len(snapshot_id.replace("-", "")) != 32: + continue + batch.append(snapshot_id) + if len(batch) >= 200: + yield from flush_batch() + batch = [] + if batch: + yield from flush_batch() + + def set_permissions_view(self, request, object_id): + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + + permissions = (request.POST.get("permissions") or "").strip().lower() + if permissions not in dict(PERMISSIONS_CHOICES): + return HttpResponseBadRequest("Invalid permissions value") + + snapshot = get_object_or_404(Snapshot.objects.select_related("crawl"), pk=object_id) + config = dict(snapshot.config or {}) + inherited_permissions = str(get_config(crawl=snapshot.crawl, resolve_plugins=False).PERMISSIONS).strip().lower() + if permissions == inherited_permissions: + config.pop("PERMISSIONS", None) + else: + config["PERMISSIONS"] = permissions + + # Keep the quick-edit write to one targeted UPDATE so SQLite only holds + # the write lock for the permission/config change itself. + Snapshot.objects.filter(pk=snapshot.pk).update(config=config, modified_at=timezone.now()) + icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions] + return JsonResponse({"permissions": permissions, "icon": icon, "label": label, "fg": fg, "bg": bg}) + def redo_failed_view(self, request, object_id): snapshot = get_object_or_404(Snapshot, pk=object_id) @@ -327,42 +829,56 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): def get_queryset(self, request): self.request = request ordering_fields = self._get_ordering_fields(request) - needs_size_sort = "size_with_stats" in ordering_fields needs_files_sort = "files" in ordering_fields needs_tags_sort = "tags_inline" in ordering_fields is_change_view = getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_change" + request.archivebox_default_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower() - prefetch_qs = ArchiveResult.objects.only( - "id", - "snapshot_id", - "plugin", - "status", - "output_size", - "output_files", - "output_str", - ) - if not is_change_view: - prefetch_qs = prefetch_qs.filter(Q(status="succeeded")) - - qs = ( - super() - .get_queryset(request) - .defer("config", "notes") - .prefetch_related( - Prefetch("crawl", queryset=Crawl.objects.select_related("created_by")), - "tags", - Prefetch("archiveresult_set", queryset=prefetch_qs), - ) - ) - - if needs_size_sort: - qs = qs.annotate( - output_size_sum=Coalesce( - Sum("archiveresult__output_size"), - 0, + prefetches = [ + Prefetch( + "crawl", + queryset=Crawl.objects.only( + "id", + "permissions", + "persona_id", + "status", + "created_by_id", + ).prefetch_related("created_by"), + ), + "tags", + ] + if is_change_view: + prefetches.append( + Prefetch( + "archiveresult_set", + queryset=ArchiveResult.objects.only( + "id", + "snapshot_id", + "plugin", + "status", + "output_size", + ), ), ) + qs = super().get_queryset(request) + if is_change_view: + qs = qs.defer("notes") + else: + qs = qs.only( + "id", + "created_at", + "url", + "timestamp", + "bookmarked_at", + "crawl_id", + "title", + "status", + "fs_version", + "output_size", + "permissions", + ) + qs = qs.prefetch_related(*prefetches) if needs_files_sort: qs = qs.annotate( ar_succeeded_count=Count( @@ -375,6 +891,66 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): return qs + @admin.display(description="πŸ‘", ordering="permissions") + def permissions_badge(self, obj): + request = self.request + permissions = getattr(obj, "snapshot_permissions", None) + if permissions is None: + if obj.permissions: + permissions = obj.permissions + elif obj.crawl.permissions: + permissions = obj.crawl.permissions + elif obj.crawl.persona_id: + persona_permissions = getattr(request, "archivebox_persona_permissions", None) + if persona_permissions is None: + persona_permissions = { + str(persona.id): persona.permissions or request.archivebox_default_permissions + for persona in Persona.objects.only("id", "permissions") + } + request.archivebox_persona_permissions = persona_permissions + permissions = persona_permissions.get(str(obj.crawl.persona_id), request.archivebox_default_permissions) + else: + permissions = request.archivebox_default_permissions + icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions] + menu_items = format_html_join( + "", + ( + '" + ), + ( + ( + " is-active" if choice_value == permissions else "", + choice_value, + choice_fg, + choice_bg, + choice_icon, + choice_label, + ) + for choice_value, choice_label in PERMISSIONS_CHOICES + for choice_icon, _choice_title, choice_fg, choice_bg in [SNAPSHOT_PERMISSION_META[choice_value]] + ), + ) + return format_html( + '' + '" + '' + "", + permissions, + reverse(f"{self.admin_site.name}:core_snapshot_set_permissions", args=[obj.pk]), + permissions, + label, + label, + fg, + bg, + icon, + menu_items, + ) + @admin.display(description="Imported Timestamp") def imported_timestamp(self, obj): context = RequestContext( @@ -555,15 +1131,15 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): @admin.display(description="Tags", ordering="tag_count") def tags_inline(self, obj): - widget = InlineTagEditorWidget(snapshot_id=str(obj.pk)) + widget = InlineTagEditorWidget(snapshot_id=str(obj.pk), editable=True) tags = self._get_prefetched_tags(obj) tags_html = widget.render( - name=f"tags_{obj.pk}", + name=f"tags_inline_{obj.pk}", value=tags if tags is not None else obj.tags.all(), - attrs={"id": f"tags_{obj.pk}"}, + attrs={"id": f"tags_inline_{obj.pk}"}, snapshot_id=str(obj.pk), ) - return mark_safe(f'{tags_html}') + return mark_safe(f'{tags_html}') @admin.display(description="Tags") def tags_badges(self, obj): @@ -716,13 +1292,13 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): def files(self, obj): results = self._get_prefetched_results(obj) if results is None: - results = obj.archiveresult_set.only("plugin", "status", "output_files", "output_str") + results = obj.archiveresult_set.only("plugin", "status", "output_size") plugins_with_output: dict[str, ArchiveResult] = {} for result in results: if result.status != ArchiveResult.StatusChoices.SUCCEEDED: continue - if not (result.output_files or str(result.output_str or "").strip()): + if not result.output_size: continue plugins_with_output.setdefault(result.plugin, result) @@ -734,15 +1310,21 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): key=lambda result: (_plugin_sort_order().get(result.plugin, 9999), result.plugin), ) visible_results = sorted_results[:14] - output = [ - format_html( - '{}', - f"/{obj.archive_path_from_db}/{result.plugin}/", - result.plugin, - get_plugin_icon(result.plugin), + output = [] + request = getattr(self, "request", None) + config = getattr(request, "archivebox_config", None) + for result in visible_results: + icon = mark_safe(get_plugin_icon(result.plugin)) + if not icon.strip(): + continue + output.append( + format_html( + '{}', + build_web_url(f"/{obj.archive_path_from_db}/{result.plugin}/", request=request, config=config), + result.plugin, + icon, + ), ) - for result in visible_results - ] if len(sorted_results) > len(visible_results): output.append( format_html( @@ -788,6 +1370,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): status_colors = { "queued": ("#f59e0b", "#fef3c7"), # amber "started": ("#3b82f6", "#dbeafe"), # blue + "paused": ("#1d4ed8", "#dbeafe"), # blue "sealed": ("#10b981", "#d1fae5"), # green "succeeded": ("#10b981", "#d1fae5"), # green "failed": ("#ef4444", "#fee2e2"), # red @@ -840,7 +1423,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): @admin.display( description="Size", - ordering="output_size_sum", + ordering="output_size", ) def size_with_stats(self, obj): """Show archive size with output size from archive results.""" @@ -902,14 +1485,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): pending = max(total - succeeded - failed - running - skipped - noresults, 0) completed = succeeded + failed + skipped + noresults percent = int((completed / total * 100) if total > 0 else 0) - is_sealed = obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED) - output_size = None - - if hasattr(obj, "output_size_sum"): - output_size = obj.output_size_sum or 0 - else: - output_size = sum(r.output_size or 0 for r in results) - + is_sealed = obj.status not in (obj.StatusChoices.QUEUED, obj.StatusChoices.STARTED, obj.StatusChoices.PAUSED) stats = { "total": total, "succeeded": succeeded, @@ -919,13 +1495,15 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): "skipped": skipped, "noresults": noresults, "percent": percent, - "output_size": output_size or 0, + "output_size": obj.output_size or 0, "is_sealed": is_sealed, } obj._admin_progress_stats = stats return stats def _get_prefetched_results(self, obj): + if "_admin_archiveresults" in obj.__dict__: + return obj.__dict__["_admin_archiveresults"] if hasattr(obj, "_prefetched_objects_cache") and "archiveresult_set" in obj._prefetched_objects_cache: return obj.archiveresult_set.all() return None @@ -1009,31 +1587,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): return format_html('{}', color, h) def grid_view(self, request, extra_context=None): - - # cl = self.get_changelist_instance(request) - - # Save before monkey patching to restore for changelist list view - admin_cls = type(self) - saved_change_list_template = admin_cls.change_list_template - saved_list_per_page = admin_cls.list_per_page - saved_list_max_show_all = admin_cls.list_max_show_all - - # Monkey patch here plus core_tags.py - admin_cls.change_list_template = "private_index_grid.html" - config = getattr(request, "archivebox_config", None) or get_config() - request.archivebox_config = config - admin_cls.list_per_page = config.SNAPSHOTS_PER_PAGE - admin_cls.list_max_show_all = admin_cls.list_per_page - - # Call monkey patched view - rendered_response = self.changelist_view(request, extra_context=extra_context) - - # Restore values - admin_cls.change_list_template = saved_change_list_template - admin_cls.list_per_page = saved_list_per_page - admin_cls.list_max_show_all = saved_list_max_show_all - - return rendered_response + extra_context = extra_context or {} + extra_context["snapshot_is_grid_view"] = True + return self.changelist_view(request, extra_context=extra_context) # for debugging, uncomment this to print all requests: # def changelist_view(self, request, extra_context=None): @@ -1123,28 +1679,24 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): messages.warning(request, "No tags specified.") return - # Parse comma-separated tag names and get/create Tag objects tag_names = [name.strip() for name in tags_str.split(",") if name.strip()] tags = [] for name in tag_names: - tag, _ = Tag.objects.get_or_create( - name__iexact=name, - defaults={"name": name}, + tag, _ = get_or_create_tag( + name, + created_by=request.user if request.user.is_authenticated else None, ) - tag = Tag.objects.filter(name__iexact=name).first() or tag tags.append(tag) # Get snapshot IDs efficiently (works with select_across for all pages) snapshot_ids = list(queryset.values_list("id", flat=True)) num_snapshots = len(snapshot_ids) - print("[+] Adding tags", [t.name for t in tags], "to", num_snapshots, "Snapshots") - - # Bulk create M2M relationships (1 query per tag, not per snapshot) for tag in tags: SnapshotTag.objects.bulk_create( - [SnapshotTag(snapshot_id=sid, tag=tag) for sid in snapshot_ids], - ignore_conflicts=True, # Skip if relationship already exists + [SnapshotTag(snapshot_id=sid, tag_id=tag.pk) for sid in snapshot_ids], + ignore_conflicts=True, + batch_size=1000, ) messages.success( @@ -1181,9 +1733,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): num_snapshots = len(snapshot_ids) tag_ids = [t.pk for t in tags] - print("[-] Removing tags", [t.name for t in tags], "from", num_snapshots, "Snapshots") - - # Bulk delete M2M relationships (1 query total, not per snapshot) deleted_count, _ = SnapshotTag.objects.filter( snapshot_id__in=snapshot_ids, tag_id__in=tag_ids, diff --git a/archivebox/core/apps.py b/archivebox/core/apps.py index 9316377c..c8f54921 100644 --- a/archivebox/core/apps.py +++ b/archivebox/core/apps.py @@ -25,18 +25,6 @@ class CoreConfig(AppConfig): if "makemigrations" not in sys.argv: from archivebox.core import models # noqa: F401 - pidfile = os.environ.get("ARCHIVEBOX_RUNSERVER_PIDFILE") - if pidfile: - should_write_pid = True - if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1": - should_write_pid = os.environ.get(DJANGO_AUTORELOAD_ENV) == "true" - if should_write_pid: - try: - with open(pidfile, "w") as handle: - handle.write(str(os.getpid())) - except Exception: - pass - def _should_prepare_runtime() -> bool: if os.environ.get("ARCHIVEBOX_RUNSERVER") == "1": if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1": @@ -45,6 +33,13 @@ class CoreConfig(AppConfig): return False if _should_prepare_runtime(): - from archivebox.machine.models import Machine + from archivebox.config import CONSTANTS + from archivebox.machine.models import Process - Machine.current() + Process.current().mark_running( + process_type=Process.TypeChoices.WORKER, + worker_type="worker_runserver", + pwd=str(CONSTANTS.DATA_DIR), + url=os.environ.get("ARCHIVEBOX_RUNSERVER_BIND_URL") or "", + timeout=0, + ) diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py index c77043d3..d8382a8f 100644 --- a/archivebox/core/forms.py +++ b/archivebox/core/forms.py @@ -1,7 +1,9 @@ __package__ = "archivebox.core" import json +import re from collections.abc import Iterable, Mapping +from decimal import Decimal, InvalidOperation, ROUND_CEILING from pathlib import Path from typing import Any @@ -13,6 +15,7 @@ from taggit.utils import edit_string_for_tags, parse_tags from archivebox.base_models.admin import KeyValueWidget from archivebox.crawls.schedule_utils import validate_schedule from archivebox.config.common import get_config, parse_delete_after +from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget from archivebox.hooks import get_plugins, discover_plugin_configs, get_plugin_icon from archivebox.personas.models import Persona @@ -146,6 +149,7 @@ HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = { "search_backend_sqlite", "ssl", } +TIMEOUT_INPUT_PATTERN = r"(0|[1-9][0-9]*|[0-9]+(?:\.[0-9]+)?\s*(?:s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours))" def get_plugin_choices(): @@ -509,6 +513,13 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form): widget=forms.Textarea( attrs={ "data-url-regex": URL_REGEX.pattern, + "placeholder": ( + "Enter URLs to archive, as one per line, CSV, JSON, or embedded in text " + "(e.g. markdown, HTML, etc.). Examples:\n" + "https://example.com\n" + "https://news.ycombinator.com,https://news.google.com\n" + "[ArchiveBox](https://github.com/ArchiveBox/ArchiveBox)" + ), }, ), required=True, @@ -526,7 +537,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form): widget=forms.RadioSelect(attrs={"class": "depth-selection"}), ) max_urls = forms.IntegerField( - label="Max URLs", + label="Max crawl URLs", required=False, min_value=0, initial=0, @@ -548,6 +559,29 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form): }, ), ) + crawl_timeout = forms.CharField( + label="Max crawl time", + required=False, + initial=0, + widget=forms.TextInput( + attrs={ + "pattern": TIMEOUT_INPUT_PATTERN, + "title": "Use 0, integer seconds, or a duration like 1.5m or 1hr. Non-zero values must be greater than 10 seconds.", + "placeholder": "0, 300, 1.5m, or 1hr", + }, + ), + ) + timeout = forms.CharField( + label="Max subtask time", + required=False, + widget=forms.TextInput( + attrs={ + "pattern": TIMEOUT_INPUT_PATTERN, + "title": "Use integer seconds or a duration like 1.5m or 1hr. Non-zero values must be greater than 10 seconds.", + "placeholder": "60, 1.5m, or 1hr", + }, + ), + ) snapshot_max_size = forms.CharField( label="Max snapshot size", required=False, @@ -569,7 +603,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form): ), ) crawl_max_concurrent_snapshots = forms.IntegerField( - label="Max concurrent snapshots", + label="Max in parallel", required=False, min_value=1, widget=forms.NumberInput( @@ -657,6 +691,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form): empty_label=None, to_field_name="name", ) + permissions = forms.ChoiceField( + label="Permissions", + choices=PERMISSIONS_CHOICES, + initial="public", + required=True, + ) index_only = forms.BooleanField( label="Index only dry run (add crawl but don't archive yet)", initial=False, @@ -670,23 +710,45 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form): ) def __init__(self, *args, **kwargs): + self.request = kwargs.pop("request", None) + self.can_override_crawl_config = bool(self.request and is_admin_user(self.request)) super().__init__(*args, **kwargs) default_persona = Persona.get_or_create_default() - default_config = get_config(persona=default_persona) - self.fields["persona"].queryset = Persona.objects.order_by("name") - self.fields["persona"].initial = default_persona.name + persona_queryset = Persona.objects.order_by("name") + if not self.can_override_crawl_config: + persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC}) + self.fields["persona"].queryset = persona_queryset + + selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first() + default_config = get_config(persona=selected_persona) if selected_persona else get_config() + if selected_persona: + self.fields["persona"].initial = selected_persona.name + self.fields["permissions"].initial = default_config.PERMISSIONS + self.fields["timeout"].initial = default_config.TIMEOUT self.fields["crawl_max_concurrent_snapshots"].initial = default_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS self.fields["delete_after"].initial = default_config.DELETE_AFTER - selected_persona = default_persona if self.is_bound: - selected_persona = Persona.objects.filter(name=str(self.data.get(self.add_prefix("persona")) or "")).first() or default_persona - self.build_plugin_groups(get_config(persona=selected_persona)) + selected_persona = persona_queryset.filter(name=str(self.data.get(self.add_prefix("persona")) or "")).first() or selected_persona + if self.can_override_crawl_config: + self.build_plugin_groups(get_config(persona=selected_persona) if selected_persona else get_config()) + else: + all_plugins = get_plugins() + for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS: + get_choice_field(self, field_name).choices = [(p, p) for p in all_plugins if p in plugin_names] + get_choice_field(self, "other_plugins").choices = [(p, p) for p in all_plugins] + self.plugin_groups = [] def clean(self): cleaned_data = super().clean() or {} + if not self.can_override_crawl_config: + cleaned_data["plugins"] = [] + cleaned_data["plugin_config"] = {} + cleaned_data["config"] = {} + return cleaned_data + # Combine all plugin groups into single list all_selected_plugins = [] for field in [ @@ -731,6 +793,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form): "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))), "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))), "same_domain_only": bool(value.get("same_domain_only")), + "subpaths_only": bool(value.get("subpaths_only")), } def clean_max_urls(self): @@ -749,6 +812,37 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form): raise forms.ValidationError("Max crawl size must be 0 or a positive number of bytes.") return value + def clean_crawl_timeout(self): + return self._clean_timeout_seconds(self.cleaned_data.get("crawl_timeout"), "Max crawl time", blank_value=0) + + def clean_timeout(self): + return self._clean_timeout_seconds(self.cleaned_data.get("timeout"), "Max subtask time", blank_value=None) + + def _clean_timeout_seconds(self, raw_value, field_label: str, *, blank_value): + raw_value = str(raw_value or "").strip().lower() + if not raw_value: + return blank_value + if raw_value.isdigit(): + value = int(raw_value) + else: + match = re.fullmatch(r"(\d+(?:\.\d+)?)\s*(s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours)", raw_value) + if not match: + raise forms.ValidationError(f"{field_label} must be seconds or a duration like 1.5m or 1hr.") + amount_str, unit = match.groups() + try: + amount = Decimal(amount_str) + except InvalidOperation as err: + raise forms.ValidationError(f"{field_label} must be seconds or a duration like 1.5m or 1hr.") from err + multiplier = 1 + if unit in {"m", "min", "mins", "minute", "minutes"}: + multiplier = 60 + elif unit in {"h", "hr", "hrs", "hour", "hours"}: + multiplier = 60 * 60 + value = int((amount * multiplier).to_integral_value(rounding=ROUND_CEILING)) + if 0 < value <= 10: + raise forms.ValidationError(f"{field_label} must be 0 or greater than 10 seconds.") + return value + def clean_snapshot_max_size(self): raw_value = str(self.cleaned_data.get("snapshot_max_size") or "").strip() if not raw_value: diff --git a/archivebox/core/host_utils.py b/archivebox/core/host_utils.py index 62e2dc4a..c5688a56 100644 --- a/archivebox/core/host_utils.py +++ b/archivebox/core/host_utils.py @@ -206,11 +206,6 @@ def get_public_base_url(request=None, config: dict[str, Any] | None = None, **co return _build_base_url_for_host(get_public_host(config=config), request=request, config=config) -# Backwards-compat aliases (archive == web) -def get_archive_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: - return get_web_base_url(request=request, config=config, **config_kwargs) - - def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) if not config.USES_SUBDOMAIN_ROUTING: @@ -233,14 +228,6 @@ def build_web_url(path: str = "", request=None, config: dict[str, Any] | None = return _build_url(get_web_base_url(request, config=config, **config_kwargs), path) -def build_api_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: - return _build_url(get_api_base_url(request, config=config, **config_kwargs), path) - - -def build_archive_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: - return _build_url(get_archive_base_url(request, config=config, **config_kwargs), path) - - def build_snapshot_url(snapshot_id: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: return _build_url(get_snapshot_base_url(snapshot_id, request=request, config=config, **config_kwargs), path) diff --git a/archivebox/core/middleware.py b/archivebox/core/middleware.py index f8275456..b7d26aec 100644 --- a/archivebox/core/middleware.py +++ b/archivebox/core/middleware.py @@ -104,7 +104,7 @@ def CacheControlMiddleware(get_response): if config is None: config = get_config(resolve_plugins=False) request.archivebox_config = config - policy = "public" if config.PUBLIC_SNAPSHOTS else "private" + policy = "private" if config.PERMISSIONS == "private" else "public" response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300" # print('Set Cache-Control header to', response['Cache-Control']) return response diff --git a/archivebox/core/migrations/0041_snapshot_permissions.py b/archivebox/core/migrations/0041_snapshot_permissions.py new file mode 100644 index 00000000..3e903984 --- /dev/null +++ b/archivebox/core/migrations/0041_snapshot_permissions.py @@ -0,0 +1,19 @@ +# Generated by Django 6.0.5 on 2026-05-28 07:25 + +import django.db.models.fields.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0040_archiveresult_delete_at_snapshot_delete_at'), + ] + + operations = [ + migrations.AddField( + model_name='snapshot', + name='permissions', + field=models.GeneratedField(db_index=True, db_persist=True, expression=django.db.models.fields.json.KeyTextTransform('PERMISSIONS', 'config'), output_field=models.CharField(max_length=16, null=True)), + ), + ] diff --git a/archivebox/core/migrations/0042_snapshot_output_size.py b/archivebox/core/migrations/0042_snapshot_output_size.py new file mode 100644 index 00000000..f6ac7bde --- /dev/null +++ b/archivebox/core/migrations/0042_snapshot_output_size.py @@ -0,0 +1,39 @@ +# Generated by Django 6.0.5 on 2026-05-28 08:22 + +from django.db import migrations, models +from django.db.models import Sum + + +def backfill_snapshot_output_size(apps, schema_editor): + Snapshot = apps.get_model("core", "Snapshot") + ArchiveResult = apps.get_model("core", "ArchiveResult") + batch = [] + rows = ArchiveResult.objects.values("snapshot_id").annotate(total_size=Sum("output_size")).order_by() + for row in rows.iterator(chunk_size=2000): + batch.append(Snapshot(id=row["snapshot_id"], output_size=row["total_size"] or 0)) + if len(batch) >= 2000: + Snapshot.objects.bulk_update(batch, ["output_size"], batch_size=2000) + batch = [] + if batch: + Snapshot.objects.bulk_update(batch, ["output_size"], batch_size=2000) + + +def clear_snapshot_output_size(apps, schema_editor): + Snapshot = apps.get_model("core", "Snapshot") + Snapshot.objects.update(output_size=0) + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0041_snapshot_permissions'), + ] + + operations = [ + migrations.AddField( + model_name='snapshot', + name='output_size', + field=models.BigIntegerField(db_index=True, default=0, editable=False, help_text='Total bytes of all ArchiveResult output files'), + ), + migrations.RunPython(backfill_snapshot_output_size, clear_snapshot_output_size), + ] diff --git a/archivebox/core/migrations/0043_archiveresult_retry_at.py b/archivebox/core/migrations/0043_archiveresult_retry_at.py new file mode 100644 index 00000000..a02146cb --- /dev/null +++ b/archivebox/core/migrations/0043_archiveresult_retry_at.py @@ -0,0 +1,18 @@ +# Generated by Django 6.0.5 on 2026-05-28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0042_snapshot_output_size"), + ] + + operations = [ + migrations.AddField( + model_name="archiveresult", + name="retry_at", + field=models.DateTimeField(blank=True, db_index=True, default=None, null=True), + ), + ] diff --git a/archivebox/core/migrations/0044_alter_archiveresult_status_alter_snapshot_status.py b/archivebox/core/migrations/0044_alter_archiveresult_status_alter_snapshot_status.py new file mode 100644 index 00000000..9ecdfdef --- /dev/null +++ b/archivebox/core/migrations/0044_alter_archiveresult_status_alter_snapshot_status.py @@ -0,0 +1,23 @@ +# Generated by Django 6.0.5 on 2026-05-28 12:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0043_archiveresult_retry_at'), + ] + + operations = [ + migrations.AlterField( + model_name='archiveresult', + name='status', + field=models.CharField(choices=[('queued', 'Queued'), ('started', 'Started'), ('paused', 'Paused'), ('backoff', 'Waiting to retry'), ('succeeded', 'Succeeded'), ('failed', 'Failed'), ('skipped', 'Skipped'), ('noresults', 'No Results')], db_index=True, default='queued', max_length=16), + ), + migrations.AlterField( + model_name='snapshot', + name='status', + field=models.CharField(choices=[('queued', 'Queued'), ('started', 'Started'), ('paused', 'Paused'), ('sealed', 'Sealed')], db_index=True, default='queued', max_length=15), + ), + ] diff --git a/archivebox/core/models.py b/archivebox/core/models.py index fb8a3c26..5b754cc2 100755 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -13,8 +13,9 @@ from urllib.parse import urlparse from statemachine import State, registry -from django.db import models -from django.db.models import Q, QuerySet +from django.db import models, transaction +from django.db.models import Q, QuerySet, Sum +from django.db.models.fields.json import KT from django.utils.functional import cached_property from django.utils.text import slugify from django.utils import timezone @@ -27,7 +28,7 @@ from django.utils.safestring import mark_safe from archivebox.config import CONSTANTS from archivebox.config.common import get_config -from archivebox.misc.system import get_dir_size, atomic_write +from archivebox.misc.system import atomic_write from archivebox.misc.util import ( MAX_URL_LENGTH, parse_date, @@ -53,7 +54,7 @@ from archivebox.base_models.models import ( ModelWithHealthStats, get_or_create_system_user_pk, ) -from archivebox.workers.models import ModelWithStateMachine, BaseStateMachine +from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine from archivebox.workers.tasks import bg_archive_snapshot from archivebox.crawls.models import Crawl from archivebox.machine.models import Binary @@ -148,6 +149,101 @@ class SnapshotTag(models.Model): class SnapshotQuerySet(models.QuerySet): """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc.""" + def paged_iterator(self, chunk_size: int = 500): + """ + Iterate snapshots using bounded keyset pages instead of one streaming cursor. + + Django's iterator(chunk_size=...) still keeps a single SQLite SELECT + cursor open until the full queryset is exhausted. That is fine for + read-only exports, but update/migration code does filesystem work and + writes while iterating; a long-lived read cursor there can stretch lock + waits across thousands of rows. This respects the queryset's existing + filters, order_by(), select_related(), and prefetch_related() state; if + no ordering is defined, it falls back to primary-key order. + """ + pk_field = self.model._meta.pk.name + raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,)) + + if any(not isinstance(term, str) or term == "?" for term in raw_ordering): + offset = 0 + while True: + batch = list(self[offset : offset + chunk_size]) + if not batch: + break + yield from batch + offset += chunk_size + return + + ordering = [] + for term in raw_ordering: + descending = term.startswith("-") + field_name = term[1:] if descending else term + if field_name == "pk": + field_name = pk_field + ordering.append(f"-{field_name}" if descending else field_name) + + ordered_field_names = [term[1:] if term.startswith("-") else term for term in ordering] + try: + if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names): + offset = 0 + while True: + batch = list(self[offset : offset + chunk_size]) + if not batch: + break + yield from batch + offset += chunk_size + return + except Exception: + offset = 0 + while True: + batch = list(self[offset : offset + chunk_size]) + if not batch: + break + yield from batch + offset += chunk_size + return + + unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if getattr(field, "unique", False))} + if not any(field_name in unique_field_names for field_name in ordered_field_names): + offset = 0 + while True: + batch = list(self[offset : offset + chunk_size]) + if not batch: + break + yield from batch + offset += chunk_size + return + + last_values = None + value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field])) + while True: + batch_qs = self.order_by(*ordering) + if last_values is not None: + page_filter = models.Q() + for idx, term in enumerate(ordering): + descending = term.startswith("-") + field_name = term[1:] if descending else term + prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)} + comparison = "lt" if descending else "gt" + page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]}) + batch_qs = batch_qs.filter(page_filter) + + batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size]) + if not batch_rows: + break + + pk_idx = value_field_names.index(pk_field) + snapshot_ids = [row[pk_idx] for row in batch_rows] + snapshots_by_id = {snapshot.pk: snapshot for snapshot in self.filter(pk__in=snapshot_ids).order_by()} + + for row in batch_rows: + snapshot_id = row[pk_idx] + snapshot = snapshots_by_id.get(snapshot_id) + if snapshot is not None: + yield snapshot + + last_values = batch_rows[-1][: len(ordered_field_names)] + # ========================================================================= # Filtering Methods # ========================================================================= @@ -345,6 +441,14 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW default=ModelWithStateMachine.StatusChoices.QUEUED, ) config = models.JSONField(default=dict, null=False, blank=False, editable=True) + permissions = models.GeneratedField( + expression=KT("config__PERMISSIONS"), + output_field=models.CharField(max_length=16, null=True), + db_persist=True, + db_index=True, + editable=False, + ) + output_size = models.BigIntegerField(default=0, db_index=True, editable=False, help_text="Total bytes of all ArchiveResult output files") notes = models.TextField(blank=True, null=False, default="") # output_dir is computed via @cached_property from fs_version and get_storage_path_for_version() @@ -389,6 +493,69 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW def __str__(self): return f"[{self.id}] {self.url[:64]}" + def update_and_requeue(self, **kwargs) -> bool: + """ + Update this Snapshot through the shared retry_at ownership path. + + Any non-final Snapshot work means the parent Crawl must also be visible + to the runner. Keep that invariant here so CLI/admin callers do not + hand-edit the parent Crawl state every time they retry a hook. + """ + updated = super().update_and_requeue(**kwargs) + if not updated: + return False + + next_status = kwargs.get("status", self.status) + if next_status not in (self.StatusChoices.QUEUED, self.StatusChoices.STARTED) or not self.crawl_id: + return True + + crawl = self.crawl + crawl_status = crawl.StatusChoices.STARTED if crawl.status == crawl.StatusChoices.STARTED else crawl.StatusChoices.QUEUED + crawl.update_and_requeue( + status=crawl_status, + retry_at=kwargs.get("retry_at") or timezone.now(), + ) + return True + + def queue_for_extraction(self, *, when=None) -> bool: + """Queue this Snapshot for the runner using the normal state path.""" + return self.update_and_requeue( + status=self.StatusChoices.QUEUED, + retry_at=when or timezone.now(), + current_step=0, + ) + + def pause(self, *, save: bool = True) -> bool: + paused = super().pause(save=save) + if paused and self.pk: + ArchiveResult.pause_queryset(self.archiveresult_set.all()) + return paused + + def resume(self, *, when: datetime | None = None, save: bool = True) -> bool: + resumed = super().resume(when=when, save=save) + if resumed and self.pk: + ArchiveResult.resume_queryset(self.archiveresult_set.all(), when=when) + return resumed + + def restore_paused_scheduler_marker(self) -> None: + """ + Keep explicit maintenance from accidentally resuming paused snapshots. + + Targeted jobs such as `archivebox update --index-only` may bump + retry_at so the orchestrator can run only queued search ArchiveResult + rows. After that maintenance pass, the lifecycle must remain PAUSED and + retry_at must go back to MAX until a real resume transition happens. + """ + type(self).objects.filter(pk=self.pk, status=self.StatusChoices.PAUSED).update( + retry_at=RETRY_AT_MAX, + modified_at=timezone.now(), + ) + + def cancel(self) -> None: + self.status = self.StatusChoices.SEALED + self.retry_at = None + self.save(update_fields=["status", "retry_at", "modified_at"]) + def get_delete_after_config_value(self): return get_config(snapshot=self).DELETE_AFTER @@ -1989,22 +2156,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW @cached_property def archive_size(self): - if hasattr(self, "output_size_sum"): - return int(self.output_size_sum or 0) - - prefetched_results = None - if hasattr(self, "_prefetched_objects_cache"): - prefetched_results = self._prefetched_objects_cache.get("archiveresult_set") - if prefetched_results: - return sum(result.output_size or result.output_size_from_files() for result in prefetched_results) - - stats = self.archiveresult_set.aggregate(result_count=models.Count("id"), total_size=models.Sum("output_size")) - if stats["result_count"]: - return int(stats["total_size"] or 0) - try: - return get_dir_size(self.output_dir)[0] - except Exception: - return 0 + return int(self.output_size or 0) def save_tags(self, tags: Iterable[str] = ()) -> None: tags_id = [Tag.objects.get_or_create(name=tag)[0].pk for tag in tags if tag.strip()] @@ -2245,9 +2397,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW update_fields = [] if queue_for_extraction: - snapshot.status = Snapshot.StatusChoices.QUEUED + if snapshot.status != Snapshot.StatusChoices.PAUSED: + snapshot.status = Snapshot.StatusChoices.QUEUED + update_fields.append("status") snapshot.retry_at = timezone.now() - update_fields.extend(["status", "retry_at"]) + update_fields.append("retry_at") # Update additional fields if provided for field_name in ("depth", "parent_snapshot_id", "crawl_id", "bookmarked_at", "created_at", "downloaded_at"): @@ -2384,6 +2538,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW ], ) legacy_result_count = retryable_results.filter(hook_name="").count() + now = timezone.now() count = retryable_results.exclude(hook_name="").update( status=ArchiveResult.StatusChoices.QUEUED, output_str="", @@ -2393,19 +2548,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW output_mimetypes="", start_ts=None, end_ts=None, + modified_at=now, ) if count + legacy_result_count > 0: - self.status = self.StatusChoices.QUEUED - self.retry_at = timezone.now() - self.current_step = 0 # Reset to step 0 for retry - self.save(update_fields=["status", "retry_at", "current_step", "modified_at"]) - - crawl = self.crawl - if crawl.status != crawl.StatusChoices.STARTED: - crawl.status = crawl.StatusChoices.QUEUED - crawl.retry_at = timezone.now() - crawl.save(update_fields=["status", "retry_at", "modified_at"]) + self.queue_for_extraction(when=now) return count + legacy_result_count @@ -2961,13 +3108,21 @@ class SnapshotMachine(BaseStateMachine): # States queued = State(value=Snapshot.StatusChoices.QUEUED, initial=True) started = State(value=Snapshot.StatusChoices.STARTED) + paused = State(value=Snapshot.StatusChoices.PAUSED) sealed = State(value=Snapshot.StatusChoices.SEALED, final=True) # Tick Event (polled by workers) - tick = queued.to.itself(unless="can_start") | queued.to(started, cond="can_start") | started.to(sealed, cond="is_finished") + tick = ( + queued.to.itself(unless="can_start") + | queued.to(started, cond="can_start") + | started.to(sealed, cond="is_finished") + | paused.to.itself() + ) # Manual event (can also be triggered by last ArchiveResult finishing) seal = started.to(sealed) + pause_requested = queued.to(paused) | started.to(paused) + resume_requested = paused.to(queued) snapshot: Snapshot @@ -2986,6 +3141,13 @@ class SnapshotMachine(BaseStateMachine): status=Snapshot.StatusChoices.QUEUED, ) + @paused.enter + def enter_paused(self): + self.snapshot.update_and_requeue( + retry_at=RETRY_AT_MAX, + status=Snapshot.StatusChoices.PAUSED, + ) + @started.enter def enter_started(self): """Just mark as started. The shared runner creates ArchiveResults and runs hooks.""" @@ -2995,8 +3157,6 @@ class SnapshotMachine(BaseStateMachine): @sealed.enter def enter_sealed(self): - import sys - # Clean up background hooks self.snapshot.cleanup() @@ -3005,19 +3165,19 @@ class SnapshotMachine(BaseStateMachine): status=Snapshot.StatusChoices.SEALED, ) - print(f"[cyan] βœ… SnapshotMachine.enter_sealed() - sealed {self.snapshot.url}[/cyan]", file=sys.stderr) - # Check if this is the last snapshot for the parent crawl - if so, seal the crawl if self.snapshot.crawl: crawl = self.snapshot.crawl remaining_active = Snapshot.objects.filter( crawl=crawl, - status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED], + status__in=[ + Snapshot.StatusChoices.QUEUED, + Snapshot.StatusChoices.STARTED, + Snapshot.StatusChoices.PAUSED, + ], ).count() if remaining_active == 0 and crawl.status == crawl.StatusChoices.STARTED: - print(f"[cyan]πŸ”’ All snapshots sealed for crawl {crawl.id}, sealing crawl[/cyan]", file=sys.stderr) - # Seal the parent crawl cast(Any, crawl).sm.seal() @@ -3025,6 +3185,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M class StatusChoices(models.TextChoices): QUEUED = "queued", "Queued" STARTED = "started", "Started" + PAUSED = "paused", "Paused" BACKOFF = "backoff", "Waiting to retry" SUCCEEDED = "succeeded", "Succeeded" FAILED = "failed", "Failed" @@ -3053,6 +3214,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M "noresults": cls.StatusChoices.NORESULTS, "queued": cls.StatusChoices.QUEUED, "started": cls.StatusChoices.STARTED, + "paused": cls.StatusChoices.PAUSED, "backoff": cls.StatusChoices.BACKOFF, }.get(str(status or "").strip().lower(), cls.StatusChoices.FAILED) @@ -3100,6 +3262,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M end_ts = models.DateTimeField(default=None, null=True, blank=True) status = models.CharField(max_length=16, choices=StatusChoices.choices, default=StatusChoices.QUEUED, db_index=True) + retry_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True) notes = models.TextField(blank=True, null=False, default="") # output_dir is computed via @property from snapshot.output_dir / plugin @@ -3123,6 +3286,22 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M def __str__(self): return f"[{self.id}] {self.snapshot.url[:64]} -> {self.plugin}" + @staticmethod + def _format_output_line_for_display(line: str) -> str: + raw_line = str(line or "") + stripped = raw_line.strip() + if not stripped or "://" in stripped or not stripped.startswith(("/", "~/")): + return raw_line + try: + data_dir = CONSTANTS.DATA_DIR.expanduser().resolve(strict=False) + rel_path = Path(stripped).expanduser().resolve(strict=False).relative_to(data_dir) + except (OSError, ValueError): + return raw_line + return f"{raw_line[: len(raw_line) - len(raw_line.lstrip())]}./{rel_path}{raw_line[len(raw_line.rstrip()):]}" + + def output_str_for_display(self) -> str: + return "\n".join(self._format_output_line_for_display(line) for line in str(self.output_str or "").splitlines()) + def get_delete_after_config_value(self): return get_config(archiveresult=self).DELETE_AFTER @@ -3225,6 +3404,13 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M return None def save(self, *args, **kwargs): + is_new = self._state.adding + update_fields = kwargs.get("update_fields") + refresh_snapshot_size = is_new or update_fields is None or "output_size" in update_fields or "snapshot" in update_fields or "snapshot_id" in update_fields + old_snapshot_id = None + if refresh_snapshot_size and not is_new: + old_snapshot_id = type(self).objects.filter(pk=self.pk).values_list("snapshot_id", flat=True).first() + update_fields = kwargs.get("update_fields") if self.delete_at is None: self.set_delete_at_from_config() @@ -3234,6 +3420,9 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M # Skip ModelWithOutputDir.save() to avoid creating index.json in plugin directories # Call the Django Model.save() directly instead models.Model.save(self, *args, **kwargs) + if refresh_snapshot_size: + snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, self.snapshot_id) if snapshot_id} + transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids)) # if is_new: # from archivebox.misc.logging_util import log_worker_event @@ -3250,6 +3439,19 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M # }, # ) + def delete(self, *args, **kwargs): + snapshot_id = self.snapshot_id + deleted = super().delete(*args, **kwargs) + if snapshot_id: + transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes({snapshot_id})) + return deleted + + @staticmethod + def refresh_snapshot_output_sizes(snapshot_ids): + for snapshot_id in snapshot_ids: + total_size = ArchiveResult.objects.filter(snapshot_id=snapshot_id).aggregate(total_size=Sum("output_size"))["total_size"] or 0 + Snapshot.objects.filter(pk=snapshot_id).update(output_size=total_size) + @cached_property def snapshot_dir(self): return Path(self.snapshot.output_dir) @@ -3267,6 +3469,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M def reset_for_retry(self, *, save: bool = True) -> None: self.status = self.StatusChoices.QUEUED + self.retry_at = None self.output_str = "" self.output_json = None self.output_files = {} @@ -3278,6 +3481,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M self.save( update_fields=[ "status", + "retry_at", "output_str", "output_json", "output_files", @@ -3289,6 +3493,48 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M ], ) + @property + def is_paused(self) -> bool: + return self.status == self.StatusChoices.PAUSED + + @classmethod + def pause_queryset(cls, queryset) -> int: + return queryset.exclude(status__in=[*cls.FINAL_STATES, cls.StatusChoices.PAUSED]).update( + status=cls.StatusChoices.PAUSED, + retry_at=RETRY_AT_MAX, + modified_at=timezone.now(), + ) + + @classmethod + def resume_queryset(cls, queryset, *, when: datetime | None = None) -> int: + return queryset.filter(status=cls.StatusChoices.PAUSED).update( + status=cls.StatusChoices.QUEUED, + retry_at=when or timezone.now(), + modified_at=timezone.now(), + ) + + def pause(self, *, save: bool = True) -> bool: + if self.status in self.FINAL_STATES: + return False + if self.is_paused: + return False + self.status = self.StatusChoices.PAUSED + self.retry_at = RETRY_AT_MAX + if save: + self.pause_queryset(type(self).objects.filter(pk=self.pk)) + self.refresh_from_db() + return True + + def resume(self, *, when: datetime | None = None, save: bool = True) -> bool: + if not self.is_paused: + return False + self.status = self.StatusChoices.QUEUED + self.retry_at = when or timezone.now() + if save: + self.resume_queryset(type(self).objects.filter(pk=self.pk), when=self.retry_at) + self.refresh_from_db() + return True + @property def plugin_module(self) -> Any | None: # Hook scripts are now used instead of Python plugin modules @@ -3370,7 +3616,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M return False snapshot_dir = Path(snapshot_dir or self.snapshot.output_dir) - exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid", "cmd.sh"} + exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"} output_files: dict[str, dict[str, Any]] = {} mime_sizes: dict[str, int] = defaultdict(int) total_size = 0 @@ -3505,7 +3751,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M plugin_name: str | None = None, output_file_map: dict[str, dict[str, Any]] | None = None, ) -> str | None: - ignored = {"stdout.log", "stderr.log", "hook.pid", "listener.pid", "cmd.sh"} + ignored = {"stdout.log", "stderr.log", "hook.pid", "listener.pid"} candidates = [ path for path in output_file_paths @@ -3731,15 +3977,14 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M self.save() return - # Read and parse JSONL output from stdout.log - stdout_file = plugin_dir / "stdout.log" records = [] process = self.process_record if process: records = extract_records_from_process(process) if not records: - stdout = stdout_file.read_text() if stdout_file.exists() else "" + stdout_file = plugin_dir / "stdout.log" + stdout = stdout_file.read_text(errors="replace") if stdout_file.exists() else "" records = Process.parse_records_from_text(stdout) # Find ArchiveResult record and update status/output from it @@ -3785,7 +4030,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M self.output_str = "Hook did not output ArchiveResult record" # Walk filesystem and populate output_files, output_size, output_mimetypes - exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid", "cmd.sh"} + exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"} mime_sizes = defaultdict(int) total_size = 0 output_files = {} diff --git a/archivebox/core/permissions.py b/archivebox/core/permissions.py new file mode 100644 index 00000000..045ad6d1 --- /dev/null +++ b/archivebox/core/permissions.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from django.db.models import Q, QuerySet +from django.http import HttpRequest + +from archivebox.config.common import get_config + +PERMISSIONS_PUBLIC = "public" +PERMISSIONS_UNLISTED = "unlisted" +PERMISSIONS_PRIVATE = "private" +PERMISSIONS_CHOICES = ( + (PERMISSIONS_PUBLIC, "Public"), + (PERMISSIONS_UNLISTED, "Unlisted"), + (PERMISSIONS_PRIVATE, "Private"), +) + + +def is_admin_user(request: HttpRequest) -> bool: + user = request.user + return bool(user.is_authenticated and user.is_active and user.is_staff) + + +def get_snapshot_permissions(snapshot) -> str: + try: + return str(get_config(snapshot=snapshot, resolve_plugins=False).PERMISSIONS).strip().lower() + except Exception: + return PERMISSIONS_PRIVATE + + +def can_view_snapshot(request: HttpRequest, snapshot) -> bool: + permissions = get_snapshot_permissions(snapshot) + return permissions in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED} or is_admin_user(request) + + +def _persona_ids_for_permissions(allowed_permissions: set[str]) -> list[str]: + from archivebox.personas.models import Persona + + fallback_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower() + personas = Persona.objects.only("id", "config") + return [ + str(persona.id) + for persona in personas + if (persona.permissions or fallback_permissions) in allowed_permissions + ] + + +def filter_personas_by_permissions(queryset: QuerySet, allowed_permissions: set[str]) -> QuerySet: + return queryset.filter(id__in=_persona_ids_for_permissions(allowed_permissions)) + + +def filter_snapshots_by_permissions(queryset: QuerySet, *, direct: bool = False, allowed_permissions: set[str] | None = None) -> QuerySet: + from archivebox.crawls.models import Crawl + from archivebox.personas.models import Persona + + allowed_permissions = allowed_permissions or ({PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED} if direct else {PERMISSIONS_PUBLIC}) + fallback_permissions = str(get_config(resolve_plugins=False).PERMISSIONS).strip().lower() + has_overrides = ( + queryset.model.objects.filter(permissions__gt="").exists() + or Crawl.objects.filter(permissions__gt="").exists() + or Persona.objects.filter(permissions__gt="").exists() + ) + if not has_overrides: + return queryset if fallback_permissions in allowed_permissions else queryset.none() + + allowed_persona_ids = _persona_ids_for_permissions(allowed_permissions) + valid_persona_ids = [str(persona_id) for persona_id in Persona.objects.values_list("id", flat=True)] + fallback_query = Q(crawl__persona_id__in=allowed_persona_ids) + if fallback_permissions in allowed_permissions: + fallback_query |= Q(crawl__persona_id__isnull=True) | ~Q(crawl__persona_id__in=valid_persona_ids) + inherited_query = Q(crawl__permissions__in=sorted(allowed_permissions)) | (Q(crawl__permissions__isnull=True) & fallback_query) + return queryset.filter( + Q(permissions__in=sorted(allowed_permissions)) | (Q(permissions__isnull=True) & inherited_query), + ) + + +def public_snapshots_queryset(queryset: QuerySet) -> QuerySet: + return filter_snapshots_by_permissions(queryset, direct=False) + + +def direct_snapshots_queryset(request: HttpRequest, queryset: QuerySet) -> QuerySet: + return queryset if is_admin_user(request) else filter_snapshots_by_permissions(queryset, direct=True) diff --git a/archivebox/core/recovery_util.py b/archivebox/core/recovery_util.py new file mode 100644 index 00000000..1cb92be0 --- /dev/null +++ b/archivebox/core/recovery_util.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +from datetime import timedelta + +from django.utils import timezone +from rich.console import Console + + +def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]: + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.machine.models import Process + from django.db.models import Exists, OuterRef, Q, Subquery, Value + from django.db.models.functions import Coalesce + + now = timezone.now() + stuck_cutoff = now - timedelta(hours=12) + recovery_console = Console(stderr=True, highlight=False) + cleaned = { + "stale_processes": Process.cleanup_stale_running(), + "orphaned_processes": Process.cleanup_orphaned_workers(), + "orphaned_chrome": Process.cleanup_orphaned_chrome() if include_chrome else 0, + "queued_crawls_unlocked": 0, + "sealed_crawl_locks_cleared": 0, + "sealed_snapshots": 0, + "unlocked_snapshots": 0, + "requeued_snapshots": 0, + "queued_snapshots_unlocked": 0, + "sealed_snapshot_locks_cleared": 0, + "requeued_archiveresults": 0, + "sealed_crawls": 0, + "unlocked_crawls": 0, + "requeued_crawls": 0, + "sealed_queued_snapshots": 0, + "sealed_queued_crawls": 0, + } + + any_archiveresults = ArchiveResult.objects.filter(snapshot_id=OuterRef("pk")) + unfinished_archiveresults = any_archiveresults.exclude(status__in=ArchiveResult.FINAL_STATES) + recent_snapshots = Snapshot.objects.filter(crawl_id=OuterRef("pk"), modified_at__gt=stuck_cutoff) + recent_archiveresults = ArchiveResult.objects.filter(snapshot__crawl_id=OuterRef("pk"), modified_at__gt=stuck_cutoff) + recent_archiveresult_processes = Process.objects.filter( + archiveresult__snapshot__crawl_id=OuterRef("pk"), + modified_at__gt=stuck_cutoff, + ) + recent_crawl_snapshots_for_snapshot = Snapshot.objects.filter(crawl_id=OuterRef("crawl_id"), modified_at__gt=stuck_cutoff) + recent_crawl_archiveresults_for_snapshot = ArchiveResult.objects.filter( + snapshot__crawl_id=OuterRef("crawl_id"), + modified_at__gt=stuck_cutoff, + ) + recent_crawl_archiveresult_processes_for_snapshot = Process.objects.filter( + archiveresult__snapshot__crawl_id=OuterRef("crawl_id"), + modified_at__gt=stuck_cutoff, + ) + + # Stale-only repair: if a queued snapshot/crawl already has only final + # projected result rows and the whole crawl has been quiet for >12hr, it + # was likely interrupted after hook completion but before state sealing. + # Never run this on fresh rows: queued work is normal during direct + # reindex/extract and while a daemon runner is active. + stale_finished_snapshot_ids = ( + Snapshot.objects.filter( + status=Snapshot.StatusChoices.QUEUED, + modified_at__lte=stuck_cutoff, + crawl__modified_at__lte=stuck_cutoff, + ) + .filter(Q(retry_at__isnull=True) | Q(retry_at__lte=stuck_cutoff)) + .annotate( + has_results=Exists(any_archiveresults), + has_unfinished_results=Exists(unfinished_archiveresults), + has_recent_snapshot=Exists(recent_crawl_snapshots_for_snapshot), + has_recent_archiveresult=Exists(recent_crawl_archiveresults_for_snapshot), + has_recent_archiveresult_process=Exists(recent_crawl_archiveresult_processes_for_snapshot), + ) + .filter( + has_results=True, + has_unfinished_results=False, + has_recent_snapshot=False, + has_recent_archiveresult=False, + has_recent_archiveresult_process=False, + ) + .values_list("id", flat=True) + ) + cleaned["sealed_queued_snapshots"] = Snapshot.objects.filter(id__in=stale_finished_snapshot_ids).update( + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + downloaded_at=Coalesce("downloaded_at", Value(now)), + ) + unrecoverable_active_child_snapshots = ( + Snapshot.objects.filter( + crawl_id=OuterRef("pk"), + status__in=[ + Snapshot.StatusChoices.QUEUED, + Snapshot.StatusChoices.STARTED, + Snapshot.StatusChoices.PAUSED, + ], + ) + .annotate( + has_results=Exists(any_archiveresults), + has_unfinished_results=Exists(unfinished_archiveresults), + ) + .filter( + Q(status=Snapshot.StatusChoices.STARTED) + | Q(modified_at__gt=stuck_cutoff) + | Q(retry_at__gt=stuck_cutoff) + | Q(has_results=False) + | Q(has_unfinished_results=True), + ) + ) + cleaned["sealed_queued_crawls"] = ( + Crawl.objects.filter( + status=Crawl.StatusChoices.QUEUED, + snapshot_set__isnull=False, + modified_at__lte=stuck_cutoff, + ) + .filter(Q(retry_at__isnull=True) | Q(retry_at__lte=stuck_cutoff)) + .annotate( + has_unrecoverable_active_child=Exists(unrecoverable_active_child_snapshots), + has_recent_snapshot=Exists(recent_snapshots), + has_recent_archiveresult=Exists(recent_archiveresults), + has_recent_archiveresult_process=Exists(recent_archiveresult_processes), + ) + .filter( + has_unrecoverable_active_child=False, + has_recent_snapshot=False, + has_recent_archiveresult=False, + has_recent_archiveresult_process=False, + ) + .update( + status=Crawl.StatusChoices.SEALED, + retry_at=None, + modified_at=now, + ) + ) + + stale_crawls = ( + Crawl.objects.filter( + status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED], + modified_at__lte=stuck_cutoff, + ) + .filter(Q(retry_at__isnull=True) | Q(retry_at__lte=now)) + .annotate( + has_recent_snapshot=Exists(recent_snapshots), + has_recent_archiveresult=Exists(recent_archiveresults), + has_recent_archiveresult_process=Exists(recent_archiveresult_processes), + ) + .filter(has_recent_snapshot=False, has_recent_archiveresult=False, has_recent_archiveresult_process=False) + .order_by("modified_at")[:10] + ) + stale_crawl_messages = [] + for crawl in stale_crawls: + if not Process.objects.filter( + pwd__contains=str(crawl.id), + status=Process.StatusChoices.RUNNING, + modified_at__gt=stuck_cutoff, + ).exists(): + stale_crawl_messages.append( + f"{crawl.id} status={crawl.status} retry_at={crawl.retry_at} modified_at={crawl.modified_at}", + ) + if stale_crawl_messages: + recovery_console.print( + "[red]❌ Orchestrator recovery found stuck active crawl invariant violation; refusing to continue.[/red]", + ) + raise RuntimeError( + "Stuck crawl invariant violated: active crawls had no crawl/snapshot/result/process changes for >12hr: " + + "; ".join(stale_crawl_messages), + ) + + running_archiveresults = ArchiveResult.objects.filter( + snapshot_id=OuterRef("pk"), + status=ArchiveResult.StatusChoices.STARTED, + process__status=Process.StatusChoices.RUNNING, + ) + unfinished_archiveresult_statuses = [ + ArchiveResult.StatusChoices.QUEUED, + ArchiveResult.StatusChoices.STARTED, + ArchiveResult.StatusChoices.PAUSED, + ArchiveResult.StatusChoices.BACKOFF, + ] + running_unfinished_archiveresults = ArchiveResult.objects.filter( + snapshot_id=OuterRef("pk"), + status__in=unfinished_archiveresult_statuses, + process__status=Process.StatusChoices.RUNNING, + ) + unfinished_without_running_archiveresults = ArchiveResult.objects.filter( + snapshot_id=OuterRef("pk"), + status__in=[ + ArchiveResult.StatusChoices.QUEUED, + ArchiveResult.StatusChoices.STARTED, + ArchiveResult.StatusChoices.BACKOFF, + ], + ).exclude( + status=ArchiveResult.StatusChoices.PAUSED, + ).exclude( + process__status=Process.StatusChoices.RUNNING, + ) + active_child_snapshots = Snapshot.objects.filter( + crawl_id=OuterRef("pk"), + status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED], + ) + due_child_snapshots = active_child_snapshots.exclude(status=Snapshot.StatusChoices.PAUSED).filter( + Q(retry_at__isnull=True) | Q(retry_at__lte=now), + ) + next_future_child_retry = Subquery( + active_child_snapshots.filter(retry_at__gt=now).order_by("retry_at").values("retry_at")[:1], + ) + + # Broken lock repair: QUEUED rows with retry_at=NULL are invisible to the + # queue. Set only the scheduling field so the runner owns the next tick. + cleaned["queued_crawls_unlocked"] = Crawl.objects.filter( + status=Crawl.StatusChoices.QUEUED, + retry_at__isnull=True, + ).update(retry_at=now, modified_at=now) + cleaned["queued_snapshots_unlocked"] = Snapshot.objects.filter( + status=Snapshot.StatusChoices.QUEUED, + retry_at__isnull=True, + ).update(retry_at=now, modified_at=now) + + # ArchiveResult has no retry_at scheduler; BACKOFF is a legacy/impossible + # persisted state here, so move it back to QUEUED for the snapshot runner. + cleaned["requeued_archiveresults"] = ArchiveResult.objects.filter( + status=ArchiveResult.StatusChoices.BACKOFF, + ).update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now) + # Impossible state repair: STARTED ArchiveResults without a live Process + # have no owner left to emit completion. Requeue only the result row; the + # snapshot/crawl schedulers will pick up normal retry processing. + cleaned["requeued_archiveresults"] += ( + ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED) + .exclude(process__status=Process.StatusChoices.RUNNING) + .update(status=ArchiveResult.StatusChoices.QUEUED, process=None, modified_at=now) + ) + + started_snapshots = Snapshot.objects.filter( + status=Snapshot.StatusChoices.STARTED, + retry_at__isnull=True, + ) + + # Normal transition: the snapshot has finished all known extractor work, + # but the process died before the state machine got to seal it. + finished_snapshot_ids = ( + started_snapshots.annotate( + has_results=Exists(any_archiveresults), + has_unfinished_results=Exists(unfinished_archiveresults), + ) + .filter(has_results=True, has_unfinished_results=False) + .values_list("id", flat=True) + ) + for snapshot in Snapshot.objects.filter(id__in=finished_snapshot_ids).select_related("crawl").iterator(chunk_size=100): + snapshot.sm.seal() + cleaned["sealed_snapshots"] += 1 + + # Broken lock repair: STARTED + retry_at=NULL means "owned by an active + # runner". If no ArchiveResult has a live process anymore, only unlock it. + # The existing runner will pick the row up through the normal queue path. + cleaned["unlocked_snapshots"] = ( + started_snapshots.annotate(has_running_results=Exists(running_archiveresults)) + .filter(has_running_results=False) + .update( + retry_at=now, + modified_at=now, + ) + ) + + # Impossible state repair: a SEALED snapshot with a still-running child is + # active, not final. Reflect that without starting duplicate work. + cleaned["requeued_snapshots"] += ( + Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED) + .annotate(has_running_unfinished_results=Exists(running_unfinished_archiveresults)) + .filter(has_running_unfinished_results=True) + .update( + status=Snapshot.StatusChoices.STARTED, + retry_at=None, + modified_at=now, + ) + ) + + # Impossible state repair: SEALED snapshots should not contain unfinished + # ArchiveResults. There is no valid state-machine transition from final + # back to queued, so repair only the fields needed for the runner to retry. + cleaned["requeued_snapshots"] += ( + Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED) + .annotate(has_unfinished_results_without_running=Exists(unfinished_without_running_archiveresults)) + .filter(has_unfinished_results_without_running=True) + .update( + status=Snapshot.StatusChoices.QUEUED, + retry_at=now, + modified_at=now, + ) + ) + + # Normal transition: a started crawl has no active snapshots left. + finished_crawl_ids = ( + Crawl.objects.filter(status=Crawl.StatusChoices.STARTED, retry_at__isnull=True) + .exclude( + snapshot_set__status__in=[ + Snapshot.StatusChoices.QUEUED, + Snapshot.StatusChoices.STARTED, + Snapshot.StatusChoices.PAUSED, + ], + ) + .values_list("id", flat=True) + ) + for crawl in Crawl.objects.filter(id__in=finished_crawl_ids).iterator(chunk_size=100): + crawl.sm.seal() + cleaned["sealed_crawls"] += 1 + + # Broken lock repair: STARTED + retry_at=NULL with unfinished snapshots is + # recoverable by unlocking the crawl. Do not create snapshots or results. + due_started_crawls = ( + Crawl.objects.filter(status=Crawl.StatusChoices.STARTED, retry_at__isnull=True) + .annotate(has_due_child=Exists(due_child_snapshots)) + .filter(has_due_child=True) + ) + cleaned["unlocked_crawls"] = due_started_crawls.update(retry_at=now, modified_at=now) + future_started_crawls = ( + Crawl.objects.filter(status=Crawl.StatusChoices.STARTED, retry_at__isnull=True) + .annotate(has_active_child=Exists(active_child_snapshots), has_due_child=Exists(due_child_snapshots), next_child_retry=next_future_child_retry) + .filter(has_active_child=True, has_due_child=False) + ) + cleaned["unlocked_crawls"] += future_started_crawls.update(retry_at=Coalesce("next_child_retry", Value(now)), modified_at=now) + + cleaned["requeued_crawls"] = 0 + + warning_recoveries = { + "stale_processes": "marked stale running Process row(s) exited", + "orphaned_processes": "marked orphaned worker/hook Process row(s) exited", + "orphaned_chrome": "terminated orphaned Chrome process(es)", + "sealed_snapshots": "sealed started Snapshot row(s) whose ArchiveResults were already final", + "unlocked_snapshots": "unlocked started Snapshot row(s) whose owner process was gone", + "sealed_crawls": "sealed started Crawl row(s) with no active Snapshots", + "unlocked_crawls": "unlocked started Crawl row(s) with pending child Snapshots", + } + error_recoveries = { + "queued_crawls_unlocked": "repaired queued Crawl row(s) with retry_at=NULL", + "queued_snapshots_unlocked": "repaired queued Snapshot row(s) with retry_at=NULL", + "requeued_archiveresults": "requeued ArchiveResult row(s) left in BACKOFF", + "requeued_snapshots": "reopened sealed Snapshot row(s) with unfinished ArchiveResults", + "requeued_crawls": "reopened sealed Crawl row(s) with active child Snapshots", + "sealed_queued_snapshots": "sealed stale queued Snapshot row(s) whose ArchiveResults were already final", + "sealed_queued_crawls": "sealed stale queued Crawl row(s) whose Snapshots were already final", + } + for key, message in warning_recoveries.items(): + if cleaned[key]: + recovery_console.print(f"[yellow]⚠️ Orchestrator recovery: {cleaned[key]} {message}.[/yellow]") + for key, message in error_recoveries.items(): + if cleaned[key]: + recovery_console.print(f"[red]❌ Orchestrator invariant repair: {cleaned[key]} {message}.[/red]") + + return cleaned diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 97970bb5..70e3c3e1 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -229,7 +229,7 @@ SQLITE_JOURNAL_MODE = os.environ.get("ARCHIVEBOX_SQLITE_JOURNAL_MODE", "WAL") SQLITE_MMAP_SIZE = os.environ.get("ARCHIVEBOX_SQLITE_MMAP_SIZE", "0" if CONSTANTS.IN_DOCKER else "134217728") SQLITE_CONNECTION_OPTIONS = { - "ENGINE": "django.db.backends.sqlite3", + "ENGINE": "archivebox.core.sqlite_backend", "TIME_ZONE": CONSTANTS.TIMEZONE, "OPTIONS": { # https://gcollazo.com/optimal-sqlite-settings-for-django/ @@ -237,7 +237,13 @@ SQLITE_CONNECTION_OPTIONS = { # https://docs.djangoproject.com/en/5.1/ref/databases/#setting-pragma-options "timeout": 30, "check_same_thread": False, - "transaction_mode": "IMMEDIATE", + # Keep SQLite on Django's default deferred transaction mode. BEGIN + # IMMEDIATE grabs the write lock as soon as atomic() opens, which is + # exactly what hurts ArchiveBox on large collections where Python code + # may do filesystem work before the actual row write. Deferred BEGIN + # keeps writes statement-scoped unless a caller explicitly opens a + # transaction around multiple writes. + "transaction_mode": None, "init_command": ( "PRAGMA foreign_keys=ON;" "PRAGMA busy_timeout = 30000;" diff --git a/archivebox/core/shutdown_util.py b/archivebox/core/shutdown_util.py new file mode 100644 index 00000000..3a392107 --- /dev/null +++ b/archivebox/core/shutdown_util.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import signal +import subprocess +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass + +import psutil + + +@dataclass +class ShutdownSignalState: + """Tracks the exact OS signal that asked a foreground command to exit.""" + + signal_name: str | None = None + + +def configured_stopwaitsecs(workers: list[dict[str, str]] | tuple[dict[str, str], ...], *, default: int = 5, buffer: int = 5) -> int: + """Return a deterministic shutdown bound from generated worker definitions.""" + + stop_grace_seconds = default + for worker in workers: + try: + stop_grace_seconds = max(stop_grace_seconds, int(worker.get("stopwaitsecs") or default) + buffer) + except (TypeError, ValueError): + continue + return stop_grace_seconds + + +def wait_popen_and_kill_children( + proc: subprocess.Popen, + children: list[psutil.Process], + *, + timeout: float, + kill_timeout: float = 2.0, +) -> None: + """Wait for a Popen parent and then hard-kill any surviving descendants.""" + + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=kill_timeout) + kill_remaining_processes(children, timeout=kill_timeout) + + +def wait_psutil_and_kill_children( + proc: psutil.Process, + children: list[psutil.Process], + *, + timeout: float, + kill_timeout: float = 2.0, +) -> None: + """Wait for a psutil parent and then hard-kill any surviving descendants.""" + + try: + if proc.status() == psutil.STATUS_ZOMBIE: + # Another ArchiveBox foreground parent owns this Popen and must reap + # it. By the time supervisord is a zombie it has already stopped + # accepting work, so the caller can clear stale pid/socket files + # without blocking for a process it cannot reap itself. + kill_remaining_processes(children, timeout=kill_timeout) + return + proc.wait(timeout=timeout) + except psutil.TimeoutExpired: + proc.kill() + kill_remaining_processes(children, timeout=kill_timeout) + try: + proc.wait(timeout=kill_timeout) + except (psutil.NoSuchProcess, psutil.TimeoutExpired): + pass + + +def kill_remaining_processes(processes: list[psutil.Process], *, timeout: float = 2.0) -> None: + _gone, alive = psutil.wait_procs(processes, timeout=timeout) + for process in alive: + try: + process.kill() + except psutil.NoSuchProcess: + pass + psutil.wait_procs(alive, timeout=timeout) + + +@contextmanager +def foreground_shutdown_signals( + handled_signals: tuple[signal.Signals, ...] = (signal.SIGHUP, signal.SIGINT, signal.SIGTERM), +) -> Iterator[ShutdownSignalState]: + """Install foreground signal handlers that print an immediate exit notice. + + Some log-tail loops intentionally swallow KeyboardInterrupt so that callers + can centralize cleanup in finally blocks. The handler writes the signal name + immediately, then raises KeyboardInterrupt to break out of the blocking read. + """ + + state = ShutdownSignalState() + previous_handlers = {sig: signal.getsignal(sig) for sig in handled_signals} + + def raise_keyboard_interrupt(signum, _frame): + state.signal_name = signal.Signals(signum).name + sys.stdout.write(f"\n[πŸ›‘] Got {state.signal_name}, stopping gracefully...\n") + sys.stdout.flush() + raise KeyboardInterrupt + + try: + for sig in handled_signals: + signal.signal(sig, raise_keyboard_interrupt) + yield state + finally: + for sig, previous_handler in previous_handlers.items(): + signal.signal(sig, previous_handler) diff --git a/archivebox/ideas/__init__.py b/archivebox/core/sqlite_backend/__init__.py similarity index 100% rename from archivebox/ideas/__init__.py rename to archivebox/core/sqlite_backend/__init__.py diff --git a/archivebox/core/sqlite_backend/base.py b/archivebox/core/sqlite_backend/base.py new file mode 100644 index 00000000..029a810e --- /dev/null +++ b/archivebox/core/sqlite_backend/base.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import sqlite3 +import time +from collections.abc import Mapping +from itertools import tee +import re + +from django.db.backends.sqlite3.base import DatabaseWrapper as DjangoSQLiteDatabaseWrapper +from django.db.backends.sqlite3.base import SQLiteCursorWrapper as DjangoSQLiteCursorWrapper + + +def _is_locked_error(error: BaseException) -> bool: + from django.db import OperationalError + + return isinstance(error, (sqlite3.OperationalError, OperationalError)) and "database is locked" in str(error).lower() + + +def _format_sql(query: str, params=None) -> str: + compact = " ".join(str(query).split()) + match = re.match(r'^(INSERT INTO|UPDATE|DELETE FROM|SELECT) "?([A-Za-z0-9_]+)"?', compact, flags=re.IGNORECASE) + if match: + compact = f"{match.group(1).upper()} {match.group(2)}" + if params is not None: + if isinstance(params, str): + params_summary = params + elif isinstance(params, (tuple, list)): + preview = ", ".join(repr(param)[:60] for param in params[:4]) + params_summary = f"{len(params)} params: {preview}" + elif isinstance(params, Mapping): + preview = ", ".join(f"{key}={repr(value)[:60]}" for key, value in list(params.items())[:4]) + params_summary = f"{len(params)} params: {preview}" + else: + params_summary = repr(params)[:120] + compact = f"{compact} ({params_summary})" + return compact[:260] + + +def _log_locked_database(query: str, params=None, *, attempt: int, elapsed: float) -> None: + from rich.console import Console + + from archivebox.misc.db import sqlite_lock_holders + + console = Console(stderr=True) + console.print(f"[yellow][*] SQLite database is locked for {elapsed:.0f}s; retrying in 5s... attempt={attempt}[/yellow]") + console.print(f"[yellow] Query: {_format_sql(query, params)}[/yellow]") + holders = sqlite_lock_holders() + if holders: + console.print("[yellow] DB holders:[/yellow]") + for holder in holders[:8]: + console.print(f"[yellow] - {holder}[/yellow]") + if len(holders) > 8: + console.print(f"[yellow] ... {len(holders) - 8} more[/yellow]") + else: + console.print("[yellow] No local process with index.sqlite3 open was visible to this user.[/yellow]") + if attempt == 1: + console.print( + "[dim] SQLite does not expose the active SQL statement from another process; only local PIDs with the DB open can be shown.[/dim]", + ) + + +def _retry_locked_database(action, query: str, params=None): + attempt = 0 + started_at = time.monotonic() + while True: + try: + return action() + except (sqlite3.OperationalError, Exception) as err: + if not _is_locked_error(err): + raise + attempt += 1 + _log_locked_database(query, params, attempt=attempt, elapsed=time.monotonic() - started_at) + time.sleep(5.0) + + +class SQLiteCursorWrapper(DjangoSQLiteCursorWrapper): + def execute(self, query, params=None): + if params is None: + return _retry_locked_database(lambda: super(SQLiteCursorWrapper, self).execute(query), query) + param_names = list(params) if isinstance(params, Mapping) else None + converted_query = self.convert_query(query, param_names=param_names) + return _retry_locked_database( + lambda: super(DjangoSQLiteCursorWrapper, self).execute(converted_query, params), + converted_query, + params, + ) + + def executemany(self, query, param_list): + peekable, param_list = tee(iter(param_list)) + if (params := next(peekable, None)) and isinstance(params, Mapping): + param_names = list(params) + else: + param_names = None + converted_query = self.convert_query(query, param_names=param_names) + param_list = tuple(param_list) + return _retry_locked_database( + lambda: super(DjangoSQLiteCursorWrapper, self).executemany(converted_query, param_list), + converted_query, + f"{len(param_list)} parameter sets", + ) + + +class DatabaseWrapper(DjangoSQLiteDatabaseWrapper): + def create_cursor(self, name=None): + return self.connection.cursor(factory=SQLiteCursorWrapper) + + def _commit(self): + return _retry_locked_database(lambda: super(DatabaseWrapper, self)._commit(), "COMMIT") + + def _rollback(self): + return _retry_locked_database(lambda: super(DatabaseWrapper, self)._rollback(), "ROLLBACK") diff --git a/archivebox/core/tag_utils.py b/archivebox/core/tag_utils.py index 9f2e23e4..6851b4f5 100644 --- a/archivebox/core/tag_utils.py +++ b/archivebox/core/tag_utils.py @@ -104,14 +104,17 @@ def get_matching_tags( return queryset -def add_snapshot_counts(tags: list[Tag]) -> None: +def add_snapshot_counts(tags: list[Tag], snapshot_queryset: QuerySet[Snapshot] | None = None) -> None: tag_ids = [tag.pk for tag in tags] if not tag_ids: return + queryset = SnapshotTag.objects.filter(tag_id__in=tag_ids) + if snapshot_queryset is not None: + queryset = queryset.filter(snapshot_id__in=snapshot_queryset.values("id")) counts = { row["tag_id"]: row["num_snapshots"] - for row in SnapshotTag.objects.filter(tag_id__in=tag_ids).values("tag_id").annotate(num_snapshots=Count("snapshot_id")) + for row in queryset.values("tag_id").annotate(num_snapshots=Count("snapshot_id")) } for tag in tags: tag.num_snapshots = counts.get(tag.pk, 0) diff --git a/archivebox/core/templatetags/core_tags.py b/archivebox/core/templatetags/core_tags.py index 597d4c11..4b024b6d 100644 --- a/archivebox/core/templatetags/core_tags.py +++ b/archivebox/core/templatetags/core_tags.py @@ -278,6 +278,14 @@ def file_size(num_bytes: int | float) -> str: return "{:3.1f} {}".format(num_bytes, "TB") +@register.filter +def intcomma(value: int | str | None) -> str: + try: + return f"{int(value or 0):,}" + except (TypeError, ValueError): + return str(value or "") + + def result_list(context, cl): """ Monkey patched result diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py index 819789f0..a7bc4cfe 100644 --- a/archivebox/core/urls.py +++ b/archivebox/core/urls.py @@ -24,6 +24,7 @@ from archivebox.core.views import ( AddView, WebAddView, HealthCheckView, + live_progress_screencast_frame_view, live_progress_view, ) @@ -69,6 +70,11 @@ urlpatterns = [ path("accounts/login/", RedirectView.as_view(url="/admin/login/")), path("accounts/logout/", RedirectView.as_view(url="/admin/logout/")), path("accounts/", include("django.contrib.auth.urls")), + re_path( + r"^admin/live-progress/screencast/(?P[0-9a-fA-F-]{8,36})\.jpg$", + archivebox_admin.admin_view(live_progress_screencast_frame_view), + name="live_progress_screencast_frame", + ), path("admin/live-progress/", archivebox_admin.admin_view(live_progress_view), name="live_progress"), path("admin/", archivebox_admin.urls), path("api/", include("archivebox.api.urls"), name="api"), diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 9654e6aa..8c4683bf 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -12,7 +12,7 @@ from pathlib import Path from urllib.parse import quote, urlparse from django.shortcuts import render, redirect -from django.http import JsonResponse, HttpRequest, HttpResponse, Http404, HttpResponseForbidden, QueryDict +from django.http import FileResponse, JsonResponse, HttpRequest, HttpResponse, Http404, HttpResponseForbidden, QueryDict from django.utils.html import format_html from django.utils.safestring import mark_safe from django.views import View @@ -22,6 +22,7 @@ from django.db.models import CharField, Count, Q, Prefetch, Sum from django.db.models.functions import Cast from django.contrib import messages from django.contrib.auth.mixins import UserPassesTestMixin +from django.core.signing import BadSignature, SignatureExpired, TimestampSigner from django.views.decorators.csrf import csrf_exempt from django.views.decorators.gzip import gzip_page from django.utils.decorators import method_decorator @@ -37,9 +38,25 @@ from archivebox.config.configset import BaseConfigSet from archivebox.misc.util import base_url, htmlencode, ts_to_date_str, urldecode, without_fragment from archivebox.misc.serve_static import serve_static_with_byterange_support from archivebox.misc.logging_util import printable_filesize -from archivebox.search import get_search_mode, prioritize_metadata_matches, query_search_index +from archivebox.search import ( + get_search_backend_display_name, + get_search_mode, + get_search_mode_backend, + get_search_mode_base, + get_search_mode_options, + prioritize_metadata_matches, + query_search_index, +) from archivebox.core.models import ArchiveResult, Snapshot +from archivebox.core.permissions import ( + PERMISSIONS_PUBLIC, + can_view_snapshot, + direct_snapshots_queryset, + filter_personas_by_permissions, + is_admin_user, + public_snapshots_queryset, +) from archivebox.core.host_utils import ( build_admin_url, build_snapshot_url, @@ -62,6 +79,7 @@ from archivebox.hooks import ( ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/" LIVE_PLUGIN_BASE_URL = "/admin/environment/plugins/" +SCREENCAST_SIGNER = TimestampSigner(salt="archivebox.live-progress.screencast") def _get_request_config(request: HttpRequest, *, resolve_plugins: bool = False): @@ -175,7 +193,7 @@ class SnapshotView(View): "USES_SUBDOMAIN_ROUTING", "ADMIN_BASE_URL", "ARCHIVE_BASE_URL", - "PUBLIC_SNAPSHOTS", + "PERMISSIONS", "SERVER_SECURITY_MODE", } scoped_config_keys = set((getattr(snapshot, "config", None) or {}).keys()) @@ -278,6 +296,12 @@ class SnapshotView(View): "size": printable_filesize(output_size) if output_size else "pending", "status": "archived" if is_archived else "not yet archived", "status_color": "success" if is_archived else "danger", + "snapshot_permissions": str(runtime_config.PERMISSIONS).strip().lower(), + "snapshot_permissions_icon": { + "public": "πŸ‘₯", + "unlisted": "πŸ”—", + "private": "πŸ”’", + }[str(runtime_config.PERMISSIONS).strip().lower()], "bookmarked_date": snapshot.bookmarked_date, "downloaded_datestr": snapshot.downloaded_datestr, "num_outputs": snapshot.num_outputs, @@ -298,10 +322,6 @@ class SnapshotView(View): return render(template_name="core/snapshot.html", request=request, context=context) def get(self, request, path): - request_config = _get_request_config(request) - if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS: - return _admin_login_redirect_or_forbidden(request) - snapshot = None try: @@ -318,6 +338,8 @@ class SnapshotView(View): try: try: snapshot = Snapshot.objects.get(Q(timestamp=slug) | Q(id__startswith=slug)) + if not can_view_snapshot(request, snapshot): + return _admin_login_redirect_or_forbidden(request) canonical_base = snapshot.url_path if canonical_base != snapshot.legacy_archive_path: target_path = f"/{canonical_base}/{archivefile or 'index.html'}" @@ -377,7 +399,7 @@ class SnapshotView(View): snap.url, snap.title_stripped[:64] or "", ) - for snap in Snapshot.objects.filter(timestamp__startswith=slug) + for snap in direct_snapshots_queryset(request, Snapshot.objects.filter(timestamp__startswith=slug)) .only("url", "timestamp", "title", "bookmarked_at") .order_by("-bookmarked_at") ) @@ -436,7 +458,7 @@ class SnapshotView(View): # slug is a URL try: try: - snapshot = SnapshotView.find_snapshots_for_url(path).get() + snapshot = direct_snapshots_queryset(request, SnapshotView.find_snapshots_for_url(path)).get() except Snapshot.DoesNotExist: raise except Snapshot.DoesNotExist: @@ -457,7 +479,7 @@ class SnapshotView(View): status=404, ) except Snapshot.MultipleObjectsReturned: - snapshots = SnapshotView.find_snapshots_for_url(path) + snapshots = direct_snapshots_queryset(request, SnapshotView.find_snapshots_for_url(path)) snapshot_hrefs = mark_safe("
").join( format_html( '{} {} {} {} {}', @@ -501,11 +523,6 @@ class SnapshotPathView(View): path: str = "", url: str | None = None, ): - request_config = _get_request_config(request) - - if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS: - return _admin_login_redirect_or_forbidden(request) - if username == "system": return redirect(request.path.replace("/system/", "/web/", 1)) @@ -517,7 +534,7 @@ class SnapshotPathView(View): requested_url = domain snapshot = None - snapshots_qs = Snapshot.objects.select_related("crawl", "crawl__created_by") + snapshots_qs = direct_snapshots_queryset(request, Snapshot.objects.select_related("crawl", "crawl__created_by")) if snapshot_id: try: snapshot = snapshots_qs.get(pk=snapshot_id) @@ -672,7 +689,27 @@ def _snapshot_sort_key(match_path: str, cache: dict[str, float]) -> tuple[float, return (cache[snapshot_id], match_path) -def _latest_response_match(domain: str, rel_path: str, *, data_root: Path) -> tuple[Path, Path] | None: +def _snapshot_id_from_replay_path(path: Path) -> str | None: + parts = path.parts + try: + responses_idx = parts.index("responses") + except ValueError: + return None + return parts[responses_idx - 1] if responses_idx > 0 else None + + +def _replay_path_visible(request: HttpRequest, path: Path) -> bool: + snapshot_id = _snapshot_id_from_replay_path(path) + if not snapshot_id: + return False + snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first() + if not snapshot or not can_view_snapshot(request, snapshot): + return False + request.archivebox_config = get_config(snapshot=snapshot, resolve_plugins=False) + return True + + +def _latest_response_match(request: HttpRequest, domain: str, rel_path: str, *, data_root: Path) -> tuple[Path, Path] | None: if not domain or not rel_path: return None domain = domain.split(":", 1)[0].lower() @@ -685,8 +722,10 @@ def _latest_response_match(domain: str, rel_path: str, *, data_root: Path) -> tu return None sort_cache: dict[str, float] = {} - best = max(matches, key=lambda match_path: _snapshot_sort_key(match_path, sort_cache)) - best_path = Path(best) + best_paths = sorted(matches, key=lambda match_path: _snapshot_sort_key(match_path, sort_cache), reverse=True) + best_path = next((Path(match_path) for match_path in best_paths if _replay_path_visible(request, Path(match_path))), None) + if best_path is None: + return None parts = best_path.parts try: responses_idx = parts.index("responses") @@ -697,7 +736,7 @@ def _latest_response_match(domain: str, rel_path: str, *, data_root: Path) -> tu return responses_root, rel_to_root -def _latest_responses_root(domain: str, *, data_root: Path) -> Path | None: +def _latest_responses_root(request: HttpRequest, domain: str, *, data_root: Path) -> Path | None: if not domain: return None domain = domain.split(":", 1)[0].lower() @@ -708,16 +747,19 @@ def _latest_responses_root(domain: str, *, data_root: Path) -> Path | None: return None sort_cache: dict[str, float] = {} - best = max(matches, key=lambda match_path: _snapshot_sort_key(match_path, sort_cache)) - return Path(best) + best_paths = sorted(matches, key=lambda match_path: _snapshot_sort_key(match_path, sort_cache), reverse=True) + return next((Path(match_path) for match_path in best_paths if _replay_path_visible(request, Path(match_path))), None) -def _latest_snapshot_for_domain(domain: str) -> Snapshot | None: +def _latest_snapshot_for_domain(request: HttpRequest, domain: str) -> Snapshot | None: if not domain: return None requested_domain = domain.split(":", 1)[0].lower() - snapshots = SnapshotView.find_snapshots_for_url(f"https://{requested_domain}").order_by("-bookmarked_at", "-created_at", "-timestamp") + snapshots = direct_snapshots_queryset( + request, + SnapshotView.find_snapshots_for_url(f"https://{requested_domain}"), + ).order_by("-bookmarked_at", "-created_at", "-timestamp") for snapshot in snapshots: if Snapshot.extract_domain_from_url(snapshot.url).lower() == requested_domain: return snapshot @@ -774,7 +816,8 @@ def _serve_responses_path(request, responses_root: Path, rel_path: str, show_ind def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""): - request_config = _get_request_config(request) + request_config = get_config(snapshot=snapshot, resolve_plugins=False) + request.archivebox_config = request_config snapshot._runtime_config = request_config rel_path = path or "" is_directory_request = bool(path) and path.endswith("/") @@ -823,13 +866,13 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = raise Http404 domain = domain.lower() - match = _latest_response_match(domain, rel_path, data_root=request_config.USERS_DIR) + match = _latest_response_match(request, domain, rel_path, data_root=request_config.USERS_DIR) if not match and "." not in Path(rel_path).name: index_path = f"{rel_path.rstrip('/')}/index.html" - match = _latest_response_match(domain, index_path, data_root=request_config.USERS_DIR) + match = _latest_response_match(request, domain, index_path, data_root=request_config.USERS_DIR) if not match and "." not in Path(rel_path).name: html_path = f"{rel_path}.html" - match = _latest_response_match(domain, html_path, data_root=request_config.USERS_DIR) + match = _latest_response_match(request, domain, html_path, data_root=request_config.USERS_DIR) show_indexes = bool(request.GET.get("files")) if match: @@ -838,14 +881,14 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = if response is not None: return response - responses_root = _latest_responses_root(domain, data_root=request_config.USERS_DIR) + responses_root = _latest_responses_root(request, domain, data_root=request_config.USERS_DIR) if responses_root: response = _serve_responses_path(request, responses_root, rel_path, show_indexes) if response is not None: return response if requested_root_index and not show_indexes: - snapshot = _latest_snapshot_for_domain(domain) + snapshot = _latest_snapshot_for_domain(request, domain) if snapshot: return SnapshotView.render_live_index(request, snapshot) @@ -861,12 +904,12 @@ class SnapshotHostView(View): def get(self, request, snapshot_id: str, path: str = ""): request_config = _get_request_config(request) - if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS: - return _admin_login_redirect_or_forbidden(request) snapshot = _find_snapshot_by_ref(snapshot_id) if not snapshot: raise Http404 + if not can_view_snapshot(request, snapshot): + return _admin_login_redirect_or_forbidden(request) canonical_host = get_snapshot_host(str(snapshot.id), config=request_config) if not host_matches(request.get_host(), canonical_host): @@ -882,13 +925,11 @@ class SnapshotReplayView(View): """Serve snapshot directory contents on a one-domain replay path.""" def get(self, request, snapshot_id: str, path: str = ""): - request_config = _get_request_config(request) - if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS: - return _admin_login_redirect_or_forbidden(request) - snapshot = _find_snapshot_by_ref(snapshot_id) if not snapshot: raise Http404 + if not can_view_snapshot(request, snapshot): + return _admin_login_redirect_or_forbidden(request) return _serve_snapshot_replay(request, snapshot, path) @@ -897,9 +938,6 @@ class OriginalDomainHostView(View): """Serve responses from the most recent snapshot when using ./.""" def get(self, request, domain: str, path: str = ""): - request_config = _get_request_config(request) - if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS: - return _admin_login_redirect_or_forbidden(request) return _serve_original_domain_replay(request, domain, path) @@ -907,9 +945,6 @@ class OriginalDomainReplayView(View): """Serve original-domain replay content on a one-domain replay path.""" def get(self, request, domain: str, path: str = ""): - request_config = _get_request_config(request) - if not request.user.is_authenticated and not request_config.PUBLIC_SNAPSHOTS: - return _admin_login_redirect_or_forbidden(request) return _serve_original_domain_replay(request, domain, path) @@ -928,6 +963,8 @@ class PublicIndexView(ListView): runtime_config = getattr(self, "runtime_config", None) if runtime_config is None: self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=True) + search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config) + search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config) context = { **super().get_context_data(**kwargs), "VERSION": VERSION, @@ -935,21 +972,27 @@ class PublicIndexView(ListView): "COMMIT_HASH": runtime_config.COMMIT_HASH, "FOOTER_INFO": runtime_config.FOOTER_INFO, "WEB_BASE_URL": build_web_url(request=self.request, config=runtime_config), - "search_mode": get_search_mode(self.request.GET.get("search_mode")), + "search_mode": search_mode, + "search_mode_options": get_search_mode_options(config=runtime_config), + "search_backend_label": get_search_backend_display_name(search_mode_backend) if search_mode_backend else "", } + context["show_search_index_hint"] = bool( + self.request.GET.get("q") + and get_search_mode_base(search_mode, config=runtime_config) == "deep" + and search_mode_backend + and getattr(context.get("paginator"), "count", 0) == 0 + ) for snapshot in context.get("object_list") or (): snapshot._icons_compact = True snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED) results = getattr(snapshot, "_prefetched_objects_cache", {}).get("archiveresult_set") if results is not None: - snapshot.output_size_sum = sum(result.output_size or 0 for result in results) snapshot.num_outputs_cached = len(results) return context def get_queryset(self, **kwargs): qs = ( - super() - .get_queryset(**kwargs) + public_snapshots_queryset(super().get_queryset(**kwargs)) .prefetch_related( Prefetch("crawl", queryset=Crawl.objects.select_related("created_by")), "tags", @@ -970,24 +1013,30 @@ class PublicIndexView(ListView): if not query: return qs - search_mode = get_search_mode(self.request.GET.get("search_mode")) + search_mode = get_search_mode(self.request.GET.get("search_mode"), config=getattr(self, "runtime_config", None)) metadata_qs = qs.filter( Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query), ) - if search_mode == "meta": + search_mode_base = get_search_mode_base(search_mode, config=getattr(self, "runtime_config", None)) + search_mode_backend = get_search_mode_backend(search_mode, config=getattr(self, "runtime_config", None)) + if search_mode_base == "meta": qs = metadata_qs else: try: - qs = prioritize_metadata_matches( - qs, - metadata_qs, - query_search_index(query, search_mode=search_mode), - ordering=self.ordering, - ) + backend_qs = query_search_index(query, search_mode=search_mode) + if search_mode_backend: + qs = qs.filter(pk__in=backend_qs.values("pk")) + else: + qs = prioritize_metadata_matches( + qs, + metadata_qs, + backend_qs, + ordering=self.ordering, + ) except Exception as err: print(f"[!] Error while using search backend: {err.__class__.__name__} {err}") - qs = metadata_qs + qs = qs.none() if search_mode_backend else metadata_qs return qs.distinct() @@ -1015,12 +1064,16 @@ class AddView(UserPassesTestMixin, FormView): return super().get_initial() + def get_form_kwargs(self): + kwargs = super().get_form_kwargs() + kwargs["request"] = self.request + return kwargs + def test_func(self): return _get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated def _can_override_crawl_config(self) -> bool: - user = self.request.user - return bool(user.is_authenticated and (getattr(user, "is_superuser", False) or getattr(user, "is_staff", False))) + return is_admin_user(self.request) def _get_custom_config_overrides(self, form: AddLinkForm) -> dict: custom_config = form.cleaned_data.get("config") or {} @@ -1034,35 +1087,55 @@ class AddView(UserPassesTestMixin, FormView): return custom_config def get_context_data(self, **kwargs): - from archivebox.personas.models import Persona - + context = super().get_context_data(**kwargs) request_config = _get_request_config(self.request, resolve_plugins=True) required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip() - plugin_configs = discover_plugin_configs() + can_override_crawl_config = self._can_override_crawl_config() + plugin_configs = discover_plugin_configs() if can_override_crawl_config else {} sensitive_keys = { str(config_key) for schema in plugin_configs.values() for config_key, prop_schema in (schema.get("properties") or {}).items() if isinstance(prop_schema, dict) and prop_schema.get("x-sensitive") } + public_persona_config_keys = { + "CRAWL_MAX_CONCURRENT_SNAPSHOTS", + "DELETE_AFTER", + "PERMISSIONS", + "TIMEOUT", + } + persona_queryset = context["form"].fields["persona"].queryset + if not can_override_crawl_config: + persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC}) persona_config_map = {} - for persona in Persona.objects.order_by("name"): - raw_config = {str(key): value for key, value in (persona.config or {}).items() if str(key) not in sensitive_keys} + for persona in persona_queryset.order_by("name"): effective_config = get_config(persona=persona) + if can_override_crawl_config: + raw_config = {str(key): value for key, value in (persona.config or {}).items() if str(key) not in sensitive_keys} + effective_config_json = {str(key): value for key, value in effective_config.items() if str(key) not in sensitive_keys} + binary_urls = get_plugin_config_binary_urls(effective_config) + else: + raw_config = {} + effective_config_json = {key: effective_config.get(key) for key in public_persona_config_keys} + binary_urls = {} persona_config_map[persona.name] = { "config": raw_config, - "effective_config": {str(key): value for key, value in effective_config.items() if str(key) not in sensitive_keys}, - "binary_urls": get_plugin_config_binary_urls(effective_config), + "effective_config": effective_config_json, + "binary_urls": binary_urls, + } + plugin_dependency_map = {} + if can_override_crawl_config: + plugin_dependency_map = { + plugin_name: [ + str(required_plugin).strip() + for required_plugin in (schema.get("required_plugins") or []) + if str(required_plugin).strip() + ] + for plugin_name, schema in plugin_configs.items() + if isinstance(schema.get("required_plugins"), list) and schema.get("required_plugins") } - plugin_dependency_map = { - plugin_name: [ - str(required_plugin).strip() for required_plugin in (schema.get("required_plugins") or []) if str(required_plugin).strip() - ] - for plugin_name, schema in plugin_configs.items() - if isinstance(schema.get("required_plugins"), list) and schema.get("required_plugins") - } return { - **super().get_context_data(**kwargs), + **context, "title": "Create Crawl", # We can't just call request.build_absolute_uri in the template, because it would include query parameters "absolute_add_path": self.request.build_absolute_uri(self.request.path), @@ -1071,6 +1144,7 @@ class AddView(UserPassesTestMixin, FormView): "required_search_plugin": required_search_plugin, "plugin_dependency_map_json": json.dumps(plugin_dependency_map, sort_keys=True), "persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str), + "can_override_crawl_config": can_override_crawl_config, "stdout": "", } @@ -1083,20 +1157,27 @@ class AddView(UserPassesTestMixin, FormView): depth = int(form.cleaned_data["depth"]) max_urls = int(form.cleaned_data.get("max_urls") or 0) crawl_max_size = int(form.cleaned_data.get("crawl_max_size") or 0) + crawl_timeout = int(form.cleaned_data.get("crawl_timeout") or 0) + timeout = form.cleaned_data.get("timeout") snapshot_max_size = int(form.cleaned_data.get("snapshot_max_size") or 0) delete_after = str(form.cleaned_data.get("delete_after") or "0").strip() or "0" crawl_max_concurrent_snapshots = int(form.cleaned_data["crawl_max_concurrent_snapshots"]) - plugins = ",".join(form.cleaned_data.get("plugins", [])) - schedule = form.cleaned_data.get("schedule", "").strip() + permissions = str(form.cleaned_data.get("permissions") or "public").strip().lower() + can_override_crawl_config = self._can_override_crawl_config() + plugins = ",".join(form.cleaned_data.get("plugins", [])) if can_override_crawl_config else "" + schedule = form.cleaned_data.get("schedule", "").strip() if can_override_crawl_config else "" persona = form.cleaned_data.get("persona") - index_only = form.cleaned_data.get("index_only", False) + index_only = form.cleaned_data.get("index_only", False) if can_override_crawl_config else False notes = form.cleaned_data.get("notes", "") url_filters = form.cleaned_data.get("url_filters") or {} plugin_config = form.cleaned_data.get("plugin_config") or {} if not isinstance(plugin_config, dict): plugin_config = {} + if not can_override_crawl_config: + plugin_config = {} custom_config = self._get_custom_config_overrides(form) custom_config.pop("DEFAULT_PERSONA", None) + custom_config.pop("PERMISSIONS", None) if persona: persona.ensure_dirs() @@ -1132,6 +1213,18 @@ class AddView(UserPassesTestMixin, FormView): config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots if delete_after != str(effective_config.DELETE_AFTER): config["DELETE_AFTER"] = delete_after + if permissions != str(effective_config.PERMISSIONS): + config["PERMISSIONS"] = permissions + if max_urls: + config["CRAWL_MAX_URLS"] = max_urls + if crawl_max_size: + config["CRAWL_MAX_SIZE"] = crawl_max_size + if crawl_timeout: + config["CRAWL_TIMEOUT"] = crawl_timeout + if timeout is not None and int(timeout) != int(effective_config.TIMEOUT): + config["TIMEOUT"] = int(timeout) + if snapshot_max_size: + config["SNAPSHOT_MAX_SIZE"] = snapshot_max_size # Merge custom config overrides config.update(plugin_config) @@ -1144,9 +1237,6 @@ class AddView(UserPassesTestMixin, FormView): crawl = Crawl.objects.create( urls=urls_content, max_depth=depth, - max_urls=max_urls, - crawl_max_size=crawl_max_size, - snapshot_max_size=snapshot_max_size, tags_str=tag, notes=notes, label=f"{created_by_name}@{HOSTNAME}{self.request.path} {timestamp}", @@ -1175,11 +1265,6 @@ class AddView(UserPassesTestMixin, FormView): ensure_background_runner() - # 4. start the Orchestrator & wait until it completes - # ... orchestrator will create the root Snapshot, which creates pending ArchiveResults, which gets run by the ArchiveResultActors ... - # from archivebox.crawls.actors import CrawlActor - # from archivebox.core.actors import SnapshotActor, ArchiveResultActor - return crawl def form_valid(self, form): @@ -1191,7 +1276,7 @@ class AddView(UserPassesTestMixin, FormView): # Build success message with schedule link if created schedule_msg = "" - if schedule: + if schedule and crawl.schedule_id: schedule_msg = f" and scheduled to repeat {schedule}" messages.success( @@ -1207,7 +1292,10 @@ class AddView(UserPassesTestMixin, FormView): class WebAddView(AddView): def _latest_snapshot_for_url(self, requested_url: str): - return SnapshotView.find_snapshots_for_url(requested_url).order_by("-bookmarked_at", "-created_at", "-timestamp").first() + return direct_snapshots_queryset( + self.request, + SnapshotView.find_snapshots_for_url(requested_url), + ).order_by("-bookmarked_at", "-created_at", "-timestamp").first() def _normalize_add_url(self, requested_url: str) -> str: if requested_url.startswith(("http://", "https://")): @@ -1263,10 +1351,13 @@ class WebAddView(AddView): "depth": defaults_form.fields["depth"].initial or "0", "max_urls": defaults_form.fields["max_urls"].initial or 0, "crawl_max_size": defaults_form.fields["crawl_max_size"].initial or "0", + "crawl_timeout": defaults_form.fields["crawl_timeout"].initial or 0, + "timeout": defaults_form.fields["timeout"].initial or 0, "snapshot_max_size": defaults_form.fields["snapshot_max_size"].initial or "0", "delete_after": defaults_form.fields["delete_after"].initial or "0", "crawl_max_concurrent_snapshots": defaults_form.fields["crawl_max_concurrent_snapshots"].initial, "persona": defaults_form.fields["persona"].initial or "Default", + "permissions": defaults_form.fields["permissions"].initial or "public", "config": "{}", }, ) @@ -1295,6 +1386,37 @@ class HealthCheckView(View): return HttpResponse("OK", content_type="text/plain", status=200) +def live_progress_screencast_frame_view(request, snapshot_id: str): + """Serve cache-only Chrome screencast frames through the admin app.""" + if not is_admin_user(request): + return HttpResponseForbidden("Permission denied") + + token = request.GET.get("token", "") + try: + if SCREENCAST_SIGNER.unsign(token, max_age=60) != str(snapshot_id): + return HttpResponseForbidden("Permission denied") + except (BadSignature, SignatureExpired): + return HttpResponseForbidden("Permission denied") + + snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first() + if not snapshot: + raise Http404 + + live_root = (CONSTANTS.CACHE_DIR / "chrome_screencast").resolve() + frame_path = live_root / str(snapshot.id) / "latest.jpg" + try: + resolved_frame_path = frame_path.resolve(strict=True) + except FileNotFoundError: + raise Http404 from None + if not resolved_frame_path.is_file() or live_root not in resolved_frame_path.parents: + raise Http404 + + response = FileResponse(resolved_frame_path.open("rb"), content_type="image/jpeg") + response["Cache-Control"] = "no-store, max-age=0" + response["X-Content-Type-Options"] = "nosniff" + return response + + @gzip_page def live_progress_view(request): """Simple JSON endpoint for live progress status - used by admin progress monitor.""" @@ -1456,18 +1578,20 @@ def live_progress_view(request): else None ) runner_worker = None - try: - from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker + orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running) + if not orchestrator_proc_running: + try: + from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker - supervisor = get_existing_supervisord_process() - runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None - except Exception: - runner_worker = None + supervisor = get_existing_supervisord_process(quiet=True) + runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None + except Exception: + runner_worker = None runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING")) runner_worker_pid = runner_worker.get("pid") if runner_worker else None - orchestrator_running = orchestrator_proc is not None or runner_worker_running - orchestrator_pid = orchestrator_proc.pid if orchestrator_proc else runner_worker_pid + orchestrator_running = orchestrator_proc_running or runner_worker_running + orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid def count_statuses(queryset, statuses) -> dict[str, int]: counts = {status: 0 for status in statuses} @@ -1476,9 +1600,13 @@ def live_progress_view(request): return counts # Get model counts by status - crawl_status_counts = count_statuses(crawl_scope, (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED)) + crawl_status_counts = count_statuses( + crawl_scope, + (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.PAUSED), + ) crawls_pending = crawl_status_counts.get(Crawl.StatusChoices.QUEUED, 0) crawls_started = crawl_status_counts.get(Crawl.StatusChoices.STARTED, 0) + crawls_paused = crawl_status_counts.get(Crawl.StatusChoices.PAUSED, 0) # Get recent crawls (last 24 hours) from datetime import timedelta @@ -1487,23 +1615,31 @@ def live_progress_view(request): recently_cancelled_after = now - timedelta(minutes=10) crawls_recent = crawl_scope.filter(created_at__gte=one_day_ago).count() - snapshot_status_counts = count_statuses(snapshot_scope, (Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED)) + snapshot_status_counts = count_statuses( + snapshot_scope, + (Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED), + ) snapshots_pending = snapshot_status_counts.get(Snapshot.StatusChoices.QUEUED, 0) snapshots_started = snapshot_status_counts.get(Snapshot.StatusChoices.STARTED, 0) + snapshots_paused = snapshot_status_counts.get(Snapshot.StatusChoices.PAUSED, 0) archiveresult_status_counts = count_statuses( archiveresult_scope, ( ArchiveResult.StatusChoices.QUEUED, ArchiveResult.StatusChoices.STARTED, + ArchiveResult.StatusChoices.PAUSED, ), ) archiveresults_pending = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0) archiveresults_started = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0) + archiveresults_paused = archiveresult_status_counts.get(ArchiveResult.StatusChoices.PAUSED, 0) archiveresults_succeeded = 0 archiveresults_failed = 0 # Build hierarchical active crawls with nested snapshots and archive results + max_progress_crawls = 3 + max_progress_snapshots = 50 active_crawl_fields = ( "id", @@ -1513,9 +1649,6 @@ def live_progress_view(request): "urls", "config", "max_depth", - "max_urls", - "crawl_max_size", - "snapshot_max_size", "tags_str", "persona_id", "status", @@ -1525,18 +1658,23 @@ def live_progress_view(request): "created_by__username", ) active_crawl_candidates = [] - for status in (Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED): + for status in (Crawl.StatusChoices.STARTED, Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.PAUSED, Crawl.StatusChoices.SEALED): status_qs = crawl_scope.filter(status=status) if status == Crawl.StatusChoices.SEALED: status_qs = status_qs.filter(modified_at__gte=recently_cancelled_after) active_crawl_candidates.extend( - status_qs.values(*active_crawl_fields).order_by("-modified_at"), + status_qs.values(*active_crawl_fields).order_by("-modified_at")[:max_progress_crawls], ) + crawl_status_priority = { + Crawl.StatusChoices.STARTED: 0, + Crawl.StatusChoices.QUEUED: 1, + Crawl.StatusChoices.PAUSED: 2, + Crawl.StatusChoices.SEALED: 3, + } active_crawls_list = sorted( {str(crawl["id"]): crawl for crawl in active_crawl_candidates}.values(), - key=lambda crawl: crawl["modified_at"], - reverse=True, - ) + key=lambda crawl: (crawl_status_priority.get(crawl["status"], 9), -(crawl["modified_at"].timestamp() if crawl["modified_at"] else 0)), + )[:max_progress_crawls] for crawl in active_crawls_list: crawl["id"] = str(crawl["id"]) if crawl["persona_id"]: @@ -1558,6 +1696,10 @@ def live_progress_view(request): persona_details_by_id[str(persona.id)] = persona_details persona_details_by_name[persona.name] = persona_details active_crawl_ids = [crawl["id"] for crawl in active_crawls_list] + active_crawl_objects = { + str(crawl.id): crawl + for crawl in Crawl.objects.filter(id__in=active_crawl_ids).select_related("created_by") + } snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids} cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids} crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids} @@ -1588,7 +1730,6 @@ def live_progress_view(request): process_records_by_crawl: dict[str, list[tuple[dict[str, object], object | None]]] = {} process_records_by_snapshot: dict[str, list[tuple[dict[str, object], object | None]]] = {} seen_process_records: set[str] = set() - active_snapshot_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED} recently_cancelled_snapshots_q = Q( status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True, @@ -1596,21 +1737,7 @@ def live_progress_view(request): ) crawls_by_id = {str(crawl["id"]): crawl for crawl in active_crawls_list} snapshots = list( - active_snapshot_scope.filter(status=Snapshot.StatusChoices.QUEUED) - .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField())) - .values( - "id_str", - "url", - "crawl_id_str", - "title", - "status", - ) - .order_by("crawl_id", "modified_at"), - ) - snapshots.extend( - active_snapshot_scope.filter( - Q(status__in=active_snapshot_statuses - {Snapshot.StatusChoices.QUEUED}) | recently_cancelled_snapshots_q, - ) + active_snapshot_scope.filter(status=Snapshot.StatusChoices.STARTED) .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField())) .values( "id_str", @@ -1625,8 +1752,42 @@ def live_progress_view(request): "fs_version", "status", ) - .order_by("crawl_id", "status", "modified_at"), + .order_by("crawl_id", "-modified_at")[:max_progress_snapshots], ) + remaining_snapshot_slots = max_progress_snapshots - len(snapshots) + if remaining_snapshot_slots > 0: + snapshots.extend( + active_snapshot_scope.filter(status=Snapshot.StatusChoices.QUEUED) + .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField())) + .values( + "id_str", + "url", + "crawl_id_str", + "title", + "status", + ) + .order_by("crawl_id", "modified_at")[:remaining_snapshot_slots], + ) + remaining_snapshot_slots = max_progress_snapshots - len(snapshots) + if remaining_snapshot_slots > 0: + snapshots.extend( + active_snapshot_scope.filter(recently_cancelled_snapshots_q) + .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField())) + .values( + "id_str", + "created_at", + "modified_at", + "url", + "timestamp", + "bookmarked_at", + "crawl_id_str", + "title", + "downloaded_at", + "fs_version", + "status", + ) + .order_by("crawl_id", "-modified_at")[:remaining_snapshot_slots], + ) def dashed_uuid(value: str) -> str: value = str(value) @@ -1762,7 +1923,7 @@ def live_progress_view(request): if proc["status"] == Process.StatusChoices.RUNNING else ( "skipped" - if proc["exit_code"] == PROCESS_EXIT_SKIPPED + if proc["exit_code"] == PROCESS_EXIT_SKIPPED or (phase == "binary" and proc["exit_code"] not in (None, 0)) else ("failed" if proc["exit_code"] not in (None, 0) else "succeeded") ) ) @@ -1839,6 +2000,8 @@ def live_progress_view(request): snapshot_favicon_url = "" snapshot_preview_url = "" snapshot_preview_link = "" + snapshot_screencast_url = "" + snapshot_screencast_link = "" snapshot_fallback_urls: list[str] = [] result_by_plugin = {result.plugin: result for result in snapshot_results} title_result = result_by_plugin.get("title") @@ -1859,6 +2022,17 @@ def live_progress_view(request): elif snapshot_favicon_url: snapshot_preview_url = snapshot_favicon_url + if snapshot["status"] == Snapshot.StatusChoices.STARTED: + live_preview_path = CONSTANTS.CACHE_DIR / "chrome_screencast" / str(snapshot["id"]) / "latest.jpg" + try: + live_preview_stat = live_preview_path.stat() + except OSError: + live_preview_stat = None + if live_preview_stat and live_preview_stat.st_size > 0: + token = SCREENCAST_SIGNER.sign(str(snapshot["id"])) + snapshot_screencast_url = f"/admin/live-progress/screencast/{snapshot['id']}.jpg?v={live_preview_stat.st_mtime_ns}&token={quote(token)}" + snapshot_screencast_link = snapshot_view_url(snapshot) + def plugin_sort_key(ar): status_order = { ArchiveResult.StatusChoices.STARTED: 0, @@ -1989,6 +2163,9 @@ def live_progress_view(request): if snapshot_preview_url: snapshot_payload["preview_url"] = snapshot_preview_url snapshot_payload["preview_link"] = snapshot_preview_link + if snapshot_screencast_url: + snapshot_payload["screencast_url"] = snapshot_screencast_url + snapshot_payload["screencast_link"] = snapshot_screencast_link if snapshot_fallback_urls: snapshot_payload["preview_fallbacks"] = snapshot_fallback_urls if snapshot_process_pids.get(str(snapshot["id"])): @@ -2005,17 +2182,25 @@ def live_progress_view(request): persona_details = persona_details or persona_details_by_name.get(persona_name) crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0) avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0 + effective_crawl_config = get_config(crawl=active_crawl_objects[crawl_id]) + max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0) + crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0) + crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0) + snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0) # Check if retry_at is in the future (would prevent worker from claiming) retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False - seconds_until_retry = int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0 + is_paused = active_crawl_objects[crawl_id].is_paused + seconds_until_retry = 0 if is_paused else int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0 crawl_worker_state = ( "running" if crawl_process_pids.get(crawl_id) or any(isinstance(snapshot, dict) and snapshot.get("worker_pid") for snapshot in active_snapshots_for_crawl) else "waiting" ) - if crawl["status"] == Crawl.StatusChoices.SEALED and cancelled_snapshots: + if is_paused: + crawl_worker_state = "paused" + elif crawl["status"] == Crawl.StatusChoices.SEALED and cancelled_snapshots: crawl_worker_state = "cancelled" elif ( crawl["status"] == Crawl.StatusChoices.STARTED @@ -2029,19 +2214,20 @@ def live_progress_view(request): "id": crawl_id, "label": (next((line.strip() for line in (crawl["urls"] or "").splitlines() if line.strip()), "") or crawl_id)[:60], "status": crawl["status"], + "is_paused": is_paused, "started": crawl["created_at"].isoformat() if crawl["created_at"] else None, "progress": crawl_progress, "created_by": crawl["created_by__username"], "persona": persona_name, "persona_admin_url": persona_details["admin_url"] if persona_details else None, "max_depth": crawl["max_depth"], - "max_urls": crawl["max_urls"], - "max_crawl_size": crawl["crawl_max_size"], - "max_snapshot_size": crawl["snapshot_max_size"], - "max_crawl_size_display": printable_filesize(crawl["crawl_max_size"]) if crawl["crawl_max_size"] else "unlimited", - "max_snapshot_size_display": printable_filesize(crawl["snapshot_max_size"]) - if crawl["snapshot_max_size"] - else "unlimited", + "max_urls": max_urls, + "max_crawl_size": crawl_max_size, + "crawl_timeout": crawl_timeout, + "max_snapshot_size": snapshot_max_size, + "max_crawl_size_display": printable_filesize(crawl_max_size) if crawl_max_size else "unlimited", + "crawl_timeout_display": f"{crawl_timeout}s" if crawl_timeout else "unlimited", + "max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited", "crawl_output_size": crawl_output_size, "avg_snapshot_size": avg_snapshot_size, "crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B", @@ -2075,11 +2261,14 @@ def live_progress_view(request): "total_workers": total_workers, "crawls_pending": crawls_pending, "crawls_started": crawls_started, + "crawls_paused": crawls_paused, "crawls_recent": crawls_recent, "snapshots_pending": snapshots_pending, "snapshots_started": snapshots_started, + "snapshots_paused": snapshots_paused, "archiveresults_pending": archiveresults_pending, "archiveresults_started": archiveresults_started, + "archiveresults_paused": archiveresults_paused, "archiveresults_succeeded": archiveresults_succeeded, "archiveresults_failed": archiveresults_failed, "active_crawls": active_crawls, @@ -2103,11 +2292,14 @@ def live_progress_view(request): "total_workers": 0, "crawls_pending": 0, "crawls_started": 0, + "crawls_paused": 0, "crawls_recent": 0, "snapshots_pending": 0, "snapshots_started": 0, + "snapshots_paused": 0, "archiveresults_pending": 0, "archiveresults_started": 0, + "archiveresults_paused": 0, "archiveresults_succeeded": 0, "archiveresults_failed": 0, "active_crawls": [], diff --git a/archivebox/core/widgets.py b/archivebox/core/widgets.py index e7d43e0f..f79e9c08 100644 --- a/archivebox/core/widgets.py +++ b/archivebox/core/widgets.py @@ -402,7 +402,7 @@ class URLFiltersWidget(forms.Widget):
- Regex patterns or domains to exclude, one pattern per line. + Regex patterns or domains to include, one pattern per line.
-
-
- - -
-
- -

- Enter domains, wildcards, or regex patterns. Denylist takes precedence over allowlist. -

- -
- ''') - - def value_from_datadict(self, data, files, name): - return { - "allowlist": data.get(f"{name}_allowlist", ""), - "denylist": data.get(f"{name}_denylist", ""), - "same_domain_only": data.get(f"{name}_same_domain_only") in ("1", "on", "true"), - } - - class URLFiltersField(forms.Field): - widget = URLFiltersWidget + widget = URLFiltersWidget(source_selector="#id_urls") def to_python(self, value): if isinstance(value, dict): return value - return {"allowlist": "", "denylist": "", "same_domain_only": False} + return {"allowlist": "", "denylist": "", "same_domain_only": False, "subpaths_only": False} class CrawlAdminForm(forms.ModelForm): @@ -416,6 +429,7 @@ class CrawlAdminForm(forms.ModelForm): "allowlist": config.get("URL_ALLOWLIST", ""), "denylist": config.get("URL_DENYLIST", ""), "same_domain_only": False, + "subpaths_only": False, } def clean_tags_editor(self): @@ -439,16 +453,18 @@ class CrawlAdminForm(forms.ModelForm): "allowlist": "\n".join(Crawl.split_filter_patterns(value.get("allowlist", ""))), "denylist": "\n".join(Crawl.split_filter_patterns(value.get("denylist", ""))), "same_domain_only": bool(value.get("same_domain_only")), + "subpaths_only": bool(value.get("subpaths_only")), } def save(self, commit=True): instance = super().save(commit=False) instance.tags_str = self.cleaned_data.get("tags_editor", "") - url_filters = self.cleaned_data.get("url_filters") or {} - instance.set_url_filters( - url_filters.get("allowlist", ""), - url_filters.get("denylist", ""), - ) + if f"{self.add_prefix('url_filters')}_allowlist" in self.data or f"{self.add_prefix('url_filters')}_denylist" in self.data: + url_filters = self.cleaned_data.get("url_filters") or {} + instance.set_url_filters( + url_filters.get("allowlist", ""), + url_filters.get("denylist", ""), + ) if commit: instance.save() instance.apply_crawl_config_filters() @@ -460,15 +476,16 @@ class CrawlAdminForm(forms.ModelForm): class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): form = CrawlAdminForm + change_form_template = "admin/crawls/crawl/change_form.html" list_select_related = () list_display = ( "id", "created_at", "created_by", "max_depth", - "max_urls", - "crawl_max_size", - "snapshot_max_size", + "stop_reason_badge", + "pause_control", + "resume_control", "label", "notes", "urls_preview", @@ -483,9 +500,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): "created_at", "created_by", "max_depth", - "max_urls", - "crawl_max_size", - "snapshot_max_size", "label", "notes", "schedule_str", @@ -496,9 +510,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): "id", "created_by__username", "max_depth", - "max_urls", - "crawl_max_size", - "snapshot_max_size", "label", "notes", "schedule_id", @@ -506,56 +517,33 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): "urls", ) - readonly_fields = ("created_at", "modified_at", "snapshots") + readonly_fields = ("created_at", "modified_at", "stop_reason_display") fieldsets = ( ( "URLs", { - "fields": ("urls",), + "fields": ("urls", "url_filters"), "classes": ("card", "wide"), }, ), ( - "Info", + "Overview", { - "fields": ("label", "notes", "tags_editor"), - "classes": ("card",), + "fields": ( + ("label", "status", "retry_at", "schedule", "created_by", "created_at", "modified_at"), + ("max_depth",), + ("stop_reason_display",), + ("notes", "tags_editor"), + ), + "classes": ("card", "wide", "crawl-admin-overview"), }, ), ( - "Settings", + "Config", { - "fields": (("max_depth", "max_urls", "crawl_max_size", "snapshot_max_size"), "url_filters", "config"), - "classes": ("card",), - }, - ), - ( - "Status", - { - "fields": ("status", "retry_at"), - "classes": ("card",), - }, - ), - ( - "Relations", - { - "fields": ("schedule", "created_by"), - "classes": ("card",), - }, - ), - ( - "Timestamps", - { - "fields": ("created_at", "modified_at"), - "classes": ("card",), - }, - ), - ( - "Snapshots", - { - "fields": ("snapshots",), - "classes": ("card", "wide"), + "fields": ("config",), + "classes": ("card", "wide", "crawl-admin-config"), }, ), ) @@ -563,36 +551,26 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): ( "URLs", { - "fields": ("urls",), + "fields": ("urls", "url_filters"), "classes": ("card", "wide"), }, ), ( - "Info", + "Overview", { - "fields": ("label", "notes", "tags_editor"), - "classes": ("card",), + "fields": ( + ("label", "status", "retry_at", "schedule", "created_by"), + ("max_depth",), + ("notes", "tags_editor"), + ), + "classes": ("card", "wide", "crawl-admin-overview"), }, ), ( - "Settings", + "Config", { - "fields": (("max_depth", "max_urls", "crawl_max_size", "snapshot_max_size"), "url_filters", "config"), - "classes": ("card",), - }, - ), - ( - "Status", - { - "fields": ("status", "retry_at"), - "classes": ("card",), - }, - ), - ( - "Relations", - { - "fields": ("schedule", "created_by"), - "classes": ("card",), + "fields": ("config",), + "classes": ("card", "wide", "crawl-admin-config"), }, ), ) @@ -600,9 +578,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): list_filter = (MaxDepthListFilter, "schedule", "created_by", "status", "retry_at") ordering = ["-created_at", "-retry_at"] list_per_page = 50 - actions = ["delete_selected_batched"] + actions = ["pause_selected_crawls", "resume_selected_crawls", "delete_selected_batched"] change_actions = ["recrawl"] + class Media: + css = {"all": ("admin/crawls/crawl_change.css",)} + js = ("admin/crawls/crawl_admin.js",) + def get_queryset(self, request): """Keep joins page-local while computing per-row snapshot counts in the page query.""" snapshot_count = ( @@ -623,6 +605,20 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): ) ) + def change_view(self, request, object_id, form_url="", extra_context=None): + self.request = request + crawl = self.get_object(request, object_id) + extra_context = { + **(extra_context or {}), + "crawl_stop_reason": crawl.limit_stop_reason() if crawl else "", + "crawl_snapshots_changelist": self.snapshots_changelist(crawl) if crawl else "", + } + return super().change_view(request, object_id, form_url, extra_context) + + def add_view(self, request, form_url="", extra_context=None): + self.request = request + return super().add_view(request, form_url, extra_context) + def get_fieldsets(self, request, obj=None): return self.fieldsets if obj else self.add_fieldsets @@ -658,6 +654,29 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): messages.success(request, f"Successfully deleted {total} crawls ({deleted_count} total objects including related records).") + @admin.action(description="Pause selected crawls") + def pause_selected_crawls(self, request, queryset): + paused = 0 + for crawl in queryset.exclude(status=Crawl.StatusChoices.SEALED).iterator(chunk_size=100): + paused += int(crawl.pause()) + if paused: + messages.success(request, f"Paused {paused} crawl(s). The runner will stop scheduling new work on the next sweep.") + else: + messages.warning(request, "No active crawls were selected to pause.") + + @admin.action(description="Resume selected crawls") + def resume_selected_crawls(self, request, queryset): + resumed = 0 + for crawl in queryset.iterator(chunk_size=100): + if crawl.status == Crawl.StatusChoices.SEALED: + crawl.status = Crawl.StatusChoices.PAUSED + crawl.save(update_fields=["status", "modified_at"]) + resumed += int(crawl.resume()) + if resumed: + messages.success(request, f"Resumed {resumed} crawl(s). The runner will pick them up on the next sweep.") + else: + messages.warning(request, "No paused or sealed crawls were selected to resume.") + @action(label="Recrawl", description="Create a new crawl with the same settings") def recrawl(self, request, obj): """Duplicate this crawl as a new crawl with the same URLs and settings.""" @@ -670,9 +689,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): new_crawl = Crawl.objects.create( urls=obj.urls, max_depth=obj.max_depth, - max_urls=obj.max_urls, - crawl_max_size=obj.crawl_max_size, - snapshot_max_size=obj.snapshot_max_size, tags_str=obj.tags_str, config=obj.config, schedule=obj.schedule, @@ -687,6 +703,39 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): return redirect("admin:crawls_crawl_change", new_crawl.id) + @admin.display(description="Stop Reason") + def stop_reason_display(self, obj): + reason = obj.limit_stop_reason() if obj else "" + if not reason: + return mark_safe('None') + return format_html('{}', reason) + + @admin.display(description="Stop Reason") + def stop_reason_badge(self, obj): + return self.stop_reason_display(obj) + + @admin.display(description="Resume") + def resume_control(self, obj): + if obj.status != Crawl.StatusChoices.SEALED and not obj.is_paused: + return mark_safe('-') + reason = "paused" if obj.is_paused else (obj.limit_stop_reason() or "sealed") + return format_html( + '', + obj.pk, + reason, + ) + + @admin.display(description="Pause") + def pause_control(self, obj): + if obj.status == Crawl.StatusChoices.SEALED: + return mark_safe('-') + if obj.is_paused: + return mark_safe('Paused') + return format_html( + '', + obj.pk, + ) + def num_snapshots(self, obj): # Use cached annotation from get_queryset to avoid N+1 count = getattr(obj, "num_snapshots_cached", None) @@ -694,8 +743,40 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): count = obj.snapshot_set.count() return count - def snapshots(self, obj): - return render_snapshots_list(obj.snapshot_set.all(), crawl=obj) + @admin.display(description="Snapshots") + def snapshots_changelist(self, obj): + request = getattr(self, "request", None) + snapshot_changelist = reverse("admin:core_snapshot_changelist") + scoped_params = {"crawl_id": str(obj.pk)} + full_url = f"{snapshot_changelist}?{urlencode(scoped_params)}" + if request is None: + return format_html('Open snapshots changelist', full_url) + + snapshot_admin = self.admin_site._registry[Snapshot] + changelist_request = copy(request) + changelist_request.method = "GET" + changelist_request.path = snapshot_changelist + changelist_request.GET = request.GET.copy() + changelist_request.GET.update( + { + **scoped_params, + "_embedded": "crawl", + "per_page": "200", + }, + ) + changelist_request.POST = request.POST.copy() + changelist_request.POST.clear() + + response = snapshot_admin.changelist_view( + changelist_request, + extra_context={"embedded_changelist": True}, + ) + context = { + **response.context_data, + "snapshot_changelist_url": full_url, + "crawl": obj, + } + return mark_safe(render_to_string("admin/crawls/crawl/snapshots_changelist.html", context, request=request)) def delete_snapshot_view(self, request: HttpRequest, object_id: str, snapshot_id: str): if request.method != "POST": @@ -832,6 +913,7 @@ class CrawlScheduleAdmin(BaseModelAdmin): actions = ["delete_selected"] def get_queryset(self, request): + self.request = request return ( super() .get_queryset(request) @@ -842,6 +924,10 @@ class CrawlScheduleAdmin(BaseModelAdmin): ) ) + def change_view(self, request, object_id, form_url="", extra_context=None): + self.request = request + return super().change_view(request, object_id, form_url, extra_context) + def get_fieldsets(self, request, obj=None): if obj is None: return tuple(fieldset for fieldset in self.fieldsets if fieldset[0] not in {"Crawls", "Snapshots"}) @@ -879,7 +965,7 @@ class CrawlScheduleAdmin(BaseModelAdmin): def snapshots(self, obj): crawl_ids = obj.crawl_set.values_list("pk", flat=True) - return render_snapshots_list(Snapshot.objects.filter(crawl_id__in=crawl_ids)) + return render_snapshots_list(Snapshot.objects.filter(crawl_id__in=crawl_ids), request=getattr(self, "request", None), prefix="schedule_snapshots") def register_admin(admin_site): diff --git a/archivebox/crawls/migrations/0011_move_crawl_limits_to_config.py b/archivebox/crawls/migrations/0011_move_crawl_limits_to_config.py new file mode 100644 index 00000000..ed6df28f --- /dev/null +++ b/archivebox/crawls/migrations/0011_move_crawl_limits_to_config.py @@ -0,0 +1,38 @@ +from django.db import migrations + + +def move_limit_fields_to_config(apps, schema_editor): + Crawl = apps.get_model("crawls", "Crawl") + rows = Crawl.objects.values("id", "config", "max_urls", "crawl_max_size", "snapshot_max_size").iterator(chunk_size=1000) + for row in rows: + config = dict(row["config"] or {}) + if row["max_urls"]: + config["CRAWL_MAX_URLS"] = row["max_urls"] + if row["crawl_max_size"]: + config["CRAWL_MAX_SIZE"] = row["crawl_max_size"] + if row["snapshot_max_size"]: + config["SNAPSHOT_MAX_SIZE"] = row["snapshot_max_size"] + if config != (row["config"] or {}): + Crawl.objects.filter(id=row["id"]).update(config=config) + + +class Migration(migrations.Migration): + dependencies = [ + ("crawls", "0010_crawl_delete_at"), + ] + + operations = [ + migrations.RunPython(move_limit_fields_to_config, migrations.RunPython.noop), + migrations.RemoveField( + model_name="crawl", + name="max_urls", + ), + migrations.RemoveField( + model_name="crawl", + name="crawl_max_size", + ), + migrations.RemoveField( + model_name="crawl", + name="snapshot_max_size", + ), + ] diff --git a/archivebox/crawls/migrations/0012_drop_stale_crawl_timeout_column.py b/archivebox/crawls/migrations/0012_drop_stale_crawl_timeout_column.py new file mode 100644 index 00000000..35393a77 --- /dev/null +++ b/archivebox/crawls/migrations/0012_drop_stale_crawl_timeout_column.py @@ -0,0 +1,22 @@ +from django.db import migrations + + +def drop_stale_crawl_timeout_column(apps, schema_editor): + table_name = "crawls_crawl" + column_name = "crawl_timeout" + connection = schema_editor.connection + with connection.cursor() as cursor: + columns = {column.name for column in connection.introspection.get_table_description(cursor, table_name)} + if column_name not in columns: + return + schema_editor.execute(f"ALTER TABLE {schema_editor.quote_name(table_name)} DROP COLUMN {schema_editor.quote_name(column_name)}") + + +class Migration(migrations.Migration): + dependencies = [ + ("crawls", "0011_move_crawl_limits_to_config"), + ] + + operations = [ + migrations.RunPython(drop_stale_crawl_timeout_column, migrations.RunPython.noop), + ] diff --git a/archivebox/crawls/migrations/0013_crawl_permissions.py b/archivebox/crawls/migrations/0013_crawl_permissions.py new file mode 100644 index 00000000..ebe554f4 --- /dev/null +++ b/archivebox/crawls/migrations/0013_crawl_permissions.py @@ -0,0 +1,19 @@ +# Generated by Django 6.0.5 on 2026-05-28 07:25 + +import django.db.models.fields.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('crawls', '0012_drop_stale_crawl_timeout_column'), + ] + + operations = [ + migrations.AddField( + model_name='crawl', + name='permissions', + field=models.GeneratedField(db_index=True, db_persist=True, expression=django.db.models.fields.json.KeyTextTransform('PERMISSIONS', 'config'), output_field=models.CharField(max_length=16, null=True)), + ), + ] diff --git a/archivebox/crawls/migrations/0014_crawl_persona_fk.py b/archivebox/crawls/migrations/0014_crawl_persona_fk.py new file mode 100644 index 00000000..0521c09d --- /dev/null +++ b/archivebox/crawls/migrations/0014_crawl_persona_fk.py @@ -0,0 +1,41 @@ +# Generated by hand on 2026-05-28 + +import django.db.models.deletion +from django.db import migrations, models + + +def clear_stale_persona_ids(apps, _schema_editor): + Crawl = apps.get_model("crawls", "Crawl") + Persona = apps.get_model("personas", "Persona") + Crawl.objects.filter(persona_id__isnull=False).exclude( + persona_id__in=Persona.objects.values_list("id", flat=True), + ).update(persona_id=None) + + +class Migration(migrations.Migration): + + dependencies = [ + ("crawls", "0013_crawl_permissions"), + ("personas", "0003_persona_permissions"), + ] + + operations = [ + migrations.RunPython(clear_stale_persona_ids, migrations.RunPython.noop), + migrations.RenameField( + model_name="crawl", + old_name="persona_id", + new_name="persona", + ), + migrations.AlterField( + model_name="crawl", + name="persona", + field=models.ForeignKey( + blank=True, + db_column="persona_id", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="crawls", + to="personas.persona", + ), + ), + ] diff --git a/archivebox/crawls/migrations/0015_alter_crawl_status.py b/archivebox/crawls/migrations/0015_alter_crawl_status.py new file mode 100644 index 00000000..4cfac303 --- /dev/null +++ b/archivebox/crawls/migrations/0015_alter_crawl_status.py @@ -0,0 +1,18 @@ +# Generated by Django 6.0.5 on 2026-05-28 12:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('crawls', '0014_crawl_persona_fk'), + ] + + operations = [ + migrations.AlterField( + model_name='crawl', + name='status', + field=models.CharField(choices=[('queued', 'Queued'), ('started', 'Started'), ('paused', 'Paused'), ('sealed', 'Sealed')], db_index=True, default='queued', max_length=15), + ), + ] diff --git a/archivebox/crawls/models.py b/archivebox/crawls/models.py index a2660590..3c2c54a6 100755 --- a/archivebox/crawls/models.py +++ b/archivebox/crawls/models.py @@ -14,6 +14,7 @@ from urllib.parse import urlparse from django.db import IntegrityError, models, transaction from django.db.models import Q +from django.db.models.fields.json import KT from django.core.exceptions import ValidationError from django.core.validators import MaxValueValidator, MinValueValidator from django.conf import settings @@ -31,7 +32,7 @@ from archivebox.base_models.models import ( ModelWithHealthStats, get_or_create_system_user_pk, ) -from archivebox.workers.models import ModelWithStateMachine, BaseStateMachine +from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine from archivebox.crawls.schedule_utils import next_run_for_schedule, validate_schedule from archivebox.misc.util import validate_url_length @@ -101,9 +102,6 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes): urls=template.urls, config=template.config or {}, max_depth=template.max_depth, - max_urls=template.max_urls, - crawl_max_size=template.crawl_max_size, - snapshot_max_size=template.snapshot_max_size, tags_str=template.tags_str, persona_id=template.persona_id, label=label, @@ -123,24 +121,23 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith urls = models.TextField(blank=False, null=False, help_text="Newline-separated list of URLs to crawl") config = models.JSONField(default=dict, null=True, blank=True) + permissions = models.GeneratedField( + expression=KT("config__PERMISSIONS"), + output_field=models.CharField(max_length=16, null=True), + db_persist=True, + db_index=True, + editable=False, + ) max_depth = models.PositiveSmallIntegerField(default=0, validators=[MinValueValidator(0), MaxValueValidator(4)]) - max_urls = models.IntegerField( - default=0, - validators=[MinValueValidator(0)], - help_text="Maximum number of URLs to snapshot for this crawl (0 = unlimited).", - ) - crawl_max_size = models.BigIntegerField( - default=0, - validators=[MinValueValidator(0)], - help_text="Maximum total archived output size in bytes for this crawl (0 = unlimited).", - ) - snapshot_max_size = models.BigIntegerField( - default=0, - validators=[MinValueValidator(0)], - help_text="Maximum archived output size in bytes for each snapshot (0 = unlimited).", - ) tags_str = models.CharField(max_length=1024, blank=True, null=False, default="") - persona_id = models.UUIDField(null=True, blank=True) + persona = models.ForeignKey( + "personas.Persona", + db_column="persona_id", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="crawls", + ) label = models.CharField(max_length=64, blank=True, null=False, default="") notes = models.TextField(blank=True, null=False, default="") schedule = models.ForeignKey(CrawlSchedule, on_delete=models.SET_NULL, null=True, blank=True, editable=True) @@ -189,6 +186,59 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith return get_config(crawl=self).DELETE_AFTER + def pause(self, *, save: bool = True) -> bool: + paused = super().pause(save=save) + if paused and self.pk: + from archivebox.core.models import ArchiveResult, Snapshot + + active_snapshots = self.snapshot_set.filter( + status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED], + ) + active_snapshots.update( + status=Snapshot.StatusChoices.PAUSED, + retry_at=RETRY_AT_MAX, + modified_at=timezone.now(), + ) + ArchiveResult.pause_queryset(ArchiveResult.objects.filter(snapshot__crawl=self)) + return paused + + def resume(self, *, when=None, save: bool = True) -> bool: + resumed = super().resume(when=when, save=save) + if resumed and self.pk: + from archivebox.core.models import ArchiveResult, Snapshot + + resume_at = when or timezone.now() + active_snapshots = self.snapshot_set.filter( + status=Snapshot.StatusChoices.PAUSED, + ) + active_snapshots.update( + status=Snapshot.StatusChoices.QUEUED, + retry_at=resume_at, + modified_at=timezone.now(), + ) + ArchiveResult.resume_queryset(ArchiveResult.objects.filter(snapshot__crawl=self), when=resume_at) + return resumed + + def cancel(self) -> None: + from archivebox.core.models import Snapshot + + cancelled_at = timezone.now() + self.status = self.StatusChoices.SEALED + self.retry_at = None + self.save(update_fields=["status", "retry_at", "modified_at"]) + Snapshot.objects.filter( + crawl=self, + status__in=[ + Snapshot.StatusChoices.QUEUED, + Snapshot.StatusChoices.STARTED, + Snapshot.StatusChoices.PAUSED, + ], + ).update( + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + modified_at=cancelled_at, + ) + @classmethod def missing_delete_at_candidates(cls): from archivebox.personas.models import Persona @@ -199,27 +249,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith def save(self, *args, **kwargs): update_fields = kwargs.get("update_fields") sync_tags = update_fields is None or "tags_str" in update_fields + old_crawl = type(self).objects.filter(pk=self.pk).first() if self.pk else None previous_tag_names = set() - if sync_tags and self.pk: - previous_tags_str = type(self).objects.filter(pk=self.pk).values_list("tags_str", flat=True).first() - previous_tag_names = set(self.parse_tag_names(previous_tags_str or "")) + if sync_tags and old_crawl is not None: + previous_tag_names = set(self.parse_tag_names(old_crawl.tags_str or "")) config = dict(self.config or {}) - if self.max_urls > 0: - config["CRAWL_MAX_URLS"] = self.max_urls - else: - config.pop("CRAWL_MAX_URLS", None) - - if self.crawl_max_size > 0: - config["CRAWL_MAX_SIZE"] = self.crawl_max_size - else: - config.pop("CRAWL_MAX_SIZE", None) - - if self.snapshot_max_size > 0: - config["SNAPSHOT_MAX_SIZE"] = self.snapshot_max_size - else: - config.pop("SNAPSHOT_MAX_SIZE", None) - if "CRAWL_MAX_CONCURRENT_SNAPSHOTS" in config: raw_concurrency = config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] if raw_concurrency in (None, ""): @@ -342,9 +377,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith "urls": self.urls, "status": self.status, "max_depth": self.max_depth, - "max_urls": self.max_urls, - "crawl_max_size": self.crawl_max_size, - "snapshot_max_size": self.snapshot_max_size, + "config": self.config or {}, "tags_str": self.tags_str, "label": self.label, "created_at": self.created_at.isoformat() if self.created_at else None, @@ -386,9 +419,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith crawl = Crawl.objects.create( urls=urls, max_depth=record.get("max_depth", record.get("depth", 0)), - max_urls=record.get("max_urls", 0), - crawl_max_size=record.get("crawl_max_size", 0), - snapshot_max_size=record.get("snapshot_max_size", 0), + config=record.get("config") or {}, tags_str=record.get("tags_str", record.get("tags", "")), label=record.get("label", ""), status=Crawl.StatusChoices.QUEUED, @@ -551,7 +582,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith filtered_snapshots = [ snapshot for snapshot in self.snapshot_set.filter( - status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED], + status__in=[ + Snapshot.StatusChoices.QUEUED, + Snapshot.StatusChoices.STARTED, + Snapshot.StatusChoices.PAUSED, + ], ).only("pk", "url", "status") if not self.url_passes_filters(snapshot.url, snapshot=snapshot, use_effective_config=False) ] @@ -603,18 +638,24 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith return len(urls) def remaining_url_capacity(self) -> int | None: - if self.max_urls <= 0: + from archivebox.config.common import get_config + + max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0) + if max_urls <= 0: return None - return max(self.max_urls - self.count_urls_for_limit(), 0) + return max(max_urls - self.count_urls_for_limit(), 0) def has_remaining_url_capacity(self) -> bool: remaining = self.remaining_url_capacity() return remaining is None or remaining > 0 def remaining_snapshot_capacity(self) -> int | None: - if self.max_urls <= 0: + from archivebox.config.common import get_config + + max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0) + if max_urls <= 0: return None - return max(self.max_urls - self.snapshot_set.count(), 0) + return max(max_urls - self.snapshot_set.count(), 0) def has_remaining_snapshot_capacity(self) -> bool: remaining = self.remaining_snapshot_capacity() @@ -687,17 +728,27 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith if self.persona_id: persona = Persona.objects.filter(id=self.persona_id).first() - if persona is None: - raise Persona.DoesNotExist(f"Crawl {self.id} references missing Persona {self.persona_id}") - return persona + if persona is not None: + return persona default_persona_name = str((self.config or {}).get("DEFAULT_PERSONA") or "").strip() if default_persona_name: persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default") + persona.ensure_dirs() return persona return None + def limit_stop_reason(self) -> str: + from abx_dl.limits import CrawlLimitState + from archivebox.config.common import get_config + + if not (self.output_dir / ".abx-dl" / "limits.json").exists(): + return "" + config = get_config(crawl=self, include_machine=False) + config["CRAWL_DIR"] = str(self.output_dir) + return CrawlLimitState.from_config(config).get_stop_reason() + def add_url(self, entry: dict) -> bool: """ Add a URL to the crawl queue if not already present. @@ -1132,17 +1183,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith The root Snapshot for this crawl, or None for system crawls that don't create snapshots """ import time - from pathlib import Path from archivebox.hooks import run_hook, discover_hooks, process_hook_records, is_finite_background_hook from archivebox.config.common import get_config from archivebox.machine.models import Binary, Machine - # Debug logging to file (since stdout/stderr redirected to /dev/null in progress mode) - debug_log = Path("/tmp/archivebox_crawl_debug.log") - with open(debug_log, "a") as f: - f.write(f"\n=== Crawl.run() starting for {self.id} at {time.time()} ===\n") - f.flush() - def get_runtime_config(): config = get_config(crawl=self) if persona_runtime_overrides: @@ -1177,9 +1221,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith self.urls.strip(), ) - with open(debug_log, "a") as f: - f.write(f"Running hook: {hook.name}\n") - f.flush() hook_start = time.time() plugin_name = hook.parent.name output_dir = self.output_dir / plugin_name @@ -1194,10 +1235,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith url=primary_url, snapshot_id=str(self.id), ) - with open(debug_log, "a") as f: - f.write(f"Hook {hook.name} completed with status={process.status}\n") - f.flush() - hook_elapsed = time.time() - hook_start if hook_elapsed > 0.5: print(f"[yellow]⏱️ Hook {hook.name} took {hook_elapsed:.2f}s[/yellow]") @@ -1213,9 +1250,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith from archivebox.hooks import extract_records_from_process records = [] - # Finite background hooks can exit before their stdout log is fully - # visible to our polling loop. Give successful hooks a brief chance - # to flush JSONL records before we move on to downstream hooks. + # Finite background hooks can exit before their completed Process + # metadata is visible. Give successful hooks a brief chance to + # flush JSONL stdout into the Process row before downstream hooks. for delay in (0.0, 0.05, 0.1, 0.25, 0.5): if delay: time.sleep(delay) @@ -1284,14 +1321,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith for hook in provider_hooks: resolved_binary_names.update(run_crawl_hook(hook)) - # Discover and run on_Crawl hooks - with open(debug_log, "a") as f: - f.write("Discovering Crawl hooks...\n") - f.flush() hooks = discover_hooks("Crawl", config=get_runtime_config()) - with open(debug_log, "a") as f: - f.write(f"Found {len(hooks)} hooks\n") - f.flush() for hook in hooks: hook_binary_names = run_crawl_hook(hook) @@ -1309,20 +1339,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith leaked_count = leaked_snapshots.count() leaked_snapshots.delete() print(f"[yellow]⚠️ Removed {leaked_count} leaked snapshot(s) created during system crawl {system_task}[/yellow]") - with open(debug_log, "a") as f: - f.write(f"Skipping snapshot creation for system crawl: {system_task}\n") - f.write("=== Crawl.run() complete ===\n\n") - f.flush() return None - with open(debug_log, "a") as f: - f.write("Creating snapshots from URLs...\n") - f.flush() - created_snapshots = self.create_snapshots_from_urls() - with open(debug_log, "a") as f: - f.write(f"Created {len(created_snapshots)} snapshots\n") - f.write("=== Crawl.run() complete ===\n\n") - f.flush() + self.create_snapshots_from_urls() # Return first snapshot for this crawl (newly created or existing) # This ensures the crawl doesn't seal if snapshots exist, even if they weren't just created @@ -1340,7 +1359,13 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith return True # If snapshots exist, check if all are sealed - if snapshots.filter(status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED]).exists(): + if snapshots.filter( + status__in=[ + Snapshot.StatusChoices.QUEUED, + Snapshot.StatusChoices.STARTED, + Snapshot.StatusChoices.PAUSED, + ], + ).exists(): return False return True @@ -1426,13 +1451,16 @@ class CrawlMachine(BaseStateMachine): # States queued = State(value=Crawl.StatusChoices.QUEUED, initial=True) started = State(value=Crawl.StatusChoices.STARTED) + paused = State(value=Crawl.StatusChoices.PAUSED) sealed = State(value=Crawl.StatusChoices.SEALED, final=True) # Tick Event (polled by workers) - tick = queued.to.itself(unless="can_start") | queued.to(started, cond="can_start") | started.to(sealed, cond="is_finished") + tick = queued.to.itself(unless="can_start") | queued.to(started, cond="can_start") | started.to(sealed, cond="is_finished") | paused.to.itself() # Manual event (triggered by last Snapshot sealing) seal = started.to(sealed) + pause_requested = queued.to(paused) | started.to(paused) + resume_requested = paused.to(queued) def can_start(self) -> bool: if not self.crawl.urls: @@ -1448,6 +1476,13 @@ class CrawlMachine(BaseStateMachine): """Check if all Snapshots for this crawl are finished.""" return self.crawl.is_finished() + @queued.enter + def enter_queued(self): + self.crawl.update_and_requeue( + retry_at=timezone.now(), + status=Crawl.StatusChoices.QUEUED, + ) + @started.enter def enter_started(self): import sys @@ -1482,6 +1517,13 @@ class CrawlMachine(BaseStateMachine): traceback.print_exc() raise + @paused.enter + def enter_paused(self): + self.crawl.update_and_requeue( + retry_at=RETRY_AT_MAX, + status=Crawl.StatusChoices.PAUSED, + ) + @sealed.enter def enter_sealed(self): # Clean up background hooks and run on_CrawlEnd hooks diff --git a/archivebox/dead/archivebox_persona.py b/archivebox/dead/archivebox_persona.py new file mode 100644 index 00000000..74ef9336 --- /dev/null +++ b/archivebox/dead/archivebox_persona.py @@ -0,0 +1,129 @@ +# ruff: noqa +NETSCAPE_COOKIE_HEADER = [ + "# Netscape HTTP Cookie File", + "# https://curl.se/docs/http-cookies.html", + "# This file was generated by ArchiveBox persona cookie extraction", + "#", + "# Format: domain\\tincludeSubdomains\\tpath\\tsecure\\texpiry\\tname\\tvalue", + "", +] + + +def _parse_netscape_cookies(path: Path) -> "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]": + cookies = OrderedDict() + if not path.exists(): + return cookies + + for line in path.read_text().splitlines(): + if not line or line.startswith("#"): + continue + parts = line.split("\t") + if len(parts) < 7: + continue + domain, include_subdomains, cookie_path, secure, expiry, name, value = parts[:7] + key = (domain, cookie_path, name) + cookies[key] = (domain, include_subdomains, cookie_path, secure, expiry, name, value) + return cookies + + +def _write_netscape_cookies(path: Path, cookies: "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]") -> None: + lines = list(NETSCAPE_COOKIE_HEADER) + for cookie in cookies.values(): + lines.append("\t".join(cookie)) + path.write_text("\n".join(lines) + "\n") + + +def _merge_netscape_cookies(existing_file: Path, new_file: Path) -> None: + existing = _parse_netscape_cookies(existing_file) + new = _parse_netscape_cookies(new_file) + for key, cookie in new.items(): + existing[key] = cookie + _write_netscape_cookies(existing_file, existing) + + +def extract_cookies_via_cdp( + user_data_dir: Path, + output_file: Path, + profile_dir: str | None = None, + chrome_binary: str | None = None, +) -> bool: + """ + Launch Chrome with the given user data dir and extract cookies via CDP. + + Returns True if successful, False otherwise. + """ + from archivebox.config.common import get_config + + # Find the cookie extraction script + chrome_plugin_dir = Path(__file__).parent.parent / "plugins" / "chrome" + extract_script = chrome_plugin_dir / "extract_cookies.js" + + if not extract_script.exists(): + rprint(f"[yellow]Cookie extraction script not found at {extract_script}[/yellow]", file=sys.stderr) + return False + + # Get node modules dir + node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules" + + # Set up environment + env = os.environ.copy() + env["NODE_MODULES_DIR"] = str(node_modules_dir) + env["CHROME_USER_DATA_DIR"] = str(user_data_dir) + env["CHROME_HEADLESS"] = "true" + if chrome_binary: + env["CHROME_BINARY"] = str(chrome_binary) + output_path = output_file + temp_output = None + temp_dir = None + if output_file.exists(): + temp_dir = Path(tempfile.mkdtemp(prefix="ab_cookies_")) + temp_output = temp_dir / "cookies.txt" + output_path = temp_output + if profile_dir: + extra_arg = f"--profile-directory={profile_dir}" + existing_extra = env.get("CHROME_ARGS_EXTRA", "").strip() + args_list = [] + if existing_extra: + if existing_extra.startswith("["): + try: + parsed = json.loads(existing_extra) + if isinstance(parsed, list): + args_list.extend(str(x) for x in parsed) + except Exception: + args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()]) + else: + args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()]) + args_list.append(extra_arg) + env["CHROME_ARGS_EXTRA"] = json.dumps(args_list) + + env["COOKIES_OUTPUT_FILE"] = str(output_path) + + try: + result = subprocess.run( + ["node", str(extract_script)], + env=env, + capture_output=True, + text=True, + timeout=60, + ) + + if result.returncode == 0: + if temp_output and temp_output.exists(): + _merge_netscape_cookies(output_file, temp_output) + return True + else: + rprint(f"[yellow]Cookie extraction failed: {result.stderr}[/yellow]", file=sys.stderr) + return False + + except subprocess.TimeoutExpired: + rprint("[yellow]Cookie extraction timed out[/yellow]", file=sys.stderr) + return False + except FileNotFoundError: + rprint("[yellow]Node.js not found. Cannot extract cookies.[/yellow]", file=sys.stderr) + return False + except Exception as e: + rprint(f"[yellow]Cookie extraction error: {e}[/yellow]", file=sys.stderr) + return False + finally: + if temp_dir and temp_dir.exists(): + shutil.rmtree(temp_dir, ignore_errors=True) diff --git a/archivebox/dead/auth.py b/archivebox/dead/auth.py new file mode 100644 index 00000000..0ae658bd --- /dev/null +++ b/archivebox/dead/auth.py @@ -0,0 +1,26 @@ +# ruff: noqa +class UsernameAndPasswordAuth(HttpBasicAuth): + """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)""" + + def authenticate(self, request: HttpRequest, username: str, password: str) -> User | None: + return _require_superuser( + auth_using_password(username=username, password=password, request=request), + request, + self.__class__.__name__, + ) + + +class DjangoSessionAuth: + """Allow authenticating with existing Django session cookies (same-origin only).""" + + def __call__(self, request: HttpRequest) -> User | None: + return self.authenticate(request) + + def authenticate(self, request: HttpRequest, **kwargs) -> User | None: + user = getattr(request, "user", None) + if isinstance(user, User) and user.is_authenticated: + setattr(request, "_api_auth_method", self.__class__.__name__) + if not user.is_superuser: + raise HttpError(403, "Valid session but User does not have permission (make sure user.is_superuser=True)") + return user + return None diff --git a/archivebox/core/templatetags/config_tags.py b/archivebox/dead/config_tags.py similarity index 97% rename from archivebox/core/templatetags/config_tags.py rename to archivebox/dead/config_tags.py index 8c305c57..16d442e2 100644 --- a/archivebox/core/templatetags/config_tags.py +++ b/archivebox/dead/config_tags.py @@ -1,3 +1,4 @@ +# ruff: noqa """Template tags for accessing config values in templates.""" from typing import Any diff --git a/archivebox/dead/db.py b/archivebox/dead/db.py new file mode 100644 index 00000000..2df1da3e --- /dev/null +++ b/archivebox/dead/db.py @@ -0,0 +1,29 @@ +# ruff: noqa +def list_migrations(out_dir: Path = DATA_DIR) -> list[tuple[bool, str]]: + """List all Django migrations and their status""" + from django.core.management import call_command + + def showmigrations() -> StringIO: + out = StringIO() + call_command("showmigrations", list=True, stdout=out) + out.seek(0) + return out + + out = retry_sqlite_locks(showmigrations, label="checking migrations") + + migrations = [] + for line in out.readlines(): + if line.strip() and "]" in line: + status_str, name_str = line.strip().split("]", 1) + is_applied = "X" in status_str + migration_name = name_str.strip() + migrations.append((is_applied, migration_name)) + + return migrations + + +def get_admins(out_dir: Path = DATA_DIR) -> list[Any]: + """Get list of superuser accounts""" + from django.contrib.auth.models import User + + return list(User.objects.filter(is_superuser=True).exclude(username="system")) diff --git a/archivebox/misc/debugging.py b/archivebox/dead/debugging.py similarity index 98% rename from archivebox/misc/debugging.py rename to archivebox/dead/debugging.py index 4ada510c..c4f0dd57 100644 --- a/archivebox/misc/debugging.py +++ b/archivebox/dead/debugging.py @@ -1,3 +1,4 @@ +# ruff: noqa from functools import wraps from time import time diff --git a/archivebox/dead/detect.py b/archivebox/dead/detect.py new file mode 100644 index 00000000..e5341351 --- /dev/null +++ b/archivebox/dead/detect.py @@ -0,0 +1,3 @@ +# ruff: noqa +def get_host_immutable_info(host_info: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in host_info.items() if key in ["guid", "net_mac", "os_family", "cpu_arch"]} diff --git a/archivebox/dead/django.py b/archivebox/dead/django.py new file mode 100644 index 00000000..c7e2ed99 --- /dev/null +++ b/archivebox/dead/django.py @@ -0,0 +1,7 @@ +# ruff: noqa +def setup_django_minimal(): + # sys.path.append(str(CONSTANTS.PACKAGE_DIR)) + # os.environ.setdefault('ARCHIVEBOX_DATA_DIR', str(CONSTANTS.DATA_DIR)) + # os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') + # django.setup() + raise Exception("dont use this anymore") diff --git a/archivebox/misc/folders.py b/archivebox/dead/folders.py similarity index 99% rename from archivebox/misc/folders.py rename to archivebox/dead/folders.py index 38d4b2ed..b3edcb41 100644 --- a/archivebox/misc/folders.py +++ b/archivebox/dead/folders.py @@ -1,3 +1,4 @@ +# ruff: noqa """ Folder utilities for ArchiveBox. diff --git a/archivebox/dead/hooks.py b/archivebox/dead/hooks.py new file mode 100644 index 00000000..db61b6b9 --- /dev/null +++ b/archivebox/dead/hooks.py @@ -0,0 +1,35 @@ +# ruff: noqa +class HookResult(TypedDict, total=False): + """Raw result from run_hook().""" + + returncode: int + stdout: str + stderr: str + output_json: dict[str, Any] | None + output_files: list[dict[str, Any]] + duration_ms: int + hook: str + plugin: str # Plugin name (directory name, e.g., 'wget', 'screenshot') + hook_name: str # Full hook filename (e.g., 'on_Snapshot__50_wget.py') + # New fields for JSONL parsing + records: list[dict[str, Any]] # Parsed JSONL records with 'type' field + + +def get_config_defaults_from_plugins() -> dict[str, Any]: + """ + Get default values for all plugin config options. + + Returns: + Dict mapping config keys to their default values. + e.g., {'SAVE_WGET': True, 'WGET_TIMEOUT': 60, ...} + """ + plugin_configs = discover_plugin_configs() + defaults = {} + + for plugin_name, schema in plugin_configs.items(): + properties = schema.get("properties", {}) + for key, prop_schema in properties.items(): + if "default" in prop_schema: + defaults[key] = prop_schema["default"] + + return defaults diff --git a/archivebox/dead/host_utils.py b/archivebox/dead/host_utils.py new file mode 100644 index 00000000..9397f78f --- /dev/null +++ b/archivebox/dead/host_utils.py @@ -0,0 +1,11 @@ +# ruff: noqa +def get_archive_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return get_web_base_url(request=request, config=config, **config_kwargs) + + +def build_api_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return _build_url(get_api_base_url(request, config=config, **config_kwargs), path) + + +def build_archive_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return _build_url(get_archive_base_url(request, config=config, **config_kwargs), path) diff --git a/archivebox/dead/jsonl.py b/archivebox/dead/jsonl.py new file mode 100644 index 00000000..f56bd488 --- /dev/null +++ b/archivebox/dead/jsonl.py @@ -0,0 +1,12 @@ +# ruff: noqa +def write_records(records: Iterator[dict[str, Any]], stream: TextIO | None = None) -> int: + """ + Write multiple JSONL records to stdout (or provided stream). + + Returns count of records written. + """ + count = 0 + for record in records: + write_record(record, stream) + count += 1 + return count diff --git a/archivebox/misc/legacy.py b/archivebox/dead/legacy.py similarity index 99% rename from archivebox/misc/legacy.py rename to archivebox/dead/legacy.py index 477c27d6..8251b794 100644 --- a/archivebox/misc/legacy.py +++ b/archivebox/dead/legacy.py @@ -1,3 +1,4 @@ +# ruff: noqa """ Legacy archive import utilities. diff --git a/archivebox/services/live_ui.py b/archivebox/dead/live_ui.py similarity index 81% rename from archivebox/services/live_ui.py rename to archivebox/dead/live_ui.py index a89f016c..c7bafee1 100644 --- a/archivebox/services/live_ui.py +++ b/archivebox/dead/live_ui.py @@ -1,3 +1,4 @@ +# ruff: noqa from abx_dl.cli import LiveBusUI __all__ = ["LiveBusUI"] diff --git a/archivebox/ideas/process_plugin.py b/archivebox/dead/process_plugin.py similarity index 99% rename from archivebox/ideas/process_plugin.py rename to archivebox/dead/process_plugin.py index aad584bb..051f8f1a 100644 --- a/archivebox/ideas/process_plugin.py +++ b/archivebox/dead/process_plugin.py @@ -1,3 +1,4 @@ +# ruff: noqa __package__ = "archivebox.ideas" import asyncio diff --git a/archivebox/dead/shutdown_util.py b/archivebox/dead/shutdown_util.py new file mode 100644 index 00000000..98682228 --- /dev/null +++ b/archivebox/dead/shutdown_util.py @@ -0,0 +1,101 @@ +# ruff: noqa +def pid_is_running(pid: int) -> bool: + """Return True when the OS still has a process for pid. + + This intentionally does not inspect ArchiveBox state. It is used only for + foreground parent processes and stale pid files; orchestrator ownership + still belongs to the database state machine and retry_at locks. + """ + + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def read_pid_file(pid_file: Path) -> int | None: + try: + return int(pid_file.read_text().strip()) + except (FileNotFoundError, ValueError): + return None + + +def unlink_pid_file_if_owner(pid_file: Path, pid: int) -> None: + """Remove a pid file only if it still points at the expected process.""" + + try: + if pid_file.read_text().strip() == str(pid): + pid_file.unlink(missing_ok=True) + except FileNotFoundError: + pass + + +def wait_for_pid_exit(pid: int, *, timeout: float, interval: float = 0.1) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not pid_is_running(pid): + return True + time.sleep(interval) + return not pid_is_running(pid) + + +def stop_pidfile_owner( + pid_file: Path, + *, + current_pid: int, + description: str, + graceful_timeout: float, + log: Callable[[str], object], + owner_matches: Callable[[int], bool] | None = None, + on_stale_pid: Callable[[], object] | None = None, + on_forced_stop: Callable[[], object] | None = None, +) -> int: + """Stop a previous foreground owner recorded in pid_file. + + This is for command-parent takeover only. It deliberately does not claim + Crawl/Snapshot work; crashed work is resumed by the existing retry_at/state + machine path after the parent process is gone. + """ + + pid = read_pid_file(pid_file) + if pid is None or pid == current_pid: + return 0 + + if not pid_is_running(pid): + pid_file.unlink(missing_ok=True) + if on_stale_pid is not None: + on_stale_pid() + return 0 + if owner_matches is not None and not owner_matches(pid): + # PIDs can be reused after an unclean exit. A stale pidfile must never + # let one ArchiveBox collection stop a process owned by another + # collection or another app entirely. + pid_file.unlink(missing_ok=True) + if on_stale_pid is not None: + on_stale_pid() + return 0 + + log(f"[yellow][*] Stopping existing {description} pid={pid}...[/yellow]") + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pid_file.unlink(missing_ok=True) + if on_stale_pid is not None: + on_stale_pid() + return 0 + + if wait_for_pid_exit(pid, timeout=graceful_timeout): + pid_file.unlink(missing_ok=True) + return 1 + + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + if on_forced_stop is not None: + on_forced_stop() + pid_file.unlink(missing_ok=True) + return 1 diff --git a/archivebox/dead/supervision_service.py b/archivebox/dead/supervision_service.py new file mode 100644 index 00000000..5a22eb9b --- /dev/null +++ b/archivebox/dead/supervision_service.py @@ -0,0 +1,41 @@ +# ruff: noqa +def ensure_single_orchestrator(*, data_dir: str | Path, takeover: bool, reason: str = ""): + from archivebox.machine.models import Machine, Process + from archivebox.workers.supervisord_util import ( + RUNNER_WORKER, + get_existing_supervisord_process, + get_or_create_supervisord_process, + start_worker, + stop_worker, + ) + + existing = healthy_orchestrator(data_dir=data_dir) + if existing and not takeover: + if reason: + pid = existing.get("pid") if isinstance(existing, dict) else existing.pid + print(f"[green][*] {reason}; existing orchestrator pid={pid} will process it.[/green]") + return existing + + supervisor = get_existing_supervisord_process() or get_or_create_supervisord_process(daemonize=False) + if existing and takeover: + print("[yellow][*] Taking over existing ArchiveBox orchestrator...[/yellow]") + try: + stop_worker(supervisor, RUNNER_WORKER["name"]) + except Exception: + pass + for proc in Process.objects.filter( + machine=Machine.current(), + process_type=Process.TypeChoices.ORCHESTRATOR, + status=Process.StatusChoices.RUNNING, + pwd=str(data_dir), + ).order_by("created_at"): + if proc.is_running: + proc.terminate(graceful_timeout=2.0) + + return start_worker(supervisor, RUNNER_WORKER) + + +def wait_until_replaced_or_signal(command, *, process_type: str, data_dir: str | Path, url: str | None = None, interval: float = 2.0) -> None: + while command_is_newest(command, process_type=process_type, data_dir=data_dir, url=url): + command.heartbeat() + time.sleep(interval) diff --git a/archivebox/dead/supervisord_util.py b/archivebox/dead/supervisord_util.py new file mode 100644 index 00000000..fe45f49b --- /dev/null +++ b/archivebox/dead/supervisord_util.py @@ -0,0 +1,90 @@ +# ruff: noqa +def follow(file, sleep_sec=0.1) -> Iterator[str]: + """Yield each line from a file as they are written. + `sleep_sec` is the time to sleep after empty reads.""" + line = "" + while True: + tmp = file.readline() + if tmp is not None and tmp != "": + line += tmp + if line.endswith("\n"): + yield line + line = "" + elif sleep_sec: + time.sleep(sleep_sec) + + +def tail_worker_logs(log_path: str): + get_or_create_supervisord_process(daemonize=False) + + from rich.live import Live + from rich.table import Table + + table = Table() + table.add_column("TS") + table.add_column("URL") + + try: + with Live(table, refresh_per_second=1) as live: # update 4 times a second to feel fluid + with open(log_path) as f: + for line in follow(f): + if "://" in line: + live.console.print(f"Working on: {line.strip()}") + # table.add_row("123124234", line.strip()) + except (KeyboardInterrupt, BrokenPipeError, OSError): + STDERR.print("\n[πŸ›‘] Got Ctrl+C, stopping gracefully...") + except SystemExit: + pass + + +def watch_worker(supervisor, daemon_name, interval=5): + """loop continuously and monitor worker's health""" + while True: + proc = get_worker(supervisor, daemon_name) + if not proc: + raise Exception("Worker disappeared while running! " + daemon_name) + + if proc["statename"] == "STOPPED": + return proc + + if proc["statename"] == "RUNNING": + time.sleep(1) + continue + + if proc["statename"] in ("STARTING", "BACKOFF", "FATAL", "EXITED", "STOPPING"): + print(f"[πŸ¦Έβ€β™‚οΈ] WARNING: Worker {daemon_name} {proc['statename']} {proc['description']}") + time.sleep(interval) + continue + + +def start_cli_workers(watch=False): + from archivebox.config.common import get_config + + supervisor = get_or_create_supervisord_process(daemonize=False) + + sonic_worker = get_sonic_supervisord_worker_from_plugin(get_config()) + workers = [(RUNNER_WORKER, False)] + if sonic_worker is not None: + workers.insert(0, (sonic_worker, False)) + + sync_supervisord_workers(supervisor, workers, prune=True) + + if watch: + try: + # Block on supervisord process - it will handle signals and stop children + if _supervisord_proc: + _supervisord_proc.wait() + else: + # Fallback to watching worker if no proc reference + watch_worker(supervisor, RUNNER_WORKER["name"]) + except (KeyboardInterrupt, BrokenPipeError, OSError): + STDERR.print("\n[πŸ›‘] Got Ctrl+C, stopping gracefully...") + except SystemExit: + pass + except BaseException as e: + STDERR.print(f"\n[πŸ›‘] Got {e.__class__.__name__} exception, stopping gracefully...") + finally: + # Ensure supervisord and all children are stopped + stop_existing_supervisord_process() + time.sleep(1.0) # Give processes time to fully terminate + return [RUNNER_WORKER] diff --git a/archivebox/dead/system.py b/archivebox/dead/system.py new file mode 100644 index 00000000..712bf052 --- /dev/null +++ b/archivebox/dead/system.py @@ -0,0 +1,78 @@ +# ruff: noqa +@enforce_types +def chmod_file(path: str, cwd: str = "", config=None, **config_kwargs) -> None: + """chmod -R /""" + + root = Path(cwd or os.getcwd()) / path + if not os.access(root, os.R_OK): + raise Exception(f"Failed to chmod: {path} does not exist (did the previous step fail?)") + + if not root.is_dir(): + # path is just a plain file + config = config or get_config(**config_kwargs) + os.chmod(root, int(config.OUTPUT_PERMISSIONS, base=8)) + else: + config = config or get_config(**config_kwargs) + for subpath in Path(path).glob("**/*"): + if subpath.is_dir(): + # directories need execute permissions to be able to list contents + os.chmod(subpath, int(config.DIR_OUTPUT_PERMISSIONS, base=8)) + else: + os.chmod(subpath, int(config.OUTPUT_PERMISSIONS, base=8)) + + +@enforce_types +def copy_and_overwrite(from_path: str | Path, to_path: str | Path): + """copy a given file or directory to a given path, overwriting the destination""" + + assert os.access(from_path, os.R_OK) + + if Path(from_path).is_dir(): + shutil.rmtree(to_path, ignore_errors=True) + shutil.copytree(from_path, to_path) + else: + with open(from_path, "rb") as src: + contents = src.read() + atomic_write(to_path, contents) + + +class suppress_output: + """ + A context manager for doing a "deep suppression" of stdout and stderr in + Python, i.e. will suppress all print, even if the print originates in a + compiled C/Fortran sub-function. + + This will not suppress raised exceptions, since exceptions are printed + to stderr just before a script exits, and after the context manager has + exited (at least, I think that is why it lets exceptions through). + + with suppress_stdout_stderr(): + rogue_function() + """ + + def __init__(self, stdout=True, stderr=True): + # Open a pair of null files + # Save the actual stdout (1) and stderr (2) file descriptors. + self.stdout, self.stderr = stdout, stderr + if stdout: + self.null_stdout = os.open(os.devnull, os.O_RDWR) + self.real_stdout = os.dup(1) + if stderr: + self.null_stderr = os.open(os.devnull, os.O_RDWR) + self.real_stderr = os.dup(2) + + def __enter__(self): + # Assign the null pointers to stdout and stderr. + if self.stdout: + os.dup2(self.null_stdout, 1) + if self.stderr: + os.dup2(self.null_stderr, 2) + + def __exit__(self, *_): + # Re-assign the real stdout/stderr back to (1) and (2) + if self.stdout: + os.dup2(self.real_stdout, 1) + os.close(self.null_stdout) + if self.stderr: + os.dup2(self.real_stderr, 2) + os.close(self.null_stderr) diff --git a/archivebox/dead/util.py b/archivebox/dead/util.py new file mode 100644 index 00000000..1bf33be3 --- /dev/null +++ b/archivebox/dead/util.py @@ -0,0 +1,114 @@ +# ruff: noqa +def short_ts(ts: Any) -> str | None: + parsed = parse_date(ts) + return None if parsed is None else str(parsed.timestamp()).split(".")[0] + + +def ts_to_iso(ts: Any) -> str | None: + parsed = parse_date(ts) + return None if parsed is None else parsed.isoformat() + + +def is_static_file(url: str): + # TODO: the proper way is with MIME type detection + ext, not only extension + return extension(url).lower() in CONSTANTS.STATICFILE_EXTENSIONS + + +@enforce_types +def str_between(string: str, start: str, end: str | None = None) -> str: + """(12345, , ) -> 12345""" + + content = string.split(start, 1)[-1] + if end is not None: + content = content.rsplit(end, 1)[0] + + return content + + +@enforce_types +def get_headers(url: str, timeout: int | None = None, config=None, **config_kwargs) -> str: + """Download the contents of a remote url and return the headers""" + # TODO: get rid of this and use an abx pluggy hook instead + + from archivebox.config.common import get_config + + config = config or get_config(**config_kwargs) + timeout = timeout or config.TIMEOUT + + try: + response = requests.head( + url, + headers={"User-Agent": config.USER_AGENT}, + verify=config.CHECK_SSL_VALIDITY, + timeout=timeout, + allow_redirects=True, + ) + if response.status_code >= 400: + raise RequestException + except ReadTimeout: + raise + except RequestException: + response = requests.get( + url, + headers={"User-Agent": config.USER_AGENT}, + verify=config.CHECK_SSL_VALIDITY, + timeout=timeout, + stream=True, + ) + + return pyjson.dumps( + { + "URL": url, + "Status-Code": response.status_code, + "Elapsed": response.elapsed.total_seconds() * 1000, + "Encoding": str(response.encoding), + "Apparent-Encoding": response.apparent_encoding, + **dict(response.headers), + }, + indent=4, + ) + + +def chrome_cleanup(config=None, **config_kwargs): + """ + Cleans up any state or runtime files that Chrome leaves behind when killed by + a timeout or other error. Handles: + - All persona chrome_profile directories (via Persona.cleanup_chrome_all()) + - Explicit CHROME_USER_DATA_DIR from config + - Legacy Docker chromium path + """ + import os + from pathlib import Path + from archivebox.config.permissions import IN_DOCKER + + # Clean up all persona chrome directories using Persona class + try: + from archivebox.personas.models import Persona + + # Clean up all personas + Persona.cleanup_chrome_all() + + # Also clean up the active persona's explicit CHROME_USER_DATA_DIR if set + # (in case it's a custom path not under PERSONAS_DIR) + from archivebox.config.common import get_config + + config = config or get_config(**config_kwargs) + chrome_user_data_dir = config.get("CHROME_USER_DATA_DIR") + if chrome_user_data_dir: + singleton_lock = Path(chrome_user_data_dir) / "SingletonLock" + if os.path.lexists(singleton_lock): + try: + singleton_lock.unlink() + except OSError: + pass + except Exception: + pass # Persona/config not available during early startup + + # Legacy Docker cleanup (for backwards compatibility) + if IN_DOCKER: + singleton_lock = "/home/archivebox/.config/chromium/SingletonLock" + if os.path.lexists(singleton_lock): + try: + os.remove(singleton_lock) + except OSError: + pass diff --git a/archivebox/personas/views.py b/archivebox/dead/views.py similarity index 66% rename from archivebox/personas/views.py rename to archivebox/dead/views.py index 60f00ef0..667554af 100644 --- a/archivebox/personas/views.py +++ b/archivebox/dead/views.py @@ -1 +1,2 @@ +# ruff: noqa # Create your views here. diff --git a/archivebox/hooks.py b/archivebox/hooks.py index 603b2c16..54c96f3d 100644 --- a/archivebox/hooks.py +++ b/archivebox/hooks.py @@ -159,22 +159,6 @@ def normalize_hook_event_name(event_name: str) -> str | None: return normalized -class HookResult(TypedDict, total=False): - """Raw result from run_hook().""" - - returncode: int - stdout: str - stderr: str - output_json: dict[str, Any] | None - output_files: list[dict[str, Any]] - duration_ms: int - hook: str - plugin: str # Plugin name (directory name, e.g., 'wget', 'screenshot') - hook_name: str # Full hook filename (e.g., 'on_Snapshot__50_wget.py') - # New fields for JSONL parsing - records: list[dict[str, Any]] # Parsed JSONL records with 'type' field - - def _model_output_dir_from_child_path(path: Path, marker: str) -> Path | None: """ Infer the model output dir from a model dir or one of its plugin subdirs. @@ -873,26 +857,6 @@ def discover_plugin_configs() -> dict[str, dict[str, Any]]: return configs -def get_config_defaults_from_plugins() -> dict[str, Any]: - """ - Get default values for all plugin config options. - - Returns: - Dict mapping config keys to their default values. - e.g., {'SAVE_WGET': True, 'WGET_TIMEOUT': 60, ...} - """ - plugin_configs = discover_plugin_configs() - defaults = {} - - for plugin_name, schema in plugin_configs.items(): - properties = schema.get("properties", {}) - for key, prop_schema in properties.items(): - if "default" in prop_schema: - defaults[key] = prop_schema["default"] - - return defaults - - def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited: set[str] | None = None) -> PluginSpecialConfig: """ Extract special config keys for a plugin following naming conventions. diff --git a/archivebox/machine/detect.py b/archivebox/machine/detect.py index 7b58f5d7..6a48e746 100644 --- a/archivebox/machine/detect.py +++ b/archivebox/machine/detect.py @@ -275,10 +275,6 @@ def get_host_stats() -> dict[str, Any]: return {} -def get_host_immutable_info(host_info: dict[str, Any]) -> dict[str, Any]: - return {key: value for key, value in host_info.items() if key in ["guid", "net_mac", "os_family", "cpu_arch"]} - - def get_host_guid() -> str: return machineid.hashed_id("archivebox") diff --git a/archivebox/machine/migrations/0017_shorten_process_progress_index_names.py b/archivebox/machine/migrations/0017_shorten_process_progress_index_names.py new file mode 100644 index 00000000..617238f0 --- /dev/null +++ b/archivebox/machine/migrations/0017_shorten_process_progress_index_names.py @@ -0,0 +1,26 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("machine", "0016_process_delete_at"), + ] + + operations = [ + migrations.RemoveIndex( + model_name="process", + name="machine_pro_progress_recent_idx", + ), + migrations.RemoveIndex( + model_name="process", + name="machine_pro_progress_running_idx", + ), + migrations.AddIndex( + model_name="process", + index=models.Index(fields=["machine", "process_type", "-modified_at"], name="mach_proc_recent_idx"), + ), + migrations.AddIndex( + model_name="process", + index=models.Index(fields=["machine", "status", "process_type"], name="mach_proc_running_idx"), + ), + ] diff --git a/archivebox/machine/migrations/0018_alter_process_process_type.py b/archivebox/machine/migrations/0018_alter_process_process_type.py new file mode 100644 index 00000000..9a93188c --- /dev/null +++ b/archivebox/machine/migrations/0018_alter_process_process_type.py @@ -0,0 +1,34 @@ +# Generated by ArchiveBox on 2026-05-28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("machine", "0017_shorten_process_progress_index_names"), + ] + + operations = [ + migrations.AlterField( + model_name="process", + name="process_type", + field=models.CharField( + choices=[ + ("supervisord", "Supervisord"), + ("orchestrator", "Orchestrator"), + ("server", "Server"), + ("update", "Update"), + ("add", "Add"), + ("search", "Search"), + ("worker", "Worker"), + ("cli", "CLI"), + ("hook", "Hook"), + ("binary", "Binary"), + ], + db_index=True, + default="cli", + help_text="Type of process (cli, worker, orchestrator, binary, supervisord)", + max_length=16, + ), + ), + ] diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py index 9368c97b..0cef7c6d 100755 --- a/archivebox/machine/models.py +++ b/archivebox/machine/models.py @@ -3,7 +3,6 @@ from __future__ import annotations __package__ = "archivebox.machine" import os -import shlex import sys import uuid import socket @@ -196,8 +195,10 @@ class Machine(ModelWithHealthStats): app_label = "machine" @classmethod - def current(cls) -> Machine: + def current(cls, refresh: bool = False) -> Machine: global _CURRENT_MACHINE + if refresh: + _CURRENT_MACHINE = None if _CURRENT_MACHINE: if timezone.now() < _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL): return _CURRENT_MACHINE @@ -335,7 +336,7 @@ class NetworkInterface(ModelWithHealthStats): @classmethod def current(cls, refresh: bool = False) -> NetworkInterface: global _CURRENT_INTERFACE - machine = Machine.current() + machine = Machine.current(refresh=refresh) if _CURRENT_INTERFACE: if ( not refresh @@ -928,6 +929,10 @@ class Process(ModelWithDeleteAfter, models.Model): class TypeChoices(models.TextChoices): SUPERVISORD = "supervisord", "Supervisord" ORCHESTRATOR = "orchestrator", "Orchestrator" + SERVER = "server", "Server" + UPDATE = "update", "Update" + ADD = "add", "Add" + SEARCH = "search", "Search" WORKER = "worker", "Worker" CLI = "cli", "CLI" HOOK = "hook", "Hook" @@ -1107,8 +1112,8 @@ class Process(ModelWithDeleteAfter, models.Model): models.Index(fields=["binary", "exit_code"]), models.Index(fields=["pid", "started_at"]), models.Index(fields=["process_type", "worker_type", "pwd", "started_at"]), - models.Index(fields=["machine", "process_type", "-modified_at"], name="machine_pro_progress_recent_idx"), - models.Index(fields=["machine", "status", "process_type"], name="machine_pro_progress_running_idx"), + models.Index(fields=["machine", "process_type", "-modified_at"], name="mach_proc_recent_idx"), + models.Index(fields=["machine", "status", "process_type"], name="mach_proc_running_idx"), ] def __str__(self) -> str: @@ -1225,7 +1230,7 @@ class Process(ModelWithDeleteAfter, models.Model): """Parse JSONL records from this process's stdout.""" stdout = self.stdout if not stdout and self.stdout_file and self.stdout_file.exists(): - stdout = self.stdout_file.read_text() + stdout = self.stdout_file.read_text(errors="replace") return self.parse_records_from_text(stdout or "") @staticmethod @@ -1259,6 +1264,50 @@ class Process(ModelWithDeleteAfter, models.Model): self.save() return True + def mark_running( + self, + *, + process_type: str | None = None, + pwd: str | Path | None = None, + url: str | None = None, + worker_type: str = "", + timeout: int | None = None, + ) -> None: + """Record the current process role without changing ownership state elsewhere.""" + updates = ["status", "retry_at", "modified_at"] + self.status = self.StatusChoices.RUNNING + self.retry_at = None + if process_type is not None and self.process_type != process_type: + self.process_type = process_type + updates.append("process_type") + if worker_type and self.worker_type != worker_type: + self.worker_type = worker_type + updates.append("worker_type") + if pwd is not None and self.pwd != str(pwd): + self.pwd = str(pwd) + updates.append("pwd") + if url is not None and self.url != url: + self.url = url + updates.append("url") + if timeout is not None and self.timeout != timeout: + self.timeout = timeout + updates.append("timeout") + self.save(update_fields=updates) + + def heartbeat(self) -> None: + """Touch modified_at so standby/leader selection can see this parent is alive.""" + self.save(update_fields=["modified_at"]) + + def mark_exited(self, *, exit_code: int = 0) -> None: + """Mark a foreground/internal process row exited after command cleanup.""" + if self.status == self.StatusChoices.EXITED and self.exit_code == exit_code: + return + self.status = self.StatusChoices.EXITED + self.exit_code = exit_code + self.ended_at = self.ended_at or timezone.now() + self.retry_at = None + self.save(update_fields=["status", "exit_code", "ended_at", "retry_at", "modified_at"]) + # ========================================================================= # Process.current() and hierarchy methods # ========================================================================= @@ -1426,6 +1475,14 @@ class Process(ModelWithDeleteAfter, models.Model): return cls.TypeChoices.SUPERVISORD elif "runner_watch" in argv_str: return cls.TypeChoices.WORKER + elif "archivebox server" in argv_str: + return cls.TypeChoices.SERVER + elif "archivebox update" in argv_str: + return cls.TypeChoices.UPDATE + elif "archivebox add" in argv_str: + return cls.TypeChoices.ADD + elif "archivebox search" in argv_str or "archivebox list" in argv_str: + return cls.TypeChoices.SEARCH elif "archivebox run" in argv_str: return cls.TypeChoices.ORCHESTRATOR elif "archivebox" in argv_str: @@ -1451,7 +1508,9 @@ class Process(ModelWithDeleteAfter, models.Model): if machine is not None: stale = stale.filter(machine=machine) - for proc in stale: + # Recovery can run against damaged DB state; stream rows so a large + # stale Process backlog cannot be materialized in memory at once. + for proc in stale.iterator(chunk_size=100): if proc.poll() is not None: cleaned += 1 continue @@ -1461,7 +1520,7 @@ class Process(ModelWithDeleteAfter, models.Model): if proc.started_at: timeout_seconds = max(int(proc.timeout or 0), 0) timeout_deadline = proc.started_at + timedelta(seconds=timeout_seconds) + PROCESS_TIMEOUT_GRACE - if timezone.now() >= timeout_deadline: + if timeout_seconds > 0 and timezone.now() >= timeout_deadline: is_stale = True # Check if too old (PID definitely reused) @@ -1483,7 +1542,7 @@ class Process(ModelWithDeleteAfter, models.Model): proc.status = cls.StatusChoices.EXITED proc.ended_at = proc.ended_at or timezone.now() proc.exit_code = proc.exit_code if proc.exit_code is not None else 0 - proc.save(update_fields=["status", "ended_at", "exit_code"]) + proc.save(update_fields=["status", "ended_at", "exit_code", "modified_at"]) cleaned += 1 return cleaned @@ -1652,18 +1711,6 @@ class Process(ModelWithDeleteAfter, models.Model): # Lifecycle methods (launch, kill, poll, wait) # ========================================================================= - @property - def pid_file(self) -> Path | None: - """Path to PID file for this process.""" - runtime_dir = self.runtime_dir - return runtime_dir / "process.pid" if runtime_dir else None - - @property - def cmd_file(self) -> Path | None: - """Path to cmd.sh script for this process.""" - runtime_dir = self.runtime_dir - return runtime_dir / "cmd.sh" if runtime_dir else None - @property def stdout_file(self) -> Path | None: """Path to stdout log.""" @@ -1694,7 +1741,7 @@ class Process(ModelWithDeleteAfter, models.Model): @property def runtime_dir(self) -> Path | None: - """Directory where this process stores runtime logs/pid/cmd metadata.""" + """Directory where this process stores runtime stdout/stderr logs.""" if not self.pwd: return None @@ -1830,32 +1877,6 @@ class Process(ModelWithDeleteAfter, models.Model): for line in self.tail_stderr(lines=lines, follow=follow): print(line, file=sys.stderr, flush=True) - def _write_pid_file(self) -> None: - """Write PID file with mtime set to process start time.""" - if self.pid and self.started_at and self.pid_file: - self.pid_file.parent.mkdir(parents=True, exist_ok=True) - # Write PID to file - self.pid_file.write_text(str(self.pid)) - # Set mtime to process start time for validation - try: - start_time = self.started_at.timestamp() - os.utime(self.pid_file, (start_time, start_time)) - except OSError: - pass # mtime optional, validation degrades gracefully - - def _write_cmd_file(self) -> None: - """Write cmd.sh script for debugging/validation.""" - if self.cmd and self.cmd_file: - self.cmd_file.parent.mkdir(parents=True, exist_ok=True) - - # Write executable shell script - script = "#!/bin/bash\n" + shlex.join(self.cmd) + "\n" - self.cmd_file.write_text(script) - try: - self.cmd_file.chmod(0o755) - except OSError: - pass - def ensure_log_files(self) -> None: """Ensure stdout/stderr log files exist for this process.""" runtime_dir = self.runtime_dir @@ -1918,9 +1939,6 @@ class Process(ModelWithDeleteAfter, models.Model): # Use provided cwd or default to pwd working_dir = cwd or self.pwd - # Write cmd.sh for debugging - self._write_cmd_file() - stdout_path = self.stdout_file stderr_path = self.stderr_file if stdout_path: @@ -1956,8 +1974,6 @@ class Process(ModelWithDeleteAfter, models.Model): self.status = self.StatusChoices.RUNNING self.save() - self._write_pid_file() - if not background: try: proc.wait(timeout=self.timeout) @@ -1971,9 +1987,9 @@ class Process(ModelWithDeleteAfter, models.Model): self.ended_at = timezone.now() if stdout_path.exists(): - self.stdout = stdout_path.read_text() + self.stdout = stdout_path.read_text(errors="replace") if stderr_path.exists(): - self.stderr = stderr_path.read_text() + self.stderr = stderr_path.read_text(errors="replace") self.status = self.StatusChoices.EXITED self.save() @@ -2013,10 +2029,6 @@ class Process(ModelWithDeleteAfter, models.Model): self.status = self.StatusChoices.EXITED self.save() - # Clean up PID file - if self.pid_file and self.pid_file.exists(): - self.pid_file.unlink(missing_ok=True) - return True except (psutil.NoSuchProcess, psutil.AccessDenied, ProcessLookupError): # Process already exited between proc check and kill @@ -2052,24 +2064,14 @@ class Process(ModelWithDeleteAfter, models.Model): pass # Process exited - read output and copy to DB if self.stdout_file and self.stdout_file.exists(): - self.stdout = self.stdout_file.read_text() + self.stdout = self.stdout_file.read_text(errors="replace") # TODO: Uncomment to cleanup (keeping for debugging for now) # self.stdout_file.unlink(missing_ok=True) if self.stderr_file and self.stderr_file.exists(): - self.stderr = self.stderr_file.read_text() + self.stderr = self.stderr_file.read_text(errors="replace") # TODO: Uncomment to cleanup (keeping for debugging for now) # self.stderr_file.unlink(missing_ok=True) - # Clean up PID file (not needed for debugging) - if self.pid_file and self.pid_file.exists(): - self.pid_file.unlink(missing_ok=True) - - # TODO: Uncomment to cleanup cmd.sh (keeping for debugging for now) - # if self.pwd: - # cmd_file = Path(self.pwd) / 'cmd.sh' - # if cmd_file.exists(): - # cmd_file.unlink(missing_ok=True) - # Try to get exit code from proc or default to unknown self.exit_code = self.exit_code if self.exit_code is not None else 0 if self.exit_code == -1: @@ -2433,12 +2435,14 @@ class Process(ModelWithDeleteAfter, models.Model): status=cls.StatusChoices.RUNNING, ) - for proc in running_children: + # Recovery can run against damaged DB state; stream rows so a large + # orphaned Process backlog cannot be materialized in memory at once. + for proc in running_children.iterator(chunk_size=100): if not proc.is_running: proc.status = cls.StatusChoices.EXITED proc.ended_at = proc.ended_at or timezone.now() proc.exit_code = proc.exit_code if proc.exit_code is not None else 0 - proc.save(update_fields=["status", "ended_at", "exit_code"]) + proc.save(update_fields=["status", "ended_at", "exit_code", "modified_at"]) cleaned += 1 continue @@ -2447,14 +2451,21 @@ class Process(ModelWithDeleteAfter, models.Model): if root.id == proc.id and root.process_type in (cls.TypeChoices.WORKER, cls.TypeChoices.HOOK): continue - # If root is an active orchestrator/cli, keep it - if root.process_type in (cls.TypeChoices.ORCHESTRATOR, cls.TypeChoices.CLI) and root.is_running: + # If root is an active ArchiveBox command/orchestrator, keep it. + if root.process_type in ( + cls.TypeChoices.ORCHESTRATOR, + cls.TypeChoices.SERVER, + cls.TypeChoices.UPDATE, + cls.TypeChoices.ADD, + cls.TypeChoices.SEARCH, + cls.TypeChoices.CLI, + ) and root.is_running: continue proc.status = cls.StatusChoices.EXITED proc.ended_at = proc.ended_at or timezone.now() proc.exit_code = proc.exit_code if proc.exit_code is not None else 0 - proc.save(update_fields=["status", "ended_at", "exit_code"]) + proc.save(update_fields=["status", "ended_at", "exit_code", "modified_at"]) cleaned += 1 if cleaned: diff --git a/archivebox/misc/checks.py b/archivebox/misc/checks.py index 2e56c891..33399b45 100644 --- a/archivebox/misc/checks.py +++ b/archivebox/misc/checks.py @@ -2,6 +2,7 @@ __package__ = "archivebox.misc" import os import sys +import time from pathlib import Path from rich import print @@ -55,23 +56,43 @@ def check_data_folder(config=None, **config_kwargs) -> None: check_data_dir_permissions(config=config) -def check_migrations(): +def check_migrations(*, blocking: bool = True, auto_apply: bool = False, cancel_delay: int = 3) -> list[str]: from archivebox import DATA_DIR - from archivebox.misc.db import list_migrations + from archivebox.misc.db import apply_migrations, pending_migrations - pending_migrations = [name for status, name in list_migrations() if not status] + pending = pending_migrations() is_migrating = any(arg in sys.argv for arg in ["makemigrations", "migrate", "init"]) - if pending_migrations and not is_migrating: - print("[red][X] This collection was created with an older version of ArchiveBox and must be upgraded first.[/red]") + if pending and not is_migrating: + print("[red][X] This collection was created with an older version of ArchiveBox and must be upgraded first.[/red]", file=sys.stderr) print(f" {DATA_DIR}", file=sys.stderr) print(file=sys.stderr) print( - f" [violet]Hint:[/violet] To upgrade it to the latest version and apply the {len(pending_migrations)} pending migrations, run:", + f" [violet]Hint:[/violet] To upgrade it to the latest version and apply the {len(pending)} pending migrations, run:", file=sys.stderr, ) print(" archivebox init", file=sys.stderr) - raise SystemExit(3) + if auto_apply: + print(file=sys.stderr) + print( + f"[yellow][*] ArchiveBox will apply migrations automatically in {cancel_delay}s. Press CTRL+C to cancel.[/yellow]", + file=sys.stderr, + ) + try: + time.sleep(cancel_delay) + except KeyboardInterrupt: + print("[red][X] Migration cancelled before any changes were applied.[/red]", file=sys.stderr) + raise SystemExit(130) from None + + # Always delegate to Django's migration executor. It records each + # migration only after it succeeds, so power loss or SIGKILL leaves + # unapplied work visible here and the next startup resumes normally. + print("[yellow][*] Applying database migrations...[/yellow]", file=sys.stderr) + apply_migrations(stdout=sys.stderr, stderr=sys.stderr, verbosity=1) + return pending_migrations() + if blocking: + raise SystemExit(3) + return pending def check_io_encoding(): diff --git a/archivebox/misc/db.py b/archivebox/misc/db.py index d9e66f3f..b9eeb853 100644 --- a/archivebox/misc/db.py +++ b/archivebox/misc/db.py @@ -6,48 +6,181 @@ __package__ = "archivebox.misc" from io import StringIO from pathlib import Path +from typing import TextIO from typing import Any +import fcntl +import importlib +import time +from collections.abc import Callable +from contextlib import contextmanager +from sqlite3 import OperationalError as SQLiteOperationalError from archivebox.config import DATA_DIR from archivebox.misc.util import enforce_types -@enforce_types -def list_migrations(out_dir: Path = DATA_DIR) -> list[tuple[bool, str]]: - """List all Django migrations and their status""" - from django.core.management import call_command +def compact_command(cmdline: list[str] | None, fallback: str = "") -> str: + parts = [str(part) for part in (cmdline or []) if str(part)] + if not parts: + return fallback + for marker in ("archivebox", "daphne", "gunicorn", "uvicorn", "supervisord", "sonic", "node"): + for idx, part in enumerate(parts): + if Path(part).name == marker or part == marker: + return " ".join([Path(parts[idx]).name, *parts[idx + 1 :]])[:220] + return " ".join([Path(parts[0]).name, *parts[1:]])[:220] - out = StringIO() - call_command("showmigrations", list=True, stdout=out) - out.seek(0) - migrations = [] - for line in out.readlines(): - if line.strip() and "]" in line: - status_str, name_str = line.strip().split("]", 1) - is_applied = "X" in status_str - migration_name = name_str.strip() - migrations.append((is_applied, migration_name)) +def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]: + import psutil - return migrations + db_path = db_path.resolve() + holders: list[str] = [] + for proc in psutil.process_iter(["pid", "ppid", "name", "cmdline", "status"]): + try: + open_files = proc.open_files() + except (psutil.AccessDenied, psutil.NoSuchProcess, psutil.ZombieProcess): + continue + for open_file in open_files: + try: + open_path = Path(open_file.path).resolve() + except (OSError, RuntimeError): + continue + if open_path == db_path or open_path.name in {f"{db_path.name}-wal", f"{db_path.name}-shm", f"{db_path.name}-journal"}: + info = proc.info + cmdline = compact_command(info.get("cmdline"), fallback=info.get("name") or "") + holders.append(f"pid={info['pid']} ppid={info['ppid']} {info['status']} {cmdline}") + break + return holders + + +def sqlite_lock_error(error: BaseException) -> bool: + return isinstance(error, SQLiteOperationalError) and "database is locked" in str(error).lower() + + +def retry_sqlite_locks(action: Callable[[], Any], *, label: str, stderr: TextIO | None = None) -> Any: + from django.db import OperationalError, connections + from rich.console import Console + + console = Console(file=stderr or None, stderr=stderr is None) + attempts = 0 + while True: + try: + return action() + except OperationalError as err: + if "database is locked" not in str(err).lower(): + raise + except SQLiteOperationalError as err: + if not sqlite_lock_error(err): + raise + + attempts += 1 + connections.close_all() + holders = sqlite_lock_holders() + console.print(f"[yellow][*] SQLite database is locked while {label}; retrying in 5s...[/yellow]") + if holders: + console.print("[yellow] DB holders:[/yellow]") + for holder in holders[:8]: + console.print(f"[yellow] - {holder}[/yellow]") + if len(holders) > 8: + console.print(f"[yellow] ... {len(holders) - 8} more[/yellow]") + else: + console.print("[yellow] No local process with index.sqlite3 open was visible to this user.[/yellow]") + if attempts == 1: + console.print("[dim] SQLite does not expose the active SQL statement from another process; only the owning local PIDs can be shown.[/dim]") + with console.status("[yellow]Waiting for SQLite database lock to clear...[/yellow]", spinner="dots"): + time.sleep(5.0) + + +@contextmanager +def migration_lock(stdout: TextIO | None = None): + from archivebox.config.paths import get_or_create_working_tmp_dir + from rich.console import Console + + lock_path = get_or_create_working_tmp_dir(autofix=True, quiet=True) / "migrate.lock" + lock_path.parent.mkdir(parents=True, exist_ok=True) + with lock_path.open("a+") as lock_file: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + # Migrations on large SQLite collections can run for hours. Use a + # kernel lock with no timeout so parallel ArchiveBox commands queue + # behind the active migrate process instead of racing it. + console = Console(file=stdout or None, stderr=stdout is None) + with console.status("[yellow]Waiting for migration lock...[/yellow]", spinner="dots"): + while True: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + time.sleep(1.0) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) @enforce_types -def apply_migrations(out_dir: Path = DATA_DIR) -> list[str]: +def pending_migrations(out_dir: Path = DATA_DIR) -> list[str]: + """Cheaply compare migration files to django_migrations without invoking migrate.""" + from django.apps import apps + from django.db import connection + from django.db.migrations.loader import MigrationLoader + + def applied_rows() -> set[tuple[str, str]]: + with connection.cursor() as cursor: + try: + cursor.execute("SELECT app, name FROM django_migrations") + except Exception as err: + if "no such table" in str(err).lower(): + return set() + raise + return {(str(app), str(name)) for app, name in cursor.fetchall()} + + applied = retry_sqlite_locks(applied_rows, label="checking applied migrations") + disk_migrations: set[tuple[str, str]] = set() + for app_config in apps.get_app_configs(): + module_name, explicit = MigrationLoader.migrations_module(app_config.label) + if module_name is None: + continue + try: + module = importlib.import_module(module_name) + except ModuleNotFoundError: + if explicit: + raise + continue + module_file = getattr(module, "__file__", None) + if not module_file: + continue + for migration_file in Path(module_file).parent.glob("[0-9][0-9][0-9][0-9]_*.py"): + disk_migrations.add((app_config.label, migration_file.stem)) + + return [f"{app}.{name}" for app, name in sorted(disk_migrations - applied)] + + +@enforce_types +def apply_migrations(out_dir: Path = DATA_DIR, stdout: TextIO | None = None, stderr: TextIO | None = None, verbosity: int = 1) -> list[str]: """Apply pending Django migrations""" from django.core.management import call_command - out1 = StringIO() + with migration_lock(stdout=stderr or stdout): + if not pending_migrations(): + return [] - call_command("migrate", interactive=False, database="default", stdout=out1) - out1.seek(0) + if stdout is not None: + retry_sqlite_locks( + lambda: call_command("migrate", interactive=False, database="default", stdout=stdout, stderr=stderr, verbosity=verbosity), + label="applying migrations", + stderr=stderr, + ) + return [] - return [line.strip() for line in out1.readlines() if line.strip()] + def migrate() -> StringIO: + out1 = StringIO() + call_command("migrate", interactive=False, database="default", stdout=out1, verbosity=verbosity) + out1.seek(0) + return out1 + out1 = retry_sqlite_locks(migrate, label="applying migrations") -@enforce_types -def get_admins(out_dir: Path = DATA_DIR) -> list[Any]: - """Get list of superuser accounts""" - from django.contrib.auth.models import User + return [line.strip() for line in out1.readlines() if line.strip()] - return list(User.objects.filter(is_superuser=True).exclude(username="system")) diff --git a/archivebox/misc/jsonl.py b/archivebox/misc/jsonl.py index e2322774..8d3201ca 100644 --- a/archivebox/misc/jsonl.py +++ b/archivebox/misc/jsonl.py @@ -164,15 +164,3 @@ def write_record(record: dict[str, Any], stream: TextIO | None = None) -> None: active_stream.write(json.dumps(record) + "\n") active_stream.flush() - -def write_records(records: Iterator[dict[str, Any]], stream: TextIO | None = None) -> int: - """ - Write multiple JSONL records to stdout (or provided stream). - - Returns count of records written. - """ - count = 0 - for record in records: - write_record(record, stream) - count += 1 - return count diff --git a/archivebox/misc/monkey_patches.py b/archivebox/misc/monkey_patches.py index 20430385..2e8f0906 100644 --- a/archivebox/misc/monkey_patches.py +++ b/archivebox/misc/monkey_patches.py @@ -35,13 +35,28 @@ warnings.filterwarnings("ignore", category=SyntaxWarning, module="sonic") class ModifiedAccessLogGenerator(access.AccessLogGenerator): """Clutge workaround until daphne uses the Python logging framework. https://github.com/django/daphne/pull/473/files""" - def write_entry(self, host, date, request, status=None, length=None, ident=None, user=None): + def __call__(self, protocol, action, details): + if protocol == "http" and action == "complete": + self.write_entry( + host=details["client"], + date=datetime.datetime.now(), + request="%(method)s %(path)s" % details, + status=details["status"], + length=details["size"], + time_taken=details.get("time_taken"), + ) + return + return super().__call__(protocol, action, details) + + def write_entry(self, host, date, request, status=None, length=None, ident=None, user=None, time_taken=None): # Ignore noisy requests to staticfiles / favicons / etc. if "GET /static/" in request: return if "GET /health/" in request: return + if "GET /admin/live-progress/" in request and (time_taken is None or time_taken < 1.0): + return if "GET /admin/jsi18n/" in request: return if request.endswith("/favicon.ico") or request.endswith("/robots.txt") or request.endswith("/screenshot.png"): diff --git a/archivebox/misc/serve_static.py b/archivebox/misc/serve_static.py index 7b602dcd..64d5c210 100644 --- a/archivebox/misc/serve_static.py +++ b/archivebox/misc/serve_static.py @@ -93,7 +93,7 @@ def _resolve_archive_path(document_root: str | Path, rel_path: str) -> tuple[Pat def _cache_policy(config=None, **config_kwargs) -> str: config = config or get_config(resolve_plugins=False, **config_kwargs) - return "public" if config.PUBLIC_SNAPSHOTS else "private" + return "private" if config.PERMISSIONS == "private" else "public" def _render_mhtml_preview_document(filename: str, output_path: str) -> str: diff --git a/archivebox/misc/system.py b/archivebox/misc/system.py index c1fdb071..e24f2667 100644 --- a/archivebox/misc/system.py +++ b/archivebox/misc/system.py @@ -3,7 +3,6 @@ __package__ = "archivebox.misc" import os import signal -import shutil import sys from json import dump @@ -124,43 +123,6 @@ def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool os.chmod(path, int(config.OUTPUT_PERMISSIONS, base=8)) -@enforce_types -def chmod_file(path: str, cwd: str = "", config=None, **config_kwargs) -> None: - """chmod -R /""" - - root = Path(cwd or os.getcwd()) / path - if not os.access(root, os.R_OK): - raise Exception(f"Failed to chmod: {path} does not exist (did the previous step fail?)") - - if not root.is_dir(): - # path is just a plain file - config = config or get_config(**config_kwargs) - os.chmod(root, int(config.OUTPUT_PERMISSIONS, base=8)) - else: - config = config or get_config(**config_kwargs) - for subpath in Path(path).glob("**/*"): - if subpath.is_dir(): - # directories need execute permissions to be able to list contents - os.chmod(subpath, int(config.DIR_OUTPUT_PERMISSIONS, base=8)) - else: - os.chmod(subpath, int(config.OUTPUT_PERMISSIONS, base=8)) - - -@enforce_types -def copy_and_overwrite(from_path: str | Path, to_path: str | Path): - """copy a given file or directory to a given path, overwriting the destination""" - - assert os.access(from_path, os.R_OK) - - if Path(from_path).is_dir(): - shutil.rmtree(to_path, ignore_errors=True) - shutil.copytree(from_path, to_path) - else: - with open(from_path, "rb") as src: - contents = src.read() - atomic_write(to_path, contents) - - @enforce_types def get_dir_size(path: str | Path, recursive: bool = True, pattern: str | None = None) -> tuple[int, int, int]: """get the total disk size of a given directory, optionally summing up @@ -187,44 +149,3 @@ def get_dir_size(path: str | Path, recursive: bool = True, pattern: str | None = pass return num_bytes, num_dirs, num_files - -class suppress_output: - """ - A context manager for doing a "deep suppression" of stdout and stderr in - Python, i.e. will suppress all print, even if the print originates in a - compiled C/Fortran sub-function. - - This will not suppress raised exceptions, since exceptions are printed - to stderr just before a script exits, and after the context manager has - exited (at least, I think that is why it lets exceptions through). - - with suppress_stdout_stderr(): - rogue_function() - """ - - def __init__(self, stdout=True, stderr=True): - # Open a pair of null files - # Save the actual stdout (1) and stderr (2) file descriptors. - self.stdout, self.stderr = stdout, stderr - if stdout: - self.null_stdout = os.open(os.devnull, os.O_RDWR) - self.real_stdout = os.dup(1) - if stderr: - self.null_stderr = os.open(os.devnull, os.O_RDWR) - self.real_stderr = os.dup(2) - - def __enter__(self): - # Assign the null pointers to stdout and stderr. - if self.stdout: - os.dup2(self.null_stdout, 1) - if self.stderr: - os.dup2(self.null_stderr, 2) - - def __exit__(self, *_): - # Re-assign the real stdout/stderr back to (1) and (2) - if self.stdout: - os.dup2(self.real_stdout, 1) - os.close(self.null_stdout) - if self.stderr: - os.dup2(self.real_stderr, 2) - os.close(self.null_stderr) diff --git a/archivebox/misc/util.py b/archivebox/misc/util.py index 05584305..3e8bde11 100644 --- a/archivebox/misc/util.py +++ b/archivebox/misc/util.py @@ -16,7 +16,6 @@ from hashlib import sha256 from urllib.parse import urlparse, quote, unquote from html import escape, unescape from datetime import datetime, timezone -from requests.exceptions import RequestException, ReadTimeout from base32_crockford import encode as base32_encode from w3lib.encoding import html_body_declared_encoding, http_content_type_encoding @@ -29,8 +28,6 @@ except ImportError: detect_encoding = lambda rawdata: "utf-8" -from archivebox.config.constants import CONSTANTS - from .logging import COLOR_DICT @@ -61,21 +58,11 @@ htmlencode = lambda s: s and escape(s, quote=True) htmldecode = lambda s: s and unescape(s) -def short_ts(ts: Any) -> str | None: - parsed = parse_date(ts) - return None if parsed is None else str(parsed.timestamp()).split(".")[0] - - def ts_to_date_str(ts: Any) -> str | None: parsed = parse_date(ts) return None if parsed is None else parsed.strftime("%Y-%m-%d %H:%M") -def ts_to_iso(ts: Any) -> str | None: - parsed = parse_date(ts) - return None if parsed is None else parsed.isoformat() - - COLOR_REGEX = re.compile(r"\[(?P\d+)(;(?P\d+)(;(?P\d+))?)?m") @@ -296,11 +283,6 @@ def parse_filesize_to_bytes(value: str | int | float | None) -> int: return int(amount * multiplier) -def is_static_file(url: str): - # TODO: the proper way is with MIME type detection + ext, not only extension - return extension(url).lower() in CONSTANTS.STATICFILE_EXTENSIONS - - def enforce_types(func): """ Enforce function arg and kwarg types at runtime using its python3 type hints @@ -355,17 +337,6 @@ def docstring(text: str | None): return decorator -@enforce_types -def str_between(string: str, start: str, end: str | None = None) -> str: - """(12345, , ) -> 12345""" - - content = string.split(start, 1)[-1] - if end is not None: - content = content.rsplit(end, 1)[0] - - return content - - @enforce_types def parse_date(date: Any) -> datetime | None: """Parse unix timestamps, iso format, and human-readable strings""" @@ -448,50 +419,6 @@ def download_url(url: str, timeout: int | None = None, config=None, **config_kwa return url.rsplit("/", 1)[-1] -@enforce_types -def get_headers(url: str, timeout: int | None = None, config=None, **config_kwargs) -> str: - """Download the contents of a remote url and return the headers""" - # TODO: get rid of this and use an abx pluggy hook instead - - from archivebox.config.common import get_config - - config = config or get_config(**config_kwargs) - timeout = timeout or config.TIMEOUT - - try: - response = requests.head( - url, - headers={"User-Agent": config.USER_AGENT}, - verify=config.CHECK_SSL_VALIDITY, - timeout=timeout, - allow_redirects=True, - ) - if response.status_code >= 400: - raise RequestException - except ReadTimeout: - raise - except RequestException: - response = requests.get( - url, - headers={"User-Agent": config.USER_AGENT}, - verify=config.CHECK_SSL_VALIDITY, - timeout=timeout, - stream=True, - ) - - return pyjson.dumps( - { - "URL": url, - "Status-Code": response.status_code, - "Elapsed": response.elapsed.total_seconds() * 1000, - "Encoding": str(response.encoding), - "Apparent-Encoding": response.apparent_encoding, - **dict(response.headers), - }, - indent=4, - ) - - @enforce_types def ansi_to_html(text: str) -> str: """ @@ -698,50 +625,3 @@ _test_url_strs = { for url_str, num_urls in _test_url_strs.items(): assert len(list(find_all_urls(url_str))) == num_urls, f"{url_str} does not contain {num_urls} urls" - -### Chrome Helpers - - -def chrome_cleanup(config=None, **config_kwargs): - """ - Cleans up any state or runtime files that Chrome leaves behind when killed by - a timeout or other error. Handles: - - All persona chrome_profile directories (via Persona.cleanup_chrome_all()) - - Explicit CHROME_USER_DATA_DIR from config - - Legacy Docker chromium path - """ - import os - from pathlib import Path - from archivebox.config.permissions import IN_DOCKER - - # Clean up all persona chrome directories using Persona class - try: - from archivebox.personas.models import Persona - - # Clean up all personas - Persona.cleanup_chrome_all() - - # Also clean up the active persona's explicit CHROME_USER_DATA_DIR if set - # (in case it's a custom path not under PERSONAS_DIR) - from archivebox.config.common import get_config - - config = config or get_config(**config_kwargs) - chrome_user_data_dir = config.get("CHROME_USER_DATA_DIR") - if chrome_user_data_dir: - singleton_lock = Path(chrome_user_data_dir) / "SingletonLock" - if os.path.lexists(singleton_lock): - try: - singleton_lock.unlink() - except OSError: - pass - except Exception: - pass # Persona/config not available during early startup - - # Legacy Docker cleanup (for backwards compatibility) - if IN_DOCKER: - singleton_lock = "/home/archivebox/.config/chromium/SingletonLock" - if os.path.lexists(singleton_lock): - try: - os.remove(singleton_lock) - except OSError: - pass diff --git a/archivebox/personas/admin.py b/archivebox/personas/admin.py index 7caee68a..bcde4c16 100644 --- a/archivebox/personas/admin.py +++ b/archivebox/personas/admin.py @@ -26,7 +26,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin): ( "Persona", { - "fields": ("name", "created_by"), + "fields": ("name", "created_by", "permissions"), "classes": ("card", "persona-card-primary"), }, ), diff --git a/archivebox/personas/forms.py b/archivebox/personas/forms.py index 7fee60e6..5520a004 100644 --- a/archivebox/personas/forms.py +++ b/archivebox/personas/forms.py @@ -7,6 +7,7 @@ from django.utils.safestring import mark_safe from archivebox.config.common import get_config from archivebox.core.forms import PluginConfigFormMixin +from archivebox.core.permissions import PERMISSIONS_CHOICES from archivebox.personas.importers import ( PersonaImportResult, PersonaImportSource, @@ -25,6 +26,12 @@ def _mode_label(title: str, description: str) -> str: class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm): + permissions = forms.ChoiceField( + label="Permissions", + choices=PERMISSIONS_CHOICES, + required=True, + help_text="Default visibility for crawls and snapshots that use this persona.", + ) import_mode = forms.ChoiceField( required=False, initial="none", @@ -102,6 +109,7 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm): self.fields["import_mode"].widget.attrs["class"] = "abx-import-mode" self.fields["import_discovered_profile"].widget.attrs["class"] = "abx-profile-picker" + self.fields["permissions"].initial = str((self.instance.config or {}).get("PERMISSIONS") or "public").strip().lower() if self.discovered_profiles: self.fields["import_discovered_profile"].choices = [ @@ -131,6 +139,7 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm): manual_config = cleaned_data.get("config") or {} if not isinstance(manual_config, dict): manual_config = {} + manual_config["PERMISSIONS"] = cleaned_data.get("permissions") or "public" plugin_config_overrides = self.clean_plugin_config_overrides(get_config()) cleaned_data["plugin_config"] = plugin_config_overrides cleaned_data["config"] = { diff --git a/archivebox/personas/migrations/0003_persona_permissions.py b/archivebox/personas/migrations/0003_persona_permissions.py new file mode 100644 index 00000000..ca3e9917 --- /dev/null +++ b/archivebox/personas/migrations/0003_persona_permissions.py @@ -0,0 +1,19 @@ +# Generated by Django 6.0.5 on 2026-05-28 07:25 + +import django.db.models.fields.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('personas', '0002_alter_persona_id'), + ] + + operations = [ + migrations.AddField( + model_name='persona', + name='permissions', + field=models.GeneratedField(db_index=True, db_persist=True, expression=django.db.models.fields.json.KeyTextTransform('PERMISSIONS', 'config'), output_field=models.CharField(max_length=16, null=True)), + ), + ] diff --git a/archivebox/personas/models.py b/archivebox/personas/models.py index a8da9fe9..93b9a930 100644 --- a/archivebox/personas/models.py +++ b/archivebox/personas/models.py @@ -19,6 +19,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any from django.db import models +from django.db.models.fields.json import KT from django.conf import settings from django.utils import timezone @@ -83,6 +84,13 @@ class Persona(ModelWithConfig): name = models.CharField(max_length=64, unique=True) created_at = models.DateTimeField(default=timezone.now, db_index=True) created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk) + permissions = models.GeneratedField( + expression=KT("config__PERMISSIONS"), + output_field=models.CharField(max_length=16, null=True), + db_persist=True, + db_index=True, + editable=False, + ) class Meta(ModelWithConfig.Meta): app_label = "personas" diff --git a/archivebox/search/__init__.py b/archivebox/search/__init__.py index 6d89b66a..9c7ad698 100644 --- a/archivebox/search/__init__.py +++ b/archivebox/search/__init__.py @@ -28,6 +28,11 @@ from archivebox.config.common import get_config # Cache discovered backends to avoid repeated filesystem scans _search_backends_cache: dict | None = None SEARCH_MODES = ("meta", "contents", "deep") +SEARCH_BACKEND_UI_NAMES = { + "rg": "ripgrep", + "sonic": "sonic", + "fts": "sqlite", +} @contextmanager @@ -54,14 +59,72 @@ def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: An os.environ[key] = value +def normalize_search_backend_name(backend_name: str | None) -> str: + return (backend_name or "").strip().lower().replace("-", "_") + + +def get_search_backend_display_name(backend_name: str) -> str: + backend_name = normalize_search_backend_name(backend_name) + return next((ui_name for ui_name, canonical_name in SEARCH_BACKEND_UI_NAMES.items() if canonical_name == backend_name), backend_name) + + def get_default_search_mode(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) - return "meta" if config.SEARCH_BACKEND_ENGINE == "ripgrep" else "contents" + backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE) + backends = get_available_backends() + if backend_name in backends: + return f"deep:{backend_name}" + if "ripgrep" in backends: + return "deep:ripgrep" + return "contents" def get_search_mode(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: - normalized = (search_mode or "").strip().lower() - return normalized if normalized in SEARCH_MODES else get_default_search_mode(config=config, **config_kwargs) + normalized = (search_mode or "").strip().lower().replace(" ", "") + if normalized in SEARCH_MODES: + return normalized + if ":" in normalized: + mode, backend_name = normalized.split(":", 1) + backend_name = normalize_search_backend_name(backend_name) + if mode == "deep" and backend_name in get_available_backends(): + return f"{mode}:{backend_name}" + return get_default_search_mode(config=config, **config_kwargs) + + +def get_search_mode_base(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return get_search_mode(search_mode, config=config, **config_kwargs).split(":", 1)[0] + + +def get_search_mode_backend(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str | None: + normalized = get_search_mode(search_mode, config=config, **config_kwargs) + if ":" not in normalized: + return None + return normalized.split(":", 1)[1] + + +def get_search_mode_options(config: dict[str, Any] | None = None, **config_kwargs: Any) -> list[dict[str, str]]: + config = config or get_config(**config_kwargs) + backends = get_available_backends() + configured_backend = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE) + backend_names = [ + *([configured_backend] if configured_backend in backends else []), + *(name for name in sorted(backends) if name != configured_backend), + ] + options = [ + {"value": "meta", "label": "meta"}, + {"value": "contents", "label": "contents"}, + ] + if backend_names: + options.extend( + { + "value": f"deep:{backend_name}", + "label": f"deep: {get_search_backend_display_name(backend_name)}", + } + for backend_name in backend_names + ) + else: + options.append({"value": "deep", "label": "deep"}) + return options def prioritize_metadata_matches( @@ -127,7 +190,7 @@ def get_backend(config: dict[str, Any] | None = None, **config_kwargs: Any) -> A Falls back to 'ripgrep' if configured backend is not found. """ config = config or get_config(**config_kwargs) - backend_name = config.SEARCH_BACKEND_ENGINE + backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE) backends = get_available_backends() if backend_name in backends: @@ -158,25 +221,50 @@ def query_search_index(query: str, search_mode: str | None = None, config: dict[ return Snapshot.objects.none() search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config) - if search_mode == "meta": + search_mode_base = get_search_mode_base(search_mode, config=config) + if search_mode_base == "meta": return Snapshot.objects.none() + from archivebox.services.supervision_service import ensure_daemon_stack + + ensure_daemon_stack(reason="search query") + snapshot_pks = list(iter_query_search_ids(query, search_mode=search_mode, config=config)) + return Snapshot.objects.filter(pk__in=list(dict.fromkeys(snapshot_pks))) + + +def iter_query_search_ids(query: str, search_mode: str | None = None, config: dict[str, Any] | None = None, **config_kwargs: Any): + """Yield snapshot IDs from configured search backends as soon as each backend produces them.""" + config = config or get_config(**config_kwargs) + if not config.USE_SEARCHING_BACKEND: + return + + search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config) + search_mode_base = get_search_mode_base(search_mode, config=config) + forced_backend = get_search_mode_backend(search_mode, config=config) + if search_mode_base == "meta": + return backends = get_available_backends() - backend_names: list[str] = [] - configured_backend = config.SEARCH_BACKEND_ENGINE - if search_mode == "deep": - if "ripgrep" in backends: - backend_names.append("ripgrep") - backend_names.extend(name for name in backends if name != "ripgrep") + configured_backend = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE) + if forced_backend: + if forced_backend not in backends: + raise RuntimeError( + f'Search backend "{forced_backend}" not found. Available backends: {list(backends) or "none"}', + ) + backend_names = [forced_backend] + elif search_mode_base == "deep": + backend_names = [ + *([configured_backend] if configured_backend in backends and configured_backend != "ripgrep" else []), + *(name for name in backends if name not in {configured_backend, "ripgrep"}), + *(["ripgrep"] if "ripgrep" in backends else []), + ] elif configured_backend in backends: - backend_names.append(configured_backend) + backend_names = [configured_backend] elif "ripgrep" in backends: - backend_names.append("ripgrep") + backend_names = ["ripgrep"] else: get_backend() - return Snapshot.objects.none() + return - snapshot_pks: list[str] = [] errors: list[Exception] = [] successful_backends = 0 try: @@ -184,14 +272,16 @@ def query_search_index(query: str, search_mode: str | None = None, config: dict[ backend = backends[backend_name] try: with search_backend_env(config=config): - if backend_name == "ripgrep": - snapshot_pks.extend(backend.search(query, search_mode=search_mode)) + if hasattr(backend, "iter_search"): + yield from backend.iter_search(query, search_mode=search_mode_base) + elif backend_name == "ripgrep": + yield from backend.search(query, search_mode=search_mode_base) else: - snapshot_pks.extend(backend.search(query)) + yield from backend.search(query) successful_backends += 1 except Exception as err: errors.append(err) - if search_mode != "deep": + if search_mode_base != "deep" or forced_backend: raise except Exception as err: stderr() @@ -201,9 +291,8 @@ def query_search_index(query: str, search_mode: str | None = None, config: dict[ ) raise else: - if not successful_backends and errors and search_mode == "deep": + if not successful_backends and errors and search_mode_base == "deep": raise errors[0] - return Snapshot.objects.filter(pk__in=list(dict.fromkeys(snapshot_pks))) @enforce_types diff --git a/archivebox/search/admin.py b/archivebox/search/admin.py index 9540870f..3c07cb2c 100644 --- a/archivebox/search/admin.py +++ b/archivebox/search/admin.py @@ -1,16 +1,69 @@ __package__ = "archivebox.search" -from django.contrib import messages -from django.contrib import admin -from django.contrib.admin.views.main import ChangeList, ORDER_VAR +import hashlib +import json -from archivebox.search import get_default_search_mode, get_search_mode, prioritize_metadata_matches, query_search_index +from django.contrib import admin +from django.contrib.admin.views.main import ChangeList +from django.core.cache import cache + +from archivebox.search import ( + get_search_backend_display_name, + get_default_search_mode, + get_search_mode, + get_search_mode_backend, + get_search_mode_base, + get_search_mode_options, + query_search_index, +) + + +SEARCH_RESULT_CACHE_TTL = 60 + + +def get_admin_search_cache_key(request, url: str | None = None) -> str: + # Search streams publish IDs for one exact changelist URL. Keeping the URL + # whole makes sidebar filters, ordering, and user scope part of the key. + payload = json.dumps( + { + "user": str(request.user.pk or "anon"), + "url": url or request.get_full_path(), + }, + sort_keys=True, + ) + return f"abx:admin-search:{hashlib.sha256(payload.encode()).hexdigest()}" + + +def get_cached_admin_search_ids(request) -> list[str] | None: + cached = cache.get(get_admin_search_cache_key(request)) + if isinstance(cached, dict): + return cached.get("ids") or [] + return None class SearchResultsChangeList(ChangeList): + def __init__(self, request, *args, **kwargs): + self.search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None)) + self.search_mode_backend = get_search_mode_backend(self.search_mode, config=getattr(request, "archivebox_config", None)) + self.search_backend_label = get_search_backend_display_name(self.search_mode_backend) if self.search_mode_backend else "" + super().__init__(request, *args, **kwargs) + self.embedded_changelist = request.GET.get("_embedded") == "crawl" + + def get_results(self, request): + super().get_results(request) + self.show_search_index_hint = bool( + self.opts.model_name == "snapshot" + and self.query + and self.result_count == 0 + and get_search_mode_base(self.search_mode, config=getattr(request, "archivebox_config", None)) == "deep" + and self.search_mode_backend + ) + def get_filters_params(self, params=None): lookup_params = super().get_filters_params(params) lookup_params.pop("search_mode", None) + lookup_params.pop("_embedded", None) + lookup_params.pop("per_page", None) return lookup_params @@ -24,37 +77,36 @@ class SearchResultsAdminMixin(admin.ModelAdmin): request = getattr(self, "request", None) return get_default_search_mode(config=getattr(request, "archivebox_config", None)) + def get_search_mode_options(self): + request = getattr(self, "request", None) + return get_search_mode_options(config=getattr(request, "archivebox_config", None)) + def get_search_results(self, request, queryset, search_term: str): """Enhances the search queryset with results from the search backend""" - qs, use_distinct = super().get_search_results(request, queryset, search_term) - search_term = search_term.strip() if not search_term: - return qs, use_distinct - search_mode = get_search_mode(request.GET.get("search_mode")) - if search_mode == "meta": - return qs, use_distinct - try: - deep_qsearch = None - if search_mode == "deep": - qsearch = query_search_index(search_term, search_mode="contents") - deep_qsearch = query_search_index(search_term, search_mode="deep") - else: - qsearch = query_search_index(search_term, search_mode=search_mode) - qs = prioritize_metadata_matches( - queryset, - qs, - qsearch, - deep_queryset=deep_qsearch, - ordering=() if not request.GET.get(ORDER_VAR) else None, - ) - except Exception as err: - print(f"[!] Error while using search backend: {err.__class__.__name__} {err}") - messages.add_message( - request, - messages.WARNING, - f"Error from the search backend, only showing results from default admin search fields - Error: {err}", - ) + return super().get_search_results(request, queryset, search_term) + search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None)) + if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl": + cached_ids = get_cached_admin_search_ids(request) + if cached_ids is not None: + return queryset.filter(pk__in=cached_ids) if cached_ids else queryset.none(), False + return queryset.none(), False - return qs, True + if get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None)) == "meta": + qs, use_distinct = super().get_search_results(request, queryset, search_term) + return qs, use_distinct + if request.GET.get("_embedded") == "crawl": + try: + return queryset.filter( + pk__in=query_search_index( + search_term, + search_mode=search_mode, + config=getattr(request, "archivebox_config", None), + ).values("pk"), + ), False + except Exception as err: + print(f"[!] Error while using search backend: {err.__class__.__name__} {err}") + return queryset.none(), False + return queryset.none(), False diff --git a/archivebox/search/sonic_daemon.py b/archivebox/search/sonic_daemon.py index b085434a..b0237cd5 100644 --- a/archivebox/search/sonic_daemon.py +++ b/archivebox/search/sonic_daemon.py @@ -21,6 +21,9 @@ def register_sonic_daemon_event_handler(bus) -> None: from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker daemon_event = SonicDaemonStartEvent.from_record(record) + if is_port_listening(daemon_event.host, daemon_event.port): + return + supervisor = get_existing_supervisord_process() if supervisor is None: raise RuntimeError("Sonic search backend is required, but ArchiveBox supervisord is not running") @@ -32,7 +35,6 @@ def register_sonic_daemon_event_handler(bus) -> None: raise RuntimeError( f"Sonic search backend worker is {worker.get('statename')}: {worker.get('description')}", ) - if not is_port_listening(daemon_event.host, daemon_event.port): - raise RuntimeError(f"Sonic search backend is not listening at {daemon_event.url}") + raise RuntimeError(f"Sonic search backend is not listening at {daemon_event.url}") bus.on(ProcessStdoutEvent, on_ProcessStdoutEvent__require_sonic_daemon) diff --git a/archivebox/services/archive_result_service.py b/archivebox/services/archive_result_service.py index 427cb114..c53b46cb 100644 --- a/archivebox/services/archive_result_service.py +++ b/archivebox/services/archive_result_service.py @@ -17,7 +17,7 @@ from .process_service import parse_event_datetime def _collect_output_metadata(plugin_dir: Path) -> tuple[dict[str, dict], int, str]: - exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid", "cmd.sh"} + exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"} output_files: dict[str, dict] = {} mime_sizes: dict[str, int] = defaultdict(int) total_size = 0 diff --git a/archivebox/services/crawl_service.py b/archivebox/services/crawl_service.py index 9b89b34f..ae410aee 100644 --- a/archivebox/services/crawl_service.py +++ b/archivebox/services/crawl_service.py @@ -22,6 +22,8 @@ class CrawlService(BaseService): from archivebox.crawls.models import Crawl crawl = await Crawl.objects.aget(id=self.crawl_id) + if crawl.is_paused: + return if crawl.status != Crawl.StatusChoices.SEALED: crawl.status = Crawl.StatusChoices.STARTED crawl.retry_at = None @@ -31,6 +33,8 @@ class CrawlService(BaseService): from archivebox.crawls.models import Crawl crawl = await Crawl.objects.aget(id=self.crawl_id) + if crawl.is_paused: + return if crawl.status != Crawl.StatusChoices.SEALED: crawl.status = Crawl.StatusChoices.STARTED crawl.retry_at = None @@ -41,8 +45,10 @@ class CrawlService(BaseService): from archivebox.core.models import Snapshot crawl = await Crawl.objects.aget(id=self.crawl_id) + if crawl.is_paused: + return is_finished = not await crawl.snapshot_set.filter( - status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED], + status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED], ).aexists() if is_finished: crawl.status = Crawl.StatusChoices.SEALED @@ -59,8 +65,10 @@ class CrawlService(BaseService): from archivebox.core.models import Snapshot crawl = await Crawl.objects.aget(id=self.crawl_id) + if crawl.is_paused: + return is_finished = not await crawl.snapshot_set.filter( - status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED], + status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.PAUSED], ).aexists() if not is_finished: if crawl.status != Crawl.StatusChoices.SEALED: diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 24ff7240..7e33b7db 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -11,7 +11,6 @@ import sys import time from collections.abc import Mapping from contextlib import nullcontext -from datetime import timedelta from pathlib import Path from tempfile import TemporaryDirectory from typing import Any @@ -50,11 +49,13 @@ from abx_dl.orchestrator import ( from abx_dl.services.process_service import ProcessService as HookProcessService from abx_dl.services.binary_service import BinaryService as HookBinaryService from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService +from abx_dl.cli import LiveBusUI from abxbus import BaseEvent from abxbus.event_bus import EventBus, get_current_event, in_handler_context from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError from archivebox.config.configset import BaseConfigSet +from archivebox.core.recovery_util import recover_orchestrator_state from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler from .archive_result_service import ArchiveResultService @@ -64,7 +65,6 @@ from .machine_service import MachineService from .process_service import ProcessService as PersistedProcessService from .snapshot_service import SnapshotService from .tag_service import TagService -from .live_ui import LiveBusUI def _bus_name(prefix: str, identifier: str) -> str: @@ -182,6 +182,7 @@ class CrawlRunner: snapshot_ids: list[str] | None = None, selected_plugins: list[str] | None = None, process_discovered_snapshots_inline: bool = True, + show_progress: bool = True, ): self.crawl = crawl self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0) @@ -194,6 +195,7 @@ class CrawlRunner: CrawlService(self.bus, crawl_id=str(crawl.id)) MachineService(self.bus) self.process_discovered_snapshots_inline = process_discovered_snapshots_inline + self.show_progress = show_progress async def ignore_snapshot(_snapshot_id: str) -> None: return None @@ -224,7 +226,7 @@ class CrawlRunner: def _install_signal_handlers(self) -> list[tuple[signal.Signals, Any, bool]]: loop = asyncio.get_running_loop() installed: list[tuple[signal.Signals, Any, bool]] = [] - for sig in (signal.SIGINT, signal.SIGTERM): + for sig in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM): previous = signal.getsignal(sig) def request_abort(sig=sig) -> None: @@ -273,6 +275,12 @@ class CrawlRunner: return True return await Crawl.objects.filter(id=self.crawl.id, status=Crawl.StatusChoices.SEALED).aexists() + async def crawl_is_paused(self) -> bool: + from archivebox.crawls.models import Crawl + + crawl = await Crawl.objects.only("status").aget(id=self.crawl.id) + return crawl.is_paused + async def watch_for_cancelled_crawl(self, parent_event: BaseEvent, *, poll_interval: float = 1.0) -> None: while True: await asyncio.sleep(poll_interval) @@ -285,6 +293,10 @@ class CrawlRunner: def runtime_plugins(self) -> dict[str, Plugin]: return filter_plugins(self.plugins, self.selected_plugins, include_providers=True) if self.selected_plugins else self.plugins + @property + def allow_paused_snapshot_maintenance(self) -> bool: + return bool(self.initial_snapshot_ids and self.selected_plugins) + async def run(self) -> None: heartbeat = CrawlHeartbeat( Path(self.crawl_output_dir), @@ -293,6 +305,7 @@ class CrawlRunner: ) installed_signal_handlers = self._install_signal_handlers() root_snapshot_id: str | None = None + bus_destroyed = False try: self._run_task = asyncio.current_task() snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)() @@ -301,24 +314,37 @@ class CrawlRunner: self.snapshot_semaphore = asyncio.Semaphore(max_concurrent_snapshots) live_ui = self._create_live_ui() with live_ui if live_ui is not None else nullcontext(): - await heartbeat.start() - await _emit_machine_config( - self.bus, - config={ - **self.base_config, - "ABX_RUNTIME": "archivebox", - }, - derived_config=self.derived_config, - ) - if snapshot_ids: - root_snapshot_id = snapshot_ids[0] - await self.run_crawl(root_snapshot_id, snapshot_ids) + try: + await heartbeat.start() + await _emit_machine_config( + self.bus, + config={ + **self.base_config, + "ABX_RUNTIME": "archivebox", + }, + derived_config=self.derived_config, + ) + if snapshot_ids: + root_snapshot_id = snapshot_ids[0] + await self.run_crawl(root_snapshot_id, snapshot_ids) + finally: + self._run_task = None + self._restore_signal_handlers(installed_signal_handlers) + await heartbeat.stop() + await self.stop_snapshot_tasks() + try: + if not self._skip_wait_until_idle: + await self.bus.wait_until_idle(timeout=30.0) + finally: + await self.bus.destroy(clear=False) + bus_destroyed = True finally: - self._run_task = None - self._restore_signal_handlers(installed_signal_handlers) - await heartbeat.stop() - if not self._skip_wait_until_idle: - await self.bus.wait_until_idle(timeout=30.0) + if not bus_destroyed: + self._run_task = None + self._restore_signal_handlers(installed_signal_handlers) + await heartbeat.stop() + await self.stop_snapshot_tasks() + await self.bus.destroy(clear=False) if self._live_stream is not None: try: self._live_stream.close() @@ -330,6 +356,8 @@ class CrawlRunner: async def enqueue_snapshot(self, snapshot_id: str, crawl_start_event: CrawlStartEvent | None = None) -> None: if await self.crawl_is_cancelled(): return + if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance: + return task = self.snapshot_tasks.get(snapshot_id) if task is not None and not task.done(): return @@ -342,6 +370,15 @@ class CrawlRunner: task = asyncio.create_task(self.run_snapshot(snapshot_id), context=_runner_task_context()) self.snapshot_tasks[snapshot_id] = task + async def stop_snapshot_tasks(self) -> None: + if not self.snapshot_tasks: + return + done, pending = await asyncio.wait(list(self.snapshot_tasks.values()), timeout=5.0) + for task in pending: + task.cancel() + await asyncio.gather(*done, *pending, return_exceptions=True) + self.snapshot_tasks.clear() + async def wait_for_snapshot_tasks(self) -> None: task_errors: list[Exception] = [] stop_scheduling = False @@ -388,7 +425,10 @@ class CrawlRunner: except Exception as err: task_errors.append(err) stop_scheduling = True - if self.snapshot_tasks and await self.crawl_is_cancelled(): + if self.snapshot_tasks and ( + await self.crawl_is_cancelled() + or (await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance) + ): stop_scheduling = True if not stop_scheduling: await self.enqueue_pending_snapshots_from_projection() @@ -422,6 +462,8 @@ class CrawlRunner: return if await self.crawl_is_cancelled(): return + if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance: + return await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)() config = await sync_to_async(lambda: get_config(crawl=self.crawl, include_machine=False), thread_sensitive=True)() @@ -433,7 +475,7 @@ class CrawlRunner: return pending_snapshot_ids = await sync_to_async( lambda: list( - self.crawl.snapshot_set.exclude(status=Snapshot.StatusChoices.SEALED) + self.crawl.snapshot_set.filter(status=Snapshot.StatusChoices.QUEUED) .exclude(id__in=active_snapshot_ids) .filter(retry_at__lte=timezone.now()) .order_by("depth", "created_at") @@ -447,6 +489,7 @@ class CrawlRunner: def load_run_state(self) -> list[str]: from archivebox.config.common import get_config + from archivebox.core.models import Snapshot from archivebox.hooks import discover_hooks from archivebox.machine.models import Machine, NetworkInterface, Process, _sanitize_machine_config @@ -480,22 +523,34 @@ class CrawlRunner: ), ) if self.initial_snapshot_ids: + # Direct snapshot maintenance paths are allowed to name paused + # snapshots explicitly. The runner still requires selected_plugins + # later, so this does not restart the crawl lifecycle. return [str(snapshot_id) for snapshot_id in self.initial_snapshot_ids] + if self.crawl.is_paused: + return [] pending_snapshots = list( - self.crawl.snapshot_set.exclude(status="sealed").order_by("depth", "created_at"), + self.crawl.snapshot_set.filter(status=Snapshot.StatusChoices.QUEUED) + .filter(retry_at__lte=timezone.now()) + .order_by("depth", "created_at"), ) if pending_snapshots: return [str(snapshot.id) for snapshot in pending_snapshots] + if self.crawl.snapshot_set.exclude(status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED]).exists(): + return [] created = self.crawl.create_snapshots_from_urls() snapshots = created or list(self.crawl.snapshot_set.filter(depth=0).order_by("created_at")) return [str(snapshot.id) for snapshot in snapshots] def finalize_run_state(self) -> None: from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot if self.persona: self.persona.cleanup_runtime_for_crawl(self.crawl) crawl = Crawl.objects.get(id=self.crawl.id) + if crawl.is_paused: + return if crawl.is_finished(): if crawl.status != Crawl.StatusChoices.SEALED: if crawl.status == Crawl.StatusChoices.STARTED: @@ -506,23 +561,33 @@ class CrawlRunner: retry_at=None, ) return + active_snapshots = crawl.snapshot_set.filter( + status__in=[ + Snapshot.StatusChoices.QUEUED, + Snapshot.StatusChoices.STARTED, + Snapshot.StatusChoices.PAUSED, + ], + ) + next_snapshot_retry = active_snapshots.order_by("retry_at", "created_at").values_list("retry_at", flat=True).first() if crawl.status == Crawl.StatusChoices.SEALED: crawl.update_and_requeue( status=Crawl.StatusChoices.QUEUED, - retry_at=timezone.now(), + retry_at=next_snapshot_retry or timezone.now(), ) return elif crawl.status != Crawl.StatusChoices.STARTED: crawl.update_and_requeue( status=Crawl.StatusChoices.STARTED, - retry_at=crawl.retry_at or timezone.now(), + retry_at=crawl.retry_at or next_snapshot_retry or timezone.now(), ) return crawl.update_and_requeue( - retry_at=crawl.retry_at or timezone.now(), + retry_at=crawl.retry_at or next_snapshot_retry or timezone.now(), ) def _create_live_ui(self) -> LiveBusUI | None: + if not self.show_progress: + return None stdout_is_tty = sys.stdout.isatty() stderr_is_tty = sys.stderr.isatty() interactive_tty = stdout_is_tty or stderr_is_tty @@ -606,6 +671,8 @@ class CrawlRunner: from archivebox.hooks import collect_urls_from_plugins await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)() + if self.crawl.is_paused and not self.allow_paused_snapshot_maintenance: + return if int(snapshot_payload["depth"]) >= self.crawl.max_depth: return @@ -623,7 +690,7 @@ class CrawlRunner: lambda: get_config(crawl=self.crawl, snapshot=parent_snapshot, include_machine=False), thread_sensitive=True, )() - if CrawlLimitState.from_config(config).get_stop_reason() == "crawl_max_size": + if CrawlLimitState.from_config(config).get_stop_reason() in ("crawl_max_size", "crawl_timeout"): return await sync_to_async(self.crawl.create_discovered_snapshots, thread_sensitive=True)( @@ -650,7 +717,7 @@ class CrawlRunner: crawl_setup_phase_timeout = compute_phase_timeout(setup_hooks, config) install_phase_timeout = compute_install_phase_timeout(get_install_plugins(plugins), config) snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")] - max_snapshot_count = max(1, int(self.crawl.max_urls or len(snapshot_ids) or 1)) + max_snapshot_count = max(1, int(config.get("CRAWL_MAX_URLS") or len(snapshot_ids) or 1)) snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config) * max_snapshot_count crawl_cleanup_phase_timeout = crawl_setup_phase_timeout crawl_lifecycle_timeout = ( @@ -723,6 +790,8 @@ class CrawlRunner: break if await self.crawl_is_cancelled(): break + if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance: + break await self.enqueue_snapshot(snapshot_id) await self.wait_for_snapshot_tasks() @@ -732,7 +801,9 @@ class CrawlRunner: cancel_watcher = asyncio.create_task(self.watch_for_cancelled_crawl(event)) try: try: - if not await self.crawl_is_cancelled(): + if not await self.crawl_is_cancelled() and ( + not await self.crawl_is_paused() or self.allow_paused_snapshot_maintenance + ): await _run_event_now( event.emit( CrawlSetupEvent( @@ -745,7 +816,9 @@ class CrawlRunner: ), crawl_setup_phase_timeout, ) - if not await self.crawl_is_cancelled(): + if not await self.crawl_is_cancelled() and ( + not await self.crawl_is_paused() or self.allow_paused_snapshot_maintenance + ): crawl_start_event = CrawlStartEvent( url=snapshot["url"], snapshot_id=snapshot["id"], @@ -838,9 +911,13 @@ class CrawlRunner: if not isinstance(crawl_start_event, CrawlStartEvent): raise RuntimeError("Snapshot events must be emitted from a CrawlStartEvent handler") snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(snapshot_id) - if snapshot["status"] == "sealed": + if snapshot["status"] == "sealed" and not self.selected_plugins: + await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id) return - if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() == "crawl_max_size": + if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() in ( + "crawl_max_size", + "crawl_timeout", + ): await sync_to_async(self.seal_snapshot_due_to_limit, thread_sensitive=True)(snapshot_id) return config = _normalize_runtime_config(snapshot["config"]) @@ -889,6 +966,9 @@ class CrawlRunner: raise RuntimeError(f"Snapshot {snapshot_id} did not complete") await completed_snapshot.wait(timeout=snapshot_phase_timeout) await completed_snapshot.event_results_list() + if snapshot["status"] == "sealed": + await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id) + return await self.enqueue_discovered_snapshots_from_outputs(snapshot) await sync_to_async( lambda: ( @@ -898,6 +978,7 @@ class CrawlRunner: status__in=[ self.crawl.snapshot_set.model.StatusChoices.QUEUED, self.crawl.snapshot_set.model.StatusChoices.STARTED, + self.crawl.snapshot_set.model.StatusChoices.PAUSED, ], ).exists() else None @@ -928,6 +1009,7 @@ def run_crawl( snapshot_ids: list[str] | None = None, selected_plugins: list[str] | None = None, process_discovered_snapshots_inline: bool = True, + show_progress: bool = True, ) -> None: from archivebox.crawls.models import Crawl @@ -938,6 +1020,7 @@ def run_crawl( snapshot_ids=snapshot_ids, selected_plugins=selected_plugins, process_discovered_snapshots_inline=process_discovered_snapshots_inline, + show_progress=show_progress, ).run(), ) @@ -994,6 +1077,148 @@ def run_binary(binary_id: str) -> None: asyncio.run(_run_binary(binary_id)) +def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None: + from archivebox.core.models import ArchiveResult + + queued_plugins = sorted( + set( + ArchiveResult.objects.filter( + snapshot_id=snapshot_id, + status=ArchiveResult.StatusChoices.QUEUED, + ) + .exclude(plugin="") + .values_list("plugin", flat=True), + ), + ) + if queued_plugins: + return queued_plugins + return None + + +def run_snapshot_maintenance(snapshot_id: str) -> bool: + from archivebox.core.models import ArchiveResult, Snapshot + + snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first() + if snapshot is None: + return False + if snapshot.archiveresult_set.filter(status=ArchiveResult.StatusChoices.QUEUED).exists(): + return False + + # retry_at is the universal "tick me" signal. For already-sealed snapshots, + # a tick with no queued ArchiveResults is maintenance-only: run normal + # save/write side effects like lazy fs migration/json rewriting, then clear + # retry_at. Paused snapshots do not reach this helper while search/index + # plugin rows are queued; run_due_snapshot restores their paused scheduler + # marker after the targeted plugin rows finish. + snapshot.retry_at = None + snapshot.save(update_fields=["retry_at", "modified_at"]) + snapshot.write_index_jsonl() + snapshot.write_json_details() + snapshot.write_html_details() + return True + + +def run_due_crawl(crawl, *, lock_seconds: int) -> bool: + if crawl.is_paused: + return True + if crawl.status in (crawl.StatusChoices.QUEUED, crawl.StatusChoices.STARTED): + from archivebox.core.models import Snapshot + + snapshot_count = crawl.snapshot_set.count() + due_nonsealed_snapshots = ( + crawl.snapshot_set.filter(status=Snapshot.StatusChoices.QUEUED, retry_at__lte=timezone.now()).exists() + ) + if snapshot_count and not due_nonsealed_snapshots: + crawl.retry_at = None + crawl.save(update_fields=["retry_at", "modified_at"]) + return True + if not crawl.claim_processing_lock(lock_seconds=lock_seconds): + return False + run_crawl(str(crawl.id), process_discovered_snapshots_inline=True) + return True + + if crawl.status == crawl.StatusChoices.SEALED: + crawl.retry_at = None + crawl.save(update_fields=["retry_at", "modified_at"]) + return True + + crawl.retry_at = None + crawl.save(update_fields=["retry_at", "modified_at"]) + return True + + +def run_due_snapshot(snapshot, *, lock_seconds: int) -> bool: + from archivebox.core.models import Snapshot + + if snapshot.is_paused: + selected_plugins = queued_plugins_for_snapshot(str(snapshot.id)) + if not selected_plugins: + # Paused is a real lifecycle state; retry_at=MAX is only the + # orchestrator selection marker. If a direct maintenance/update + # command bumps retry_at on a paused snapshot but there are no + # targeted ArchiveResult rows to run, restore the scheduler marker + # without changing status. + snapshot.restore_paused_scheduler_marker() + return True + if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds): + return False + try: + # Explicit maintenance, e.g. `archivebox update --index-only`, may + # need to run search/index hooks for a paused snapshot. That should + # not resume the crawl or make unrelated queued work runnable, so + # selected_plugins is required and the paused state is restored in + # the finally block below. + run_crawl( + str(snapshot.crawl_id), + snapshot_ids=[str(snapshot.id)], + selected_plugins=selected_plugins, + process_discovered_snapshots_inline=True, + ) + finally: + # Targeted plugin rows can complete while the Snapshot remains + # paused. Put retry_at back at MAX so the orchestrator leaves the + # paused lifecycle alone until an explicit resume transition. + snapshot.restore_paused_scheduler_marker() + return True + if snapshot.status == Snapshot.StatusChoices.SEALED: + if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds): + return False + snapshot.refresh_from_db() + selected_plugins = queued_plugins_for_snapshot(str(snapshot.id)) + if selected_plugins: + run_crawl( + str(snapshot.crawl_id), + snapshot_ids=[str(snapshot.id)], + selected_plugins=selected_plugins, + process_discovered_snapshots_inline=True, + ) + return True + return run_snapshot_maintenance(str(snapshot.id)) + + if not snapshot.claim_processing_lock(lock_seconds=lock_seconds): + return False + run_crawl( + str(snapshot.crawl_id), + snapshot_ids=[str(snapshot.id)], + selected_plugins=queued_plugins_for_snapshot(str(snapshot.id)), + process_discovered_snapshots_inline=True, + ) + return True + + +def run_due_binary(binary, *, lock_seconds: int) -> bool: + binary_name = str(binary.name or "") + binary_path = Path(binary_name).expanduser() + if (binary_path.is_absolute() or binary_name.startswith("~")) and not binary_path.exists(): + binary.retry_at = None + binary.save(update_fields=["retry_at", "modified_at"]) + return True + if not binary.claim_processing_lock(lock_seconds=lock_seconds): + return False + run_binary(str(binary.id)) + return True + + async def _run_install(plugin_names: list[str] | None = None) -> None: from archivebox.config.common import get_config from archivebox.machine.models import Machine, _sanitize_machine_config @@ -1012,6 +1237,7 @@ async def _run_install(plugin_names: list[str] | None = None) -> None: MachineService(bus) await _emit_machine_config(bus, config=config, derived_config=derived_config) live_stream = None + bus_destroyed = False try: selected_plugins = filter_plugins(plugins, list(plugin_names), include_providers=True) if plugin_names else plugins @@ -1068,20 +1294,28 @@ async def _run_install(plugin_names: list[str] | None = None) -> None: plugins_label=plugins_label, ) with live_ui if live_ui is not None else nullcontext(): - await abx_install_plugins( - plugin_names=plugin_names, - plugins=plugins, - output_dir=output_dir, - config_overrides=config, - derived_config_overrides=derived_config, - emit_jsonl=False, - bus=bus, - MachineService=None, - ) + try: + await abx_install_plugins( + plugin_names=plugin_names, + plugins=plugins, + output_dir=output_dir, + config_overrides=config, + derived_config_overrides=derived_config, + emit_jsonl=False, + bus=bus, + MachineService=None, + ) + finally: + try: + await bus.wait_until_idle() + finally: + await bus.destroy(clear=False) + bus_destroyed = True if live_ui is not None: live_ui.print_summary(output_dir=output_dir) finally: - await bus.wait_until_idle() + if not bus_destroyed: + await bus.destroy(clear=False) try: if live_stream is not None: live_stream.close() @@ -1093,219 +1327,16 @@ def run_install(*, plugin_names: list[str] | None = None) -> None: asyncio.run(_run_install(plugin_names=plugin_names)) -def recover_orphaned_crawls() -> int: - from archivebox.crawls.models import Crawl - from archivebox.core.models import Snapshot - from archivebox.machine.models import Process - - active_crawl_ids: set[str] = set() - orphaned_crawls = list( - Crawl.objects.filter( - status=Crawl.StatusChoices.STARTED, - retry_at__isnull=True, - ).prefetch_related("snapshot_set"), - ) - running_processes = ( - Process.get_running() - .filter( - process_type__in=[ - Process.TypeChoices.WORKER, - Process.TypeChoices.HOOK, - Process.TypeChoices.BINARY, - ], - ) - .only("pwd") - ) - - for proc in running_processes: - if not proc.pwd: - continue - proc_pwd = Path(proc.pwd) - for crawl in orphaned_crawls: - matched_snapshot = None - for snapshot in crawl.snapshot_set.all(): - try: - proc_pwd.relative_to(snapshot.output_dir) - matched_snapshot = snapshot - break - except ValueError: - continue - if matched_snapshot is not None: - active_crawl_ids.add(str(crawl.id)) - break - - recovered = 0 - now = timezone.now() - for crawl in orphaned_crawls: - if str(crawl.id) in active_crawl_ids: - continue - - snapshots = list(crawl.snapshot_set.all()) - if not snapshots or all(snapshot.status == Snapshot.StatusChoices.SEALED for snapshot in snapshots): - if crawl.status == Crawl.StatusChoices.STARTED: - crawl.sm.seal() - else: - crawl.update_and_requeue( - status=Crawl.StatusChoices.SEALED, - retry_at=None, - ) - recovered += 1 - continue - - crawl.update_and_requeue( - status=Crawl.StatusChoices.STARTED, - retry_at=now, - ) - recovered += 1 - - return recovered - - -def recover_orphaned_snapshots() -> int: - from archivebox.crawls.models import Crawl - from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.machine.models import Process - from django.db.models import Exists, OuterRef - - active_snapshot_ids: set[str] = set() - now = timezone.now() - orphaned_snapshots = list( - Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED, retry_at__isnull=True) - .select_related("crawl") - .prefetch_related("archiveresult_set"), - ) - - queued_result_snapshot_ids = list( - ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED).values_list("snapshot_id", flat=True).distinct(), - ) - if queued_result_snapshot_ids: - orphaned_snapshots.extend( - snapshot - for snapshot in Snapshot.objects.filter(id__in=queued_result_snapshot_ids) - .select_related("crawl") - .prefetch_related("archiveresult_set") - if snapshot.status == Snapshot.StatusChoices.SEALED - ) - - recent_active_crawl_ids = list( - Crawl.objects.filter( - status__in=[Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED], - modified_at__gte=now - timedelta(days=1), - ) - .order_by("-modified_at") - .values_list("id", flat=True)[:1000], - ) - if recent_active_crawl_ids: - orphaned_snapshots.extend( - Snapshot.objects.filter( - crawl_id__in=recent_active_crawl_ids, - status=Snapshot.StatusChoices.SEALED, - downloaded_at__isnull=False, - ) - .annotate(has_results=Exists(ArchiveResult.objects.filter(snapshot_id=OuterRef("pk")))) - .filter(has_results=False) - .select_related("crawl") - .prefetch_related("archiveresult_set") - .order_by("-modified_at")[:1000], - ) - running_processes = ( - Process.get_running() - .filter( - process_type__in=[ - Process.TypeChoices.WORKER, - Process.TypeChoices.HOOK, - Process.TypeChoices.BINARY, - ], - ) - .only("pwd") - ) - - for proc in running_processes: - if not proc.pwd: - continue - proc_pwd = Path(proc.pwd) - for snapshot in orphaned_snapshots: - try: - proc_pwd.relative_to(snapshot.output_dir) - active_snapshot_ids.add(str(snapshot.id)) - break - except ValueError: - continue - - recovered = 0 - for snapshot in orphaned_snapshots: - if str(snapshot.id) in active_snapshot_ids: - continue - - results = list(snapshot.archiveresult_set.all()) - if results and all(result.status in ArchiveResult.FINAL_STATES for result in results): - snapshot.downloaded_at = snapshot.downloaded_at or now - snapshot.save(update_fields=["downloaded_at", "modified_at"]) - if snapshot.status == Snapshot.StatusChoices.STARTED: - snapshot.sm.seal() - else: - snapshot.update_and_requeue( - status=Snapshot.StatusChoices.SEALED, - retry_at=None, - ) - - crawl = snapshot.crawl - if crawl.is_finished() and crawl.status != Crawl.StatusChoices.SEALED: - if crawl.status == Crawl.StatusChoices.STARTED: - crawl.sm.seal() - else: - crawl.update_and_requeue( - status=Crawl.StatusChoices.SEALED, - retry_at=None, - ) - recovered += 1 - continue - - snapshot.update_and_requeue( - status=Snapshot.StatusChoices.QUEUED, - retry_at=now, - ) - - crawl = snapshot.crawl - crawl_status = crawl.status if crawl.status == Crawl.StatusChoices.STARTED else Crawl.StatusChoices.QUEUED - crawl.update_and_requeue( - status=crawl_status, - retry_at=now, - ) - recovered += 1 - - return recovered - - -def cleanup_orchestrator_state(*, recover: bool = True, include_chrome: bool = False) -> dict[str, int]: - from archivebox.machine.models import Process - - cleaned = { - "stale_processes": Process.cleanup_stale_running(), - "orphaned_processes": Process.cleanup_orphaned_workers(), - "orphaned_chrome": Process.cleanup_orphaned_chrome() if include_chrome else 0, - "orphaned_snapshots": 0, - "orphaned_crawls": 0, - } - if recover: - cleaned["orphaned_snapshots"] = recover_orphaned_snapshots() - cleaned["orphaned_crawls"] = recover_orphaned_crawls() - return cleaned - - def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> int: from archivebox.crawls.models import Crawl, CrawlSchedule from archivebox.core.models import ArchiveResult, Snapshot from archivebox.machine.models import Binary, Process + crawl_claim_lock_seconds = 10 last_recovery_at = 0.0 last_retention_at = 0.0 while True: now_monotonic = time.monotonic() - if daemon: - if now_monotonic - last_recovery_at >= 30.0: - cleanup_orchestrator_state() - last_recovery_at = now_monotonic if now_monotonic - last_retention_at >= (60.0 if daemon else 1.0): for model in (ArchiveResult, Snapshot, Crawl, Process): model.delete_expired(batch_size=100) @@ -1317,77 +1348,41 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> if schedule.is_due(now): schedule.enqueue(queued_at=now) - queued_crawls = Crawl.objects.filter( - retry_at__lte=timezone.now(), - status=Crawl.StatusChoices.QUEUED, - ) + due_crawls = Crawl.objects.filter(retry_at__lte=timezone.now()) if crawl_id: - queued_crawls = queued_crawls.filter(id=crawl_id) - queued_crawls = queued_crawls.order_by("retry_at", "created_at") - - queued_crawl = queued_crawls.first() - if queued_crawl is not None: - if not queued_crawl.claim_processing_lock(lock_seconds=60): + due_crawls = due_crawls.filter(id=crawl_id) + due_crawl = due_crawls.order_by("retry_at", "created_at").first() + if due_crawl is not None: + if not run_due_crawl(due_crawl, lock_seconds=crawl_claim_lock_seconds): continue - run_crawl(str(queued_crawl.id), process_discovered_snapshots_inline=True) continue - pending = Crawl.objects.filter( - retry_at__lte=timezone.now(), - status=Crawl.StatusChoices.STARTED, - ) + due_snapshots = Snapshot.objects.filter(retry_at__lte=timezone.now()).select_related("crawl") if crawl_id: - pending = pending.filter(id=crawl_id) - pending = pending.order_by("retry_at", "created_at") - - crawl = pending.first() - if crawl is not None: - if not crawl.claim_processing_lock(lock_seconds=60): + due_snapshots = due_snapshots.filter(crawl_id=crawl_id) + due_snapshot = due_snapshots.order_by("retry_at", "created_at").first() + if due_snapshot is not None: + if not run_due_snapshot(due_snapshot, lock_seconds=60): continue - run_crawl(str(crawl.id), process_discovered_snapshots_inline=True) continue if crawl_id is None: - snapshot = ( - Snapshot.objects.filter(retry_at__lte=timezone.now()) - .exclude(status=Snapshot.StatusChoices.SEALED) - .exclude(crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED]) - .select_related("crawl") - .order_by("retry_at", "created_at") - .first() - ) - if snapshot is not None: - if not snapshot.claim_processing_lock(lock_seconds=60): - continue - run_crawl( - str(snapshot.crawl_id), - snapshot_ids=[str(snapshot.id)], - process_discovered_snapshots_inline=True, - ) - continue - - if crawl_id is None: - # Standalone binary backlog should not starve queued crawls or snapshots. - # Crawl.run() already claims and installs crawl-declared Binary rows as needed. - binary = ( + due_binary = ( Binary.objects.filter(retry_at__lte=timezone.now()) .exclude(status=Binary.StatusChoices.INSTALLED) .order_by("retry_at", "created_at") .first() ) - if binary is not None: - binary_name = str(binary.name or "") - binary_path = Path(binary_name).expanduser() - if (binary_path.is_absolute() or binary_name.startswith("~")) and not binary_path.exists(): - binary.retry_at = None - binary.save(update_fields=["retry_at", "modified_at"]) + if due_binary is not None: + if not run_due_binary(due_binary, lock_seconds=60): continue - if not binary.claim_processing_lock(lock_seconds=60): - continue - run_binary(str(binary.id)) continue if daemon: + now_monotonic = time.monotonic() + if now_monotonic - last_recovery_at >= 30.0: + recover_orchestrator_state() + last_recovery_at = now_monotonic time.sleep(2.0) continue return 0 diff --git a/archivebox/services/snapshot_service.py b/archivebox/services/snapshot_service.py index 5b259554..e48b8188 100644 --- a/archivebox/services/snapshot_service.py +++ b/archivebox/services/snapshot_service.py @@ -24,6 +24,8 @@ class SnapshotService(BaseService): snapshot = await Snapshot.objects.filter(id=event.snapshot_id, crawl_id=self.crawl_id).afirst() if snapshot is not None: + if snapshot.is_paused: + return if snapshot.status == Snapshot.StatusChoices.QUEUED: await sync_to_async(snapshot.sm.tick, thread_sensitive=True)() await sync_to_async(snapshot.refresh_from_db, thread_sensitive=True)() @@ -42,7 +44,7 @@ class SnapshotService(BaseService): snapshot.downloaded_at = snapshot.downloaded_at or timezone.now() await snapshot.asave(update_fields=["downloaded_at", "modified_at"]) stop_reason = await sync_to_async(self._crawl_limit_stop_reason, thread_sensitive=True)(snapshot.crawl) - if snapshot.crawl_id and stop_reason == "crawl_max_size": + if snapshot.crawl_id and stop_reason in ("crawl_max_size", "crawl_timeout"): await ( Snapshot.objects.filter( crawl_id=snapshot.crawl_id, diff --git a/archivebox/services/supervision_service.py b/archivebox/services/supervision_service.py new file mode 100644 index 00000000..ada118f3 --- /dev/null +++ b/archivebox/services/supervision_service.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import time +from pathlib import Path + +from django.utils import timezone +from rich import print + + +def runtime_stack_owner_types(): + from archivebox.machine.models import Process + + return ( + Process.TypeChoices.UPDATE, + Process.TypeChoices.SERVER, + Process.TypeChoices.ORCHESTRATOR, + Process.TypeChoices.ADD, + ) + + +def current_command(process_type: str, *, data_dir: str | Path, url: str | None = None): + from archivebox.machine.models import Process + + proc = Process.current() + proc.mark_running(process_type=process_type, pwd=str(data_dir), url=url, timeout=0) + return proc + + +def live_processes(*, process_type: str, data_dir: str | Path, url: str | None = None): + from archivebox.machine.models import Machine, Process + + Process.cleanup_stale_running(machine=Machine.current()) + qs = Process.objects.filter( + machine=Machine.current(), + process_type=process_type, + status=Process.StatusChoices.RUNNING, + pwd=str(data_dir), + ) + if url is not None: + qs = qs.filter(url=url) + return [proc for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50) if proc.is_running] + + +def newest_live_process(*, process_type: str, data_dir: str | Path, url: str | None = None): + processes = live_processes(process_type=process_type, data_dir=data_dir, url=url) + return processes[0] if processes else None + + +def command_is_newest(command, *, process_type: str, data_dir: str | Path, url: str | None = None) -> bool: + leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url) + return bool(leader and leader.id == command.id) + + +def runtime_stack_owner(*, data_dir: str | Path): + from archivebox.machine.models import Machine, Process + + Process.cleanup_stale_running(machine=Machine.current()) + base_qs = Process.objects.filter( + machine=Machine.current(), + status=Process.StatusChoices.RUNNING, + pwd=str(data_dir), + process_type__in=runtime_stack_owner_types(), + ) + for process_types in ( + (Process.TypeChoices.UPDATE,), + (Process.TypeChoices.SERVER, Process.TypeChoices.ADD), + (Process.TypeChoices.ORCHESTRATOR,), + ): + qs = base_qs.filter(process_type__in=process_types) + for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50): + if proc.is_running: + return proc + return None + + +def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool: + owner = runtime_stack_owner(data_dir=data_dir) + return bool(owner and owner.id == command.id) + + +def ensure_daemon_stack(*, reason: str = ""): + from archivebox.config.common import get_config + from archivebox.workers.supervisord_util import ( + get_existing_supervisord_process, + get_or_create_supervisord_process, + get_sonic_supervisord_worker_from_plugin, + get_worker, + start_worker, + ) + + config = get_config() + sonic_worker = get_sonic_supervisord_worker_from_plugin(config) + if sonic_worker is None: + return None + + from abx_plugins.plugins.search_backend_sonic.daemon import is_port_listening, prepare_sonic_daemon + + sonic_event = prepare_sonic_daemon(config) + if is_port_listening(sonic_event.host, sonic_event.port): + return { + "name": sonic_event.worker_name, + "statename": "RUNNING", + "description": f"existing Sonic daemon at {sonic_event.url}", + } + + supervisor = get_existing_supervisord_process() or get_or_create_supervisord_process(daemonize=False) + worker = get_worker(supervisor, sonic_worker["name"]) + if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"): + return worker + + if reason: + print(f"[yellow][*] Starting daemon stack for {reason}...[/yellow]") + return start_worker(supervisor, sonic_worker) + + +def healthy_orchestrator(*, data_dir: str | Path): + from archivebox.machine.models import Machine, Process + from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker + + Process.cleanup_stale_running(machine=Machine.current()) + supervisor = get_existing_supervisord_process() + worker = get_worker(supervisor, "worker_runner") if supervisor else None + if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"): + return worker + + for proc in Process.objects.filter( + machine=Machine.current(), + process_type=Process.TypeChoices.ORCHESTRATOR, + status=Process.StatusChoices.RUNNING, + pwd=str(data_dir), + ).order_by("-created_at"): + if proc.is_running: + return proc + return None + + +def standby_until_leader_needed(command, *, process_type: str, data_dir: str | Path, url: str | None = None, interval: float = 2.0) -> None: + from archivebox.workers.supervisord_util import reap_foreground_supervisord_process + + announced = False + while not command_is_newest(command, process_type=process_type, data_dir=data_dir, url=url): + reap_foreground_supervisord_process() + if not announced: + leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url) + leader_pid = leader.pid if leader else "unknown" + print(f"[yellow][*] Standing by; newer ArchiveBox parent pid={leader_pid} is running the orchestrator and server.[/yellow]") + announced = True + command.heartbeat() + time.sleep(interval) + command.modified_at = timezone.now() + command.save(update_fields=["modified_at"]) + + +def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> None: + from archivebox.workers.supervisord_util import reap_foreground_supervisord_process + + announced = False + while not command_owns_runtime_stack(command, data_dir=data_dir): + reap_foreground_supervisord_process() + if not announced: + owner = runtime_stack_owner(data_dir=data_dir) + owner_pid = owner.pid if owner else "unknown" + owner_type = owner.process_type if owner else "unknown" + print(f"[yellow][*] Standing by; ArchiveBox {owner_type} pid={owner_pid} owns the runtime stack.[/yellow]") + announced = True + command.heartbeat() + time.sleep(interval) + command.modified_at = timezone.now() + command.save(update_fields=["modified_at"]) diff --git a/archivebox/templates/admin/actions.html b/archivebox/templates/admin/actions.html index 66dc93ae..9c4134b3 100644 --- a/archivebox/templates/admin/actions.html +++ b/archivebox/templates/admin/actions.html @@ -1,5 +1,5 @@ -{% load i18n %} -
+{% load i18n core_tags %} +
{% block actions %} {% block actions-form %} @@ -16,50 +16,42 @@ {% endblock %} {% block actions-counter %} {% if actions_selection_counter %} - {{ selection_note }} + + 0 / {{ cl.result_list|length|intcomma }} selected + + {% if cl.opts.model_name == 'snapshot' %} + + {% if cl.full_result_count and cl.full_result_count != cl.result_count %} + + {{ cl.result_count|intcomma }} + / + {{ cl.full_result_count|intcomma }} + total + {% else %} + + {{ cl.result_count|intcomma }} + total + {% endif %} + + {% endif %} {% if cl.result_count != cl.result_list|length %} + {% if cl.opts.model_name != 'snapshot' %} + {% endif %} + {% if cl.opts.model_name != 'snapshot' %} {% endif %} + {% endif %} + {% endif %} {% endblock %} {% endblock %}
- {% if action_index|default:0 == 0 %} - {% if cl.has_filters or opts.model_name == 'snapshot' %} -
- {% if cl.has_filters %} - - {% endif %} - {% if request.resolver_match.url_name == 'grid' %} - - {% elif opts.model_name == 'snapshot' %} - - {% endif %} -
- {% endif %} - {% endif %}
diff --git a/archivebox/templates/admin/actions_as_select.html b/archivebox/templates/admin/actions_as_select.html deleted file mode 100644 index e69de29b..00000000 diff --git a/archivebox/templates/admin/base.html b/archivebox/templates/admin/base.html index 2c8b4198..8927ce03 100644 --- a/archivebox/templates/admin/base.html +++ b/archivebox/templates/admin/base.html @@ -1216,6 +1216,28 @@ font-size: 15px; } + .search-empty-state { + padding: 28px 18px; + text-align: center; + color: #475569; + background: #f8fafc; + border-top: 1px solid #e2e8f0; + } + + .search-empty-state p { + margin: 0; + } + + .search-empty-state code { + display: inline-block; + margin-left: 4px; + padding: 2px 6px; + border-radius: 5px; + background: #e2e8f0; + color: #0f172a; + font-size: 13px; + } + /* Date hierarchy */ .xfull { padding: 12px 16px; @@ -1457,9 +1479,63 @@ .actions .tag-inline-input { min-width: 40px; padding: 0; + margin: 0; + height: auto; + min-height: 0; + border: 0; + border-radius: 0; + box-shadow: none; + background: transparent; font-size: 11px; } + .model-snapshot.change-list #changelist .actions-tags-with-buttons { + display: grid; + grid-template-columns: minmax(0, 1fr) 34px 34px; + gap: 0; + align-items: stretch; + border: 1px solid #cbd5e1; + border-radius: 8px; + background: #fff; + overflow: hidden; + } + + .model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-editor-container { + width: auto; + max-width: none; + height: 34px; + min-height: 34px; + padding: 4px 9px; + border: 0; + border-radius: 0; + box-shadow: none; + overflow: hidden; + } + + .model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-inline-input { + height: auto; + min-height: 0; + padding: 0; + margin: 0; + border: 0; + border-radius: 0; + box-shadow: none; + background: transparent; + } + + .model-snapshot.change-list #changelist .actions-tags-with-buttons .button[name="add_tags"], + .model-snapshot.change-list #changelist .actions-tags-with-buttons .button[name="remove_tags"] { + width: 34px; + min-width: 34px; + height: 34px; + padding: 0; + margin: 0; + border: 0; + border-left: 1px solid #cbd5e1; + border-radius: 0; + justify-content: center; + } + /* Container in list view title column */ .tags-inline-editor { @@ -1675,18 +1751,41 @@ // change the admin actions button from a dropdown to buttons across function fix_actions() { const container = $('div.actions') + if (container.find('.action-buttons').length) return // too many actions to turn into buttons if (container.find('select[name=action] option').length >= 11) return // hide the empty default option thats just a placeholder with no value - container.find('label:nth-child(1), button[value=0]').hide() + container.find('label:nth-child(1), button[type=submit][name=index][value=0]').hide() const buttons = $('
') .insertAfter('div.actions button[type=submit]') .css('display', 'inline') .addClass('action-buttons'); + function flushPendingActionTags() { + const tagContainer = document.querySelector('.actions-tags') + const input = tagContainer?.querySelector('.tag-inline-input') + const hidden = tagContainer?.querySelector('input[type="hidden"][name="tags"]') + const pending = (input?.value || '').trim() + if (!pending || !hidden) return + const seen = new Set() + const tags = (hidden.value ? hidden.value.split(',') : []) + .concat(pending.split(',')) + .map((tag) => tag.trim()) + .filter((tag) => { + const key = tag.toLowerCase() + if (!tag || seen.has(key)) return false + seen.add(key) + return true + }) + hidden.value = tags.join(',') + input.value = '' + hidden.dispatchEvent(new Event('input', { bubbles: true })) + hidden.dispatchEvent(new Event('change', { bubbles: true })) + } + // for each action in the dropdown, turn it into a button instead container.find('select[name=action] option:gt(0)').each(function () { const action_type = this.value @@ -1699,7 +1798,11 @@ e.preventDefault() e.stopPropagation() - const num_selected = document.querySelector('.action-counter').innerText.split(' ')[0] + const num_selected = ( + document.querySelector('.action-selected-count')?.textContent.split('/')[0].trim() + || document.querySelector('.action-counter')?.textContent.split(' ')[0] + || '0' + ) if (action_type === 'overwrite_snapshots') { const message = ( @@ -1715,6 +1818,9 @@ ) if (!window.confirm(message)) return false } + if (action_type === 'add_tags' || action_type === 'remove_tags') { + flushPendingActionTags() + } // select the action from the original Django admin dropdown container.find('select[name=action]') @@ -1729,6 +1835,14 @@ .appendTo(buttons) }) console.log('Converted', buttons.children().length, 'admin actions from dropdown to buttons') + const tagContainer = document.querySelector('.actions-tags') + if (tagContainer) { + const tagButtons = buttons.find('button[name="add_tags"], button[name="remove_tags"]') + if (tagButtons.length) { + tagContainer.classList.add('actions-tags-with-buttons') + tagButtons.appendTo(tagContainer) + } + } if (window.jQuery && window.jQuery.fn.select2) { window.jQuery('select[multiple]').select2(); } @@ -1737,7 +1851,188 @@ const tagContainer = document.querySelector('.actions-tags'); if (!tagContainer) return; const checked = document.querySelectorAll('#changelist-form input.action-select:checked').length; - tagContainer.style.display = checked > 0 ? 'inline-flex' : 'none'; + tagContainer.style.display = tagContainer.classList.contains('actions-tags-with-buttons') || checked > 0 ? 'inline-flex' : 'none'; + } + function setupActionSummary() { + const summary = document.querySelector('.action-summary') + if (!summary || summary.dataset.summaryReady) return + summary.dataset.summaryReady = '1' + const selectedCount = summary.querySelector('.action-selected-count') + const counter = summary.querySelector('.action-counter') + const formatter = new Intl.NumberFormat() + const update = function() { + if (!selectedCount || !counter) return + const match = counter.textContent.match(/(\d+)\s+of\s+(\d+)\s+selected/) + const selectAcross = document.querySelector('div.actions input.select-across')?.value === '1' + const selected = selectAcross + ? Number(summary.dataset.resultCount || 0) + : (match ? Number(match[1]) : document.querySelectorAll('#changelist-form input.action-select:checked').length) + const pageCount = match ? Number(match[2]) : Number(summary.dataset.pageCount || 0) + const selectedLimit = selectAcross ? Number(summary.dataset.resultCount || pageCount) : pageCount + selectedCount.textContent = formatter.format(selected) + ' / ' + formatter.format(selectedLimit) + ' selected' + summary.classList.toggle('action-summary-has-selection', selected > 0) + summary.classList.toggle('action-summary-select-across', selectAcross) + const totalCount = summary.querySelector('.action-total-count') + if (totalCount) { + totalCount.hidden = selectAcross + } + } + new MutationObserver(update).observe(counter, { childList: true, characterData: true, subtree: true }) + document.querySelector('#changelist-form')?.addEventListener('change', function() { + window.setTimeout(update, 0) + }) + summary.addEventListener('click', function(event) { + const explicitSelectAll = event.target.closest('.action-total-select, .action-total-count .question a') + const selectedTotalClick = event.target.closest('.action-total-reset') && summary.classList.contains('action-summary-has-selection') + if (explicitSelectAll || selectedTotalClick) { + const questionLink = summary.querySelector('.action-total-count .question a') + const allToggle = document.getElementById('action-toggle') + const pageCount = Number(summary.dataset.pageCount || 0) + const resultCount = Number(summary.dataset.resultCount || pageCount) + event.preventDefault() + if (allToggle && !allToggle.checked) { + allToggle.click() + } + if (questionLink && resultCount > pageCount) { + questionLink.click() + } + } + window.setTimeout(update, 0) + }) + update() + } + function setupSearchModeSelect() { + const storageKey = 'archivebox-admin-search-mode' + const params = new URLSearchParams(window.location.search) + document.querySelectorAll('.search-mode-select').forEach(function(select) { + if (select.dataset.searchModeReady) return + select.dataset.searchModeReady = '1' + const values = Array.from(select.options).map(option => option.value) + const stored = localStorage.getItem(storageKey) + if (!params.has('search_mode') && values.includes(stored)) { + select.value = stored + if ((params.get('q') || '').trim()) { + params.set('search_mode', stored) + params.delete('p') + window.location.replace(window.location.pathname + '?' + params.toString() + window.location.hash) + return + } + } + select.addEventListener('change', function() { + localStorage.setItem(storageKey, select.value) + const search = select.closest('#changelist-search') + if (search && search.dataset.embeddedSearch === '1') { + submitEmbeddedChangelistSearch(search) + return + } + if (search && search.tagName === 'FORM' && document.activeElement === select) { + search.requestSubmit() + } + }) + }) + } + function submitEmbeddedChangelistSearch(search) { + const params = new URLSearchParams(window.location.search) + search.querySelectorAll('[name]').forEach(function(input) { + const value = (input.value || '').trim() + if (value) { + params.set(input.name, value) + } else { + params.delete(input.name) + } + }) + params.delete('p') + const query = params.toString() + window.location.href = window.location.pathname + (query ? '?' + query : '') + window.location.hash + } + function setupEmbeddedChangelistSearch() { + document.querySelectorAll('#changelist-search[data-embedded-search="1"]').forEach(function(search) { + if (search.dataset.searchReady) return + search.dataset.searchReady = '1' + search.querySelector('.changelist-search-submit')?.addEventListener('click', function() { + submitEmbeddedChangelistSearch(search) + }) + search.querySelector('#searchbar')?.addEventListener('keydown', function(event) { + if (event.key === 'Enter') { + event.preventDefault() + submitEmbeddedChangelistSearch(search) + } + }) + }) + } + function setupSnapshotPermissionsQuickEdit() { + if (document.body.dataset.snapshotPermissionsReady) return + document.body.dataset.snapshotPermissionsReady = '1' + document.addEventListener('click', function(event) { + const toggle = event.target.closest('.snapshot-permissions-button') + const item = event.target.closest('.snapshot-permissions-menu-item') + document.querySelectorAll('.snapshot-permissions-quick.is-open').forEach(function(openMenu) { + if (!openMenu.contains(event.target)) { + openMenu.classList.remove('is-open') + openMenu.querySelector('.snapshot-permissions-button')?.setAttribute('aria-expanded', 'false') + const menu = openMenu.querySelector('.snapshot-permissions-menu') + if (menu) menu.hidden = true + } + }) + if (toggle) { + event.preventDefault() + const wrapper = toggle.closest('.snapshot-permissions-quick') + const menu = wrapper?.querySelector('.snapshot-permissions-menu') + if (!wrapper || !menu) return + const isOpen = wrapper.classList.toggle('is-open') + toggle.setAttribute('aria-expanded', isOpen ? 'true' : 'false') + menu.hidden = !isOpen + return + } + if (!item) return + event.preventDefault() + const wrapper = item.closest('.snapshot-permissions-quick') + const permissions = item.dataset.permissions + const csrf = document.querySelector('input[name="csrfmiddlewaretoken"]')?.value + if (!wrapper || !permissions || !csrf || wrapper.dataset.saving === '1') return + wrapper.dataset.saving = '1' + const body = new URLSearchParams({permissions: permissions, csrfmiddlewaretoken: csrf}) + fetch(wrapper.dataset.permissionsUrl, { + method: 'POST', + headers: {'X-CSRFToken': csrf, 'Content-Type': 'application/x-www-form-urlencoded'}, + body: body.toString(), + }).then(function(response) { + if (!response.ok) throw new Error('Failed to update permissions') + return response.json() + }).then(function(data) { + wrapper.dataset.currentPermissions = data.permissions + wrapper.querySelectorAll('.snapshot-permissions-menu-item').forEach(function(button) { + button.classList.toggle('is-active', button.dataset.permissions === data.permissions) + }) + const icon = wrapper.querySelector('.snapshot-permissions-icon') + if (icon) { + icon.textContent = data.icon + icon.style.color = data.fg + icon.style.background = data.bg + } + const toggle = wrapper.querySelector('.snapshot-permissions-button') + if (toggle) { + toggle.className = 'snapshot-permissions-button snapshot-permissions-' + data.permissions + toggle.title = data.label + toggle.setAttribute('aria-label', 'Change snapshot permissions: ' + data.label) + toggle.setAttribute('aria-expanded', 'false') + } + wrapper.classList.remove('is-open') + const menu = wrapper.querySelector('.snapshot-permissions-menu') + if (menu) menu.hidden = true + }).catch(function(error) { + window.alert(error.message) + }).finally(function() { + wrapper.dataset.saving = '' + }) + }) + } + function setupChangelistFormHandlers() { + const form = document.querySelector('#changelist-form') + if (form && !form.dataset.archiveboxActionsReady) { + form.dataset.archiveboxActionsReady = '1' + form.addEventListener('change', updateTagWidgetVisibility) + } } function fixInlineAddRow() { $('#id_snapshottag-MAX_NUM_FORMS').val('1000') @@ -1858,28 +2153,30 @@ }) return false } - if ($) { - $(document).ready(function() { + window.archiveboxInitAdminChangelist = function() { + if (window.jQuery) { fix_actions() - updateTagWidgetVisibility() - const form = document.querySelector('#changelist-form') - if (form) { - form.addEventListener('change', updateTagWidgetVisibility) - } fixInlineAddRow() setupSnapshotGridListToggle() + } + updateTagWidgetVisibility() + setupActionSummary() + setupSearchModeSelect() + setupEmbeddedChangelistSearch() + setupChangelistFormHandlers() + selectSnapshotIfHotlinked() + } + if ($) { + $(document).ready(function() { + window.archiveboxInitAdminChangelist() + setupSnapshotPermissionsQuickEdit() setTimeOffset() - selectSnapshotIfHotlinked() }) } else { document.addEventListener('DOMContentLoaded', function() { - updateTagWidgetVisibility() - const form = document.querySelector('#changelist-form') - if (form) { - form.addEventListener('change', updateTagWidgetVisibility) - } + window.archiveboxInitAdminChangelist() + setupSnapshotPermissionsQuickEdit() setTimeOffset() - selectSnapshotIfHotlinked() }) } diff --git a/archivebox/templates/admin/change_list.html b/archivebox/templates/admin/change_list.html index a1c8eafc..3c711d95 100644 --- a/archivebox/templates/admin/change_list.html +++ b/archivebox/templates/admin/change_list.html @@ -46,7 +46,7 @@ {% endblock %} -{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-list{% endblock %} +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-list{% if embedded_changelist %} embedded-change-list{% endif %}{% endblock %} {% if not is_popup %} {% block breadcrumbs %} @@ -63,11 +63,13 @@ {% block content %}
{% block object-tools %} + {% if not embedded_changelist %}
    {% block object-tools-items %} {% change_list_object_tools %} {% endblock %}
+ {% endif %} {% endblock %} {% if cl.formset and cl.formset.errors %}

@@ -75,101 +77,6 @@

{{ cl.formset.non_form_errors }} {% endif %} - {% if cl.model_admin.show_search_mode_selector %} - {% with current_search_mode=cl.params.search_mode|default:cl.model_admin.get_default_search_mode %} -
- {% endwith %} - {% else %} -
- {% endif %} -
-
- {% block search %}{% search_form cl %}{% endblock %} - {% block date_hierarchy %}{% if cl.date_hierarchy %}{% date_hierarchy cl %}{% endif %}{% endblock %} - -
{% csrf_token %} - {% if cl.formset %} -
{{ cl.formset.management_form }}
- {% endif %} - - {% block result_list %} - {% if action_form and actions_on_top and cl.show_admin_actions %}{% admin_actions %}{% endif %} - {% result_list cl %} - {% if action_form and actions_on_bottom and cl.show_admin_actions %}{% admin_actions %}{% endif %} - {% endblock %} - {% block pagination %} -
- {% pagination cl %} - {% if cl.formset and cl.result_count %}{% endif %} -
- {% endblock %} -
-
-
- {% block filters %} - {% if cl.has_filters %} -
-

- {% translate 'Filter' %} - -

- {% if cl.is_facets_optional or cl.has_active_filters %}
- {% if cl.is_facets_optional %}

- {% if cl.add_facets %}{% translate "Hide counts" %} - {% else %}{% translate "Show counts" %}{% endif %} -

{% endif %} - {% if cl.has_active_filters %}

- ✖ {% translate "Clear all filters" %} -

{% endif %} -
{% endif %} - {% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %} -
- {% endif %} - {% endblock %} -
+ {% include "admin/change_list_panel.html" %}
- {% if cl.has_filters %} - - {% endif %} {% endblock %} diff --git a/archivebox/templates/admin/change_list_panel.html b/archivebox/templates/admin/change_list_panel.html new file mode 100644 index 00000000..eb262793 --- /dev/null +++ b/archivebox/templates/admin/change_list_panel.html @@ -0,0 +1,107 @@ +{% load i18n admin_list core_tags %} + +{% if cl.model_admin.show_search_mode_selector %} + {% with current_search_mode=cl.params.search_mode|default:cl.model_admin.get_default_search_mode %} +
+ {% endwith %} +{% else %} +
+{% endif %} +
+
+ {% search_form cl %} + {% if cl.date_hierarchy %}{% date_hierarchy cl %}{% endif %} + +
{% csrf_token %} + {% if cl.formset %} +
{{ cl.formset.management_form }}
+ {% endif %} + + {% if action_form and actions_on_top and cl.show_admin_actions %}{% admin_actions %}{% endif %} + {% if cl.snapshot_is_grid_view %} + {% snapshots_grid cl %} + {% else %} + {% result_list cl %} + {% endif %} + {% if action_form and actions_on_bottom and cl.show_admin_actions %}{% admin_actions %}{% endif %} +
+ {% pagination cl %} + {% if cl.formset and cl.result_count %}{% endif %} +
+
+
+
+ {% if cl.has_filters and not embedded_changelist %} +
+

+ {% translate 'Filter' %} + +

+ {% if cl.is_facets_optional or cl.has_active_filters %}
+ {% if cl.is_facets_optional %}

+ {% if cl.add_facets %}{% translate "Hide counts" %} + {% else %}{% translate "Show counts" %}{% endif %} +

{% endif %} + {% if cl.has_active_filters %}

+ ✖ {% translate "Clear all filters" %} +

{% endif %} +
{% endif %} + {% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %} +
+ {% endif %} +
+ +{% if cl.has_filters and not embedded_changelist %} + +{% endif %} +{% include "admin/snapshot_search_stream.html" %} diff --git a/archivebox/templates/admin/change_list_results.html b/archivebox/templates/admin/change_list_results.html index 71f410e3..5893c2cf 100644 --- a/archivebox/templates/admin/change_list_results.html +++ b/archivebox/templates/admin/change_list_results.html @@ -7,6 +7,14 @@ {% if results %}
+{% if cl.opts.model_name == "snapshot" %} ++ + + + + +{% endif %} {% for header in result_headers %} @@ -35,4 +43,12 @@
+{% elif cl.show_search_index_hint %} +
+

+ 0 results from deep: {{ cl.search_backend_label }}. + If this looks wrong, the search index may need to be updated: + archivebox update --index-only +

+
{% endif %} diff --git a/archivebox/templates/admin/core/archiveresult/change_list.html b/archivebox/templates/admin/core/archiveresult/change_list.html index b44e9211..440d4ec1 100644 --- a/archivebox/templates/admin/core/archiveresult/change_list.html +++ b/archivebox/templates/admin/core/archiveresult/change_list.html @@ -118,8 +118,8 @@ toggle.textContent = collapsed ? toggle.dataset.showLabel : toggle.dataset.hideLabel; toggle.setAttribute('aria-expanded', collapsed ? 'false' : 'true'); if (toolbarToggle) { - toolbarToggle.textContent = toggle.dataset.showLabel; - toolbarToggle.style.display = collapsed ? 'inline-block' : 'none'; + toolbarToggle.textContent = collapsed ? 'Filters β–Έ' : 'Filters β—‚'; + toolbarToggle.style.display = 'inline-flex'; } } diff --git a/archivebox/templates/admin/crawls/crawl/change_form.html b/archivebox/templates/admin/crawls/crawl/change_form.html new file mode 100644 index 00000000..dcf4e27b --- /dev/null +++ b/archivebox/templates/admin/crawls/crawl/change_form.html @@ -0,0 +1,42 @@ +{% extends "admin/change_form.html" %} + +{% block object-tools-items %} +{% if original %} +
  • + + Stop reason: + {% if crawl_stop_reason %} + {{ crawl_stop_reason }} + {% else %} + none + {% endif %} + + {% if original.status != "sealed" and not original.is_paused %} +
    + {% csrf_token %} + + + + +
    + {% endif %} + {% if original.status == "sealed" or original.is_paused %} +
    + {% csrf_token %} + + + + +
    + {% endif %} +
    • +{% endif %} +{{ block.super }} +{% endblock %} + +{% block content %} +{{ block.super }} +{% if crawl_snapshots_changelist %} + {{ crawl_snapshots_changelist }} +{% endif %} +{% endblock %} diff --git a/archivebox/templates/admin/crawls/crawl/snapshots_changelist.html b/archivebox/templates/admin/crawls/crawl/snapshots_changelist.html new file mode 100644 index 00000000..925d243b --- /dev/null +++ b/archivebox/templates/admin/crawls/crawl/snapshots_changelist.html @@ -0,0 +1,9 @@ +{% load i18n %} + +
    +
    + {% translate "Snapshots in this crawl" %} + {% translate "Open full changelist" %} +
    + {% include "admin/change_list_panel.html" with changelist_form_action=snapshot_changelist_url %} +
    diff --git a/archivebox/templates/admin/private_index.html b/archivebox/templates/admin/private_index.html deleted file mode 100644 index 7db75b30..00000000 --- a/archivebox/templates/admin/private_index.html +++ /dev/null @@ -1,138 +0,0 @@ -{% extends "admin/base_site.html" %} -{% load i18n admin_urls static admin_list %} -{% load core_tags %} - -{% block extrastyle %} - {{ block.super }} - - {% if cl.formset %} - - {% endif %} - {% if cl.formset or action_form %} - - {% endif %} - {{ media.css }} - {% if not actions_on_top and not actions_on_bottom %} - - {% endif %} -{% endblock %} - -{% block extrahead %} -{{ block.super }} -{{ media.js }} -{% endblock %} - -{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-list{% endblock %} - -{% if not is_popup %} -{% block breadcrumbs %} -
    -{% translate 'Home' %} -› {{ cl.opts.app_config.verbose_name }} -› {{ cl.opts.verbose_name_plural|capfirst }} -
    -{% endblock %} -{% endif %} - -{% block coltype %}{% endblock %} - -{% block content %} -
    - {% block object-tools %} -
      - {% block object-tools-items %} - {% change_list_object_tools %} - {% endblock %} -
    - {% endblock %} - {% if cl.formset and cl.formset.errors %} -

    - {% if cl.formset.total_error_count == 1 %}{% translate "Please correct the error below." %}{% else %}{% translate "Please correct the errors below." %}{% endif %} -

    - {{ cl.formset.non_form_errors }} - {% endif %} -
    -
    - {% block search %}{% search_form cl %}{% endblock %} - {% block date_hierarchy %}{% if cl.date_hierarchy %}{% date_hierarchy cl %}{% endif %}{% endblock %} - -
    {% csrf_token %} - {% if cl.formset %} -
    {{ cl.formset.management_form }}
    - {% endif %} - - {% block result_list %} - {% if action_form and actions_on_top and cl.show_admin_actions %}{% admin_actions %}{% endif %} - {% comment %} - Table grid - {% result_list cl %} - {% endcomment %} - {% snapshots_grid cl %} - {% if action_form and actions_on_bottom and cl.show_admin_actions %}{% admin_actions %}{% endif %} - {% endblock %} - {% block pagination %}{% pagination cl %}{% endblock %} -
    -
    - {% block filters %} - {% if cl.has_filters %} -
    -

    - {% translate 'Filter' %} - -

    - {% if cl.has_active_filters %}

    - ✖ {% translate "Clear all filters" %} -

    {% endif %} - {% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %} -
    - {% endif %} - {% endblock %} -
    -
    - {% if cl.has_filters %} - - {% endif %} -{% endblock %} diff --git a/archivebox/templates/admin/private_index_grid.html b/archivebox/templates/admin/private_index_grid.html deleted file mode 100644 index 7db75b30..00000000 --- a/archivebox/templates/admin/private_index_grid.html +++ /dev/null @@ -1,138 +0,0 @@ -{% extends "admin/base_site.html" %} -{% load i18n admin_urls static admin_list %} -{% load core_tags %} - -{% block extrastyle %} - {{ block.super }} - - {% if cl.formset %} - - {% endif %} - {% if cl.formset or action_form %} - - {% endif %} - {{ media.css }} - {% if not actions_on_top and not actions_on_bottom %} - - {% endif %} -{% endblock %} - -{% block extrahead %} -{{ block.super }} -{{ media.js }} -{% endblock %} - -{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-list{% endblock %} - -{% if not is_popup %} -{% block breadcrumbs %} -
    -{% translate 'Home' %} -› {{ cl.opts.app_config.verbose_name }} -› {{ cl.opts.verbose_name_plural|capfirst }} -
    -{% endblock %} -{% endif %} - -{% block coltype %}{% endblock %} - -{% block content %} -
    - {% block object-tools %} -
      - {% block object-tools-items %} - {% change_list_object_tools %} - {% endblock %} -
    - {% endblock %} - {% if cl.formset and cl.formset.errors %} -

    - {% if cl.formset.total_error_count == 1 %}{% translate "Please correct the error below." %}{% else %}{% translate "Please correct the errors below." %}{% endif %} -

    - {{ cl.formset.non_form_errors }} - {% endif %} -
    -
    - {% block search %}{% search_form cl %}{% endblock %} - {% block date_hierarchy %}{% if cl.date_hierarchy %}{% date_hierarchy cl %}{% endif %}{% endblock %} - -
    {% csrf_token %} - {% if cl.formset %} -
    {{ cl.formset.management_form }}
    - {% endif %} - - {% block result_list %} - {% if action_form and actions_on_top and cl.show_admin_actions %}{% admin_actions %}{% endif %} - {% comment %} - Table grid - {% result_list cl %} - {% endcomment %} - {% snapshots_grid cl %} - {% if action_form and actions_on_bottom and cl.show_admin_actions %}{% admin_actions %}{% endif %} - {% endblock %} - {% block pagination %}{% pagination cl %}{% endblock %} -
    -
    - {% block filters %} - {% if cl.has_filters %} -
    -

    - {% translate 'Filter' %} - -

    - {% if cl.has_active_filters %}

    - ✖ {% translate "Clear all filters" %} -

    {% endif %} - {% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %} -
    - {% endif %} - {% endblock %} -
    -
    - {% if cl.has_filters %} - - {% endif %} -{% endblock %} diff --git a/archivebox/templates/admin/progress_monitor.html b/archivebox/templates/admin/progress_monitor.html index c703bc21..7c08b9af 100644 --- a/archivebox/templates/admin/progress_monitor.html +++ b/archivebox/templates/admin/progress_monitor.html @@ -16,6 +16,12 @@ max-height: 350px; overflow-y: auto; } + #progress-monitor .progress-content { + display: flex; + align-items: flex-start; + gap: 12px; + padding: 12px 16px; + } /* Header Bar */ #progress-monitor .header-bar { @@ -33,11 +39,13 @@ display: flex; align-items: center; gap: 16px; + min-width: 0; } #progress-monitor .header-right { display: flex; align-items: center; gap: 12px; + flex-shrink: 0; } /* Orchestrator Status */ @@ -130,10 +138,10 @@ color: #c9d1d9; border-color: #8b949e; } + #progress-monitor .crawl-action-btn, #progress-monitor .cancel-item-btn { background: transparent; border: 1px solid #30363d; - color: #f85149; cursor: pointer; padding: 2px 6px; border-radius: 6px; @@ -142,11 +150,23 @@ transition: all 0.2s; flex-shrink: 0; } + #progress-monitor .pause-item-btn { + color: #d29922; + } + #progress-monitor .pause-item-btn:hover { + background: rgba(210, 153, 34, 0.12); + border-color: #d29922; + color: #f0b72f; + } + #progress-monitor .cancel-item-btn { + color: #f85149; + } #progress-monitor .cancel-item-btn:hover { background: rgba(248, 81, 73, 0.12); border-color: #f85149; color: #ff7b72; } + #progress-monitor .crawl-action-btn.is-busy, #progress-monitor .cancel-item-btn.is-busy { opacity: 0.6; cursor: wait; @@ -156,12 +176,88 @@ /* Tree Container */ #progress-monitor .tree-container { - padding: 12px 16px; + flex: 1 1 auto; + min-width: 0; + padding: 0; } - #progress-monitor.collapsed .tree-container { + #progress-monitor.collapsed .progress-content { display: none; } + /* Chrome Screencast */ + #progress-monitor .screencast-panel { + display: none; + flex: 0 0 336px; + width: 336px; + position: sticky; + top: 49px; + border: 1px solid rgba(88, 166, 255, 0.5); + border-radius: 8px; + overflow: hidden; + background: #0d1117; + box-shadow: 0 12px 34px rgba(1, 4, 9, 0.4), 0 0 0 1px rgba(88, 166, 255, 0.12); + } + #progress-monitor .screencast-panel.visible { + display: block; + } + #progress-monitor .screencast-frame { + display: block; + width: 100%; + aspect-ratio: 16 / 10; + background: #010409; + color: #6e7681; + text-decoration: none; + overflow: hidden; + } + #progress-monitor .screencast-frame img { + display: block; + width: 100%; + height: 100%; + object-fit: cover; + object-position: top center; + } + #progress-monitor .screencast-caption { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 10px; + border-top: 1px solid #21262d; + background: rgba(13, 17, 23, 0.96); + color: inherit; + text-decoration: none; + } + #progress-monitor .screencast-dot { + width: 7px; + height: 7px; + border-radius: 50%; + flex: 0 0 7px; + background: #3fb950; + box-shadow: 0 0 8px rgba(63, 185, 80, 0.9); + } + #progress-monitor .screencast-text { + min-width: 0; + } + #progress-monitor .screencast-title { + display: block; + color: #f0f6fc; + font-size: 12px; + font-weight: 600; + line-height: 1.25; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + #progress-monitor .screencast-url { + display: block; + color: #8b949e; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + font-size: 10px; + line-height: 1.25; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + /* Idle Message */ #progress-monitor .idle-message { color: #8b949e; @@ -563,6 +659,10 @@ background: rgba(210, 153, 34, 0.2); color: #d29922; } + #progress-monitor .status-badge.paused { + background: rgba(88, 166, 255, 0.18); + color: #58a6ff; + } #progress-monitor .status-badge.sealed, #progress-monitor .status-badge.succeeded { background: rgba(63, 185, 80, 0.2); @@ -711,9 +811,182 @@ font-size: 9px; } + @media (max-width: 900px) { + #progress-monitor .header-bar { + align-items: flex-start; + gap: 8px; + } + #progress-monitor .header-left { + flex: 1; + min-width: 0; + flex-wrap: wrap; + gap: 8px 14px; + } + #progress-monitor .stats { + flex-wrap: wrap; + gap: 8px 12px; + } + #progress-monitor .tree-container { + max-height: 310px; + } + #progress-monitor .progress-content { + flex-direction: column; + padding: 10px 8px; + } + #progress-monitor .screencast-panel { + position: static; + order: -1; + width: min(100%, 336px); + flex-basis: auto; + align-self: center; + } + #progress-monitor .crawl-header { + align-items: flex-start; + gap: 8px; + padding: 8px; + } + #progress-monitor .crawl-header-link { + display: grid; + grid-template-columns: minmax(0, 1fr) auto auto auto; + align-items: start; + gap: 8px; + width: 100%; + } + #progress-monitor .crawl-info { + grid-column: 1 / -1; + width: 100%; + } + #progress-monitor .crawl-label { + max-width: 100%; + } + #progress-monitor .crawl-badges, + #progress-monitor .crawl-stats { + justify-content: flex-start; + min-width: 0; + } + #progress-monitor .crawl-badge { + max-width: min(100%, 260px); + } + #progress-monitor .crawl-stats { + grid-column: 1 / 2; + } + #progress-monitor .crawl-body { + padding: 0 8px 10px; + } + #progress-monitor .crawl-progress { + padding: 8px; + } + #progress-monitor .snapshot-header { + align-items: flex-start; + padding: 8px; + } + #progress-monitor .snapshot-header-link { + min-width: 0; + } + } + + @media (max-width: 520px) { + #progress-monitor { + font-size: 11px; + } + #progress-monitor .header-bar { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + align-items: start; + padding: 7px 8px; + } + #progress-monitor .header-left { + gap: 6px 10px; + } + #progress-monitor .header-right { + justify-self: end; + } + #progress-monitor .toggle-btn { + white-space: nowrap; + } + #progress-monitor .orchestrator-status { + max-width: 100%; + } + #progress-monitor .stats { + flex: 1 0 100%; + gap: 6px 10px; + } + #progress-monitor .stat-label { + font-size: 9px; + } + #progress-monitor .tree-container { + max-height: 300px; + } + #progress-monitor .progress-content { + padding: 8px; + } + #progress-monitor .screencast-panel { + width: 100%; + align-self: stretch; + } + #progress-monitor .crawl-header { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + padding: 8px; + } + #progress-monitor .crawl-header-link { + grid-template-columns: minmax(0, 1fr); + } + #progress-monitor .crawl-stats { + grid-column: auto; + } + #progress-monitor .crawl-title-link { + white-space: normal; + line-height: 1.35; + } + #progress-monitor .crawl-badges { + gap: 4px; + } + #progress-monitor .crawl-badge { + max-width: 100%; + min-width: 0; + font-size: 10px; + white-space: nowrap; + } + #progress-monitor .crawl-stats { + gap: 4px; + } + #progress-monitor .status-badge, + #progress-monitor .duration-badge, + #progress-monitor .pid-label { + justify-self: start; + } + #progress-monitor .snapshot-header-link { + display: grid; + grid-template-columns: 56px minmax(0, 1fr); + gap: 8px; + } + #progress-monitor .snapshot-preview { + width: 56px; + height: 42px; + flex-basis: 56px; + } + #progress-monitor .snapshot-title-line { + align-items: flex-start; + } + #progress-monitor .snapshot-title, + #progress-monitor .snapshot-url { + white-space: normal; + overflow-wrap: anywhere; + } + #progress-monitor .extractor-list { + padding: 7px 8px; + gap: 4px; + } + #progress-monitor .extractor-badge { + max-width: 100%; + white-space: nowrap; + } + } + -
    +
    @@ -1046,6 +1365,12 @@ Crawl cannot start: ${crawl.urls_preview ? 'unknown error' : 'no URLs'}
    `; + } else if (crawl.is_paused) { + warningHtml = ` +
    + Crawl is paused. Resume it to continue processing queued snapshots. +
    + `; } else if (crawl.status === 'queued' && crawl.retry_at_future) { // Queued but retry_at is in future (was claimed by worker, will retry) warningHtml = ` @@ -1085,12 +1410,14 @@ const maxUrlsText = (crawl.max_urls || 0) > 0 ? crawl.max_urls : 'unlimited'; const urlLimitText = `${currentUrlCount} / ${maxUrlsText}`; const crawlSizeLimitText = `${crawl.crawl_output_size_display || '0 B'} / ${crawl.max_crawl_size_display || 'unlimited'}`; + const crawlTimeoutText = (crawl.crawl_timeout || 0) > 0 ? `${crawl.crawl_timeout}s` : 'unlimited'; const snapshotSizeLimitText = `${crawl.avg_snapshot_size_display || '0 B'} / ${crawl.max_snapshot_size_display || 'unlimited'}`; const crawlBadges = [ crawlBadge('persona', 'Persona Config', crawl.persona || 'Default', crawl.persona_admin_url, 'Edit persona config'), crawlBadge('limit', 'depth', crawl.max_depth || 0, adminFieldUrl('max_depth'), 'Edit crawl depth'), crawlBadge('limit', 'urls', urlLimitText, adminFieldUrl('max_urls'), 'Edit max URLs'), crawlBadge('size', 'crawl size', crawlSizeLimitText, adminFieldUrl('crawl_max_size'), 'Edit max crawl size'), + crawlBadge('limit', 'time', crawlTimeoutText, adminFieldUrl('crawl_timeout'), 'Edit max crawl time'), crawlBadge('size', 'avg snap', snapshotSizeLimitText, adminFieldUrl('snapshot_max_size'), 'Edit max snapshot size'), ...(crawl.tags || []).map(tag => `#${escapeHtml(tag)}`), ].join(''); @@ -1114,8 +1441,9 @@
    ${crawlPidHtml} ${crawlDurationHtml} - ${crawl.status || 'unknown'} + ${crawl.is_paused ? 'paused' : (crawl.status || 'unknown')}
    + ${pauseBtn} ${cancelBtn}
    @@ -1200,6 +1528,7 @@ document.getElementById('total-succeeded').textContent = data.archiveresults_succeeded; document.getElementById('total-failed').textContent = data.archiveresults_failed; + updateScreencastPanel(data); // Render crawl tree if (data.active_crawls.length > 0) { @@ -1268,14 +1597,16 @@ isCollapsed = collapsed; if (isCollapsed) { monitor.classList.add('collapsed'); - collapseBtn.textContent = 'Expand'; + collapseBtn.textContent = 'Details'; + collapseBtn.setAttribute('aria-expanded', 'false'); } else { monitor.classList.remove('collapsed'); - collapseBtn.textContent = 'Details'; + collapseBtn.textContent = 'Hide'; + collapseBtn.setAttribute('aria-expanded', 'true'); } } - function setCancelButtonState(btn, busy) { + function setActionButtonState(btn, busy) { if (!btn) return; const label = btn.dataset.label || 'βœ•'; btn.disabled = !!busy; @@ -1287,15 +1618,15 @@ if (!crawlId) return; if (!getApiKey()) { console.warn('API key unavailable for this session.'); - setCancelButtonState(btn, false); + setActionButtonState(btn, false); return; } - setCancelButtonState(btn, true); + setActionButtonState(btn, true); fetch(buildApiUrl(`/api/v1/crawls/crawl/${crawlId}`), { method: 'PATCH', headers: buildApiHeaders(), - body: JSON.stringify({ status: 'sealed', retry_at: null }), + body: JSON.stringify({ action: 'cancel' }), }) .then(response => response.json()) .then(data => { @@ -1306,7 +1637,34 @@ }) .catch(error => { console.error('Cancel crawl failed:', error); - setCancelButtonState(btn, false); + setActionButtonState(btn, false); + }); + } + + function setCrawlPaused(crawlId, action, btn) { + if (!crawlId) return; + if (!getApiKey()) { + console.warn('API key unavailable for this session.'); + setActionButtonState(btn, false); + return; + } + setActionButtonState(btn, true); + + fetch(buildApiUrl(`/api/v1/crawls/crawl/${crawlId}`), { + method: 'PATCH', + headers: buildApiHeaders(), + body: JSON.stringify({ action: action }), + }) + .then(response => response.json()) + .then(data => { + if (data.error) { + console.error('Crawl action error:', data.error); + } + fetchProgress(); + }) + .catch(error => { + console.error('Crawl action failed:', error); + setActionButtonState(btn, false); }); } @@ -1314,15 +1672,15 @@ if (!snapshotId) return; if (!getApiKey()) { console.warn('API key unavailable for this session.'); - setCancelButtonState(btn, false); + setActionButtonState(btn, false); return; } - setCancelButtonState(btn, true); + setActionButtonState(btn, true); fetch(buildApiUrl(`/api/v1/core/snapshot/${snapshotId}`), { method: 'PATCH', headers: buildApiHeaders(), - body: JSON.stringify({ status: 'sealed', retry_at: null }), + body: JSON.stringify({ action: 'cancel' }), }) .then(response => response.json()) .then(data => { @@ -1333,7 +1691,7 @@ }) .catch(error => { console.error('Cancel snapshot failed:', error); - setCancelButtonState(btn, false); + setActionButtonState(btn, false); }); } @@ -1343,6 +1701,13 @@ }); crawlTree.addEventListener('click', function(event) { + const actionBtn = event.target.closest('.crawl-action-btn'); + if (actionBtn) { + event.preventDefault(); + event.stopPropagation(); + setCrawlPaused(actionBtn.dataset.crawlId, actionBtn.dataset.crawlAction, actionBtn); + return; + } const btn = event.target.closest('.cancel-item-btn'); if (!btn) return; event.preventDefault(); diff --git a/archivebox/templates/admin/search_form.html b/archivebox/templates/admin/search_form.html index e386041c..dc5c3ae3 100644 --- a/archivebox/templates/admin/search_form.html +++ b/archivebox/templates/admin/search_form.html @@ -2,33 +2,58 @@ {% if cl.search_fields %}
    +
    +{% if cl.opts.model_name == 'snapshot' and not cl.embedded_changelist %} + {% if cl.snapshot_is_grid_view %} + + {% else %} + + {% endif %} +{% endif %} +{% if cl.embedded_changelist %} + +{% else %} + +{% endif %} +{% if cl.has_filters and not cl.embedded_changelist %} + +{% endif %} +
    +
    {% endif %} diff --git a/archivebox/templates/admin/snapshot_search_stream.html b/archivebox/templates/admin/snapshot_search_stream.html new file mode 100644 index 00000000..4d192480 --- /dev/null +++ b/archivebox/templates/admin/snapshot_search_stream.html @@ -0,0 +1,122 @@ +{% if opts.app_label == "core" and opts.model_name == "snapshot" and not embedded_changelist %} + +{% endif %} diff --git a/archivebox/templates/core/add.html b/archivebox/templates/core/add.html index 65c70df1..d5dc7592 100644 --- a/archivebox/templates/core/add.html +++ b/archivebox/templates/core/add.html @@ -75,21 +75,49 @@ {% if form.url.errors %}
    {{ form.url.errors }}
    {% endif %} -
    - Enter URLs to archive, as one per line, CSV, JSON, or embedded in text (e.g. markdown, HTML, etc.). Examples:
    - https://example.com
    - https://news.ycombinator.com,https://news.google.com
    - [ArchiveBox](https://github.com/ArchiveBox/ArchiveBox) -
    -
    - {{ form.tag.label_tag }} - {{ form.tag }} - {% if form.tag.errors %} -
    {{ form.tag.errors }}
    - {% endif %} -
    Tags will be applied to all snapshots created by this crawl.
    +
    +
    +
    + + {{ form.tag.label_tag }} +
    + {{ form.tag }} + {% if form.tag.errors %} +
    {{ form.tag.errors }}
    + {% endif %} +
    Tags will be applied to all snapshots created by this crawl.
    +
    + +
    +
    + + {{ form.persona.label_tag }} +
    + {{ form.persona }} + {% if form.persona.errors %} +
    {{ form.persona.errors }}
    + {% endif %} +
    + Authentication profile (Chrome profile, cookies, etc.) to use when accessing URLs. + {% if can_override_crawl_config %} + Create new persona / import from Chrome β†’ + {% endif %} +
    +
    + +
    +
    + + {{ form.permissions.label_tag }} +
    + {{ form.permissions }} + {% if form.permissions.errors %} +
    {{ form.permissions.errors }}
    + {% endif %} +
    Public lists it. Unlisted only serves direct links. Private requires admin login.
    +
    @@ -100,53 +128,7 @@
    {{ form.depth.errors }}
    {% endif %}
    Controls how many links deep the crawl will follow from the starting URLs.
    - -
    -
    - {{ form.max_urls.label_tag }} - {{ form.max_urls }} - {% if form.max_urls.errors %} -
    {{ form.max_urls.errors }}
    - {% endif %} -
    0 means unlimited. When set, only the first N filtered URLs will be snapshotted.
    -
    - -
    - {{ form.crawl_max_size.label_tag }} - {{ form.crawl_max_size }} - {% if form.crawl_max_size.errors %} -
    {{ form.crawl_max_size.errors }}
    - {% endif %} -
    0 means unlimited across the whole crawl. Accepts bytes or units like 45mb and 1gb.
    -
    - -
    - {{ form.snapshot_max_size.label_tag }} - {{ form.snapshot_max_size }} - {% if form.snapshot_max_size.errors %} -
    {{ form.snapshot_max_size.errors }}
    - {% endif %} -
    0 means unlimited per snapshot. Accepts bytes or units like 45mb and 1gb.
    -
    - -
    - {{ form.delete_after.label_tag }} - {{ form.delete_after }} - {% if form.delete_after.errors %} -
    {{ form.delete_after.errors }}
    - {% endif %} -
    0 keeps rows forever. Use 1h, 7d, 4w, 6mo, or 1y.
    -
    - -
    - {{ form.crawl_max_concurrent_snapshots.label_tag }} - {{ form.crawl_max_concurrent_snapshots }} - {% if form.crawl_max_concurrent_snapshots.errors %} -
    {{ form.crawl_max_concurrent_snapshots.errors }}
    - {% endif %} -
    Caps how many snapshots from this crawl archive at the same time.
    -
    -
    +
    @@ -157,6 +139,92 @@
    +
    +
    +
    + + {{ form.max_urls.label_tag }} +
    + {{ form.max_urls }} + {% if form.max_urls.errors %} +
    {{ form.max_urls.errors }}
    + {% endif %} +
    0 = unlimited. Whole numbers, e.g. 25, 300.
    +
    + +
    +
    + + {{ form.crawl_max_size.label_tag }} +
    + {{ form.crawl_max_size }} + {% if form.crawl_max_size.errors %} +
    {{ form.crawl_max_size.errors }}
    + {% endif %} +
    0 = unlimited. Sizes: 45mb, 1.5gb, 2tb.
    +
    + +
    +
    + + {{ form.crawl_timeout.label_tag }} +
    + {{ form.crawl_timeout }} + {% if form.crawl_timeout.errors %} +
    {{ form.crawl_timeout.errors }}
    + {% endif %} +
    0 = unlimited. Must be >10s: 11, 1.5m, 1hr.
    +
    + +
    +
    + + {{ form.timeout.label_tag }} +
    + {{ form.timeout }} + {% if form.timeout.errors %} +
    {{ form.timeout.errors }}
    + {% endif %} +
    Must be >10s: 11, 1.5m, 1hr.
    +
    + +
    +
    + + {{ form.snapshot_max_size.label_tag }} +
    + {{ form.snapshot_max_size }} + {% if form.snapshot_max_size.errors %} +
    {{ form.snapshot_max_size.errors }}
    + {% endif %} +
    0 = unlimited. Sizes: 45mb, 1.5gb, 2tb.
    +
    + +
    +
    + + {{ form.delete_after.label_tag }} +
    + {{ form.delete_after }} + {% if form.delete_after.errors %} +
    {{ form.delete_after.errors }}
    + {% endif %} +
    0 = keep forever. Durations: 1hr, 30d, 3mo.
    +
    + +
    +
    + + {{ form.crawl_max_concurrent_snapshots.label_tag }} +
    + {{ form.crawl_max_concurrent_snapshots }} + {% if form.crawl_max_concurrent_snapshots.errors %} +
    {{ form.crawl_max_concurrent_snapshots.errors }}
    + {% endif %} +
    Whole numbers, e.g. 1, 4, 12.
    +
    +
    +
    {{ form.notes.label_tag }} {{ form.notes }} @@ -166,19 +234,9 @@
    Optional description for this crawl (visible in the admin interface).
    -
    - {{ form.persona.label_tag }} - {{ form.persona }} - {% if form.persona.errors %} -
    {{ form.persona.errors }}
    - {% endif %} -
    - Authentication profile (Chrome profile, cookies, etc.) to use when accessing URLs. - Create new persona / import from Chrome β†’ -
    -
    + {% if can_override_crawl_config %}

    Crawl Plugins

    @@ -239,6 +297,7 @@
    + {% endif %}
    @@ -279,6 +338,11 @@ { bg: 'rgba(210, 105, 30, 0.16)', border: 'rgba(210, 105, 30, 0.5)' }, ]; const requiredSearchPlugin = '{{ required_search_plugin|default:""|escapejs }}'; + const sameDomainToggleSlot = document.getElementById('same-domain-toggle-slot'); + const shortcutToggles = document.querySelectorAll('.url-filters-toggle'); + if (sameDomainToggleSlot && shortcutToggles.length) { + shortcutToggles.forEach((toggle) => sameDomainToggleSlot.appendChild(toggle)); + } const pluginDependencyMap = JSON.parse('{{ plugin_dependency_map_json|default:"{}"|escapejs }}'); const personaConfigMap = JSON.parse('{{ persona_config_map_json|default:"{}"|escapejs }}'); const personaSelect = document.querySelector('select[name="persona"]'); @@ -413,6 +477,16 @@ if (deleteAfterInput && deleteAfterValue !== undefined && deleteAfterValue !== null) { deleteAfterInput.value = deleteAfterValue || '0'; } + const timeoutInput = document.querySelector('input[name="timeout"]'); + const timeoutValue = personaData.effective_config?.TIMEOUT; + if (timeoutInput && timeoutValue !== undefined && timeoutValue !== null) { + timeoutInput.value = timeoutValue || '0'; + } + const permissionsSelect = document.querySelector('select[name="permissions"]'); + const permissionsValue = personaData.effective_config?.PERMISSIONS; + if (permissionsSelect && permissionsValue) { + permissionsSelect.value = permissionsValue; + } if (typeof window.archiveboxSetPluginConfigValues === 'function') { window.archiveboxSetPluginConfigValues(personaData.effective_config || {}, personaData.binary_urls || {}); } @@ -1046,6 +1120,7 @@ event.target.matches('textarea[name="url_filters_denylist"]') || event.target.matches('input[name="max_urls"]') || event.target.matches('input[name="url_filters_same_domain_only"]') || + event.target.matches('input[name="url_filters_subpaths_only"]') || event.target.matches('#id_config_rows .kv-key') || event.target.matches('#id_config_rows .kv-value') ) { diff --git a/archivebox/templates/core/public_index.html b/archivebox/templates/core/public_index.html index 046ee20c..fc45a558 100644 --- a/archivebox/templates/core/public_index.html +++ b/archivebox/templates/core/public_index.html @@ -34,23 +34,90 @@ margin: 0; } + .public-search-input { + position: relative; + display: grid; + grid-template-columns: 46px minmax(0, 1fr); + align-items: center; + flex: 1 1 420px; + min-width: 220px; + height: 34px; + border: 1px solid #cbd5e1; + border-radius: 7px; + background: #fff; + overflow: hidden; + } + + .public-search-addon { + position: relative; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 2px; + box-sizing: border-box; + width: 46px; + height: 100%; + padding: 0; + border-right: 1px solid #e2e8f0; + background: #f8fafc; + cursor: pointer; + } + .public-snapshot-toolbar label.search-icon { display: inline-flex; align-items: center; justify-content: center; - width: 24px; - height: 24px; + width: 16px; + height: 16px; margin: 0; opacity: 0.65; } + .public-search-mode-select { + appearance: none; + -webkit-appearance: none; + position: absolute; + inset: 0 auto 0 0; + width: 46px; + min-width: 46px; + max-width: 46px; + height: 100%; + padding: 0; + border: 0; + background: transparent; + color: #111827; + font-size: 14px; + opacity: 1; + outline: 0; + text-indent: -9999px; + cursor: pointer; + z-index: 3; + } + + .public-search-mode-select option { + background: #ffffff; + color: #111827; + font-size: 14px; + line-height: 1.4; + text-indent: 0; + } + + .public-search-addon .search-mode-caret { + color: #64748b; + font-size: 10px; + line-height: 1; + pointer-events: none; + transform: translateY(1px); + } + .public-snapshot-toolbar #searchbar { - flex: 1 1 320px; + width: 100%; min-width: 180px; - height: 34px; + height: 100%; + box-sizing: border-box; padding: 6px 10px; - border: 1px solid #cbd5e1; - border-radius: 7px; + border: 0; + border-radius: 0; font-size: 13px; } @@ -61,40 +128,6 @@ border-radius: 7px; } - .public-search-modes { - display: inline-flex; - align-items: center; - gap: 2px; - padding: 2px; - border: 1px solid #dbe3ee; - border-radius: 8px; - background: #f8fafc; - white-space: nowrap; - } - - .public-search-modes label { - display: inline-flex; - align-items: center; - gap: 5px; - margin: 0; - padding: 5px 8px; - border-radius: 6px; - color: #475569; - font-size: 12px; - line-height: 1; - cursor: pointer; - } - - .public-search-modes label:has(input:checked) { - background: #fff; - color: #0f172a; - box-shadow: 0 1px 2px rgba(15, 23, 42, 0.08); - } - - .public-search-modes input { - margin: 0; - } - .public-snapshot-count { color: #64748b; font-size: 12px; @@ -340,6 +373,23 @@ color: #cbd5e1; } + .public-search-empty { + padding: 28px 18px; + color: #475569; + text-align: center; + background: #f8fafc; + } + + .public-search-empty code { + display: inline-block; + margin-left: 4px; + padding: 2px 6px; + border-radius: 5px; + background: #e2e8f0; + color: #0f172a; + font-size: 12px; + } + .public-pagination { margin: 18px 0 0; text-align: center; @@ -365,9 +415,9 @@ align-self: flex-start; } - .public-search-modes { - flex-wrap: wrap; - white-space: normal; + .public-search-input { + width: 100%; + flex-basis: auto; } } @@ -388,14 +438,19 @@
    + {% endblock %} diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html index bdb6bad9..af13669d 100644 --- a/archivebox/templates/core/snapshot.html +++ b/archivebox/templates/core/snapshot.html @@ -243,6 +243,10 @@ line-height: 1.2; white-space: nowrap; } + .header-tags .permission-pill { + border-style: dashed; + text-transform: lowercase; + } .header-badges { display: flex; flex-wrap: wrap; @@ -1034,10 +1038,15 @@ {{title|truncatechars:120|safe}} {% if title_tags %} + {{ snapshot_permissions_icon }} [{{ snapshot_permissions }}] {% for tag in title_tags %} {{ tag.name }} {% endfor %} + {% else %} + + {{ snapshot_permissions_icon }} [{{ snapshot_permissions }}] + {% endif %} β–Ύ
    diff --git a/archivebox/templates/static/add.css b/archivebox/templates/static/add.css index 20611802..7498f1da 100755 --- a/archivebox/templates/static/add.css +++ b/archivebox/templates/static/add.css @@ -72,7 +72,7 @@ ul#id_depth { } -textarea, select, input[type="text"] { +textarea, select, input[type="text"], input[type="number"] { border-radius: 4px; border: 2px solid #004882; box-shadow: 4px 4px 4px rgba(0,0,0,0.02); @@ -86,7 +86,7 @@ textarea { min-height: 300px; } -input[type="text"] { +input[type="text"], input[type="number"] { min-height: 42px; } @@ -158,11 +158,42 @@ select { margin-bottom: 20px; } +.tags-persona-row { + display: grid; + grid-template-columns: minmax(0, 6fr) minmax(240px, 3fr) minmax(170px, 2fr); + gap: 18px; + align-items: stretch; +} + +.tags-persona-row .form-field { + display: flex; + flex-direction: column; + min-width: 0; +} + +.tags-persona-row .tag-editor-container, +.tags-persona-row select[name="persona"], +.tags-persona-row select[name="permissions"] { + box-sizing: border-box; + min-height: 46px; +} + +.tags-persona-row select[name="persona"], +.tags-persona-row select[name="permissions"] { + height: 62px; +} + +.persona-field .help-text a { + display: inline-block; + margin-top: 2px; +} + .settings-row { display: grid; - grid-template-columns: minmax(260px, 340px) minmax(420px, 1fr); + grid-template-columns: minmax(220px, 300px) minmax(420px, 1fr); gap: 18px; align-items: start; + margin-bottom: 12px; } .form-field label { @@ -172,15 +203,76 @@ select { margin-bottom: 8px; } +.field-label-with-icon { + display: flex; + align-items: center; + gap: 7px; + margin-bottom: 8px; +} + +.field-label-with-icon label { + margin-bottom: 0; +} + +.field-label-icon { + font-size: 17px; + line-height: 1; +} + +#same-domain-toggle-slot { + margin-top: 12px; + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 5px; +} + .crawl-limit-grid { display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 12px; - margin-top: 14px; + grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); + gap: 10px; + margin: 0 0 20px; } .crawl-limit-field { margin-bottom: 0; + padding: 10px; + border: 1px solid #dbe3ea; + border-radius: 6px; + background: #fff; + min-width: 0; +} + +.crawl-limit-field label { + margin-bottom: 6px; + font-size: 13px; + font-weight: 700; + color: #1f2937; +} + +.crawl-limit-field .field-label-with-icon { + gap: 5px; + margin-bottom: 6px; +} + +.crawl-limit-field .field-label-with-icon label { + margin-bottom: 0; +} + +.crawl-limit-field .field-label-icon { + font-size: 14px; +} + +.crawl-limit-field input { + min-height: 34px; + padding: 6px 8px; + font-size: 13px; +} + +.crawl-limit-field .help-text { + margin-top: 5px; + font-size: 11px; + line-height: 1.25; } .field-header { @@ -257,6 +349,7 @@ select { .detected-urls-panel { display: flex; flex-direction: column; + margin-top: 38px; min-height: 240px; padding: 12px 14px; background: linear-gradient(180deg, #fff 0%, #f6f8fb 100%); @@ -550,7 +643,7 @@ select { display: inline-flex !important; align-items: center; gap: 8px; - margin-top: 10px; + margin-top: 0; font-size: 14px !important; font-weight: 600; } @@ -595,6 +688,10 @@ select { } @media (max-width: 1020px) { + .tags-persona-row { + grid-template-columns: 1fr; + } + .settings-row { grid-template-columns: 1fr; } @@ -603,6 +700,10 @@ select { grid-template-columns: 1fr; } + .detected-urls-panel { + margin-top: 0; + } + .url-filters-grid { grid-template-columns: 1fr; } @@ -707,7 +808,7 @@ input:focus, select:focus, textarea:focus, button:focus { /* Responsive layout */ @media (max-width: 768px) { .crawl-limit-grid { - grid-template-columns: 1fr; + grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); } .plugin-presets { diff --git a/archivebox/templates/static/admin.css b/archivebox/templates/static/admin.css index c4b58b74..d4a89433 100755 --- a/archivebox/templates/static/admin.css +++ b/archivebox/templates/static/admin.css @@ -88,8 +88,8 @@ div.breadcrumbs { line-height: 1; } -body.model-snapshot.change-list div.breadcrumbs, -body.model-snapshot.change-list #content .object-tools { +.model-snapshot.change-list div.breadcrumbs, +.model-snapshot.change-list #content .object-tools { display: none; } @@ -228,6 +228,420 @@ body.change-list #content .object-tools { margin-top: 3px; } +body.change-list:not(.model-snapshot) #changelist .changelist-form-container > div { + display: flex; + flex-wrap: wrap; + align-items: flex-start; + gap: 10px 12px; + min-width: 0; +} + +body.change-list:not(.model-snapshot) #changelist #toolbar { + flex: 2 1 520px; + min-width: min(100%, 420px); + margin: 0 !important; +} + +body.change-list:not(.model-snapshot) #changelist #toolbar #changelist-search, +body.change-list:not(.model-snapshot) #changelist #toolbar #changelist-search > div { + display: flex !important; + flex-wrap: wrap !important; + align-items: center; + gap: 8px; + width: 100%; + min-width: 0; + max-width: 100%; + white-space: normal !important; +} + +body.change-list:not(.model-snapshot) #changelist #toolbar #searchbar { + flex: 1 1 220px; + min-width: 180px; + width: auto; +} + +body.change-list:not(.model-snapshot) #changelist #toolbar .search-mode-selector { + flex: 1 1 300px !important; + flex-wrap: wrap !important; + margin-left: 0 !important; + min-width: 260px; + white-space: normal !important; +} + +body.change-list #changelist #changelist-form { + display: contents; +} + +body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top { + flex: 3 1 560px; + width: auto; + min-width: 0; + max-width: 100%; + box-sizing: border-box; + margin-left: auto; + flex-wrap: wrap !important; + overflow: visible; + align-items: flex-start; + align-content: flex-start; +} + +body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .actions-left { + flex: 0 1 auto; + flex-wrap: wrap !important; + min-width: 0; + align-items: flex-start; +} + +body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .actions-right { + margin-left: 0; + flex-wrap: wrap; + align-items: flex-start; +} + +body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .action-buttons { + display: flex !important; + flex-wrap: wrap; + align-items: center; + gap: 4px; + min-width: 0; +} + +body.change-list:not(.model-snapshot) #changelist .actions-tags-with-buttons { + display: inline-flex; + align-items: center; + gap: 4px; + flex: 1 1 280px; + min-width: 220px; + max-width: 420px; +} + +body.change-list:not(.model-snapshot) #changelist .actions-tags-with-buttons .tag-editor-container { + flex: 1 1 180px; + width: auto; + max-width: none; +} + +body.change-list:not(.model-snapshot) #content #changelist .actions-tags-with-buttons .button[name="add_tags"], +body.change-list:not(.model-snapshot) #content #changelist .actions-tags-with-buttons .button[name="remove_tags"] { + flex: 0 0 30px; + width: 30px; + min-width: 30px; + padding-left: 0; + padding-right: 0; + margin: 0; + justify-content: center; +} + +body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-bottom, +body.change-list:not(.model-snapshot) #changelist #changelist-form > .results, +body.change-list:not(.model-snapshot) #changelist #changelist-form > .changelist-footer, +body.change-list:not(.model-snapshot) #changelist .xfull { + flex: 1 0 100%; + width: 100%; + max-width: 100%; +} + +body.change-list:not(.model-snapshot) #changelist #changelist-form > .results, +body.change-list:not(.model-snapshot) #changelist #changelist-form > .changelist-footer, +body.change-list:not(.model-snapshot) #changelist #toolbar, +body.change-list:not(.model-snapshot) #changelist .xfull { + margin-right: 0 !important; +} + +body.change-list:not(.model-snapshot) #changelist #changelist-form > .results { + overflow-x: auto; + -webkit-overflow-scrolling: touch; +} + +.model-snapshot.change-list #result_list { + min-width: 1060px; +} + +@media (max-width: 1180px) { + body.change-list #changelist { + grid-template-columns: minmax(0, 1fr) !important; + } + + body.change-list #changelist-filter { + width: auto !important; + min-width: 0 !important; + max-width: none !important; + } + + body.change-list:not(.model-snapshot) #changelist #toolbar, + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top { + flex: 1 0 100%; + width: 100%; + margin-left: 0; + } + + body.change-list:not(.model-snapshot) #content #changelist #toolbar .small.quiet { + flex: 1 0 100%; + width: 100%; + white-space: normal; + text-align: left; + } + + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top { + max-width: 100%; + box-sizing: border-box; + } +} + +@media (max-width: 1180px) { + body.change-list #content, + body.change-list #content-main, + body.change-list #changelist, + body.change-list #changelist .changelist-form-container, + body.change-list:not(.model-snapshot) #changelist .changelist-form-container > div { + max-width: 100%; + min-width: 0; + box-sizing: border-box; + } + + body.change-list #changelist .changelist-form-container > div { + gap: 8px; + } + + body.change-list:not(.model-snapshot) #changelist #toolbar #changelist-search > div { + display: grid !important; + grid-template-columns: 24px minmax(0, 1fr) auto; + gap: 7px; + align-items: center; + } + + body.change-list:not(.model-snapshot) #content #changelist #toolbar label[for="searchbar"] { + grid-column: 1; + justify-content: center; + } + + body.change-list:not(.model-snapshot) #content #changelist #toolbar #searchbar { + grid-column: 2; + flex: none !important; + width: 100% !important; + min-width: 0; + } + + body.change-list:not(.model-snapshot) #content #changelist #toolbar form input[type="submit"] { + grid-column: 3; + flex: none !important; + width: auto !important; + min-width: 76px; + white-space: nowrap; + } + + body.change-list:not(.model-snapshot) #content #changelist #toolbar .search-mode-selector { + grid-column: 1 / -1; + display: grid !important; + grid-template-columns: repeat(3, minmax(0, 1fr)); + flex: none !important; + width: 100% !important; + max-width: 100%; + min-width: 0; + gap: 6px !important; + } + + body.change-list:not(.model-snapshot) #content #changelist #toolbar .small.quiet { + grid-column: 1 / -1; + flex: none !important; + width: 100% !important; + max-width: 100%; + min-width: 0; + white-space: normal; + text-align: left; + } + + body.change-list:not(.model-snapshot) #content #changelist #toolbar .search-mode-selector label { + justify-content: center; + min-width: 0 !important; + width: 100%; + padding-left: 8px !important; + padding-right: 8px !important; + } + + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top { + padding: 8px; + gap: 8px; + overflow-x: visible !important; + } + + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .actions-left, + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .actions-right, + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .action-buttons { + width: 100%; + } + + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .actions-right { + justify-content: flex-start; + } + + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .actions-left { + display: grid !important; + grid-template-columns: 1fr; + gap: 8px; + white-space: normal !important; + } + + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .action-buttons { + display: flex !important; + flex-wrap: wrap; + gap: 6px; + } + + body.change-list:not(.model-snapshot) #content #changelist .actions .button { + flex: 1 1 135px; + width: auto; + min-height: 30px; + margin: 0; + justify-content: center; + text-align: center; + white-space: nowrap; + } + + body.change-list:not(.model-snapshot) #content #changelist .actions .button[name="add_tags"], + body.change-list:not(.model-snapshot) #content #changelist .actions .button[name="remove_tags"] { + flex: 0 0 48px; + width: 48px; + } + + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .action-buttons .button[name="resnapshot_snapshot"], + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .action-buttons .button[name="update_snapshots"], + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .action-buttons .button[name="overwrite_snapshots"], + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .action-buttons .button[name="delete_snapshots"] { + margin-left: 0; + margin-right: 0; + } + + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .actions-right { + display: grid !important; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 6px; + } + + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .actions-tags-with-buttons { + width: 100% !important; + max-width: 100% !important; + flex-basis: 100% !important; + min-width: 0; + } + + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .actions-tags-with-buttons .tag-editor-container { + width: auto !important; + max-width: none !important; + flex: 1 1 160px !important; + } + + body.change-list:not(.model-snapshot) #changelist #changelist-form > .actions-top .action-counter { + width: 100%; + white-space: normal; + } + + .model-snapshot.change-list #result_list { + display: block; + min-width: 0; + width: 100%; + border-collapse: separate; + } + + .model-snapshot.change-list #result_list thead { + display: none; + } + + .model-snapshot.change-list #result_list tbody { + display: block; + } + + .model-snapshot.change-list #result_list tbody tr { + display: grid; + grid-template-columns: 28px minmax(0, 1fr); + gap: 5px 8px; + margin: 8px; + padding: 8px; + border: 1px solid #e2e8f0; + border-radius: 8px; + background: #fff; + } + + .model-snapshot.change-list #result_list tbody tr.selected { + background: #fffde7; + border-color: #facc15; + } + + .model-snapshot.change-list #result_list tbody th, + .model-snapshot.change-list #result_list tbody td { + display: block; + width: auto !important; + min-width: 0 !important; + max-width: none !important; + padding: 0 !important; + border: 0 !important; + background: transparent !important; + box-sizing: border-box; + overflow: visible; + } + + .model-snapshot.change-list #result_list tbody td.action-checkbox { + grid-column: 1; + grid-row: 1 / 5; + align-self: start; + padding-top: 3px !important; + } + + .model-snapshot.change-list #result_list tbody th.field-created_at, + .model-snapshot.change-list #result_list tbody td.field-created_at { + grid-column: 2; + grid-row: 1; + font-size: 12px; + color: #64748b; + } + + .model-snapshot.change-list #result_list tbody td.field-preview_icon { + display: none; + } + + .model-snapshot.change-list #result_list tbody td.field-title_str { + grid-column: 2; + grid-row: 2; + } + + .model-snapshot.change-list #result_list tbody td.field-title_str a { + display: block; + max-width: 100%; + overflow-wrap: anywhere; + } + + .model-snapshot.change-list #result_list tbody td.field-tags_inline { + grid-column: 2; + grid-row: 3; + } + + .model-snapshot.change-list #result_list tbody td.field-status_with_progress, + .model-snapshot.change-list #result_list tbody td.field-files, + .model-snapshot.change-list #result_list tbody td.field-size_with_stats { + grid-column: 2; + font-size: 12px; + } + + .model-snapshot.change-list #result_list tbody td.field-status_with_progress { + grid-row: 4; + } + + .model-snapshot.change-list #result_list tbody td.field-files { + grid-row: 5; + white-space: normal; + } + + .model-snapshot.change-list #result_list tbody td.field-size_with_stats { + grid-row: 6; + color: #64748b; + } + + .model-snapshot.change-list #result_list tbody td.field-tags_inline .tag-editor-inline, + .model-snapshot.change-list #result_list tbody td.field-tags_inline .tag-pills-inline { + max-width: 100%; + } +} + /* Filter Sidebar - Improved Layout */ #content #changelist-filter { background: #fff; @@ -410,6 +824,103 @@ body.change-list #content .object-tools { padding-right: 2px; } +body.model-snapshot.change-list #result_list col.snapshot-permissions-col, +body.model-snapshot.change-list #result_list th.column-permissions_badge, +body.model-snapshot.change-list #result_list td.field-permissions_badge { + width: 22px !important; + min-width: 22px !important; + max-width: 22px !important; +} + +body.model-snapshot.change-list #result_list th.column-permissions_badge, +body.model-snapshot.change-list #result_list td.field-permissions_badge { + padding-left: 0 !important; + padding-right: 0 !important; + text-align: center !important; + overflow: visible; +} + +body.model-snapshot.change-list #result_list th.column-permissions_badge .text, +body.model-snapshot.change-list #result_list th.column-permissions_badge .text span { + display: inline-flex !important; + width: 22px !important; + min-width: 22px !important; + justify-content: center !important; + padding: 0 !important; +} + +.snapshot-permissions-quick { + position: relative; + display: inline-flex; + align-items: center; + justify-content: center; +} + +.snapshot-permissions-button { + display: inline-flex; + align-items: center; + justify-content: center; + width: 24px; + height: 24px; + padding: 0; + border: 0; + border-radius: 999px; + background: transparent; + cursor: pointer; +} + +.snapshot-permissions-button:hover .snapshot-permissions-icon, +.snapshot-permissions-button:focus .snapshot-permissions-icon { + box-shadow: 0 0 0 2px #bfdbfe; +} + +.snapshot-permissions-icon { + display: inline-flex; + align-items: center; + justify-content: center; + width: 18px; + height: 18px; + border-radius: 999px; + font-size: 11px; + line-height: 1; +} + +.snapshot-permissions-menu { + position: absolute; + top: 26px; + left: 0; + z-index: 30; + min-width: 126px; + padding: 4px; + border: 1px solid #cbd5e1; + border-radius: 8px; + background: #ffffff; + box-shadow: 0 10px 24px rgba(15, 23, 42, 0.18); +} + +.snapshot-permissions-menu-item { + display: flex; + align-items: center; + gap: 8px; + width: 100%; + height: 30px; + padding: 0 8px; + border: 0; + border-radius: 6px; + background: transparent; + color: #334155; + font-size: 12px; + font-weight: 600; + text-align: left; + cursor: pointer; +} + +.snapshot-permissions-menu-item:hover, +.snapshot-permissions-menu-item.is-active { + background: #eff6ff; + color: #1d4ed8; +} + #content th.column-created_at, #content tbody th.field-created_at { padding-left: 6px; @@ -604,10 +1115,15 @@ body.change-list #content .object-tools { display: inline-flex; align-items: center; justify-content: center; - border-radius: 0; - color: #1f2937; - background: transparent; + border-radius: 3px; + color: #475569; + background: #f8fafc; + border: 1px solid #e2e8f0; box-shadow: none; + font-size: 7px; + font-weight: 700; + line-height: 1; + letter-spacing: 0; } .files-icons .abx-output-icon svg { @@ -925,3 +1441,1847 @@ tbody .output-link:hover {opacity: 1;} transition: width 0.3s ease; border-radius: 4px; } + +/* Snapshot changelist responsive controls/results. Keep this at the end so it wins over Django admin defaults. */ +.model-snapshot.change-list #changelist .changelist-form-container > div { + align-items: stretch; + display: grid !important; + grid-template-columns: minmax(0, 1fr); + gap: 8px; +} + +.model-snapshot.change-list #changelist #toolbar, +.model-snapshot.change-list #changelist #changelist-form > .actions-top { + border: 1px solid #e2e8f0; + background: #ffffff; +} + +.model-snapshot.change-list #changelist #toolbar { + width: 100%; + max-width: 100%; + min-width: 0; + padding: 8px !important; + border-radius: 8px; + box-sizing: border-box; +} + +.model-snapshot.change-list #changelist #toolbar #changelist-search > div { + display: grid !important; + grid-template-columns: minmax(220px, 1fr) auto; + align-items: center; + gap: 8px; + position: relative; +} + +.model-snapshot.change-list #changelist #toolbar label[for="searchbar"] { + grid-column: 1; + grid-row: 1; + position: relative; + left: 10px; + top: 0; + z-index: 1; + width: 18px; + height: 18px; + align-self: center; + opacity: 0.72; + pointer-events: none; +} + +.model-snapshot.change-list #changelist #toolbar label[for="searchbar"] img { + display: block; + width: 18px; + height: 18px; +} + +.model-snapshot.change-list #changelist #toolbar #searchbar { + grid-column: 1; + grid-row: 1; + width: 100% !important; + min-width: 0 !important; + height: 34px; + box-sizing: border-box; + padding-left: 34px; +} + +.model-snapshot.change-list #changelist #toolbar form input[type="submit"] { + grid-column: 2; + height: 34px; + min-width: 78px; + margin: 0; + white-space: nowrap; +} + +.model-snapshot.change-list #changelist #toolbar .search-mode-selector { + grid-column: 1 / -1; + display: flex !important; + flex-wrap: nowrap !important; + gap: 6px !important; + width: 100% !important; + min-width: 0 !important; + margin: 0 !important; +} + +.model-snapshot.change-list #changelist #toolbar .search-mode-selector label { + flex: 1 1 0; + justify-content: center; + min-width: 0 !important; + height: 28px; + padding: 4px 8px !important; + overflow: hidden; +} + +.model-snapshot.change-list #changelist #toolbar .small.quiet { + grid-column: 1 / -1; + padding-left: 1px; + line-height: 1.4; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top { + width: 100%; + max-width: 100%; + min-width: 0; + display: grid !important; + grid-template-columns: minmax(0, 1fr) auto; + gap: 8px; + align-items: start; + padding: 8px !important; + border-radius: 8px; + overflow: visible !important; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-left { + display: grid !important; + grid-template-columns: minmax(0, 1fr); + gap: 8px; + align-items: center; + min-width: 0; + white-space: normal !important; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons { + display: grid !important; + grid-template-columns: minmax(0, 1fr) 34px 34px; + align-items: stretch; + gap: 5px; + min-width: 0 !important; + max-width: none !important; + width: 100% !important; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-editor-container { + min-width: 0 !important; + width: 100% !important; + max-width: none !important; + height: 34px !important; + min-height: 34px !important; + padding: 4px 9px !important; + overflow: hidden; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-inline-input { + min-width: 120px; +} + +.model-snapshot.change-list #content #changelist .actions-tags-with-buttons .button[name="add_tags"], +.model-snapshot.change-list #content #changelist .actions-tags-with-buttons .button[name="remove_tags"] { + width: 34px !important; + min-width: 34px !important; + height: 34px; + padding: 0 !important; + margin: 0 !important; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons { + display: grid !important; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 6px; + min-width: 0; +} + +.model-snapshot.change-list #content #changelist .actions-top .action-buttons .button { + width: 100%; + min-width: 0; + height: 34px; + margin: 0 !important; + justify-content: center; + overflow: hidden; + text-overflow: ellipsis; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary { + display: inline-flex; + align-items: center; + gap: 18px; + min-width: 0; + white-space: nowrap; + color: #64748b; + font-size: 13px; + font-weight: 500; + line-height: 1.3; + letter-spacing: 0; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary a { + font: inherit; + font-weight: 600; + color: #2563eb; + text-decoration: underline; + text-underline-offset: 2px; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary .action-selected-count, +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary .action-total-count { + font: inherit; + color: inherit; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary .all, +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary .clear { + font: inherit; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary .action-counter, +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary .all { + display: none !important; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-total-count .question:not(.hidden) + .action-match-count-static, +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-total-count .question:not(.hidden) ~ .action-total-reset { + display: none; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-right { + display: grid !important; + grid-template-columns: repeat(2, minmax(88px, auto)); + gap: 6px; + margin-left: 0; +} + +.model-snapshot.change-list #content #changelist .actions-top .actions-right .button { + width: 100%; + height: 34px; + margin: 0 !important; + justify-content: center; +} + +@media (max-width: 1180px) { + .model-snapshot.change-list #changelist .changelist-form-container > div { + display: grid !important; + grid-template-columns: minmax(0, 1fr); + gap: 8px; + } + + .model-snapshot.change-list #changelist #toolbar, + .model-snapshot.change-list #changelist #changelist-form > .actions-top, + .model-snapshot.change-list #changelist #changelist-form > .results, + .model-snapshot.change-list #changelist #changelist-form > .changelist-footer { + width: 100%; + max-width: 100%; + box-sizing: border-box; + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top { + grid-template-columns: minmax(0, 1fr); + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-left { + grid-template-columns: minmax(0, 1fr); + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-right { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .model-snapshot.change-list #result_list { + display: block; + min-width: 0 !important; + width: 100% !important; + border-collapse: separate; + } + + .model-snapshot.change-list #result_list thead { + display: none; + } + + .model-snapshot.change-list #result_list tbody { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 8px; + padding: 8px; + } + + .model-snapshot.change-list #result_list tbody tr { + display: grid; + grid-template-columns: 24px minmax(0, 1fr) auto; + grid-template-areas: + "check date status" + "check title title" + "check tags tags" + "check files size"; + gap: 5px 8px; + align-items: start; + min-width: 0; + margin: 0 !important; + padding: 10px !important; + border: 1px solid #e2e8f0; + border-radius: 8px; + background: #ffffff; + } + + .model-snapshot.change-list #result_list tbody tr.selected { + border-color: #facc15; + background: #fffde7; + } + + .model-snapshot.change-list #result_list tbody th, + .model-snapshot.change-list #result_list tbody td { + display: block; + width: auto !important; + min-width: 0 !important; + max-width: none !important; + padding: 0 !important; + border: 0 !important; + background: transparent !important; + box-sizing: border-box; + overflow: visible; + } + + .model-snapshot.change-list #result_list tbody td.action-checkbox { + grid-area: check; + padding-top: 2px !important; + } + + .model-snapshot.change-list #result_list tbody th.field-created_at, + .model-snapshot.change-list #result_list tbody td.field-created_at { + grid-area: date; + font-size: 12px; + color: #64748b; + white-space: nowrap; + } + + .model-snapshot.change-list #result_list tbody td.field-preview_icon { + display: none; + } + + .model-snapshot.change-list #result_list tbody td.field-title_str { + grid-area: title; + } + + .model-snapshot.change-list #result_list tbody td.field-title_str a { + display: block; + max-width: 100%; + overflow-wrap: anywhere; + } + + .model-snapshot.change-list #result_list tbody td.field-tags_inline { + grid-area: tags; + } + + .model-snapshot.change-list #result_list tbody td.field-status_with_progress { + grid-area: status; + justify-self: end; + } + + .model-snapshot.change-list #result_list tbody td.field-files { + grid-area: files; + white-space: normal; + } + + .model-snapshot.change-list #result_list tbody td.field-files .files-icons, + .model-snapshot.change-list #result_list tbody td.field-files .files-icons--compact { + display: flex !important; + flex-wrap: wrap; + max-width: 100%; + gap: 4px; + } + + .model-snapshot.change-list #result_list tbody td.field-size_with_stats { + grid-area: size; + justify-self: end; + color: #64748b; + font-size: 12px; + text-align: right; + white-space: nowrap; + } +} + +@media (max-width: 720px) { + .model-snapshot.change-list #changelist #toolbar #changelist-search > div { + grid-template-columns: minmax(0, 1fr) auto; + } + + .model-snapshot.change-list #changelist #toolbar label[for="searchbar"] { + top: 0; + } + + .model-snapshot.change-list #changelist #toolbar .search-mode-selector { + gap: 5px !important; + } + + .model-snapshot.change-list #changelist #toolbar .search-mode-selector label { + font-size: 12px !important; + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .model-snapshot.change-list #result_list tbody { + grid-template-columns: minmax(0, 1fr); + padding: 8px; + } +} + +.model-snapshot.change-list .cards { + grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); + gap: 12px; + padding: 10px 8px; +} + +.model-snapshot.change-list .cards .card { + min-width: 0; +} + +.model-snapshot.change-list .cards .card .card-info { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 5px 8px; + align-items: center; + padding: 6px 8px; + text-align: left; +} + +.model-snapshot.change-list .cards .card .card-info > a { + grid-column: 1 / -1; + justify-self: center; + max-width: 100%; +} + +.model-snapshot.change-list .cards .card .card-info .timestamp { + display: block; + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.model-snapshot.change-list .cards .card .card-info > div { + min-width: 0; +} + +.model-snapshot.change-list .cards .card .card-info .files-icons { + display: inline-flex !important; + flex-flow: row wrap !important; + justify-content: center !important; + align-items: center !important; + gap: 2px !important; + max-width: 100%; +} + +.model-snapshot.change-list .cards .card .card-info .files-icons a, +.model-snapshot.change-list .cards .card .card-info .files-icons .abx-output-icon { + flex: 0 0 16px; +} + +.model-snapshot.change-list .cards .card .card-info label { + display: inline-grid; + grid-template-columns: auto 18px; + gap: 8px; + align-items: center; + justify-self: end; + width: auto; + height: auto; + margin: 0; +} + +.model-snapshot.change-list .cards .card .card-info input[type=checkbox] { + float: none; + margin: 0; +} + +.model-snapshot.change-list .cards .card .card-thumbnail { + background: #fbfafb; +} + +.model-snapshot.change-list #content #changelist #toolbar #changelist-search > div { + grid-template-columns: minmax(0, 1fr) auto !important; +} + +.model-snapshot.change-list #content #changelist #toolbar label[for="searchbar"] { + grid-column: 1 !important; + grid-row: 1 !important; + width: 18px !important; + min-width: 18px !important; + justify-content: flex-start !important; +} + +.model-snapshot.change-list #content #changelist #toolbar #searchbar { + grid-column: 1 !important; + grid-row: 1 !important; + margin: 0 !important; +} + +.model-snapshot.change-list #content #changelist #toolbar form input[type="submit"] { + grid-column: 2 !important; + grid-row: 1 !important; +} + +@media (max-width: 900px) { + .model-snapshot.change-list .cards { + grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); + } +} + +@media (max-width: 430px) { + .model-snapshot.change-list .cards { + grid-template-columns: minmax(0, 1fr); + padding: 8px 6px; + } + + .model-snapshot.change-list .cards .card .card-info { + grid-template-columns: minmax(0, 1fr) auto; + padding: 6px; + } +} + +@media (max-width: 900px) { + .model-snapshot.change-list #result_list tbody { + grid-template-columns: minmax(0, 1fr); + } +} + +@media (max-width: 430px) { + .model-snapshot.change-list #content { + padding-left: 6px; + padding-right: 6px; + } + + .model-snapshot.change-list #changelist #toolbar, + .model-snapshot.change-list #changelist #changelist-form > .actions-top { + padding: 7px !important; + } + + .model-snapshot.change-list #changelist #toolbar form input[type="submit"] { + min-width: 72px; + padding-left: 10px; + padding-right: 10px; + } + + .model-snapshot.change-list #changelist .actions-tags-with-buttons { + grid-template-columns: minmax(0, 1fr) 34px 34px; + } + + .model-snapshot.change-list #result_list tbody tr { + grid-template-columns: 24px minmax(0, 1fr); + grid-template-areas: + "check date" + "check title" + "check tags" + "check status" + "check files" + "check size"; + } + + .model-snapshot.change-list #result_list tbody td.field-status_with_progress, + .model-snapshot.change-list #result_list tbody td.field-size_with_stats { + justify-self: start; + text-align: left; + } +} + +/* Snapshot changelist toolbar. The controls live in search_form.html so the CSS does not have to move unrelated admin nodes around. */ +.model-snapshot.change-list #toolbar .changelist-toolbar-row { + display: grid; + grid-template-columns: 36px minmax(240px, 1fr) auto; + gap: 8px; + align-items: start; +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon-toggle, +.model-snapshot.change-list #toolbar .filter-pane-toggle { + height: 34px; + margin: 0; + justify-content: center; +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon-toggle { + width: 36px; + min-width: 36px; + padding: 0; + flex: 0 0 36px; +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon { + width: 15px; + height: 15px; + display: grid; + gap: 2px; +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon span { + display: block; + border: 1px solid #64748b; + background: #f8fafc; + border-radius: 2px; +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon-grid { + grid-template-columns: repeat(2, 1fr); + grid-template-rows: repeat(2, 1fr); +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon-list { + grid-template-rows: repeat(3, 1fr); +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon-list span { + border-radius: 999px; +} + +.model-snapshot.change-list #toolbar .filter-pane-toggle { + width: 96px; + min-width: 96px; + padding: 0 10px; + white-space: nowrap; +} + +.model-snapshot.change-list #toolbar #changelist-search { + min-width: 0; +} + +.model-snapshot.change-list #toolbar #changelist-search > div { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 6px; + align-items: center; + position: relative; +} + +.model-snapshot.change-list #toolbar label[for="searchbar"] { + grid-column: 1; + grid-row: 1; + position: absolute; + left: 10px; + z-index: 1; + width: 18px; + height: 18px; + opacity: 0.72; + pointer-events: none; +} + +.model-snapshot.change-list #toolbar label[for="searchbar"] img { + display: block; + width: 18px; + height: 18px; +} + +.model-snapshot.change-list #toolbar #searchbar { + grid-column: 1; + grid-row: 1; + width: 100%; + min-width: 0; + height: 34px; + box-sizing: border-box; + padding-left: 34px; +} + +.model-snapshot.change-list #toolbar form input[type="submit"] { + grid-column: 2; + grid-row: 1; + height: 34px; + margin: 0; + white-space: nowrap; +} + +.model-snapshot.change-list #toolbar .search-mode-selector, +.model-snapshot.change-list #toolbar .small.quiet { + grid-column: 1 / -1; + margin-left: 0 !important; +} + +.model-snapshot.change-list #toolbar .search-mode-selector { + display: flex !important; + flex-wrap: nowrap !important; + gap: 6px !important; + min-width: 0; + width: 100%; + overflow: visible; +} + +.model-snapshot.change-list #toolbar .search-mode-selector .search-mode-option { + flex: 0 0 auto; + justify-content: center; + padding-left: 8px !important; + padding-right: 8px !important; + overflow: visible; + white-space: nowrap; +} + +.model-snapshot.change-list #changelist #toolbar .search-mode-selector .search-mode-meta, +.model-snapshot.change-list #changelist #toolbar .search-mode-selector .search-mode-deep { + min-width: 78px !important; +} + +.model-snapshot.change-list #changelist #toolbar .search-mode-selector .search-mode-contents { + min-width: 112px !important; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top { + padding: 8px; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-left { + display: grid; + grid-template-columns: minmax(220px, 0.8fr) minmax(360px, 1.2fr) auto; + gap: 8px; + align-items: center; + min-width: 0; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons { + display: grid !important; + grid-template-columns: minmax(0, 1fr) 34px 34px; + gap: 0; + align-items: stretch; + min-width: 0; + width: 100%; + border: 1px solid #cbd5e1; + border-radius: 8px; + background: #ffffff; + overflow: hidden; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-editor-container { + width: auto; + max-width: none; + min-height: 32px; + height: 32px; + padding: 3px 9px; + border: 0; + border-radius: 0; + box-shadow: none; + overflow: hidden; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons .button[name="add_tags"], +.model-snapshot.change-list #changelist .actions-tags-with-buttons .button[name="remove_tags"] { + width: 34px; + min-width: 34px; + height: 32px; + padding: 0; + margin: 0; + border: 0; + border-left: 1px solid #cbd5e1; + border-radius: 0; + justify-content: center; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 6px; + min-width: 0; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons .button { + width: 100%; + min-width: 0; + height: 34px; + margin: 0; + justify-content: center; +} + +.model-snapshot.change-list .cards .card .card-info .files-icons { + display: inline-flex !important; + flex-flow: row wrap !important; + justify-content: center !important; + align-items: center !important; + gap: 2px !important; + max-width: 100%; +} + +body.embedded-change-list #content { + padding: 0; +} + +body.embedded-change-list #header, +body.embedded-change-list .breadcrumbs, +body.embedded-change-list #progress, +body.embedded-change-list #progress-monitor, +body.embedded-change-list #footer { + display: none; +} + +body.embedded-change-list #content-main { + width: 100%; +} + +body.embedded-change-list #changelist { + margin: 0; + border-radius: 0; + box-shadow: none; +} + +body.embedded-change-list #changelist-filter { + display: none; +} + +body.embedded-change-list.model-snapshot.change-list #changelist .changelist-form-container { + display: block; +} + +body.embedded-change-list.model-snapshot.change-list #toolbar, +body.embedded-change-list.model-snapshot.change-list #changelist .actions { + border-left: 0; + border-right: 0; + border-radius: 0; +} + +@media (min-width: 760px) and (max-width: 1180px) { + .model-snapshot.change-list:not(.embedded-change-list) #changelist .changelist-form-container { + display: grid; + grid-template-columns: minmax(330px, 0.9fr) minmax(390px, 1.1fr); + gap: 8px; + align-items: stretch; + } + + .model-snapshot.change-list:not(.embedded-change-list) #changelist #toolbar { + grid-column: 1; + grid-row: 1; + align-self: stretch; + box-sizing: border-box; + } + + .model-snapshot.change-list:not(.embedded-change-list) #changelist #changelist-form > .actions-top { + grid-column: 2; + grid-row: 1; + align-self: stretch; + margin: 0; + box-sizing: border-box; + } + + .model-snapshot.change-list:not(.embedded-change-list) #changelist #changelist-form > .cards, + .model-snapshot.change-list:not(.embedded-change-list) #changelist #changelist-form > .results, + .model-snapshot.change-list:not(.embedded-change-list) #changelist #changelist-form > .changelist-footer, + .model-snapshot.change-list:not(.embedded-change-list) #changelist .paginator, + .model-snapshot.change-list:not(.embedded-change-list) #changelist .xfull { + grid-column: 1 / -1; + } + + .model-snapshot.change-list #toolbar .changelist-toolbar-row { + grid-template-columns: 36px minmax(0, 1fr) 96px; + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-left { + grid-template-columns: minmax(0, 1fr); + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-counter { + grid-column: 1 / -1; + font-size: 12px; + } +} + +@media (min-width: 760px) and (max-width: 900px) { + .model-snapshot.change-list:not(.embedded-change-list) #changelist .changelist-form-container { + grid-template-columns: minmax(380px, 1fr) minmax(340px, 0.9fr); + } + + .model-snapshot.change-list #toolbar .changelist-toolbar-row { + grid-template-columns: 36px minmax(0, 1fr) 84px; + } + + .model-snapshot.change-list #toolbar .filter-pane-toggle { + width: 84px; + min-width: 84px; + padding-left: 6px; + padding-right: 6px; + } + + .model-snapshot.change-list #toolbar form input[type="submit"] { + min-width: 72px; + padding-left: 10px; + padding-right: 10px; + } + + .model-snapshot.change-list #toolbar .search-mode-selector label span[aria-hidden="true"] { + display: none; + } + + .model-snapshot.change-list #toolbar .search-mode-selector .search-mode-meta, + .model-snapshot.change-list #toolbar .search-mode-selector .search-mode-deep { + min-width: 72px !important; + } + + .model-snapshot.change-list #toolbar .search-mode-selector .search-mode-contents { + min-width: 98px !important; + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons { + grid-template-columns: repeat(4, minmax(0, 1fr)); + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons .button { + padding-left: 5px; + padding-right: 5px; + font-size: 11px; + } +} + +@media (max-width: 759px) { + .model-snapshot.change-list #toolbar .changelist-toolbar-row { + grid-template-columns: 36px minmax(0, 1fr) 84px; + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-left { + grid-template-columns: minmax(0, 1fr); + } + + .model-snapshot.change-list #toolbar .filter-pane-toggle { + width: 100%; + } + + .model-snapshot.change-list #toolbar .snapshot-view-icon-toggle { + width: 36px; + min-width: 36px; + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} + +/* Snapshot changelist final layout pass. + The toolbar, action bar, results, and footer are children of the wrapper inside + .changelist-form-container, so the row layout has to be applied there. */ +.model-snapshot.change-list #changelist .changelist-form-container > div { + display: grid !important; + grid-template-columns: minmax(480px, 0.8fr) minmax(720px, 1.2fr); + gap: 8px; + align-items: stretch; +} + +.model-snapshot.change-list #changelist #toolbar { + grid-column: 1; + grid-row: 1; + margin: 0 !important; + box-sizing: border-box; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top { + grid-column: 2; + grid-row: 1; + margin: 0; + align-self: stretch; + box-sizing: border-box; +} + +.model-snapshot.change-list #changelist #changelist-form > .cards, +.model-snapshot.change-list #changelist #changelist-form > .results, +.model-snapshot.change-list #changelist #changelist-form > .changelist-footer, +.model-snapshot.change-list #changelist .paginator, +.model-snapshot.change-list #changelist .xfull { + grid-column: 1 / -1; +} + +.model-snapshot.change-list #toolbar .changelist-toolbar-row { + display: grid; + grid-template-columns: 36px minmax(0, 1fr) 96px; + gap: 8px; + align-items: start; +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon-toggle { + display: inline-flex; + width: 36px; + min-width: 36px; + height: 34px; + padding: 0; + margin: 0; + align-items: center; + justify-content: center; +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon { + display: grid; + width: 15px; + height: 15px; + gap: 2px; +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon span { + display: block; + min-width: 0; + min-height: 0; + border: 1px solid #64748b; + border-radius: 2px; + background: #f8fafc; +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon-grid { + grid-template-columns: repeat(2, 1fr); + grid-template-rows: repeat(2, 1fr); +} + +.model-snapshot.change-list #toolbar .snapshot-view-icon-list { + grid-template-rows: repeat(3, 1fr); +} + +.model-snapshot.change-list #toolbar .filter-pane-toggle { + display: inline-flex; + width: 96px; + min-width: 96px; + height: 34px; + padding: 0 10px; + margin: 0; + align-items: center; + justify-content: center; + white-space: nowrap; +} + +.model-snapshot.change-list #toolbar #changelist-search { + min-width: 0; +} + +.model-snapshot.change-list #toolbar #changelist-search > div { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 6px; + align-items: center; + position: relative; +} + +.model-snapshot.change-list #toolbar label[for="searchbar"] { + grid-column: 1; + grid-row: 1; + position: absolute; + left: 10px; + top: 50%; + width: 18px; + height: 18px; + transform: translateY(-50%); + opacity: 0.72; + pointer-events: none; + z-index: 1; +} + +.model-snapshot.change-list #toolbar #searchbar { + grid-column: 1; + grid-row: 1; + width: 100%; + min-width: 0; + height: 34px; + margin: 0; + box-sizing: border-box; + padding-left: 34px; +} + +.model-snapshot.change-list #toolbar form input[type="submit"] { + grid-column: 2; + grid-row: 1; + height: 34px; + margin: 0; + white-space: nowrap; +} + +.model-snapshot.change-list #toolbar .search-mode-selector { + grid-column: 1 / -1; + display: flex !important; + flex-wrap: nowrap !important; + gap: 6px !important; + width: 100%; + min-width: 0; + margin: 0 !important; + overflow: visible; +} + +.model-snapshot.change-list #toolbar .search-mode-selector .search-mode-option { + flex: 0 0 auto; + height: 28px; + justify-content: center; + padding: 4px 8px !important; + white-space: nowrap; + overflow: visible; +} + +.model-snapshot.change-list #toolbar .search-mode-selector .search-mode-meta, +.model-snapshot.change-list #toolbar .search-mode-selector .search-mode-deep { + min-width: 78px !important; +} + +.model-snapshot.change-list #toolbar .search-mode-selector .search-mode-contents { + min-width: 112px !important; +} + +.model-snapshot.change-list #toolbar .small.quiet { + grid-column: 1 / -1; + margin: 0 !important; + line-height: 1.35; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-left { + display: grid !important; + grid-template-columns: minmax(170px, 0.65fr) minmax(280px, 1.35fr) auto; + gap: 8px; + align-items: center; + min-width: 0; + width: 100%; + white-space: normal !important; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons { + display: grid !important; + grid-template-columns: minmax(0, 1fr) 34px 34px; + gap: 0; + align-items: stretch; + min-width: 0 !important; + width: 100% !important; + max-width: none !important; + height: 34px; + box-sizing: border-box; + border: 1px solid #cbd5e1; + border-radius: 8px; + background: #ffffff; + overflow: hidden; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-editor-container { + width: auto !important; + max-width: none !important; + height: 32px !important; + min-height: 0 !important; + padding: 0 9px !important; + box-sizing: border-box; + flex: 1 1 auto !important; + border: 0; + border-radius: 0; + box-shadow: none; + overflow: hidden; + flex-wrap: nowrap; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-inline-input { + min-width: 0; + height: auto; + min-height: 0; + padding: 0; + margin: 0; + border: 0; + border-radius: 0; + box-shadow: none; + background: transparent; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons .button[name="add_tags"], +.model-snapshot.change-list #changelist .actions-tags-with-buttons .button[name="remove_tags"] { + width: 34px; + min-width: 34px; + height: 32px; + padding: 0; + margin: 0; + border: 0; + border-left: 1px solid #cbd5e1; + border-radius: 0; + justify-content: center; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons { + display: grid !important; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 5px; + min-width: 0; + width: 100%; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons .button { + width: 100%; + min-width: 0; + height: 34px; + padding-left: 6px; + padding-right: 6px; + font-size: 12px; + margin: 0; + justify-content: center; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-counter { + align-self: center; + justify-self: start; + white-space: nowrap; +} + +@media (max-width: 1420px) { + .model-snapshot.change-list #changelist .changelist-form-container > div { + grid-template-columns: minmax(420px, 0.8fr) minmax(560px, 1.2fr); + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-left { + grid-template-columns: minmax(170px, 0.55fr) minmax(340px, 1.45fr); + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons .button { + font-size: 11px; + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary { + grid-column: 1 / -1; + } +} + +@media (max-width: 900px) { + .model-snapshot.change-list #changelist .changelist-form-container > div { + grid-template-columns: minmax(0, 1fr); + } + + .model-snapshot.change-list #changelist #toolbar, + .model-snapshot.change-list #changelist #changelist-form > .actions-top { + grid-column: 1; + grid-row: auto; + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} + +@media (max-width: 520px) { + .model-snapshot.change-list #toolbar .changelist-toolbar-row { + grid-template-columns: 36px minmax(0, 1fr) 84px; + } + + .model-snapshot.change-list #toolbar .filter-pane-toggle { + width: 84px; + min-width: 84px; + padding-left: 6px; + padding-right: 6px; + } + + .model-snapshot.change-list #toolbar .search-mode-selector label span[aria-hidden="true"] { + display: none; + } +} + +.snapshot-changelist-widget.model-snapshot.change-list #changelist .changelist-form-container > div { + grid-template-columns: minmax(0, 1fr); +} + +.snapshot-changelist-widget.model-snapshot.change-list #changelist #toolbar, +.snapshot-changelist-widget.model-snapshot.change-list #changelist #changelist-form > .actions-top, +.snapshot-changelist-widget.model-snapshot.change-list #changelist #changelist-form > .cards, +.snapshot-changelist-widget.model-snapshot.change-list #changelist #changelist-form > .results, +.snapshot-changelist-widget.model-snapshot.change-list #changelist #changelist-form > .changelist-footer, +.snapshot-changelist-widget.model-snapshot.change-list #changelist .paginator, +.snapshot-changelist-widget.model-snapshot.change-list #changelist .xfull { + grid-column: 1; + grid-row: auto; +} + +.snapshot-changelist-widget.model-snapshot.change-list #changelist { + display: block; + width: 100%; + max-width: 100%; + min-width: 0; + flex: none; + overflow: hidden; +} + +.snapshot-changelist-widget.model-snapshot.change-list #changelist #changelist-form > .results { + overflow-x: auto; +} + +.snapshot-changelist-widget.model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-left { + grid-template-columns: minmax(0, 1fr); +} + +.snapshot-changelist-widget.model-snapshot.change-list #toolbar .search-mode-selector { + flex-wrap: wrap !important; +} + +.model-snapshot.change-list #toolbar .search-input-wrap { + grid-column: 1; + grid-row: 1; + position: relative; + display: block; + width: 100%; + min-width: 0; +} + +.model-snapshot.change-list #toolbar .search-input-wrap label[for="searchbar"] { + position: absolute; + left: 10px; + top: 50%; + width: 18px; + height: 18px; + margin: 0; + transform: translateY(-50%); + opacity: 0.72; + pointer-events: none; + z-index: 1; +} + +.model-snapshot.change-list #toolbar .search-input-wrap label[for="searchbar"] img { + display: block; + width: 18px; + height: 18px; +} + +.model-snapshot.change-list #toolbar .search-input-wrap #searchbar { + width: 100%; + min-width: 0; + height: 34px; + margin: 0; + box-sizing: border-box; + padding-left: 34px; +} + +.model-snapshot.change-list #content #changelist #toolbar .search-input-wrap label[for="searchbar"] { + grid-column: auto !important; + grid-row: auto !important; + position: absolute !important; + left: 10px !important; + top: 50% !important; + width: 18px !important; + min-width: 18px !important; + height: 18px !important; + margin: 0 !important; + transform: translateY(-50%); + justify-content: center !important; +} + +.model-snapshot.change-list #content #changelist #toolbar .search-input-wrap #searchbar { + grid-column: auto !important; + grid-row: auto !important; + width: 100% !important; + padding-left: 34px !important; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary { + grid-column: 3; + grid-row: 1; + align-self: center; + justify-self: start; +} + +@media (max-width: 1420px) { + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary { + grid-column: 1 / -1; + grid-row: auto; + } +} + +@media (max-width: 900px) { + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary { + overflow-x: auto; + max-width: 100%; + } +} + +#toolbar .search-input-wrap { + position: relative; + display: block; + min-width: 0; +} + +#toolbar .search-input-addon { + position: absolute; + left: 1px; + top: 1px; + bottom: 1px; + z-index: 2; + display: inline-flex; + align-items: center; + gap: 4px; + box-sizing: border-box; + max-width: calc(100% - 2px); + padding: 0 8px; + border-right: 1px solid #cbd5e1; + border-radius: 7px 0 0 7px; + background: #f8fafc; +} + +#toolbar .search-input-addon label[for="searchbar"], +.model-snapshot.change-list #content #changelist #toolbar .search-input-wrap .search-input-addon label[for="searchbar"] { + position: static !important; + display: inline-flex; + align-items: center; + justify-content: center !important; + width: 16px !important; + min-width: 16px !important; + height: 16px !important; + margin: 0 !important; + transform: none !important; + opacity: 0.72; + pointer-events: none; +} + +#toolbar .search-input-addon label[for="searchbar"] img, +.model-snapshot.change-list #content #changelist #toolbar .search-input-wrap .search-input-addon label[for="searchbar"] img { + display: block; + width: 16px; + height: 16px; +} + +#toolbar .search-mode-select { + max-width: 82px; + height: 28px; + padding: 0 18px 0 0; + border: 0; + outline: 0; + background: transparent; + color: #475569; + font: 600 12px/1.2 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; + letter-spacing: 0; +} + +#toolbar .search-mode-select:focus { + color: #0f172a; +} + +#toolbar .search-input-wrap #searchbar, +.model-snapshot.change-list #content #changelist #toolbar .search-input-wrap #searchbar { + padding-left: 42px !important; +} + +#toolbar .search-input-wrap-with-mode #searchbar, +.model-snapshot.change-list #content #changelist #toolbar .search-input-wrap-with-mode #searchbar { + padding-left: 120px !important; +} + +.model-snapshot.change-list #content #changelist #toolbar #changelist-search > div { + grid-template-columns: minmax(0, 1fr) auto !important; +} + +.model-snapshot.change-list #content #changelist #toolbar .search-input-wrap { + grid-column: 1 !important; + grid-row: 1 !important; +} + +.model-snapshot.change-list #content #changelist #toolbar form input[type="submit"] { + grid-column: 2 !important; + grid-row: 1 !important; +} + +#content form .search-input-wrap { + display: block !important; + max-width: none !important; + box-sizing: border-box; +} + +#content form .search-input-addon { + position: absolute !important; + left: 1px !important; + top: 1px !important; + bottom: 1px !important; + width: 112px !important; + padding: 0 8px !important; +} + +#content form .search-mode-select { + position: static !important; + width: auto !important; + max-width: 82px !important; + min-width: 0 !important; + height: 28px !important; + padding-left: 0 !important; +} + +#content form .search-input-wrap #searchbar { + width: 100% !important; + max-width: none !important; + box-sizing: border-box; +} + +.snapshot-changelist-widget.model-snapshot.change-list #toolbar .changelist-toolbar-row { + grid-template-columns: 36px minmax(0, 1fr) auto 96px; +} + +.snapshot-changelist-widget.model-snapshot.change-list #toolbar .search-input-wrap { + grid-column: 2 !important; + grid-row: 1 !important; +} + +.snapshot-changelist-widget.model-snapshot.change-list #toolbar input[type="submit"] { + grid-column: 3 !important; + grid-row: 1 !important; +} + +.snapshot-changelist-widget.model-snapshot.change-list #toolbar .filter-pane-toggle { + grid-column: 4; + grid-row: 1; +} + +/* Snapshot action row visual polish. Keep this last so older Django admin button + and tag-widget rules cannot leak inconsistent type styles into the row. */ +.model-snapshot.change-list #changelist #changelist-form > .actions-top, +.model-snapshot.change-list #changelist #changelist-form > .actions-top * { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; + letter-spacing: 0; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top { + padding: 10px 14px !important; + background: #ffffff; + border-color: #e2e8f0; + border-radius: 8px; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-left { + align-items: center; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons { + height: 40px; + border: 1px solid #cbd5e1; + border-radius: 8px; + background: #ffffff; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-editor-container { + height: 38px !important; + min-height: 0 !important; + padding: 0 12px !important; + align-items: center; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-inline-input { + font-size: 14px; + font-weight: 400; + color: #334155; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons .tag-inline-input::placeholder { + color: #94a3b8; +} + +.model-snapshot.change-list #changelist .actions-tags-with-buttons .button[name="add_tags"], +.model-snapshot.change-list #changelist .actions-tags-with-buttons .button[name="remove_tags"] { + height: 38px; + width: 40px; + min-width: 40px; + font-size: 17px; + font-weight: 600; + line-height: 1; + color: #1f2937; + background: #f6e46f; + border-left: 1px solid #d9c759; + box-shadow: none; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons { + grid-template-columns: repeat(4, minmax(84px, 1fr)); + gap: 6px; + align-items: center; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons .button { + height: 40px; + min-width: 0; + padding: 0 10px; + border-radius: 6px; + border: 0; + box-shadow: none; + font-size: 14px !important; + font-weight: 600; + line-height: 1.2; + color: #102a2c; + text-align: center; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons .button[name="resnapshot_snapshot"] { + background: #4fb5ad; + color: #082f2c; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons .button[name="update_snapshots"] { + background: #a8df57; + color: #17310b; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons .button[name="overwrite_snapshots"] { + background: #f7ad43; + color: #3b2405; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons .button[name="delete_snapshots"] { + background: #e43d79; + color: #fff7fb; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary { + display: inline-flex; + flex-direction: column; + gap: 3px; + align-items: flex-start; + color: #475569; + font-size: 14px; + font-weight: 500; + line-height: 1.2; + white-space: nowrap; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary .action-selected-count, +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary .action-total-count, +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary .clear { + color: #475569; + font-size: 14px; + font-weight: 500; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary.action-summary-select-across .action-total-count { + display: none; +} + +.model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary a { + color: #2563eb; + font-size: 14px; + font-weight: 600; + text-decoration-thickness: 1px; + text-underline-offset: 2px; +} + +@media (max-width: 1180px) { + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons { + grid-template-columns: repeat(4, minmax(78px, 1fr)); + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons .button { + padding: 0 7px; + font-size: 13px !important; + } +} + +/* Snapshot search input group: mode dropdown + search icon inside the field. */ +.model-snapshot.change-list #toolbar #changelist-search > div { + display: grid !important; + grid-template-columns: minmax(0, 1fr) auto; + gap: 8px; + align-items: center; +} + +.model-snapshot.change-list #toolbar .search-input-wrap { + grid-column: 1; + grid-row: 1; + position: relative; + display: grid; + grid-template-columns: auto minmax(0, 1fr); + align-items: center; + width: 100%; + min-width: 0; + height: 40px; + border: 1px solid #cbd5e1; + border-radius: 8px; + background: #ffffff; + overflow: hidden; + box-sizing: border-box; +} + +.model-snapshot.change-list #toolbar .search-input-addon { + position: relative !important; + left: auto !important; + top: auto !important; + bottom: auto !important; + z-index: auto; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 2px; + height: 100%; + width: 46px; + min-width: 46px; + max-width: 46px; + padding: 0; + border-right: 1px solid #e2e8f0; + border-radius: 0; + background: #f8fafc; + box-sizing: border-box; + cursor: pointer; +} + +.model-snapshot.change-list #toolbar .search-input-wrap-with-mode .search-input-addon::before { + content: ""; + position: absolute; + left: 12px; + top: 10px; + z-index: 4; + display: block; + width: 11px; + height: 11px; + border: 2px solid #64748b; + border-radius: 999px; + background: transparent; + pointer-events: none; +} + +.model-snapshot.change-list #toolbar .search-input-wrap-with-mode .search-input-addon::after { + content: ""; + position: absolute; + left: 25px; + top: 24px; + z-index: 4; + display: block; + width: 7px; + height: 2px; + border-radius: 999px; + background: #64748b; + transform: rotate(45deg); + pointer-events: none; +} + +.model-snapshot.change-list #toolbar .search-input-wrap-with-mode .search-input-addon label[for="searchbar"], +.model-snapshot.change-list #content #changelist #toolbar .search-input-wrap-with-mode .search-input-addon label[for="searchbar"] { + position: absolute !important; + z-index: 0; + display: none !important; + align-items: center !important; + justify-content: center !important; + width: 17px !important; + min-width: 17px !important; + height: 17px !important; + margin: 0 !important; + transform: none !important; + opacity: 0; + pointer-events: none; +} + +.model-snapshot.change-list #toolbar .search-input-wrap-with-mode .search-input-addon label[for="searchbar"] img { + width: 17px; + height: 17px; + display: block; +} + +.model-snapshot.change-list #toolbar .search-mode-select { + appearance: none; + -webkit-appearance: none; + position: absolute; + inset: 0 auto 0 0; + width: 46px !important; + min-width: 46px !important; + max-width: 46px !important; + height: 100%; + padding: 0; + border: 0; + border-radius: 0; + background: transparent; + color: #111827; + font-size: 14px; + font-weight: 500; + line-height: 1; + outline: 0; + opacity: 1; + text-indent: -9999px; + cursor: pointer; + z-index: 3; +} + +.model-snapshot.change-list #toolbar .search-mode-select option { + background: #ffffff; + color: #111827; + font-size: 14px; + line-height: 1.4; + text-indent: 0; +} + +.model-snapshot.change-list #toolbar .search-mode-caret { + position: absolute; + right: 9px; + top: 50%; + display: block; + z-index: 2; + color: #64748b; + font-size: 10px; + line-height: 1; + pointer-events: none; + transform: translateY(-45%); +} + +.model-snapshot.change-list #toolbar .search-input-wrap #searchbar, +.model-snapshot.change-list #content #changelist #toolbar .search-input-wrap #searchbar { + grid-column: 2; + grid-row: 1; + width: 100% !important; + min-width: 0; + height: 100%; + margin: 0 !important; + padding: 0 12px !important; + border: 0; + border-radius: 0; + box-shadow: none; + background: #ffffff; + color: #111827; + font-size: 14px; + font-weight: 400; + box-sizing: border-box; +} + +.model-snapshot.change-list #toolbar .search-input-wrap #searchbar:focus { + outline: 0; + box-shadow: inset 0 0 0 1px #93c5fd; +} + +.model-snapshot.change-list #toolbar form input[type="submit"] { + grid-column: 2; + grid-row: 1; + height: 40px; + min-width: 86px; + padding: 0 16px; + margin: 0; + border-radius: 8px; + font-size: 14px; + font-weight: 600; +} + +#content #changelist #toolbar .changelist-search-submit { + border-color: #aa1e55; +} + +body.change-list:not(.model-snapshot) #changelist #toolbar .changelist-search-submit { + flex: 0 0 auto; +} + +.model-snapshot.change-list #toolbar .changelist-search-submit { + grid-column: 2; + grid-row: 1; + height: 40px; + min-width: 86px; + padding: 0 16px; + margin: 0; + border-radius: 8px; + font-size: 14px; + font-weight: 600; + white-space: nowrap; +} + +.snapshot-changelist-widget.model-snapshot.change-list #toolbar .changelist-toolbar-row { + grid-template-columns: minmax(0, 1fr); +} + +.snapshot-changelist-widget.model-snapshot.change-list #toolbar #changelist-search { + grid-column: 1; + grid-row: 1; + display: block !important; + width: 100%; + min-width: 0; +} + +.snapshot-changelist-widget.model-snapshot.change-list #toolbar #changelist-search > div { + display: grid !important; + grid-template-columns: minmax(0, 1fr) auto; + gap: 8px; + align-items: center; + width: 100% !important; + min-width: 0; +} + +.snapshot-changelist-widget.model-snapshot.change-list #toolbar .search-input-wrap { + grid-column: 1 !important; + grid-row: 1 !important; + width: 100% !important; +} + +.snapshot-changelist-widget.model-snapshot.change-list #toolbar .changelist-search-submit { + grid-column: 2 !important; + grid-row: 1 !important; +} diff --git a/archivebox/templates/static/admin/crawls/crawl_admin.js b/archivebox/templates/static/admin/crawls/crawl_admin.js new file mode 100644 index 00000000..6a258c38 --- /dev/null +++ b/archivebox/templates/static/admin/crawls/crawl_admin.js @@ -0,0 +1,16 @@ +(function() { + document.addEventListener('click', function(event) { + var button = event.target.closest('.crawl-resume-row, .crawl-pause-row'); + if (!button) return; + + var form = document.getElementById('changelist-form'); + var actionSelect = form ? form.querySelector('select[name="action"]') : null; + if (!form || !actionSelect) return; + + form.querySelectorAll('input[name="_selected_action"]').forEach(function(checkbox) { + checkbox.checked = checkbox.value === button.getAttribute('data-crawl-id'); + }); + actionSelect.value = button.classList.contains('crawl-pause-row') ? 'pause_selected_crawls' : 'resume_selected_crawls'; + form.submit(); + }); +})(); diff --git a/archivebox/templates/static/admin/crawls/crawl_change.css b/archivebox/templates/static/admin/crawls/crawl_change.css new file mode 100644 index 00000000..c3fc87a6 --- /dev/null +++ b/archivebox/templates/static/admin/crawls/crawl_change.css @@ -0,0 +1,227 @@ +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config { + flex: 1 1 100% !important; + max-width: 100% !important; + min-width: 100% !important; +} + +body.model-crawl.change-form #content { + max-width: none; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview > div { + padding: 14px; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-row { + padding: 10px 0; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-multiline { + display: grid !important; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 14px; + align-items: start; + width: 100%; + max-width: 100% !important; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-multiline > div { + min-width: 0; + margin: 0 !important; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-multiline input, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-multiline select, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-multiline textarea, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-multiline .readonly { + width: 100% !important; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-row.field-max_depth.field-max_urls.field-crawl_max_size .help, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .timezonewarning { + display: none !important; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-row.field-stop_reason_display { + padding: 4px 0; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-row.field-stop_reason_display .readonly { + min-height: 0; + padding: 6px 8px !important; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .field-notes, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .field-tags_editor { + min-width: 0; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-row.field-notes.field-tags_editor .form-multiline { + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .field-url_filters > div, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .field-url_filters #id_url_filters_container { + width: 100%; + max-width: none; + min-width: 0; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config > div, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .form-row, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .field-config, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .field-config > div, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .key-value-editor { + width: 100% !important; + max-width: none !important; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .key-value-rows { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 12px; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .key-value-row { + margin: 0 !important; + padding: 12px; + border: 1px solid #e2e8f0; + border-radius: 8px; + background: #fbfdff; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .kv-inputs { + display: grid !important; + grid-template-columns: minmax(180px, 0.42fr) minmax(260px, 1fr) 34px; + gap: 10px !important; + align-items: center !important; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .kv-key, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .kv-value { + width: 100% !important; + min-width: 0; + flex: none !important; + font-size: 13.5px !important; + line-height: 1.45 !important; + padding: 8px 10px !important; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .kv-help { + margin-top: 7px !important; + color: #526070 !important; + font-size: 12px !important; + line-height: 1.35; +} + +.archivebox-crawl-resume-tool { + display: inline-flex; + align-items: center; + gap: 8px; +} + +.crawl-resume-action-form { + display: inline-flex; + margin: 0; +} + +.crawl-stop-reason, +.crawl-stop-reason-inline { + display: inline-flex; + align-items: center; + gap: 5px; + padding: 3px 8px; + border: 1px solid #cbd5e1; + border-radius: 6px; + background: #f8fafc; + color: #334155; + font-size: 12px; + line-height: 1.25; + white-space: nowrap; +} + +.crawl-stop-reason--empty, +.crawl-resume-muted { + color: #94a3b8; +} + +.crawl-resume-row, +.crawl-resume-submit, +.crawl-pause-row, +.crawl-pause-submit { + cursor: pointer; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots > div, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots .form-row, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots .field-snapshots_changelist, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots .field-snapshots_changelist > div { + flex: 1 1 100% !important; + max-width: 100% !important; + min-width: 100% !important; + width: 100% !important; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots > div { + padding: 0; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots .form-row, +body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots .field-snapshots_changelist { + padding: 0; + border: 0; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots .field-snapshots_changelist > label { + display: none !important; +} + +body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots .readonly { + padding: 0 !important; + border: 0 !important; + background: transparent !important; + font-family: inherit !important; + line-height: inherit !important; +} + +.crawl-snapshots-embed { + width: 100%; + overflow: hidden; + border-radius: 0 0 12px 12px; + background: #fff; +} + +.crawl-snapshots-embed__toolbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + padding: 10px 14px; + border-bottom: 1px solid #e2e8f0; + background: #f8fafc; +} + +.crawl-snapshots-embed__toolbar strong { + color: #334155; + font-size: 13px; +} + +.crawl-snapshots-embed__toolbar .button { + padding: 6px 10px; + font-size: 12px; +} + +@media (max-width: 900px) { + body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-multiline, + body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-row.field-notes.field-tags_editor .form-multiline, + body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .key-value-rows { + grid-template-columns: 1fr; + } + + body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .kv-inputs { + grid-template-columns: 1fr; + align-items: stretch !important; + } +} diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py index f2a4f2bf..19228806 100644 --- a/archivebox/tests/conftest.py +++ b/archivebox/tests/conftest.py @@ -144,6 +144,14 @@ def isolate_test_runtime(tmp_path, monkeypatch): original_popen = subprocess.Popen os.chdir(tmp_path) + def reset_machine_model_caches() -> None: + import archivebox.machine.models as machine_models + + machine_models._CURRENT_MACHINE = None + machine_models._CURRENT_INTERFACE = None + machine_models._CURRENT_PROCESS = None + machine_models._CURRENT_BINARIES.clear() + def guarded_chdir(path: os.PathLike[str] | str) -> None: _assert_not_repo_path(Path(path), label="cwd") original_chdir(path) @@ -163,10 +171,12 @@ def isolate_test_runtime(tmp_path, monkeypatch): os.environ.pop("USERS_DIR", None) os.environ.pop("CRAWL_DIR", None) os.environ.pop("SNAP_DIR", None) + reset_machine_model_caches() try: _assert_safe_runtime_paths(cwd=Path.cwd(), env=os.environ) yield finally: + reset_machine_model_caches() original_chdir(original_cwd) os.environ.clear() os.environ.update(original_env) @@ -325,7 +335,7 @@ def wait_for_archive_outputs( rel_path = candidate.relative_to(snapshot_dir) if rel_path.parts and rel_path.parts[0] == 'responses': continue - if rel_path.name in {"stdout.log", "stderr.log", "cmd.sh"}: + if rel_path.name in {"stdout.log", "stderr.log"}: continue output_rel = str(rel_path) break @@ -436,7 +446,6 @@ def real_archive_with_example(tmp_path_factory, request): "--set", "LISTEN_HOST=archivebox.localhost:8000", "PUBLIC_INDEX=True", - "PUBLIC_SNAPSHOTS=True", "PUBLIC_ADD_VIEW=True", ], cwd=tmp_path, diff --git a/archivebox/tests/orm_helpers.py b/archivebox/tests/orm_helpers.py new file mode 100644 index 00000000..a322e6a2 --- /dev/null +++ b/archivebox/tests/orm_helpers.py @@ -0,0 +1,39 @@ +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +from django.conf import settings +from django.db import connections + + +def archivebox_db_path(path: str | Path = ".") -> Path: + path = Path(path) + return path if path.name == "index.sqlite3" else path / "index.sqlite3" + + +@contextmanager +def use_archivebox_db(path: str | Path = ".") -> Iterator[None]: + connection = connections["default"] + original_name = connection.settings_dict["NAME"] + original_database_name = connections.databases["default"]["NAME"] + original_setting_name = settings.DATABASES["default"]["NAME"] + original_connection = getattr(connections._connections, "default", None) + db_path = str(archivebox_db_path(path)) + + connection.close() + connection.settings_dict["NAME"] = db_path + connections.databases["default"]["NAME"] = db_path + settings.DATABASES["default"]["NAME"] = db_path + if original_connection is not None: + delattr(connections._connections, "default") + try: + yield + finally: + connections["default"].close() + connections.databases["default"]["NAME"] = original_database_name + settings.DATABASES["default"]["NAME"] = original_setting_name + if hasattr(connections._connections, "default"): + delattr(connections._connections, "default") + if original_connection is not None: + original_connection.settings_dict["NAME"] = original_name + setattr(connections._connections, "default", original_connection) diff --git a/archivebox/tests/test_add_view.py b/archivebox/tests/test_add_view.py index 7f20073e..d4b7bf45 100644 --- a/archivebox/tests/test_add_view.py +++ b/archivebox/tests/test_add_view.py @@ -1,12 +1,10 @@ import json -import re - import pytest from django.contrib.auth import get_user_model from django.urls import reverse from archivebox.config.common import get_config -from archivebox.core.models import Tag +from archivebox.core.models import Snapshot, Tag from archivebox.crawls.models import Crawl from archivebox.personas.models import Persona @@ -31,39 +29,45 @@ def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, m monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") response = client.get(reverse("add"), HTTP_HOST=WEB_HOST) - body = response.content.decode() + form = response.context["form"] assert response.status_code == 200 - assert "tag-editor-container" in body - assert 'name="url_filters_allowlist"' in body - assert 'name="url_filters_denylist"' in body - assert "Same domain only" in body - assert 'name="persona"' in body - assert "Overwrite existing snapshots" not in body - assert "Update/retry previously failed URLs" not in body - assert "Index only dry run (add crawl but don't archive yet)" in body - assert 'name="notes"' in body - assert 'name="max_urls"' in body - assert 'name="crawl_max_size"' in body - assert 'name="snapshot_max_size"' in body - assert 'name="delete_after"' in body - assert 'name="crawl_max_concurrent_snapshots"' in body - assert 'Crawl Plugins") - assert "data-url-regex=" in body - assert 'id="url-highlight-layer"' in body - assert 'id="detected-urls-list"' in body - assert "detected-url-toggle-btn" in body - assert "plugin-config-details" in body - assert 'name="plugin_config__wget__WGET_TIMEOUT"' in body - assert 'name="plugin_config__chrome__CHROME_HEADLESS"' in body - assert "personaConfigMap" in body - assert "archiveboxSetPluginConfigValues" in body - assert "el.name === 'config' || el.name.startsWith('plugin_config__')" in body + assert response.context["can_override_crawl_config"] is False + assert form.plugin_groups == [] + assert { + "url", + "tag", + "url_filters", + "persona", + "permissions", + "depth", + "max_urls", + "crawl_max_size", + "crawl_timeout", + "timeout", + "snapshot_max_size", + "delete_after", + "crawl_max_concurrent_snapshots", + "notes", + }.issubset(form.fields) + + +def test_add_view_admin_renders_plugin_config_grid(client, admin_user, monkeypatch): + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") + client.force_login(admin_user) + + response = client.get(reverse("add"), HTTP_HOST=ADMIN_HOST) + form = response.context["form"] + + assert response.status_code == 200 + assert response.context["can_override_crawl_config"] is True + assert form.plugin_groups + assert any(card["config_fields"] for group in form.plugin_groups for card in group["plugins"]) def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_user, monkeypatch): monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") + client.force_login(admin_user) default_persona = Persona.get_or_create_default() default_persona.config = { "COOKIES_FILE": "/tmp/archivebox-default-cookies.txt", @@ -76,36 +80,42 @@ def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_ config={"WGET_TIMEOUT": 88, "CHROME_HEADLESS": False, "COOKIES_FILE": "/tmp/archivebox-private-cookies.txt"}, ) - response = client.get(reverse("add"), HTTP_HOST=WEB_HOST) - body = response.content.decode() - + response = client.get(reverse("add"), HTTP_HOST=ADMIN_HOST) assert response.status_code == 200 - assert "Private" in body - assert "WGET_TIMEOUT" in body - assert "88" in body - assert "CHROME_HEADLESS" in body - assert "YTDLP_COOKIES_FILE" in body - assert "/tmp/archivebox-default-cookies.txt" in body - assert "Default: {COOKIES_FILE}" in body - assert "/admin/environment/binaries/yt-dlp/" in body or "/admin/machine/binary/" in body persona_config_map = json.loads(response.context["persona_config_map_json"]) assert persona_config_map["Default"]["effective_config"]["YTDLP_COOKIES_FILE"] == "/tmp/archivebox-default-cookies.txt" assert persona_config_map["Private"]["effective_config"]["YTDLP_COOKIES_FILE"] == "/tmp/archivebox-private-cookies.txt" +def test_add_view_public_only_lists_public_personas(client, admin_user, monkeypatch): + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") + secret_value = "SHOULD_NOT_LEAK_PUBLIC_PERSONA_SECRET" + default_persona = Persona.get_or_create_default() + default_persona.config = {"PERMISSIONS": "public", "NODE_BINARY": "/secret/node", "TWOCAPTCHA_API_KEY": secret_value} + default_persona.save(update_fields=["config"]) + Persona.objects.create(name="Unlisted", created_by=admin_user, config={"PERMISSIONS": "unlisted"}) + Persona.objects.create(name="Private", created_by=admin_user, config={"PERMISSIONS": "private"}) + + response = client.get(reverse("add"), HTTP_HOST=WEB_HOST) + form = response.context["form"] + persona_config_map = json.loads(response.context["persona_config_map_json"]) + + assert response.status_code == 200 + assert set(form.fields["persona"].queryset.values_list("name", flat=True)) == {"Default"} + assert secret_value.encode() not in response.content + assert set(persona_config_map.keys()) == {"Default"} + assert {"NODE_BINARY", "TWOCAPTCHA_API_KEY"}.isdisjoint(persona_config_map["Default"]["effective_config"]) + + def test_add_view_hides_search_backend_plugins(client, monkeypatch): monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "sqlite") response = client.get(reverse("add"), HTTP_HOST=WEB_HOST) - body = response.content.decode() + form = response.context["form"] assert response.status_code == 200 - assert not re.search(r']*value="search_backend_sqlite"', body) - assert 'data-plugin-name="search_backend_ripgrep"' not in body - assert 'data-plugin-name="search_backend_sonic"' not in body - assert 'data-plugin-name="search_backend_sqlite"' not in body - assert "const requiredSearchPlugin = 'search_backend_sqlite';" in body + assert form.plugin_groups == [] def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_user, monkeypatch): @@ -120,6 +130,8 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_ "depth": "1", "max_urls": "3", "crawl_max_size": "45mb", + "crawl_timeout": "120", + "timeout": "1.5m", "snapshot_max_size": "5mb", "delete_after": "2h", "crawl_max_concurrent_snapshots": "5", @@ -128,23 +140,23 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_ "notes": "Created from /add/", "schedule": "", "persona": "Default", + "permissions": "public", "index_only": "", "config": "{}", }, - HTTP_HOST=WEB_HOST, + HTTP_HOST=ADMIN_HOST, ) - assert response.status_code == 302 + assert response.status_code == 302, response.context["form"].errors if response.context else response.content.decode() crawl = Crawl.objects.order_by("-created_at").first() assert crawl is not None assert crawl.tags_str == "alpha,beta" assert crawl.notes == "Created from /add/" - assert crawl.max_urls == 3 - assert crawl.crawl_max_size == 45 * 1024 * 1024 - assert crawl.snapshot_max_size == 5 * 1024 * 1024 assert crawl.config["CRAWL_MAX_URLS"] == 3 assert crawl.config["CRAWL_MAX_SIZE"] == 45 * 1024 * 1024 + assert crawl.config["CRAWL_TIMEOUT"] == 120 + assert crawl.config["TIMEOUT"] == 90 assert crawl.config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024 assert crawl.config["DELETE_AFTER"] == "2h" assert crawl.delete_at is not None @@ -177,13 +189,14 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin "notes": "", "schedule": "", "persona": "Private", + "permissions": "public", "index_only": "", "config": '{"DEFAULT_PERSONA": "Default"}', }, - HTTP_HOST=WEB_HOST, + HTTP_HOST=ADMIN_HOST, ) - assert response.status_code == 302 + assert response.status_code == 302, response.context["form"].errors if response.context else response.content.decode() crawl = Crawl.objects.order_by("-created_at").first() assert crawl is not None @@ -213,13 +226,14 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatc "notes": "", "schedule": "", "persona": "Default", + "permissions": "public", "index_only": "", "main_plugins": ["wget"], "plugin_config__wget__WGET_TIMEOUT": "77", "plugin_config__wget__WGET_WARC_ENABLED": "false", "config": "{}", }, - HTTP_HOST=WEB_HOST, + HTTP_HOST=ADMIN_HOST, ) assert response.status_code == 302 @@ -231,6 +245,58 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatc assert crawl.config["WGET_WARC_ENABLED"] is False +def test_add_view_public_submission_ignores_plugin_and_custom_config(client, admin_user, monkeypatch): + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") + monkeypatch.setattr("archivebox.services.runner.ensure_background_runner", lambda: True) + + response = client.post( + reverse("add"), + data={ + "url": "https://example.com/public-safe", + "tag": "", + "depth": "0", + "max_urls": "10", + "crawl_max_size": "45mb", + "crawl_timeout": "120", + "timeout": "1.5m", + "snapshot_max_size": "5mb", + "delete_after": "2h", + "crawl_max_concurrent_snapshots": "2", + "url_filters_allowlist": "example.com", + "url_filters_denylist": "cdn.example.com", + "notes": "public add", + "schedule": "daily", + "persona": "Default", + "permissions": "public", + "index_only": "on", + "main_plugins": ["wget"], + "plugin_config__twocaptcha__TWOCAPTCHA_API_KEY": "posted-token", + "plugin_config__wget__WGET_TIMEOUT": "77", + "config": '{"NODE_BINARY": "/tmp/node", "TWOCAPTCHA_API_KEY": "posted-token", "URL_ALLOWLIST": "bad.example.com"}', + }, + HTTP_HOST=WEB_HOST, + ) + + assert response.status_code == 302, response.context["form"].errors if response.context else response.content.decode() + crawl = Crawl.objects.order_by("-created_at").first() + assert crawl is not None + assert crawl.config["CRAWL_MAX_URLS"] == 10 + assert crawl.config["CRAWL_MAX_SIZE"] == 45 * 1024 * 1024 + assert crawl.config["CRAWL_TIMEOUT"] == 120 + assert crawl.config["TIMEOUT"] == 90 + assert crawl.config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024 + assert crawl.config["DELETE_AFTER"] == "2h" + assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 2 + assert crawl.config["URL_ALLOWLIST"] == "example.com" + assert crawl.config["URL_DENYLIST"] == "cdn.example.com" + assert "PLUGINS" not in crawl.config + assert "WGET_TIMEOUT" not in crawl.config + assert "NODE_BINARY" not in crawl.config + assert "TWOCAPTCHA_API_KEY" not in crawl.config + assert "INDEX_ONLY" not in crawl.config + assert crawl.schedule is None + + def test_add_view_starts_background_runner_after_creating_crawl(client, admin_user, monkeypatch): monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") client.force_login(admin_user) @@ -252,10 +318,11 @@ def test_add_view_starts_background_runner_after_creating_crawl(client, admin_us "notes": "", "schedule": "", "persona": "Default", + "permissions": "public", "index_only": "", "config": "{}", }, - HTTP_HOST=WEB_HOST, + HTTP_HOST=ADMIN_HOST, ) assert response.status_code == 302 @@ -288,10 +355,11 @@ def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkey "notes": "", "schedule": "", "persona": "Default", + "permissions": "public", "index_only": "", "config": "{}", }, - HTTP_HOST=WEB_HOST, + HTTP_HOST=ADMIN_HOST, ) assert response.status_code == 302 @@ -334,10 +402,11 @@ def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_us "notes": "", "schedule": "", "persona": "Default", + "permissions": "public", "index_only": "", "config": "{}", }, - HTTP_HOST=WEB_HOST, + HTTP_HOST=ADMIN_HOST, ) assert response.status_code == 302 @@ -356,16 +425,22 @@ def test_add_view_exposes_api_token_for_tag_widget_autocomplete(client, admin_us monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") client.force_login(admin_user) - response = client.get(reverse("add"), HTTP_HOST=WEB_HOST) + response = client.get(reverse("add"), HTTP_HOST=ADMIN_HOST) assert response.status_code == 200 assert b"window.ARCHIVEBOX_API_KEY" in response.content -def test_tags_autocomplete_requires_auth_when_public_snapshots_list_disabled(client, monkeypatch): - monkeypatch.setenv("PUBLIC_SNAPSHOTS_LIST", "false") +def _create_tagged_snapshot(user, *, permissions="public"): + crawl = Crawl.objects.create(urls="https://example.com", created_by=user, config={"PERMISSIONS": permissions}) + snapshot = Snapshot.from_json({"url": "https://example.com", "tags": "archive"}, overrides={"crawl": crawl}) + assert snapshot is not None + return snapshot + + +def test_tags_autocomplete_requires_auth_when_public_index_disabled(client, admin_user, monkeypatch): monkeypatch.setenv("PUBLIC_INDEX", "false") - Tag.objects.create(name="archive") + _create_tagged_snapshot(admin_user) response = client.get( reverse("api-1:tags_autocomplete"), @@ -376,10 +451,11 @@ def test_tags_autocomplete_requires_auth_when_public_snapshots_list_disabled(cli assert response.status_code == 401 -def test_tags_autocomplete_allows_public_access_when_public_snapshots_list_enabled(client, monkeypatch): - monkeypatch.setenv("PUBLIC_SNAPSHOTS_LIST", "true") - monkeypatch.setenv("PUBLIC_INDEX", "false") - Tag.objects.create(name="archive") +def test_tags_autocomplete_lists_only_public_snapshot_tags(client, admin_user, monkeypatch): + monkeypatch.setenv("PUBLIC_INDEX", "true") + _create_tagged_snapshot(admin_user) + _create_tagged_snapshot(admin_user, permissions="unlisted") + Tag.objects.create(name="private-empty") response = client.get( reverse("api-1:tags_autocomplete"), @@ -391,8 +467,7 @@ def test_tags_autocomplete_allows_public_access_when_public_snapshots_list_enabl assert response.json()["tags"][0]["name"] == "archive" -def test_tags_autocomplete_allows_authenticated_user_when_public_snapshots_list_disabled(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_SNAPSHOTS_LIST", "false") +def test_tags_autocomplete_allows_authenticated_user_when_public_index_disabled(client, admin_user, monkeypatch): monkeypatch.setenv("PUBLIC_INDEX", "false") Tag.objects.create(name="archive") client.force_login(admin_user) diff --git a/archivebox/tests/test_admin_views.py b/archivebox/tests/test_admin_views.py index ce531865..5941a4f2 100644 --- a/archivebox/tests/test_admin_views.py +++ b/archivebox/tests/test_admin_views.py @@ -28,6 +28,7 @@ pytestmark = pytest.mark.django_db User = get_user_model() ADMIN_HOST = "admin.archivebox.localhost:8000" PUBLIC_HOST = "public.archivebox.localhost:8000" +WEB_HOST = "web.archivebox.localhost:8000" @pytest.fixture @@ -254,9 +255,11 @@ class TestSnapshotProgressStats: {"name": "video.mp4", "path": "ytdlp/video.mp4", "size": 111}, ] - def test_discover_outputs_falls_back_to_hashes_index_without_filesystem_walk(self, snapshot, monkeypatch): + def test_discover_outputs_falls_back_to_hashes_index_without_filesystem_walk(self, snapshot): """Older snapshots can still render cards from hashes.json when DB output_files are missing.""" - from archivebox.core.models import ArchiveResult, Snapshot + import json + + from archivebox.core.models import ArchiveResult ArchiveResult.objects.create( snapshot=snapshot, @@ -266,31 +269,27 @@ class TestSnapshotProgressStats: output_files={}, ) - monkeypatch.setattr( - Snapshot, - "hashes_index", - property( - lambda self: { + hashes_dir = Path(snapshot.output_dir) / "hashes" + hashes_dir.mkdir(parents=True, exist_ok=True) + (hashes_dir / "hashes.json").write_text( + json.dumps( + { "responses/index.jsonl": {"size": 456}, "responses/all/20260323T073504__GET__example.com__.html": {"size": 789}, "responses/all/20260323T073504__GET__example.com__app.js": {"size": 123}, }, ), - raising=False, + encoding="utf-8", ) - outputs = snapshot.discover_outputs(include_filesystem_fallback=False) + outputs = snapshot.discover_outputs(include_filesystem_fallback=True) assert next(output for output in outputs if output["name"] == "responses")["path"] == ( "responses/all/20260323T073504__GET__example.com__.html" ) - def test_discover_outputs_falls_back_to_filesystem_for_missing_db_and_hashes(self, snapshot, monkeypatch): + def test_discover_outputs_falls_back_to_filesystem_for_missing_db_and_hashes(self, snapshot): """Snapshot page can still recover cards from plugin dirs when DB metadata is missing.""" - from archivebox.core.models import Snapshot - - monkeypatch.setattr(Snapshot, "hashes_index", property(lambda self: {}), raising=False) - responses_dir = Path(snapshot.output_dir) / "responses" (responses_dir / "all").mkdir(parents=True, exist_ok=True) (responses_dir / "index.jsonl").write_text("{}", encoding="utf-8") @@ -1250,7 +1249,7 @@ class TestLiveProgressView: import archivebox.workers.supervisord_util as supervisord_util machine_models._CURRENT_MACHINE = None - monkeypatch.setattr(supervisord_util, "get_existing_supervisord_process", lambda: object()) + monkeypatch.setattr(supervisord_util, "get_existing_supervisord_process", lambda **_kwargs: object()) monkeypatch.setattr( supervisord_util, "get_worker", @@ -1696,6 +1695,41 @@ class TestPublicIndexSearch: response = client.get("/public/", {"q": "public-example.com"}, HTTP_HOST=PUBLIC_HOST) assert response.status_code == 200 + @override_settings(PUBLIC_INDEX=True) + def test_public_index_lists_only_public_snapshots(self, client, admin_user): + from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl + + public_crawl = Crawl.objects.create(urls="https://public.example", created_by=admin_user, config={"PERMISSIONS": "public"}) + unlisted_crawl = Crawl.objects.create(urls="https://unlisted.example", created_by=admin_user, config={"PERMISSIONS": "unlisted"}) + private_crawl = Crawl.objects.create(urls="https://private.example", created_by=admin_user, config={"PERMISSIONS": "private"}) + Snapshot.objects.create(url="https://public.example", title="Public Snapshot", crawl=public_crawl, status=Snapshot.StatusChoices.SEALED) + Snapshot.objects.create(url="https://unlisted.example", title="Unlisted Snapshot", crawl=unlisted_crawl, status=Snapshot.StatusChoices.SEALED) + Snapshot.objects.create(url="https://private.example", title="Private Snapshot", crawl=private_crawl, status=Snapshot.StatusChoices.SEALED) + + response = client.get("/public/", HTTP_HOST=PUBLIC_HOST) + + assert response.status_code == 200 + assert b"Public Snapshot" in response.content + assert b"Unlisted Snapshot" not in response.content + assert b"Private Snapshot" not in response.content + + def test_direct_snapshot_urls_allow_unlisted_but_not_private_for_guests(self, client, admin_user): + from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl + + unlisted_crawl = Crawl.objects.create(urls="https://unlisted.example", created_by=admin_user, config={"PERMISSIONS": "unlisted"}) + private_crawl = Crawl.objects.create(urls="https://private.example", created_by=admin_user, config={"PERMISSIONS": "private"}) + unlisted_snapshot = Snapshot.objects.create(url="https://unlisted.example", crawl=unlisted_crawl, status=Snapshot.StatusChoices.SEALED) + private_snapshot = Snapshot.objects.create(url="https://private.example", crawl=private_crawl, status=Snapshot.StatusChoices.SEALED) + + unlisted_response = client.get(f"/snapshot/{unlisted_snapshot.id}/", HTTP_HOST=WEB_HOST) + private_response = client.get(f"/snapshot/{private_snapshot.id}/", HTTP_HOST=WEB_HOST) + + assert unlisted_response.status_code == 200 + assert private_response.status_code == 302 + assert private_response["Location"].startswith("/admin/login/") + @override_settings(PUBLIC_INDEX=True) def test_public_search_mode_selector_defaults_to_meta_for_ripgrep(self, client, monkeypatch): monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep") diff --git a/archivebox/tests/test_archive_result_service.py b/archivebox/tests/test_archive_result_service.py index e6ba3688..7e276c04 100644 --- a/archivebox/tests/test_archive_result_service.py +++ b/archivebox/tests/test_archive_result_service.py @@ -2,7 +2,6 @@ from pathlib import Path from uuid import uuid4 import pytest -from django.db import connection from abx_dl.events import ArchiveResultEvent, BinaryRequestEvent, ProcessEvent, ProcessStartedEvent @@ -14,8 +13,9 @@ pytestmark = pytest.mark.django_db(transaction=True) def _cleanup_machine_process_rows() -> None: - with connection.cursor() as cursor: - cursor.execute("DELETE FROM machine_process") + from archivebox.machine.models import Process + + Process.objects.all().delete() def _create_snapshot(): diff --git a/archivebox/tests/test_cli_add.py b/archivebox/tests/test_cli_add.py index dc6e144a..018970df 100644 --- a/archivebox/tests/test_cli_add.py +++ b/archivebox/tests/test_cli_add.py @@ -5,10 +5,17 @@ Verify add creates snapshots in DB, crawls, source files, and archive directorie """ import os -import sqlite3 import subprocess from pathlib import Path +import pytest + +from archivebox.core.models import Snapshot +from archivebox.crawls.models import Crawl +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + def _find_snapshot_dir(data_dir: Path, snapshot_id: str) -> Path | None: candidates = {snapshot_id} @@ -35,13 +42,11 @@ def test_add_single_url_creates_snapshot_in_db(tmp_path, process, disable_extrac assert result.returncode == 0 - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshots = c.execute("SELECT url FROM core_snapshot").fetchall() - conn.close() + with use_archivebox_db(tmp_path): + snapshots = list(Snapshot.objects.values_list("url", flat=True)) assert len(snapshots) == 1 - assert snapshots[0][0] == "https://example.com" + assert snapshots[0] == "https://example.com" def test_add_bg_creates_root_snapshot_rows_immediately(tmp_path, process, disable_extractors_dict): @@ -55,14 +60,11 @@ def test_add_bg_creates_root_snapshot_rows_immediately(tmp_path, process, disabl assert result.returncode == 0 - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshots = c.execute("SELECT url, status FROM core_snapshot").fetchall() - conn.close() + with use_archivebox_db(tmp_path): + snapshots = list(Snapshot.objects.values_list("url", "status")) assert len(snapshots) == 1 - assert snapshots[0][0] == "https://example.com" - assert snapshots[0][1] == "queued" + assert snapshots[0] == ("https://example.com", "queued") def test_add_creates_crawl_record(tmp_path, process, disable_extractors_dict): @@ -74,10 +76,8 @@ def test_add_creates_crawl_record(tmp_path, process, disable_extractors_dict): env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - crawl_count = c.execute("SELECT COUNT(*) FROM crawls_crawl").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + crawl_count = Crawl.objects.count() assert crawl_count == 1 @@ -112,15 +112,13 @@ def test_add_multiple_urls_single_command(tmp_path, process, disable_extractors_ assert result.returncode == 0 - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_count = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] - urls = c.execute("SELECT url FROM core_snapshot ORDER BY url").fetchall() - conn.close() + with use_archivebox_db(tmp_path): + snapshot_count = Snapshot.objects.count() + urls = list(Snapshot.objects.order_by("url").values_list("url", flat=True)) assert snapshot_count == 2 - assert urls[0][0] == "https://example.com" - assert urls[1][0] == "https://example.org" + assert urls[0] == "https://example.com" + assert urls[1] == "https://example.org" def test_add_from_file(tmp_path, process, disable_extractors_dict): @@ -143,11 +141,9 @@ def test_add_from_file(tmp_path, process, disable_extractors_dict): assert result.returncode == 0 - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - crawl_count = c.execute("SELECT COUNT(*) FROM crawls_crawl").fetchone()[0] - snapshot_count = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + crawl_count = Crawl.objects.count() + snapshot_count = Snapshot.objects.count() # The file is parsed into two input URLs. assert crawl_count == 1 @@ -208,10 +204,8 @@ def test_add_with_tags(tmp_path, process, disable_extractors_dict): env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - tags_str = c.execute("SELECT tags_str FROM crawls_crawl").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + tags_str = Crawl.objects.values_list("tags_str", flat=True).get() # Tags are stored as a comma-separated string in crawl assert "test" in tags_str or "example" in tags_str @@ -228,15 +222,11 @@ def test_add_records_selected_persona_on_crawl(tmp_path, process, disable_extrac assert result.returncode == 0 - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - persona_id, default_persona = c.execute( - "SELECT persona_id, json_extract(config, '$.DEFAULT_PERSONA') FROM crawls_crawl LIMIT 1", - ).fetchone() - conn.close() + with use_archivebox_db(tmp_path): + crawl = Crawl.objects.get() - assert persona_id - assert default_persona is None + assert crawl.persona_id + assert crawl.config.get("DEFAULT_PERSONA") is None assert (tmp_path / "personas" / "Default" / "chrome_profile").is_dir() @@ -258,15 +248,11 @@ def test_add_records_url_filter_overrides_on_crawl(tmp_path, process, disable_ex assert result.returncode == 0 - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - allowlist, denylist = c.execute( - "SELECT json_extract(config, '$.URL_ALLOWLIST'), json_extract(config, '$.URL_DENYLIST') FROM crawls_crawl LIMIT 1", - ).fetchone() - conn.close() + with use_archivebox_db(tmp_path): + crawl = Crawl.objects.get() - assert allowlist == "example.com,*.example.com" - assert denylist == "static.example.com" + assert crawl.config["URL_ALLOWLIST"] == "example.com,*.example.com" + assert crawl.config["URL_DENYLIST"] == "static.example.com" assert (tmp_path / "personas" / "Default" / "chrome_extensions").is_dir() @@ -292,11 +278,9 @@ def test_add_duplicate_url_creates_separate_crawls(tmp_path, process, disable_ex env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_count = c.execute("SELECT COUNT(*) FROM core_snapshot WHERE url='https://example.com'").fetchone()[0] - crawl_count = c.execute("SELECT COUNT(*) FROM crawls_crawl").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + snapshot_count = Snapshot.objects.filter(url="https://example.com").count() + crawl_count = Crawl.objects.count() # Each add creates a new crawl with its own snapshot assert crawl_count == 2 @@ -334,10 +318,8 @@ def test_add_creates_snapshot_output_directory(tmp_path, process, disable_extrac env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_id = str(c.execute("SELECT id FROM core_snapshot").fetchone()[0]) - conn.close() + with use_archivebox_db(tmp_path): + snapshot_id = str(Snapshot.objects.values_list("id", flat=True).get()) snapshot_dir = _find_snapshot_dir(tmp_path, snapshot_id) assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}" @@ -358,6 +340,7 @@ def test_add_help_shows_depth_and_tag_options(tmp_path, process): assert "--depth" in result.stdout assert "--max-urls" in result.stdout assert "--crawl-max-size" in result.stdout + assert "--crawl-timeout" in result.stdout assert "--snapshot-max-size" in result.stdout assert "--tag" in result.stdout @@ -372,6 +355,7 @@ def test_add_records_max_url_and_size_limits_on_crawl(tmp_path, process, disable "--depth=1", "--max-urls=3", "--crawl-max-size=45mb", + "--crawl-timeout=120", "--snapshot-max-size=5mb", "https://example.com", ], @@ -381,19 +365,15 @@ def test_add_records_max_url_and_size_limits_on_crawl(tmp_path, process, disable assert result.returncode == 0 - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - max_urls, crawl_max_size, snapshot_max_size, config_max_urls, config_crawl_max_size, config_snapshot_max_size = c.execute( - "SELECT max_urls, crawl_max_size, snapshot_max_size, json_extract(config, '$.CRAWL_MAX_URLS'), json_extract(config, '$.CRAWL_MAX_SIZE'), json_extract(config, '$.SNAPSHOT_MAX_SIZE') FROM crawls_crawl LIMIT 1", - ).fetchone() - conn.close() + columns = {field.name for field in Crawl._meta.local_fields} + with use_archivebox_db(tmp_path): + config = Crawl.objects.values_list("config", flat=True).get() or {} - assert max_urls == 3 - assert crawl_max_size == 45 * 1024 * 1024 - assert snapshot_max_size == 5 * 1024 * 1024 - assert config_max_urls == 3 - assert config_crawl_max_size == 45 * 1024 * 1024 - assert config_snapshot_max_size == 5 * 1024 * 1024 + assert {"max_urls", "crawl_max_size", "crawl_timeout", "snapshot_max_size"}.isdisjoint(columns) + assert config["CRAWL_MAX_URLS"] == 3 + assert config["CRAWL_MAX_SIZE"] == 45 * 1024 * 1024 + assert config["CRAWL_TIMEOUT"] == 120 + assert config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024 def test_add_without_args_shows_usage(tmp_path, process): @@ -424,10 +404,8 @@ def test_add_index_only_skips_extraction(tmp_path, process, disable_extractors_d assert result.returncode == 0 # Snapshot should exist but archive results should be minimal - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_count = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + snapshot_count = Snapshot.objects.count() assert snapshot_count == 1 @@ -441,16 +419,9 @@ def test_add_links_snapshot_to_crawl(tmp_path, process, disable_extractors_dict) env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - # Get crawl id - crawl_id = c.execute("SELECT id FROM crawls_crawl").fetchone()[0] - - # Get snapshot's crawl_id - snapshot_crawl = c.execute("SELECT crawl_id FROM core_snapshot").fetchone()[0] - - conn.close() + with use_archivebox_db(tmp_path): + crawl_id = Crawl.objects.values_list("id", flat=True).get() + snapshot_crawl = Snapshot.objects.values_list("crawl_id", flat=True).get() assert snapshot_crawl == crawl_id @@ -464,10 +435,8 @@ def test_add_sets_snapshot_timestamp(tmp_path, process, disable_extractors_dict) env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - timestamp = c.execute("SELECT timestamp FROM core_snapshot").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + timestamp = Snapshot.objects.values_list("timestamp", flat=True).get() assert timestamp is not None assert len(str(timestamp)) > 0 diff --git a/archivebox/tests/test_cli_extract.py b/archivebox/tests/test_cli_extract.py index 0d1e5b00..d0e7eee5 100644 --- a/archivebox/tests/test_cli_extract.py +++ b/archivebox/tests/test_cli_extract.py @@ -5,9 +5,15 @@ Verify extract re-runs extractors on existing snapshots. """ import os -import sqlite3 import subprocess +import pytest + +from archivebox.core.models import Snapshot +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + def test_extract_runs_on_existing_snapshots(tmp_path, process, disable_extractors_dict): """Test that extract command runs on existing snapshots.""" @@ -43,10 +49,8 @@ def test_extract_preserves_snapshot_count(tmp_path, process, disable_extractors_ env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - count_before = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + count_before = Snapshot.objects.count() # Run extract subprocess.run( @@ -56,9 +60,7 @@ def test_extract_preserves_snapshot_count(tmp_path, process, disable_extractors_ timeout=30, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - count_after = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + count_after = Snapshot.objects.count() assert count_after == count_before diff --git a/archivebox/tests/test_cli_extract_input.py b/archivebox/tests/test_cli_extract_input.py index 481d6b85..cfee5d9a 100644 --- a/archivebox/tests/test_cli_extract_input.py +++ b/archivebox/tests/test_cli_extract_input.py @@ -2,9 +2,20 @@ import os import subprocess -import sqlite3 import json +import pytest + +from archivebox.core.models import ArchiveResult, Snapshot +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + + +def _snapshot_id(data_dir): + with use_archivebox_db(data_dir): + return Snapshot.objects.values_list("id", flat=True).first() + def test_extract_runs_on_snapshot_id(tmp_path, process, disable_extractors_dict): """Test that extract command accepts a snapshot ID.""" @@ -17,11 +28,7 @@ def test_extract_runs_on_snapshot_id(tmp_path, process, disable_extractors_dict) env=disable_extractors_dict, ) - # Get the snapshot ID - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_id = c.execute("SELECT id FROM core_snapshot LIMIT 1").fetchone()[0] - conn.close() + snapshot_id = _snapshot_id(tmp_path) # Run extract on the snapshot result = subprocess.run( @@ -46,11 +53,7 @@ def test_extract_with_enabled_extractor_creates_archiveresult(tmp_path, process, env=disable_extractors_dict, ) - # Get the snapshot ID - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_id = c.execute("SELECT id FROM core_snapshot LIMIT 1").fetchone()[0] - conn.close() + snapshot_id = _snapshot_id(tmp_path) # Run extract with title extractor enabled env = disable_extractors_dict.copy() @@ -63,14 +66,8 @@ def test_extract_with_enabled_extractor_creates_archiveresult(tmp_path, process, env=env, ) - # Check for archiveresults (may be queued, not completed with --no-wait) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - count = c.execute( - "SELECT COUNT(*) FROM core_archiveresult WHERE snapshot_id = ?", - (snapshot_id,), - ).fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + count = ArchiveResult.objects.filter(snapshot_id=snapshot_id).count() # May or may not have results depending on timing assert count >= 0 @@ -87,11 +84,7 @@ def test_extract_plugin_option_accepted(tmp_path, process, disable_extractors_di env=disable_extractors_dict, ) - # Get the snapshot ID - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_id = c.execute("SELECT id FROM core_snapshot LIMIT 1").fetchone()[0] - conn.close() + snapshot_id = _snapshot_id(tmp_path) result = subprocess.run( ["archivebox", "extract", "--plugin=title", "--no-wait", str(snapshot_id)], @@ -114,11 +107,7 @@ def test_extract_stdin_snapshot_id(tmp_path, process, disable_extractors_dict): env=disable_extractors_dict, ) - # Get the snapshot ID - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_id = c.execute("SELECT id FROM core_snapshot LIMIT 1").fetchone()[0] - conn.close() + snapshot_id = _snapshot_id(tmp_path) result = subprocess.run( ["archivebox", "extract", "--no-wait"], @@ -143,11 +132,7 @@ def test_extract_stdin_jsonl_input(tmp_path, process, disable_extractors_dict): env=disable_extractors_dict, ) - # Get the snapshot ID - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_id = c.execute("SELECT id FROM core_snapshot LIMIT 1").fetchone()[0] - conn.close() + snapshot_id = _snapshot_id(tmp_path) jsonl_input = json.dumps({"type": "Snapshot", "id": str(snapshot_id)}) + "\n" @@ -185,14 +170,8 @@ def test_extract_pipeline_from_snapshot(tmp_path, process, disable_extractors_di snapshot_proc.wait() - # Check database for snapshot - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot = c.execute( - "SELECT id, url FROM core_snapshot WHERE url = ?", - ("https://example.com",), - ).fetchone() - conn.close() + with use_archivebox_db(tmp_path): + snapshot = Snapshot.objects.filter(url="https://example.com").first() assert snapshot is not None, "Snapshot should be created by pipeline" @@ -213,16 +192,13 @@ def test_extract_multiple_snapshots(tmp_path, process, disable_extractors_dict): env=disable_extractors_dict, ) - # Get all snapshot IDs - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_ids = c.execute("SELECT id FROM core_snapshot").fetchall() - conn.close() + with use_archivebox_db(tmp_path): + snapshot_ids = list(Snapshot.objects.values_list("id", flat=True)) assert len(snapshot_ids) >= 2, "Should have at least 2 snapshots" # Extract from all snapshots - ids_input = "\n".join(str(s[0]) for s in snapshot_ids) + "\n" + ids_input = "\n".join(str(snapshot_id) for snapshot_id in snapshot_ids) + "\n" result = subprocess.run( ["archivebox", "extract", "--no-wait"], input=ids_input, @@ -232,11 +208,8 @@ def test_extract_multiple_snapshots(tmp_path, process, disable_extractors_dict): ) assert result.returncode == 0, result.stderr - # Should not error - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - count = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + count = Snapshot.objects.count() assert count >= 2, "Both snapshots should still exist after extraction" diff --git a/archivebox/tests/test_cli_init.py b/archivebox/tests/test_cli_init.py index eb24d477..21eaec87 100644 --- a/archivebox/tests/test_cli_init.py +++ b/archivebox/tests/test_cli_init.py @@ -5,10 +5,18 @@ Verify init creates correct database schema, filesystem structure, and config. """ import os -import sqlite3 import subprocess +import pytest +from django.db.migrations.recorder import MigrationRecorder + from archivebox.config.common import get_config +from archivebox.core.models import ArchiveResult, Snapshot +from archivebox.crawls.models import Crawl +from archivebox.machine.models import Machine +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) DIR_PERMISSIONS = get_config().OUTPUT_PERMISSIONS.replace("6", "7").replace("4", "5") @@ -87,38 +95,20 @@ def test_init_runs_migrations(tmp_path): os.chdir(tmp_path) subprocess.run(["archivebox", "init"], capture_output=True) - # Check that migrations were applied - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() + with use_archivebox_db(tmp_path): + migration_count = MigrationRecorder.Migration.objects.count() - # Check django_migrations table exists - migrations = c.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name='django_migrations'", - ).fetchall() - assert len(migrations) == 1 - - # Check that some migrations were applied - migration_count = c.execute("SELECT COUNT(*) FROM django_migrations").fetchone()[0] assert migration_count > 0 - conn.close() - def test_init_creates_core_snapshot_table(tmp_path): """Test that init creates core_snapshot table.""" os.chdir(tmp_path) subprocess.run(["archivebox", "init"], capture_output=True) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - # Check core_snapshot table exists - tables = c.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name='core_snapshot'", - ).fetchall() - assert len(tables) == 1 - - conn.close() + assert Snapshot._meta.db_table == "core_snapshot" + with use_archivebox_db(tmp_path): + assert Snapshot.objects.count() == 0 def test_init_creates_crawls_crawl_table(tmp_path): @@ -126,16 +116,9 @@ def test_init_creates_crawls_crawl_table(tmp_path): os.chdir(tmp_path) subprocess.run(["archivebox", "init"], capture_output=True) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - # Check crawls_crawl table exists - tables = c.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name='crawls_crawl'", - ).fetchall() - assert len(tables) == 1 - - conn.close() + assert Crawl._meta.db_table == "crawls_crawl" + with use_archivebox_db(tmp_path): + assert Crawl.objects.count() == 0 def test_init_creates_core_archiveresult_table(tmp_path): @@ -143,16 +126,9 @@ def test_init_creates_core_archiveresult_table(tmp_path): os.chdir(tmp_path) subprocess.run(["archivebox", "init"], capture_output=True) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - # Check core_archiveresult table exists - tables = c.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name='core_archiveresult'", - ).fetchall() - assert len(tables) == 1 - - conn.close() + assert ArchiveResult._meta.db_table == "core_archiveresult" + with use_archivebox_db(tmp_path): + assert ArchiveResult.objects.count() == 0 def test_init_sets_correct_file_permissions(tmp_path): @@ -184,11 +160,9 @@ def test_init_is_idempotent(tmp_path): assert "updating existing ArchiveBox" in result2.stdout or "up-to-date" in result2.stdout.lower() # Database should still be valid - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - count = c.execute("SELECT COUNT(*) FROM django_migrations").fetchone()[0] + with use_archivebox_db(tmp_path): + count = MigrationRecorder.Migration.objects.count() assert count > 0 - conn.close() def test_init_with_existing_data_preserves_snapshots(tmp_path, process, disable_extractors_dict): @@ -203,22 +177,18 @@ def test_init_with_existing_data_preserves_snapshots(tmp_path, process, disable_ ) # Check snapshot was created - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - count_before = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] + with use_archivebox_db(tmp_path): + count_before = Snapshot.objects.count() assert count_before == 1 - conn.close() # Run init again result = subprocess.run(["archivebox", "init"], capture_output=True) assert result.returncode == 0 # Snapshot should still exist - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - count_after = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] + with use_archivebox_db(tmp_path): + count_after = Snapshot.objects.count() assert count_after == count_before - conn.close() def test_init_quick_flag_skips_checks(tmp_path): @@ -238,16 +208,9 @@ def test_init_creates_machine_table(tmp_path): os.chdir(tmp_path) subprocess.run(["archivebox", "init"], capture_output=True) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - # Check machine_machine table exists - tables = c.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name='machine_machine'", - ).fetchall() - conn.close() - - assert len(tables) == 1 + assert Machine._meta.db_table == "machine_machine" + with use_archivebox_db(tmp_path): + Machine.objects.count() def test_init_output_shows_collection_info(tmp_path): diff --git a/archivebox/tests/test_cli_install.py b/archivebox/tests/test_cli_install.py index 3d64c916..adb74ffe 100644 --- a/archivebox/tests/test_cli_install.py +++ b/archivebox/tests/test_cli_install.py @@ -5,10 +5,18 @@ Verify install detects and records binary dependencies in DB. """ import os -import sqlite3 import subprocess from pathlib import Path +import pytest + +from archivebox.core.models import Snapshot +from archivebox.crawls.models import Crawl +from archivebox.machine.models import Binary +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + def test_install_runs_successfully(tmp_path, process): """Test that install command runs without error.""" @@ -34,17 +42,8 @@ def test_install_creates_binary_records_in_db(tmp_path, process): timeout=60, ) - # Check that binary records were created - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - # Check machine_binary table exists - tables = c.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name='machine_binary'", - ).fetchall() - conn.close() - - assert len(tables) == 1 + with use_archivebox_db(tmp_path): + Binary.objects.count() def test_install_dry_run_does_not_install(tmp_path, process): @@ -170,22 +169,14 @@ def test_install_updates_binary_table(tmp_path, process): output = result.stdout + result.stderr assert result.returncode == 0, output - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - binary_counts = dict( - c.execute( - "SELECT status, COUNT(*) FROM machine_binary GROUP BY status", - ).fetchall(), - ) - snapshot_count = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] - sealed_crawls = c.execute( - "SELECT COUNT(*) FROM crawls_crawl WHERE status='sealed'", - ).fetchone()[0] - installed_python = c.execute( - "SELECT COUNT(*) FROM machine_binary WHERE status='installed' AND name='python'", - ).fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + binary_counts = { + status: Binary.objects.filter(status=status).count() + for status in Binary.objects.values_list("status", flat=True).distinct() + } + snapshot_count = Snapshot.objects.count() + sealed_crawls = Crawl.objects.filter(status="sealed").count() + installed_python = Binary.objects.filter(status="installed", name="python").count() assert sealed_crawls == 0 assert snapshot_count == 0 diff --git a/archivebox/tests/test_cli_list.py b/archivebox/tests/test_cli_list.py index 927b2b38..499c090f 100644 --- a/archivebox/tests/test_cli_list.py +++ b/archivebox/tests/test_cli_list.py @@ -6,9 +6,15 @@ Verify list emits snapshot JSONL and applies the documented filters. import json import os -import sqlite3 import subprocess +import pytest + +from archivebox.core.models import Snapshot +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + def _parse_jsonl(stdout: str) -> list[dict]: return [json.loads(line) for line in stdout.splitlines() if line.strip().startswith("{")] @@ -75,15 +81,8 @@ def test_list_filters_by_crawl_id_and_limit(tmp_path, process, disable_extractor check=True, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - crawl_id = str( - c.execute( - "SELECT crawl_id FROM core_snapshot WHERE url = ?", - ("https://example.com",), - ).fetchone()[0], - ) - conn.close() + with use_archivebox_db(tmp_path): + crawl_id = str(Snapshot.objects.values_list("crawl_id", flat=True).get(url="https://example.com")) result = subprocess.run( ["archivebox", "list", "--crawl-id", crawl_id, "--limit", "1"], @@ -109,10 +108,8 @@ def test_list_filters_by_status(tmp_path, process, disable_extractors_dict): check=True, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - status = c.execute("SELECT status FROM core_snapshot LIMIT 1").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + status = Snapshot.objects.values_list("status", flat=True).get() result = subprocess.run( ["archivebox", "list", "--status", status], diff --git a/archivebox/tests/test_cli_piping.py b/archivebox/tests/test_cli_piping.py index 721f82b3..3f9d36a4 100644 --- a/archivebox/tests/test_cli_piping.py +++ b/archivebox/tests/test_cli_piping.py @@ -6,17 +6,22 @@ This file covers both: - subprocess integration for the supported records `archivebox run` consumes """ -import sqlite3 import sys import uuid from io import StringIO -from pathlib import Path +import pytest + +from archivebox.core.models import Snapshot +from archivebox.machine.models import Binary from archivebox.tests.conftest import ( create_test_url, parse_jsonl_output, run_archivebox_cmd, ) +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) PIPE_TEST_ENV = { @@ -46,22 +51,8 @@ def _assert_stdout_is_jsonl_only(stdout: str) -> None: assert all(line.lstrip().startswith("{") for line in lines), stdout -def _sqlite_param(value: object) -> object: - if not isinstance(value, str): - return value - try: - return uuid.UUID(value).hex - except ValueError: - return value - - -def _db_value(data_dir: Path, sql: str, params: tuple[object, ...] = ()) -> object | None: - conn = sqlite3.connect(data_dir / "index.sqlite3") - try: - row = conn.execute(sql, tuple(_sqlite_param(param) for param in params)).fetchone() - finally: - conn.close() - return row[0] if row else None +def _uuid(value: str) -> uuid.UUID: + return uuid.UUID(value) def test_parse_line_accepts_supported_piping_inputs(): @@ -227,11 +218,8 @@ def test_crawl_create_stdout_pipes_into_run(initialized_archive): run_records = parse_jsonl_output(run_stdout) assert any(record.get("type") == "Crawl" and record.get("id") == crawl["id"] for record in run_records) - snapshot_count = _db_value( - initialized_archive, - "SELECT COUNT(*) FROM core_snapshot WHERE crawl_id = ?", - (crawl["id"],), - ) + with use_archivebox_db(initialized_archive): + snapshot_count = Snapshot.objects.filter(crawl_id=_uuid(crawl["id"])).count() assert isinstance(snapshot_count, int) assert snapshot_count >= 1 @@ -272,11 +260,8 @@ def test_snapshot_list_stdout_pipes_into_run(initialized_archive): run_records = parse_jsonl_output(run_stdout) assert any(record.get("type") == "Snapshot" and record.get("id") == snapshot["id"] for record in run_records) - snapshot_status = _db_value( - initialized_archive, - "SELECT status FROM core_snapshot WHERE id = ?", - (snapshot["id"],), - ) + with use_archivebox_db(initialized_archive): + snapshot_status = Snapshot.objects.values_list("status", flat=True).get(pk=_uuid(snapshot["id"])) assert snapshot_status == "sealed" @@ -351,11 +336,8 @@ def test_binary_create_stdout_pipes_into_run(initialized_archive): run_records = parse_jsonl_output(run_stdout) assert any(record.get("type") in {"BinaryRequest", "Binary"} and record.get("id") == binary["id"] for record in run_records) - status = _db_value( - initialized_archive, - "SELECT status FROM machine_binary WHERE id = ?", - (binary["id"],), - ) + with use_archivebox_db(initialized_archive): + status = Binary.objects.values_list("status", flat=True).get(pk=_uuid(binary["id"])) assert status in {"queued", "installed"} @@ -400,9 +382,6 @@ def test_multi_stage_pipeline_into_run(initialized_archive): snapshot = next(record for record in run_records if record.get("type") == "Snapshot") assert any(record.get("type") == "ArchiveResult" for record in run_records) - snapshot_status = _db_value( - initialized_archive, - "SELECT status FROM core_snapshot WHERE id = ?", - (snapshot["id"],), - ) + with use_archivebox_db(initialized_archive): + snapshot_status = Snapshot.objects.values_list("status", flat=True).get(pk=_uuid(snapshot["id"])) assert snapshot_status == "sealed" diff --git a/archivebox/tests/test_cli_real_flows.py b/archivebox/tests/test_cli_real_flows.py index 6a4cc908..7d15f559 100644 --- a/archivebox/tests/test_cli_real_flows.py +++ b/archivebox/tests/test_cli_real_flows.py @@ -3,16 +3,25 @@ import json import os +import re import signal -import sqlite3 +import socket import subprocess import sys import time +from pathlib import Path import pytest +from archivebox.core.models import ArchiveResult, Snapshot +from archivebox.crawls.models import Crawl +from archivebox.machine.models import Process +from archivebox.tests.orm_helpers import use_archivebox_db + from .conftest import _find_system_browser +pytestmark = pytest.mark.django_db(transaction=True) + def _pid_is_alive(pid: int) -> bool: try: @@ -52,6 +61,256 @@ def _cleanup_process_group(group_pid: int | None, *child_pids: int | None) -> No pass +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _live_exit_env(data_dir, *, plugins_root=None, extra=None): + env = os.environ.copy() + env.update( + { + "DATA_DIR": str(data_dir), + "USE_COLOR": "false", + "SHOW_PROGRESS": "false", + "SAVE_ARCHIVEDOTORG": "false", + "SAVE_FAVICON": "false", + "SAVE_HEADERS": "false", + "SAVE_TITLE": "false", + "SAVE_READABILITY": "false", + "SAVE_SINGLEFILE": "false", + "SAVE_MERCURY": "false", + "SAVE_SCREENSHOT": "false", + "SAVE_PDF": "false", + "SAVE_DOM": "false", + "SAVE_GIT": "false", + "SAVE_YTDLP": "false", + "TIMEOUT": "60", + "WGET_TIMEOUT": "45", + "CRAWL_MAX_CONCURRENT_SNAPSHOTS": "1", + "PARSE_HTML_URLS_ENABLED": "true", + "PARSE_DOM_OUTLINKS_ENABLED": "false", + "SEARCH_BACKEND_ENGINE": "sqlite", + }, + ) + if plugins_root is not None: + env["ABX_PLUGINS_DIR"] = str(plugins_root) + if extra: + env.update(extra) + return env + + +def _wait_for_port(host: str, port: int, *, timeout: float = 30.0) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection((host, port), timeout=0.25): + return + except OSError: + time.sleep(0.1) + raise AssertionError(f"server did not listen on {host}:{port}") + + +def _wait_for_log(log_path: Path, text: str, *, timeout: float = 30.0) -> str: + deadline = time.time() + timeout + content = "" + while time.time() < deadline: + if log_path.exists(): + content = log_path.read_text(encoding="utf-8", errors="replace") + if text in content: + return content + time.sleep(0.1) + raise AssertionError(f"timed out waiting for {text!r} in {log_path}:\n{content}") + + +def _wait_for_log_count(log_path: Path, text: str, count: int, *, timeout: float = 30.0) -> str: + deadline = time.time() + timeout + content = "" + while time.time() < deadline: + if log_path.exists(): + content = log_path.read_text(encoding="utf-8", errors="replace") + if content.count(text) >= count: + return content + time.sleep(0.1) + raise AssertionError(f"timed out waiting for {count} occurrences of {text!r} in {log_path}:\n{content}") + + +def _wait_for_pid_to_disappear(pid: int, *, timeout: float = 20.0) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + if not _pid_is_alive(pid): + return + time.sleep(0.1) + raise AssertionError(f"PID {pid} is still running") + + +def _supervisor_pid_from_log(log_path: Path) -> int: + content = log_path.read_text(encoding="utf-8", errors="replace") + matches = re.findall(r"Supervisord connected \(pid=(\d+)\)", content) + assert matches, content + return int(matches[-1]) + + +def _worker_pid_from_log(log_path: Path, worker_name: str) -> int: + content = log_path.read_text(encoding="utf-8", errors="replace") + matches = re.findall(rf"Worker {re.escape(worker_name)}: started RUNNING \(pid (\d+),", content) + assert matches, content + return int(matches[-1]) + + +def _pgrep_data_dir(data_dir) -> list[str]: + result = subprocess.run(["pgrep", "-af", str(data_dir)], capture_output=True, text=True, timeout=5) + return [line for line in result.stdout.splitlines() if "pgrep -af" not in line] + + +def _assert_no_processes_for_data_dir(data_dir, *, timeout: float = 10.0) -> None: + deadline = time.time() + timeout + remaining: list[str] = [] + while time.time() < deadline: + remaining = _pgrep_data_dir(data_dir) + if not remaining: + return + time.sleep(0.25) + raise AssertionError("processes still reference test DATA_DIR:\n" + "\n".join(remaining)) + + +def _kill_processes_for_data_dir(data_dir) -> None: + for line in _pgrep_data_dir(data_dir): + try: + pid = int(line.split(None, 1)[0]) + except (IndexError, ValueError): + continue + if pid != os.getpid(): + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def _start_server(data_dir, *, port: int, log_name: str, env: dict[str, str] | None = None) -> tuple[subprocess.Popen[str], Path]: + log_path = data_dir / log_name + log = log_path.open("w", encoding="utf-8") + proc = subprocess.Popen( + [sys.executable, "-m", "archivebox", "server", f"127.0.0.1:{port}"], + cwd=data_dir, + env=env or _live_exit_env(data_dir), + stdout=log, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + log.close() + _wait_for_port("127.0.0.1", port) + _wait_for_log(log_path, "Tailing worker logs", timeout=30.0) + return proc, log_path + + +def _stop_process(proc: subprocess.Popen[str], sig=signal.SIGTERM, *, timeout: float = 15.0) -> str: + if proc.poll() is None: + try: + os.killpg(proc.pid, sig) + except (ProcessLookupError, OSError): + try: + os.kill(proc.pid, sig) + except ProcessLookupError: + pass + try: + stdout, _stderr = proc.communicate(timeout=timeout) + return stdout or "" + except subprocess.TimeoutExpired: + try: + os.killpg(proc.pid, signal.SIGKILL) + except (ProcessLookupError, OSError): + try: + os.kill(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, _stderr = proc.communicate(timeout=5) + return stdout or "" + + +def _write_slow_snapshot_plugin(plugins_root, marker_dir): + plugin_dir = plugins_root / "slow_exit" + plugin_dir.mkdir(parents=True, exist_ok=True) + hook = plugin_dir / "on_Snapshot__09_slow_exit.finite.bg.sh" + hook.write_text( + "\n".join( + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + f"marker_dir={str(marker_dir)!r}", + "mkdir -p \"$marker_dir\"", + "echo $$ >> \"$marker_dir/hook-pids.txt\"", + "touch \"$marker_dir/hook-started\"", + "trap 'touch \"$marker_dir/hook-stopped\"; exit 0' TERM INT HUP", + "while true; do sleep 1; done", + "", + ], + ), + encoding="utf-8", + ) + hook.chmod(0o755) + return plugin_dir + + +def _wait_for_crawl_state(data_dir, predicate, *, timeout: float = 30.0): + deadline = time.time() + timeout + last = None + while time.time() < deadline: + with use_archivebox_db(data_dir): + last = { + "crawls": list(Crawl.objects.order_by("created_at").values("id", "status", "retry_at")), + "snapshots": list(Snapshot.objects.order_by("created_at").values("id", "url", "status", "retry_at")), + "results": list(ArchiveResult.objects.order_by("created_at").values("id", "plugin", "status")), + "processes": list(Process.objects.order_by("created_at").values("id", "process_type", "status", "pid", "cmd")), + } + if predicate(last): + return last + time.sleep(0.25) + raise AssertionError(f"timed out waiting for crawl state, last={last}") + + +def _wait_for_hook_runs(marker_dir: Path, count: int, *, timeout: float = 45.0) -> list[int]: + pid_file = marker_dir / "hook-pids.txt" + deadline = time.time() + timeout + pids: list[int] = [] + while time.time() < deadline: + if pid_file.exists(): + pids = [int(line.strip()) for line in pid_file.read_text().splitlines() if line.strip()] + if len(pids) >= count: + return pids + time.sleep(0.25) + raise AssertionError(f"timed out waiting for {count} slow hook runs, got {pids}") + + +def _start_live_add(data_dir, env, *, url="https://example.com", max_urls="2", log_name="archivebox-add.log") -> tuple[subprocess.Popen[str], Path]: + log_path = data_dir / log_name + log = log_path.open("w", encoding="utf-8") + urls = [url] if isinstance(url, str) else url + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "archivebox", + "add", + "--depth=1", + f"--max-urls={max_urls}", + "--crawl-max-size=50mb", + "--plugins=wget,parse_html_urls,slow_exit", + *urls, + ], + cwd=data_dir, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + log.close() + return proc, log_path + + @pytest.mark.timeout(90) def test_cli_run_signal_cleans_background_hook_process_group(tmp_path, process): os.chdir(tmp_path) @@ -175,6 +434,320 @@ def test_cli_run_signal_cleans_background_hook_process_group(tmp_path, process): _cleanup_process_group(foreground_pid) +@pytest.mark.timeout(300) +@pytest.mark.parametrize( + ("stop_signal", "expected_notice"), + [ + (signal.SIGHUP, "Got SIGHUP"), + (signal.SIGINT, "Got SIGINT"), + (signal.SIGTERM, "Got SIGTERM"), + (signal.SIGKILL, None), + ], +) +def test_live_server_signal_exit_and_resume_uses_existing_supervisor_state(tmp_path, process, stop_signal, expected_notice): + os.chdir(tmp_path) + assert process.returncode == 0, process.stderr + + env = _live_exit_env(tmp_path) + port = _free_port() + server = None + resumed = None + try: + server, server_log = _start_server(tmp_path, port=port, log_name=f"server-{stop_signal.name}.log", env=env) + + os.kill(server.pid, stop_signal) + try: + server.wait(timeout=20 if stop_signal != signal.SIGKILL else 5) + except subprocess.TimeoutExpired: + os.kill(server.pid, signal.SIGKILL) + server.wait(timeout=5) + + if expected_notice: + log_text = server_log.read_text(encoding="utf-8", errors="replace") + assert expected_notice in log_text + assert "ArchiveBox server shut down gracefully" in log_text + _assert_no_processes_for_data_dir(tmp_path, timeout=12) + + resumed, resumed_log = _start_server(tmp_path, port=port, log_name=f"server-{stop_signal.name}-resumed.log", env=env) + status = subprocess.run( + [sys.executable, "-m", "archivebox", "status"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=60, + ) + assert status.returncode == 0, status.stderr or status.stdout + + os.kill(resumed.pid, signal.SIGTERM) + resumed.wait(timeout=20) + resumed_text = resumed_log.read_text(encoding="utf-8", errors="replace") + assert "Got SIGTERM" in resumed_text + assert "ArchiveBox server shut down gracefully" in resumed_text + _assert_no_processes_for_data_dir(tmp_path, timeout=12) + finally: + for proc in (server, resumed): + if proc is not None and proc.poll() is None: + _stop_process(proc, signal.SIGKILL) + _kill_processes_for_data_dir(tmp_path) + + +@pytest.mark.timeout(240) +def test_live_second_server_takes_over_existing_server_parent(tmp_path, process): + os.chdir(tmp_path) + assert process.returncode == 0, process.stderr + + env = _live_exit_env(tmp_path) + port = _free_port() + first = None + second = None + try: + first, first_log = _start_server(tmp_path, port=port, log_name="server-first.log", env=env) + second, second_log = _start_server(tmp_path, port=port, log_name="server-second.log", env=env) + + assert first.poll() is None + first_text = first_log.read_text(encoding="utf-8", errors="replace") + second_text = second_log.read_text(encoding="utf-8", errors="replace") + assert "Newer ArchiveBox server parent took over; standing by." in first_text + assert "is now running the orchestrator and server" in second_text + + status = subprocess.run( + [sys.executable, "-m", "archivebox", "status"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=60, + ) + assert status.returncode == 0, status.stderr or status.stdout + + first_takeovers = first_log.read_text(encoding="utf-8", errors="replace").count("is now running the orchestrator and server") + _stop_process(second, signal.SIGTERM) + second = None + _wait_for_log_count(first_log, "is now running the orchestrator and server", first_takeovers + 1, timeout=35) + assert first.poll() is None + finally: + if second is not None and second.poll() is None: + _stop_process(second, signal.SIGTERM) + if first is not None and first.poll() is None: + _stop_process(first, signal.SIGKILL) + _kill_processes_for_data_dir(tmp_path) + _assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(420) +def test_live_repeated_server_startups_take_over_cleanly(tmp_path, process): + os.chdir(tmp_path) + assert process.returncode == 0, process.stderr + + env = _live_exit_env(tmp_path) + port = _free_port() + servers: list[subprocess.Popen[str]] = [] + server_pids: list[int] = [] + daphne_pids: list[int] = [] + runner_pids: list[int] = [] + try: + for index in range(5): + server, log_path = _start_server(tmp_path, port=port, log_name=f"server-chaos-{index}.log", env=env) + servers.append(server) + server_pids.append(server.pid) + daphne_pids.append(_worker_pid_from_log(log_path, "worker_daphne")) + runner_pids.append(_worker_pid_from_log(log_path, "worker_runner")) + + if index > 0: + previous_server = servers[index - 1] + previous_log = (tmp_path / f"server-chaos-{index - 1}.log").read_text(encoding="utf-8", errors="replace") + current_log = log_path.read_text(encoding="utf-8", errors="replace") + assert previous_server.poll() is None + assert _pid_is_alive(server_pids[index - 1]) + assert "Newer ArchiveBox server parent took over; standing by." in previous_log + assert "is now running the orchestrator and server" in current_log + _wait_for_pid_to_disappear(daphne_pids[index - 1], timeout=15) + _wait_for_pid_to_disappear(runner_pids[index - 1], timeout=15) + + status = subprocess.run( + [sys.executable, "-m", "archivebox", "status"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=60, + ) + assert status.returncode == 0, status.stderr or status.stdout + time.sleep(5) + + assert servers[-1].poll() is None + assert all(server.poll() is None for server in servers) + listener = subprocess.run( + ["lsof", "-nP", f"-iTCP:{port}", "-sTCP:LISTEN"], + capture_output=True, + text=True, + timeout=10, + ) + assert listener.returncode == 0, listener.stderr or listener.stdout + assert listener.stdout.count(f":{port} (LISTEN)") == 1 + + previous_log_path = tmp_path / "server-chaos-3.log" + previous_takeovers = previous_log_path.read_text(encoding="utf-8", errors="replace").count("is now running the orchestrator and server") + _stop_process(servers[-1], signal.SIGTERM) + _wait_for_log_count(previous_log_path, "is now running the orchestrator and server", previous_takeovers + 1, timeout=35) + assert servers[3].poll() is None + finally: + for server in reversed(servers): + if server.poll() is None: + _stop_process(server, signal.SIGTERM) + _kill_processes_for_data_dir(tmp_path) + _assert_no_processes_for_data_dir(tmp_path, timeout=12) + + +@pytest.mark.timeout(240) +def test_live_servers_in_different_data_dirs_do_not_interfere(tmp_path, process): + os.chdir(tmp_path) + assert process.returncode == 0, process.stderr + + first_data_dir = tmp_path + second_data_dir = tmp_path.parent / f"{tmp_path.name}-second" + second_data_dir.mkdir() + second_env = _live_exit_env(second_data_dir) + second_init = subprocess.run( + [sys.executable, "-m", "archivebox", "init"], + cwd=second_data_dir, + env=second_env, + capture_output=True, + text=True, + timeout=90, + ) + assert second_init.returncode == 0, second_init.stderr or second_init.stdout + + first_port = _free_port() + second_port = _free_port() + first = None + second = None + first_resumed = None + try: + first = _start_server(first_data_dir, port=first_port, log_name="server-first-data-dir.log", env=_live_exit_env(first_data_dir))[0] + second = _start_server(second_data_dir, port=second_port, log_name="server-second-data-dir.log", env=second_env)[0] + + first_status = subprocess.run( + [sys.executable, "-m", "archivebox", "status"], + cwd=first_data_dir, + env=_live_exit_env(first_data_dir), + capture_output=True, + text=True, + timeout=60, + ) + second_status = subprocess.run( + [sys.executable, "-m", "archivebox", "status"], + cwd=second_data_dir, + env=second_env, + capture_output=True, + text=True, + timeout=60, + ) + assert first_status.returncode == 0, first_status.stderr or first_status.stdout + assert second_status.returncode == 0, second_status.stderr or second_status.stdout + + _stop_process(first, signal.SIGTERM) + first = None + assert second.poll() is None, "stopping one DATA_DIR server must not stop another DATA_DIR server" + + first_resumed = _start_server(first_data_dir, port=first_port, log_name="server-first-data-dir-resumed.log", env=_live_exit_env(first_data_dir))[0] + assert second.poll() is None, "restarting one DATA_DIR server must not take over another DATA_DIR supervisor" + finally: + for proc in (first, first_resumed, second): + if proc is not None and proc.poll() is None: + _stop_process(proc, signal.SIGTERM) + _kill_processes_for_data_dir(first_data_dir) + _kill_processes_for_data_dir(second_data_dir) + _assert_no_processes_for_data_dir(first_data_dir, timeout=12) + _assert_no_processes_for_data_dir(second_data_dir, timeout=12) + + +@pytest.mark.timeout(420) +def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, process): + os.chdir(tmp_path) + assert process.returncode == 0, process.stderr + + plugins_root = tmp_path / "runtime_plugins" + marker_dir = tmp_path / "slow-plugin-markers" + _write_slow_snapshot_plugin(plugins_root, marker_dir) + env = _live_exit_env(tmp_path, plugins_root=plugins_root) + port = _free_port() + server = None + server2 = None + server3 = None + add_proc = None + add_proc2 = None + try: + server, server_log = _start_server(tmp_path, port=port, log_name="server-add-owner-1.log", env=env) + supervisor_pid_before = _supervisor_pid_from_log(server_log) + + update_result = subprocess.run( + [sys.executable, "-m", "archivebox", "update", "--index-only", "--batch-size=10"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=90, + ) + assert update_result.returncode == 0, update_result.stderr or update_result.stdout + assert server.poll() is None + assert _pid_is_alive(supervisor_pid_before) + assert _supervisor_pid_from_log(server_log) == supervisor_pid_before + + add_proc, add_log = _start_live_add(tmp_path, env, url=["https://example.com", "https://blog.sweeting.me"], log_name="archivebox-add-1.log") + _wait_for_hook_runs(marker_dir, 1) + _wait_for_crawl_state( + tmp_path, + lambda state: any(snapshot["status"] == Snapshot.StatusChoices.STARTED for snapshot in state["snapshots"]), + timeout=30, + ) + + os.kill(server.pid, signal.SIGTERM) + server.wait(timeout=20) + assert add_proc.poll() is None, "foreground add should keep owning its crawl after the server exits" + assert "Got SIGTERM" in server_log.read_text(encoding="utf-8", errors="replace") + + server2, _server2_log = _start_server(tmp_path, port=port, log_name="server-add-owner-2.log", env=env) + os.killpg(add_proc.pid, signal.SIGTERM) + add_proc.wait(timeout=30) + add_output = add_log.read_text(encoding="utf-8", errors="replace") + assert "Runner error" not in add_output + _wait_for_hook_runs(marker_dir, 2, timeout=60) + _wait_for_crawl_state( + tmp_path, + lambda state: any(crawl["status"] in (Crawl.StatusChoices.STARTED, Crawl.StatusChoices.QUEUED) for crawl in state["crawls"]) + and any(result["plugin"] == "slow_exit" for result in state["results"]), + timeout=30, + ) + + add_proc2, add_log2 = _start_live_add(tmp_path, env, url="https://example.com/?exit-resume=2", max_urls="1", log_name="archivebox-add-2.log") + _wait_for_hook_runs(marker_dir, 3, timeout=60) + os.killpg(add_proc2.pid, signal.SIGTERM) + os.kill(server2.pid, signal.SIGTERM) + add_proc2.wait(timeout=30) + add_output2 = add_log2.read_text(encoding="utf-8", errors="replace") + server2.wait(timeout=20) + assert "Runner error" not in add_output2 + + server3, _server3_log = _start_server(tmp_path, port=port, log_name="server-add-owner-3.log", env=env) + _wait_for_hook_runs(marker_dir, 4, timeout=70) + + with use_archivebox_db(tmp_path): + crawls = list(Crawl.objects.order_by("created_at").values_list("status", "retry_at")) + snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "status", "retry_at")) + failed_results = list(ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.FAILED).values_list("plugin", "output_str")) + assert crawls + assert snapshots + assert not failed_results + finally: + for proc in (add_proc, add_proc2, server, server2, server3): + if proc is not None and proc.poll() is None: + _stop_process(proc, signal.SIGTERM, timeout=10) + _kill_processes_for_data_dir(tmp_path) + _assert_no_processes_for_data_dir(tmp_path, timeout=12) + + @pytest.mark.timeout(180) def test_cli_add_real_urls_with_options_writes_inspectable_outputs(tmp_path, process): os.chdir(tmp_path) @@ -284,39 +857,28 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(tmp_path, pro listed = [json.loads(line) for line in list_result.stdout.splitlines() if line.strip()] assert {item["url"] for item in listed} >= set(wget_urls) - conn = sqlite3.connect(tmp_path / "index.sqlite3") - try: - crawl = conn.execute( - "SELECT max_depth, max_urls, crawl_max_size, snapshot_max_size, tags_str, config FROM crawls_crawl ORDER BY created_at DESC LIMIT 1", - ).fetchone() - real_flow_crawl = conn.execute( - "SELECT max_depth, max_urls, crawl_max_size, snapshot_max_size, tags_str, config FROM crawls_crawl WHERE tags_str = 'real-flow,challenge'", - ).fetchone() - snapshots = conn.execute( - "SELECT id, url, depth, status, title FROM core_snapshot ORDER BY url", - ).fetchall() - archive_results = conn.execute( - "SELECT s.url, ar.plugin, ar.status, ar.output_files, ar.output_size, ar.output_str " - "FROM core_archiveresult ar " - "JOIN core_snapshot s ON s.id = ar.snapshot_id " - "ORDER BY s.url, ar.plugin", - ).fetchall() - processes = conn.execute( - "SELECT process_type, status, exit_code, pwd, cmd FROM machine_process WHERE process_type = 'hook'", - ).fetchall() - finally: - conn.close() + with use_archivebox_db(tmp_path): + crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first() + real_flow_crawl = Crawl.objects.filter(tags_str="real-flow,challenge").values_list("max_depth", "tags_str", "config").first() + snapshots = list(Snapshot.objects.order_by("url").values_list("id", "url", "depth", "status", "title")) + archive_results = list( + ArchiveResult.objects.select_related("snapshot") + .order_by("snapshot__url", "plugin") + .values_list("snapshot__url", "plugin", "status", "output_files", "output_size", "output_str"), + ) + processes = list(Process.objects.filter(process_type="hook").values_list("process_type", "status", "exit_code", "pwd", "cmd")) assert real_flow_crawl is not None assert real_flow_crawl[0] == 0 - assert real_flow_crawl[1] == 2 - assert real_flow_crawl[2] == 10 * 1024 * 1024 - assert real_flow_crawl[3] == 0 - assert real_flow_crawl[4] == "real-flow,challenge" - assert "wget" in real_flow_crawl[5] + assert real_flow_crawl[1] == "real-flow,challenge" + real_flow_config = real_flow_crawl[2] or {} + assert real_flow_config["CRAWL_MAX_URLS"] == 2 + assert real_flow_config["CRAWL_MAX_SIZE"] == 10 * 1024 * 1024 + assert real_flow_config.get("SNAPSHOT_MAX_SIZE", 0) == 0 + assert "wget" in real_flow_config["PLUGINS"] assert crawl is not None - assert crawl[4] == "chrome-flow" - assert "wget,headers,title" in crawl[5] + assert crawl[1] == "chrome-flow" + assert "wget,headers,title" in json.dumps(crawl[2] or {}) snapshot_urls = {url for _id, url, _depth, _status, _title in snapshots} assert snapshot_urls >= {*wget_urls, chrome_url} @@ -403,24 +965,21 @@ def test_cli_recursive_crawl_processes_discovered_html_urls(tmp_path, process): ) assert result.returncode == 0, result.stderr or result.stdout - conn = sqlite3.connect(tmp_path / "index.sqlite3") - try: - crawl = conn.execute( - "SELECT max_depth, max_urls, crawl_max_size, snapshot_max_size, tags_str FROM crawls_crawl ORDER BY created_at DESC LIMIT 1", - ).fetchone() - snapshots = conn.execute( - "SELECT url, depth, status FROM core_snapshot ORDER BY depth, url", - ).fetchall() - archive_results = conn.execute( - "SELECT s.url, ar.plugin, ar.status, ar.output_files " - "FROM core_archiveresult ar " - "JOIN core_snapshot s ON s.id = ar.snapshot_id " - "ORDER BY s.depth, s.url, ar.plugin", - ).fetchall() - finally: - conn.close() + with use_archivebox_db(tmp_path): + crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first() + snapshots = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "status")) + archive_results = list( + ArchiveResult.objects.select_related("snapshot") + .order_by("snapshot__depth", "snapshot__url", "plugin") + .values_list("snapshot__url", "plugin", "status", "output_files"), + ) - assert crawl == (2, 2, 50 * 1024 * 1024, 0, "recursive-flow") + assert crawl[0] == 2 + assert crawl[1] == "recursive-flow" + crawl_config = crawl[2] or {} + assert crawl_config["CRAWL_MAX_URLS"] == 2 + assert crawl_config["CRAWL_MAX_SIZE"] == 50 * 1024 * 1024 + assert crawl_config.get("SNAPSHOT_MAX_SIZE", 0) == 0 assert ("https://example.com", 0, "sealed") in snapshots assert any(url == "https://iana.org/domains/example" and depth == 1 and status == "sealed" for url, depth, status in snapshots) diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index 8bc9d8d2..32eef3f3 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -302,12 +302,12 @@ class TestRunDaemonMode: @pytest.mark.django_db -class TestRecoverOrphanedCrawls: - def test_recover_orphaned_crawl_requeues_started_crawl_without_active_processes(self): +class TestRecoverOrchestratorState: + def test_recover_orchestrator_state_unlocks_started_crawl_with_pending_snapshot(self): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orphaned_crawls + from archivebox.services.runner import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://example.com", @@ -322,58 +322,18 @@ class TestRecoverOrphanedCrawls: retry_at=None, ) - recovered = recover_orphaned_crawls() + recovered = recover_orchestrator_state() crawl.refresh_from_db() - assert recovered == 1 + assert recovered["unlocked_crawls"] == 1 assert crawl.status == Crawl.StatusChoices.STARTED assert crawl.retry_at is not None - def test_recover_orphaned_crawl_skips_active_child_processes(self): - import archivebox.machine.models as machine_models - from django.utils import timezone - + def test_recover_orchestrator_state_seals_started_crawl_with_finished_snapshots(self): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.machine.models import Machine, Process - from archivebox.services.runner import recover_orphaned_crawls - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by_id=get_or_create_system_user_pk(), - status=Crawl.StatusChoices.STARTED, - retry_at=None, - ) - snapshot = Snapshot.objects.create( - url="https://example.com", - crawl=crawl, - status=Snapshot.StatusChoices.QUEUED, - retry_at=None, - ) - - machine_models._CURRENT_MACHINE = None - machine = Machine.current() - Process.objects.create( - machine=machine, - process_type=Process.TypeChoices.HOOK, - status=Process.StatusChoices.RUNNING, - pwd=str(snapshot.output_dir / "chrome"), - cmd=["/plugins/chrome/on_CrawlSetup__91_chrome_wait.js"], - started_at=timezone.now(), - ) - - recovered = recover_orphaned_crawls() - - crawl.refresh_from_db() - assert recovered == 0 - assert crawl.retry_at is None - - def test_recover_orphaned_crawl_seals_when_all_snapshots_are_already_sealed(self): - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl - from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orphaned_crawls + from archivebox.services.runner import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://example.com", @@ -388,21 +348,215 @@ class TestRecoverOrphanedCrawls: retry_at=None, ) - recovered = recover_orphaned_crawls() + recovered = recover_orchestrator_state() crawl.refresh_from_db() - assert recovered == 1 + assert recovered["sealed_crawls"] == 1 assert crawl.status == Crawl.StatusChoices.SEALED assert crawl.retry_at is None + def test_recover_orchestrator_state_repairs_retry_at_status_invariants(self): + from django.utils import timezone -@pytest.mark.django_db -class TestRecoverOrphanedSnapshots: - def test_recover_orphaned_snapshot_requeues_started_snapshot_without_active_processes(self): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot - from archivebox.services.runner import recover_orphaned_snapshots + from archivebox.services.runner import recover_orchestrator_state + + user_id = get_or_create_system_user_pk() + queued_crawl = Crawl.objects.create( + urls="https://example.com/queued-crawl", + created_by_id=user_id, + status=Crawl.StatusChoices.QUEUED, + retry_at=None, + ) + sealed_crawl = Crawl.objects.create( + urls="https://example.com/sealed-crawl", + created_by_id=user_id, + status=Crawl.StatusChoices.SEALED, + retry_at=timezone.now(), + ) + queued_snapshot = Snapshot.objects.create( + url="https://example.com/queued-snapshot", + crawl=queued_crawl, + status=Snapshot.StatusChoices.QUEUED, + retry_at=None, + ) + sealed_snapshot = Snapshot.objects.create( + url="https://example.com/sealed-snapshot", + crawl=sealed_crawl, + status=Snapshot.StatusChoices.SEALED, + retry_at=timezone.now(), + ) + + recovered = recover_orchestrator_state() + + queued_crawl.refresh_from_db() + sealed_crawl.refresh_from_db() + queued_snapshot.refresh_from_db() + sealed_snapshot.refresh_from_db() + + assert recovered["queued_crawls_unlocked"] == 1 + assert recovered["sealed_crawl_locks_cleared"] == 0 + assert recovered["queued_snapshots_unlocked"] == 1 + assert recovered["sealed_snapshot_locks_cleared"] == 0 + assert queued_crawl.status == Crawl.StatusChoices.QUEUED + assert queued_crawl.retry_at is not None + assert sealed_crawl.status == Crawl.StatusChoices.SEALED + assert sealed_crawl.retry_at is not None + assert queued_snapshot.status == Snapshot.StatusChoices.QUEUED + assert queued_snapshot.retry_at is not None + assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED + assert sealed_snapshot.retry_at is not None + + def test_recover_orchestrator_state_requeues_backoff_archiveresults(self): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.services.runner import recover_orchestrator_state + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + ) + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="title", + hook_name="on_Snapshot__01_title", + status=ArchiveResult.StatusChoices.BACKOFF, + ) + + recovered = recover_orchestrator_state() + + result.refresh_from_db() + snapshot.refresh_from_db() + crawl.refresh_from_db() + + assert recovered["requeued_archiveresults"] == 1 + assert recovered["requeued_snapshots"] == 1 + assert recovered["requeued_crawls"] == 1 + assert result.status == ArchiveResult.StatusChoices.QUEUED + assert snapshot.status == Snapshot.StatusChoices.QUEUED + assert crawl.status == Crawl.StatusChoices.QUEUED + + def test_recover_orchestrator_state_leaves_due_queued_snapshot_for_runner_even_with_final_results(self): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.services.runner import recover_orchestrator_state + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.QUEUED, + retry_at=None, + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.QUEUED, + retry_at=None, + ) + ArchiveResult.objects.create( + snapshot=snapshot, + plugin="title", + hook_name="on_Snapshot__01_title", + status=ArchiveResult.StatusChoices.SUCCEEDED, + ) + + recovered = recover_orchestrator_state() + + snapshot.refresh_from_db() + crawl.refresh_from_db() + + assert recovered["sealed_queued_snapshots"] == 0 + assert recovered["sealed_queued_crawls"] == 0 + assert snapshot.status == Snapshot.StatusChoices.QUEUED + assert snapshot.retry_at is not None + assert snapshot.downloaded_at is None + assert crawl.status == Crawl.StatusChoices.QUEUED + assert crawl.retry_at is not None + + def test_recover_orchestrator_state_seals_stale_queued_snapshot_with_final_results(self): + from datetime import timedelta + + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.services.runner import recover_orchestrator_state + + old = timezone.now() - timedelta(hours=13) + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.QUEUED, + retry_at=old, + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.QUEUED, + retry_at=old, + ) + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="title", + hook_name="on_Snapshot__01_title", + status=ArchiveResult.StatusChoices.SUCCEEDED, + ) + Crawl.objects.filter(pk=crawl.pk).update(modified_at=old) + Snapshot.objects.filter(pk=snapshot.pk).update(modified_at=old) + ArchiveResult.objects.filter(pk=result.pk).update(modified_at=old) + + recovered = recover_orchestrator_state() + + snapshot.refresh_from_db() + crawl.refresh_from_db() + + assert recovered["sealed_queued_snapshots"] == 1 + assert recovered["sealed_queued_crawls"] == 1 + assert snapshot.status == Snapshot.StatusChoices.SEALED + assert snapshot.retry_at is None + assert snapshot.downloaded_at is not None + assert crawl.status == Crawl.StatusChoices.SEALED + assert crawl.retry_at is None + + def test_recover_orchestrator_state_raises_on_stale_active_crawl(self): + from datetime import timedelta + + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.services.runner import recover_orchestrator_state + + old = timezone.now() - timedelta(hours=13) + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.QUEUED, + retry_at=old, + ) + Crawl.objects.filter(id=crawl.id).update(modified_at=old, retry_at=old) + + with pytest.raises(RuntimeError, match="Stuck crawl invariant violated"): + recover_orchestrator_state() + + def test_recover_orchestrator_state_unlocks_started_snapshot_without_running_result(self): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://example.com", @@ -417,22 +571,22 @@ class TestRecoverOrphanedSnapshots: retry_at=None, ) - recovered = recover_orphaned_snapshots() + recovered = recover_orchestrator_state() snapshot.refresh_from_db() crawl.refresh_from_db() - assert recovered == 1 - assert snapshot.status == Snapshot.StatusChoices.QUEUED + assert recovered["unlocked_snapshots"] == 1 + assert snapshot.status == Snapshot.StatusChoices.STARTED assert snapshot.retry_at is not None assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is not None - def test_recover_orphaned_snapshot_requeues_sealed_snapshot_with_queued_results(self): + def test_recover_orchestrator_state_requeues_sealed_snapshot_with_queued_results(self): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.services.runner import recover_orphaned_snapshots + from archivebox.services.runner import recover_orchestrator_state crawl = Crawl.objects.create( urls="https://example.com", @@ -453,13 +607,455 @@ class TestRecoverOrphanedSnapshots: status=ArchiveResult.StatusChoices.QUEUED, ) - recovered = recover_orphaned_snapshots() + recovered = recover_orchestrator_state() snapshot.refresh_from_db() crawl.refresh_from_db() - assert recovered == 1 + assert recovered["requeued_snapshots"] == 1 + assert recovered["requeued_crawls"] == 1 assert snapshot.status == Snapshot.StatusChoices.QUEUED assert snapshot.retry_at is not None assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is not None + + def test_recover_orchestrator_state_ignores_sealed_downloaded_snapshot_without_results(self): + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import recover_orchestrator_state + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + downloaded_at=timezone.now(), + retry_at=None, + ) + + recovered = recover_orchestrator_state() + + snapshot.refresh_from_db() + crawl.refresh_from_db() + + assert recovered["requeued_snapshots"] == 0 + assert recovered["unlocked_snapshots"] == 0 + assert snapshot.status == Snapshot.StatusChoices.SEALED + assert snapshot.retry_at is None + assert crawl.status == Crawl.StatusChoices.SEALED + assert crawl.retry_at is None + + def test_recover_orchestrator_state_seals_started_snapshot_with_final_results(self): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.services.runner import recover_orchestrator_state + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.STARTED, + retry_at=None, + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.STARTED, + retry_at=None, + ) + ArchiveResult.objects.create( + snapshot=snapshot, + plugin="title", + hook_name="on_Snapshot__01_title", + status=ArchiveResult.StatusChoices.SUCCEEDED, + ) + + recovered = recover_orchestrator_state() + + snapshot.refresh_from_db() + assert recovered["sealed_snapshots"] == 1 + assert snapshot.status == Snapshot.StatusChoices.SEALED + assert snapshot.retry_at is None + + +@pytest.mark.django_db +class TestRecoverOrchestratorStateRedFailureModes: + def test_recovery_does_not_seal_queued_snapshot_waiting_for_future_retry_even_with_final_results(self): + from datetime import timedelta + + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.services.runner import recover_orchestrator_state + + future = timezone.now() + timedelta(days=1) + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.STARTED, + retry_at=future, + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.QUEUED, + retry_at=future, + ) + ArchiveResult.objects.create( + snapshot=snapshot, + plugin="title", + hook_name="on_Snapshot__01_title", + status=ArchiveResult.StatusChoices.SUCCEEDED, + ) + + recover_orchestrator_state() + + snapshot.refresh_from_db() + assert snapshot.status == Snapshot.StatusChoices.QUEUED + assert snapshot.retry_at == future + + def test_recovery_does_not_seal_queued_crawl_waiting_for_future_retry_even_with_finished_snapshots(self): + from datetime import timedelta + + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import recover_orchestrator_state + + future = timezone.now() + timedelta(days=1) + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.QUEUED, + retry_at=future, + ) + Snapshot.objects.create(url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.SEALED, retry_at=None) + + recover_orchestrator_state() + + crawl.refresh_from_db() + assert crawl.status == Crawl.StatusChoices.QUEUED + assert crawl.retry_at == future + + def test_recovery_requeues_sealed_parent_without_making_future_retry_child_due_now(self): + from datetime import timedelta + + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import recover_orchestrator_state + + future = timezone.now() + timedelta(days=1) + crawl = Crawl.objects.create( + urls="https://blog.sweeting.me", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + snapshot = Snapshot.objects.create(url="https://blog.sweeting.me", crawl=crawl, status=Snapshot.StatusChoices.QUEUED, retry_at=future) + + recover_orchestrator_state() + + crawl.refresh_from_db() + snapshot.refresh_from_db() + assert crawl.status == Crawl.StatusChoices.QUEUED + assert crawl.retry_at == future + assert snapshot.retry_at == future + + def test_recovery_unlocks_started_parent_to_future_retry_child_not_now(self): + from datetime import timedelta + + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import recover_orchestrator_state + + future = timezone.now() + timedelta(days=1) + crawl = Crawl.objects.create( + urls="https://www.mathjax.org/", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.STARTED, + retry_at=None, + ) + Snapshot.objects.create(url="https://www.mathjax.org/", crawl=crawl, status=Snapshot.StatusChoices.QUEUED, retry_at=future) + + recover_orchestrator_state() + + crawl.refresh_from_db() + assert crawl.status == Crawl.StatusChoices.STARTED + assert crawl.retry_at == future + + def test_recovery_requeues_started_archiveresult_without_process(self): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.services.runner import recover_orchestrator_state + + crawl = Crawl.objects.create( + urls="https://www.mathjax.org/", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.STARTED, + retry_at=None, + ) + snapshot = Snapshot.objects.create(url="https://www.mathjax.org/", crawl=crawl, status=Snapshot.StatusChoices.STARTED, retry_at=None) + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="title", + hook_name="on_Snapshot__01_title", + status=ArchiveResult.StatusChoices.STARTED, + ) + + recover_orchestrator_state() + + result.refresh_from_db() + assert result.status == ArchiveResult.StatusChoices.QUEUED + + def test_recovery_requeues_started_archiveresult_with_exited_process(self): + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.machine.models import Machine, NetworkInterface, Process + from archivebox.services.runner import recover_orchestrator_state + + crawl = Crawl.objects.create( + urls="https://revealjs.com/", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.STARTED, + retry_at=None, + ) + snapshot = Snapshot.objects.create(url="https://revealjs.com/", crawl=crawl, status=Snapshot.StatusChoices.STARTED, retry_at=None) + process = Process.objects.create( + machine=Machine.current(refresh=True), + iface=NetworkInterface.current(refresh=True), + process_type=Process.TypeChoices.HOOK, + worker_type="archiveresult", + pwd=str(snapshot.output_dir / "title"), + cmd=["python", "--version"], + status=Process.StatusChoices.EXITED, + retry_at=None, + exit_code=0, + ended_at=timezone.now(), + ) + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="title", + hook_name="on_Snapshot__01_title", + status=ArchiveResult.StatusChoices.STARTED, + process=process, + ) + + recover_orchestrator_state() + + result.refresh_from_db() + assert result.status == ArchiveResult.StatusChoices.QUEUED + + def test_recovery_requeues_sealed_snapshot_started_result_with_exited_process_result_too(self): + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.machine.models import Machine, NetworkInterface, Process + from archivebox.services.runner import recover_orchestrator_state + + crawl = Crawl.objects.create( + urls="https://pdfobject.com/pdf/sample-3pp.pdf", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + snapshot = Snapshot.objects.create( + url="https://pdfobject.com/pdf/sample-3pp.pdf", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + ) + process = Process.objects.create( + machine=Machine.current(refresh=True), + iface=NetworkInterface.current(refresh=True), + process_type=Process.TypeChoices.HOOK, + worker_type="archiveresult", + pwd=str(snapshot.output_dir / "pdf"), + cmd=["python", "--version"], + status=Process.StatusChoices.EXITED, + retry_at=None, + exit_code=0, + ended_at=timezone.now(), + ) + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="pdf", + hook_name="on_Snapshot__50_pdf", + status=ArchiveResult.StatusChoices.STARTED, + process=process, + ) + + recover_orchestrator_state() + + snapshot.refresh_from_db() + result.refresh_from_db() + assert snapshot.status == Snapshot.StatusChoices.QUEUED + assert result.status == ArchiveResult.StatusChoices.QUEUED + + def test_recovery_requeues_started_snapshot_result_before_unlocking_snapshot(self): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.services.runner import recover_orchestrator_state + + crawl = Crawl.objects.create( + urls="https://mermaid-js.github.io/mermaid/", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.STARTED, + retry_at=None, + ) + snapshot = Snapshot.objects.create( + url="https://mermaid-js.github.io/mermaid/", + crawl=crawl, + status=Snapshot.StatusChoices.STARTED, + retry_at=None, + ) + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="title", + hook_name="on_Snapshot__01_title", + status=ArchiveResult.StatusChoices.STARTED, + ) + + recover_orchestrator_state() + + snapshot.refresh_from_db() + result.refresh_from_db() + assert result.status == ArchiveResult.StatusChoices.QUEUED + assert snapshot.retry_at is not None + + def test_crawl_runner_load_run_state_does_not_return_future_retry_snapshots(self): + from datetime import timedelta + + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import CrawlRunner + + future = timezone.now() + timedelta(days=1) + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.STARTED, + retry_at=future, + ) + Snapshot.objects.create(url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.QUEUED, retry_at=future) + + runner = CrawlRunner(crawl, selected_plugins=[]) + + assert runner.load_run_state() == [] + + def test_crawl_runner_finalize_run_state_preserves_next_future_snapshot_retry(self): + from datetime import timedelta + + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import CrawlRunner + + future = timezone.now() + timedelta(days=1) + crawl = Crawl.objects.create( + urls="https://blog.sweeting.me", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.STARTED, + retry_at=None, + ) + Snapshot.objects.create(url="https://blog.sweeting.me", crawl=crawl, status=Snapshot.StatusChoices.QUEUED, retry_at=future) + + runner = CrawlRunner(crawl, selected_plugins=[]) + runner.finalize_run_state() + + crawl.refresh_from_db() + assert crawl.status == Crawl.StatusChoices.STARTED + assert crawl.retry_at == future + + def test_recovery_raises_stale_due_crawl_even_with_recent_unrelated_process_path_containing_crawl_id(self): + from datetime import timedelta + + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.machine.models import Machine, NetworkInterface, Process + from archivebox.services.runner import recover_orchestrator_state + + old = timezone.now() - timedelta(hours=13) + crawl = Crawl.objects.create( + urls="https://github.com/nodeca/pica", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.QUEUED, + retry_at=old, + ) + Crawl.objects.filter(id=crawl.id).update(modified_at=old, retry_at=old) + Process.objects.create( + machine=Machine.current(refresh=True), + iface=NetworkInterface.current(refresh=True), + process_type=Process.TypeChoices.HOOK, + worker_type="archiveresult", + pwd=f"/tmp/not-an-archivebox-child/{crawl.id}/title", + cmd=["python", "--version"], + status=Process.StatusChoices.EXITED, + retry_at=None, + exit_code=0, + ended_at=timezone.now(), + ) + + with pytest.raises(RuntimeError, match="Stuck crawl invariant violated"): + recover_orchestrator_state() + + def test_recovery_does_not_crash_on_invalid_utf8_process_logs(self, tmp_path): + from datetime import timedelta + + from django.utils import timezone + + from archivebox.machine.models import Machine, NetworkInterface, Process + from archivebox.services.runner import recover_orchestrator_state + + runtime_dir = tmp_path / "https_example_com" / ".hooks" / "on_Snapshot__01_title.py" + runtime_dir.mkdir(parents=True) + (runtime_dir / "stdout.log").write_bytes(b"\\xff\\xfe\\xfa") + process = Process.objects.create( + machine=Machine.current(refresh=True), + iface=NetworkInterface.current(refresh=True), + process_type=Process.TypeChoices.HOOK, + worker_type="archiveresult", + pwd=str(tmp_path / "https_example_com"), + cmd=["on_Snapshot__01_title.py"], + status=Process.StatusChoices.RUNNING, + retry_at=None, + pid=999999, + started_at=timezone.now() - timedelta(hours=1), + timeout=1, + ) + + recover_orchestrator_state() + + process.refresh_from_db() + assert process.status == Process.StatusChoices.EXITED diff --git a/archivebox/tests/test_cli_schedule.py b/archivebox/tests/test_cli_schedule.py index 1922312e..074b8d22 100644 --- a/archivebox/tests/test_cli_schedule.py +++ b/archivebox/tests/test_cli_schedule.py @@ -2,9 +2,15 @@ """CLI-specific tests for archivebox schedule.""" import os -import sqlite3 import subprocess +import pytest + +from archivebox.crawls.models import Crawl +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + def test_schedule_run_all_enqueues_scheduled_crawl(tmp_path, process, disable_extractors_dict): os.chdir(tmp_path) @@ -26,12 +32,9 @@ def test_schedule_run_all_enqueues_scheduled_crawl(tmp_path, process, disable_ex assert result.returncode == 0 assert "Enqueued 1 scheduled crawl" in result.stdout - conn = sqlite3.connect(tmp_path / "index.sqlite3") - try: - crawl_count = conn.execute("SELECT COUNT(*) FROM crawls_crawl").fetchone()[0] - queued_count = conn.execute("SELECT COUNT(*) FROM crawls_crawl WHERE status = 'queued'").fetchone()[0] - finally: - conn.close() + with use_archivebox_db(tmp_path): + crawl_count = Crawl.objects.count() + queued_count = Crawl.objects.filter(status="queued").count() assert crawl_count >= 2 assert queued_count >= 1 @@ -49,12 +52,7 @@ def test_schedule_without_import_path_creates_maintenance_schedule(tmp_path, pro assert result.returncode == 0 assert "Created scheduled maintenance update" in result.stdout - conn = sqlite3.connect(tmp_path / "index.sqlite3") - try: - row = conn.execute( - "SELECT urls, status FROM crawls_crawl ORDER BY created_at DESC LIMIT 1", - ).fetchone() - finally: - conn.close() + with use_archivebox_db(tmp_path): + row = Crawl.objects.order_by("-created_at").values_list("urls", "status").first() assert row == ("archivebox://update", "sealed") diff --git a/archivebox/tests/test_cli_server.py b/archivebox/tests/test_cli_server.py index 99ce4152..459ceeb2 100644 --- a/archivebox/tests/test_cli_server.py +++ b/archivebox/tests/test_cli_server.py @@ -65,17 +65,17 @@ def test_runner_worker_uses_current_interpreter(): def test_reload_workers_use_current_interpreter_and_supervisord_managed_runner(): from archivebox.workers.supervisord_util import RUNNER_WATCH_WORKER, RUNSERVER_WORKER - runserver = RUNSERVER_WORKER("127.0.0.1", "8000", reload=True, pidfile="/tmp/runserver.pid") - watcher = RUNNER_WATCH_WORKER("/tmp/runserver.pid") + runserver = RUNSERVER_WORKER("127.0.0.1", "8000", reload=True) + watcher = RUNNER_WATCH_WORKER("http://127.0.0.1:8000") assert runserver["name"] == "worker_runserver" assert runserver["command"] == f"{sys.executable} -m archivebox manage runserver 127.0.0.1:8000" assert 'ARCHIVEBOX_RUNSERVER="1"' in runserver["environment"] assert 'ARCHIVEBOX_AUTORELOAD="1"' in runserver["environment"] - assert 'ARCHIVEBOX_RUNSERVER_PIDFILE="/tmp/runserver.pid"' in runserver["environment"] + assert 'ARCHIVEBOX_RUNSERVER_BIND_URL="http://127.0.0.1:8000"' in runserver["environment"] assert watcher["name"] == "worker_runner_watch" - assert watcher["command"] == f"{sys.executable} -m archivebox manage runner_watch --pidfile=/tmp/runserver.pid" + assert watcher["command"] == f"{sys.executable} -m archivebox manage runner_watch --bind-url=http://127.0.0.1:8000" def test_start_server_workers_starts_plugin_owned_sonic_worker(monkeypatch): @@ -164,99 +164,3 @@ def test_sonic_daemon_event_handler_requires_running_supervised_worker(monkeypat asyncio.run(run_test()) - -def test_stop_existing_background_runner_stops_orchestrators_without_row_healing(): - from archivebox.cli.archivebox_server import stop_existing_background_runner - - runner_a = Mock() - runner_a.kill_tree = Mock() - runner_a.terminate = Mock() - runner_b = Mock() - runner_b.kill_tree = Mock(side_effect=RuntimeError("boom")) - runner_b.terminate = Mock() - - process_model = Mock() - process_model.StatusChoices.RUNNING = "running" - process_model.TypeChoices.ORCHESTRATOR = "orchestrator" - queryset = Mock() - queryset.order_by.return_value = [runner_a, runner_b] - process_model.objects.filter.return_value = queryset - - supervisor = Mock() - stop_worker = Mock() - log = Mock() - - stopped = stop_existing_background_runner( - machine=Mock(), - process_model=process_model, - supervisor=supervisor, - stop_worker_fn=stop_worker, - log=log, - ) - - assert stopped == 2 - process_model.cleanup_stale_running.assert_not_called() - process_model.cleanup_orphaned_workers.assert_not_called() - stop_worker.assert_any_call(supervisor, "worker_runner") - stop_worker.assert_any_call(supervisor, "worker_runner_watch") - runner_a.kill_tree.assert_called_once_with(graceful_timeout=2.0) - runner_b.terminate.assert_called_once_with(graceful_timeout=2.0) - log.assert_called_once() - - -def test_stop_existing_server_workers_takes_over_same_runserver_port(monkeypatch): - from archivebox.cli.archivebox_server import stop_existing_server_workers - - supervisor = Mock() - supervisor.getProcessInfo.side_effect = lambda name: { - "worker_runserver": {"statename": "RUNNING"}, - "worker_daphne": {"statename": "STOPPED"}, - }.get(name, None) - stop_worker = Mock() - log = Mock() - - monkeypatch.setattr( - "archivebox.cli.archivebox_server._read_supervisor_worker_command", - lambda worker_name: f"{sys.executable} -m archivebox manage runserver 0.0.0.0:8000" if worker_name == "worker_runserver" else "", - ) - - stopped = stop_existing_server_workers( - supervisor=supervisor, - stop_worker_fn=stop_worker, - host="0.0.0.0", - port="8000", - log=log, - ) - - assert stopped == 1 - stop_worker.assert_called_once_with(supervisor, "worker_runserver") - log.assert_called_once() - - -def test_stop_existing_server_workers_leaves_different_port_running(monkeypatch): - from archivebox.cli.archivebox_server import stop_existing_server_workers - - supervisor = Mock() - supervisor.getProcessInfo.side_effect = lambda name: { - "worker_runserver": {"statename": "RUNNING"}, - "worker_daphne": {"statename": "STOPPED"}, - }.get(name, None) - stop_worker = Mock() - log = Mock() - - monkeypatch.setattr( - "archivebox.cli.archivebox_server._read_supervisor_worker_command", - lambda worker_name: f"{sys.executable} -m archivebox manage runserver 127.0.0.1:9000" if worker_name == "worker_runserver" else "", - ) - - stopped = stop_existing_server_workers( - supervisor=supervisor, - stop_worker_fn=stop_worker, - host="0.0.0.0", - port="8000", - log=log, - ) - - assert stopped == 0 - stop_worker.assert_not_called() - log.assert_not_called() diff --git a/archivebox/tests/test_cli_status.py b/archivebox/tests/test_cli_status.py index 9f77dbea..7b033c0c 100644 --- a/archivebox/tests/test_cli_status.py +++ b/archivebox/tests/test_cli_status.py @@ -5,10 +5,16 @@ Verify status reports accurate collection state from DB and filesystem. """ import os -import sqlite3 import subprocess from pathlib import Path +import pytest + +from archivebox.core.models import Snapshot +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + def _find_snapshot_dir(data_dir: Path, snapshot_id: str) -> Path | None: candidates = {snapshot_id} @@ -58,10 +64,8 @@ def test_status_shows_correct_snapshot_count(tmp_path, process, disable_extracto result = subprocess.run(["archivebox", "status"], capture_output=True, text=True) # Verify DB has 3 snapshots - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - db_count = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + db_count = Snapshot.objects.count() assert db_count == 3 # Status output should show 3 @@ -143,10 +147,8 @@ def test_status_counts_new_snapshot_output_dirs_as_archived(tmp_path, process, d check=True, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_id = c.execute("SELECT id FROM core_snapshot WHERE url = ?", ("https://example.com",)).fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + snapshot_id = Snapshot.objects.values_list("id", flat=True).get(url="https://example.com") snapshot_dir = _find_snapshot_dir(tmp_path, str(snapshot_id)) assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}" @@ -183,10 +185,8 @@ def test_status_reads_from_db_not_filesystem(tmp_path, process, disable_extracto ) # Verify DB has snapshot - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - db_count = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + db_count = Snapshot.objects.count() assert db_count == 1 diff --git a/archivebox/tests/test_cli_update.py b/archivebox/tests/test_cli_update.py index 8c6f5db1..164b98af 100644 --- a/archivebox/tests/test_cli_update.py +++ b/archivebox/tests/test_cli_update.py @@ -5,9 +5,15 @@ Verify update drains old dirs, reconciles DB, and queues snapshots. """ import os -import sqlite3 import subprocess +import pytest + +from archivebox.core.models import Snapshot +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + def test_update_runs_successfully_on_empty_archive(tmp_path, process): """Test that update runs without error on empty archive.""" @@ -88,10 +94,8 @@ def test_update_preserves_snapshot_count(tmp_path, process, disable_extractors_d ) # Count before update - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - count_before = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + count_before = Snapshot.objects.count() assert count_before == 1 @@ -104,10 +108,8 @@ def test_update_preserves_snapshot_count(tmp_path, process, disable_extractors_d ) # Count after update - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - count_after = c.execute("SELECT COUNT(*) FROM core_snapshot").fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + count_after = Snapshot.objects.count() # Snapshot count should remain the same assert count_after == count_before @@ -135,10 +137,8 @@ def test_update_seals_migrated_snapshots(tmp_path, process, disable_extractors_d assert result.returncode == 0 # Check that snapshot remains archived instead of being queued for a full re-crawl. - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - status, retry_at = c.execute("SELECT status, retry_at FROM core_snapshot").fetchone() - conn.close() + with use_archivebox_db(tmp_path): + status, retry_at = Snapshot.objects.values_list("status", "retry_at").get() assert status == "sealed" assert retry_at is None diff --git a/archivebox/tests/test_crawl.py b/archivebox/tests/test_crawl.py index e1c1a746..360e720a 100644 --- a/archivebox/tests/test_crawl.py +++ b/archivebox/tests/test_crawl.py @@ -3,10 +3,15 @@ import os import subprocess -import sqlite3 import pytest +from archivebox.core.models import Snapshot +from archivebox.crawls.models import Crawl +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + def test_crawl_creates_crawl_object(tmp_path, process, disable_extractors_dict): """Test that crawl command creates a Crawl object.""" @@ -19,10 +24,8 @@ def test_crawl_creates_crawl_object(tmp_path, process, disable_extractors_dict): env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - crawl = c.execute("SELECT id, max_depth FROM crawls_crawl ORDER BY created_at DESC LIMIT 1").fetchone() - conn.close() + with use_archivebox_db(tmp_path): + crawl = Crawl.objects.order_by("-created_at").first() assert crawl is not None, "Crawl object should be created" @@ -38,13 +41,11 @@ def test_crawl_depth_sets_max_depth_in_crawl(tmp_path, process, disable_extracto env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - crawl = c.execute("SELECT max_depth FROM crawls_crawl ORDER BY created_at DESC LIMIT 1").fetchone() - conn.close() + with use_archivebox_db(tmp_path): + crawl = Crawl.objects.order_by("-created_at").first() assert crawl is not None - assert crawl[0] == 2, "Crawl max_depth should match --depth=2" + assert crawl.max_depth == 2, "Crawl max_depth should match --depth=2" def test_crawl_creates_snapshot_for_url(tmp_path, process, disable_extractors_dict): @@ -58,13 +59,8 @@ def test_crawl_creates_snapshot_for_url(tmp_path, process, disable_extractors_di env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot = c.execute( - "SELECT url FROM core_snapshot WHERE url = ?", - ("https://example.com",), - ).fetchone() - conn.close() + with use_archivebox_db(tmp_path): + snapshot = Snapshot.objects.filter(url="https://example.com").first() assert snapshot is not None, "Snapshot should be created for input URL" @@ -80,23 +76,13 @@ def test_crawl_links_snapshot_to_crawl(tmp_path, process, disable_extractors_dic env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - # Get the crawl ID - crawl = c.execute("SELECT id FROM crawls_crawl ORDER BY created_at DESC LIMIT 1").fetchone() - assert crawl is not None - crawl_id = crawl[0] - - # Check snapshot has correct crawl_id - snapshot = c.execute( - "SELECT crawl_id FROM core_snapshot WHERE url = ?", - ("https://example.com",), - ).fetchone() - conn.close() + with use_archivebox_db(tmp_path): + crawl = Crawl.objects.order_by("-created_at").first() + assert crawl is not None + snapshot = Snapshot.objects.filter(url="https://example.com").first() assert snapshot is not None - assert snapshot[0] == crawl_id, "Snapshot should be linked to Crawl" + assert snapshot.crawl_id == crawl.id, "Snapshot should be linked to Crawl" def test_crawl_multiple_urls_creates_multiple_snapshots(tmp_path, process, disable_extractors_dict): @@ -116,12 +102,9 @@ def test_crawl_multiple_urls_creates_multiple_snapshots(tmp_path, process, disab env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - urls = c.execute("SELECT url FROM core_snapshot ORDER BY url").fetchall() - conn.close() + with use_archivebox_db(tmp_path): + urls = list(Snapshot.objects.order_by("url").values_list("url", flat=True)) - urls = [u[0] for u in urls] assert "https://example.com" in urls assert "https://iana.org" in urls @@ -141,10 +124,8 @@ def test_crawl_from_file_creates_snapshot(tmp_path, process, disable_extractors_ env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot = c.execute("SELECT url FROM core_snapshot").fetchone() - conn.close() + with use_archivebox_db(tmp_path): + snapshot = Snapshot.objects.first() # Should create at least one snapshot (the source file or the URL) assert snapshot is not None, "Should create at least one snapshot" @@ -161,13 +142,11 @@ def test_crawl_persists_input_urls_on_crawl(tmp_path, process, disable_extractor env=disable_extractors_dict, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - crawl_urls = c.execute("SELECT urls FROM crawls_crawl ORDER BY created_at DESC LIMIT 1").fetchone() - conn.close() + with use_archivebox_db(tmp_path): + crawl = Crawl.objects.order_by("-created_at").first() - assert crawl_urls is not None, "Crawl should be created for crawl input" - assert "https://example.com" in crawl_urls[0], "Crawl should persist input URLs" + assert crawl is not None, "Crawl should be created for crawl input" + assert "https://example.com" in crawl.urls, "Crawl should persist input URLs" class TestCrawlCLI: diff --git a/archivebox/tests/test_crawl_admin.py b/archivebox/tests/test_crawl_admin.py index 58964f65..e8134b63 100644 --- a/archivebox/tests/test_crawl_admin.py +++ b/archivebox/tests/test_crawl_admin.py @@ -62,6 +62,7 @@ def test_crawl_admin_add_view_renders_url_filter_alias_fields(client, admin_user assert b'name="url_filters_allowlist"' in response.content assert b'name="url_filters_denylist"' in response.content assert b"Same domain only" in response.content + assert b"Subpaths only" in response.content def test_crawl_admin_form_saves_tags_editor_to_tags_str(crawl, admin_user): @@ -69,11 +70,8 @@ def test_crawl_admin_form_saves_tags_editor_to_tags_str(crawl, admin_user): data={ "created_at": crawl.created_at.strftime("%Y-%m-%d %H:%M:%S"), "urls": crawl.urls, - "config": "{}", + "config": '{"CRAWL_MAX_URLS": 3, "CRAWL_MAX_SIZE": 47185920, "CRAWL_TIMEOUT": 120, "SNAPSHOT_MAX_SIZE": 5242880}', "max_depth": "0", - "max_urls": "3", - "crawl_max_size": str(45 * 1024 * 1024), - "snapshot_max_size": str(5 * 1024 * 1024), "tags_editor": "alpha, beta, Alpha, gamma", "url_filters_allowlist": "example.com\n*.example.com", "url_filters_denylist": "static.example.com", @@ -95,16 +93,38 @@ def test_crawl_admin_form_saves_tags_editor_to_tags_str(crawl, admin_user): updated = form.save() updated.refresh_from_db() assert updated.tags_str == "alpha,beta,gamma" - assert updated.max_urls == 3 - assert updated.crawl_max_size == 45 * 1024 * 1024 - assert updated.snapshot_max_size == 5 * 1024 * 1024 assert updated.config["CRAWL_MAX_URLS"] == 3 assert updated.config["CRAWL_MAX_SIZE"] == 45 * 1024 * 1024 + assert updated.config["CRAWL_TIMEOUT"] == 120 assert updated.config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024 assert updated.config["URL_ALLOWLIST"] == "example.com\n*.example.com" assert updated.config["URL_DENYLIST"] == "static.example.com" +def test_crawl_admin_resume_action_updates_only_status(client, admin_user, crawl): + crawl.status = Crawl.StatusChoices.SEALED + crawl.retry_at = None + crawl.notes = "unsaved-change-guard" + crawl.save(update_fields=["status", "retry_at", "notes", "modified_at"]) + + client.login(username="crawladmin", password="testpassword") + response = client.post( + reverse("admin:crawls_crawl_changelist"), + data={ + "action": "resume_selected_crawls", + "_selected_action": str(crawl.pk), + "index": "0", + }, + HTTP_HOST=ADMIN_HOST, + ) + + assert response.status_code == 302 + crawl.refresh_from_db() + assert crawl.status == Crawl.StatusChoices.STARTED + assert crawl.retry_at is not None + assert crawl.notes == "unsaved-change-guard" + + @pytest.mark.django_db(transaction=True) def test_crawl_tag_changes_sync_existing_snapshot_tags(crawl): snapshots = crawl.create_snapshots_from_urls() @@ -256,7 +276,7 @@ def test_create_snapshots_from_urls_respects_max_urls(admin_user): "https://example.com/contact", ], ), - max_urls=2, + config={"CRAWL_MAX_URLS": 2}, created_by=admin_user, ) diff --git a/archivebox/tests/test_machine_models.py b/archivebox/tests/test_machine_models.py index 0aaa4b7c..8d713ae1 100644 --- a/archivebox/tests/test_machine_models.py +++ b/archivebox/tests/test_machine_models.py @@ -529,7 +529,7 @@ class TestProcessCurrent(TestCase): """runner_watch should be classified as a worker, not the orchestrator itself.""" old_argv = sys.argv try: - sys.argv = ["archivebox", "manage", "runner_watch", "--pidfile=/tmp/runserver.pid"] + sys.argv = ["archivebox", "manage", "runner_watch", "--bind-url=http://127.0.0.1:8000"] result = Process._detect_process_type() self.assertEqual(result, Process.TypeChoices.WORKER) finally: diff --git a/archivebox/tests/test_migrations_fresh.py b/archivebox/tests/test_migrations_fresh.py index 4d2a23da..74421ceb 100644 --- a/archivebox/tests/test_migrations_fresh.py +++ b/archivebox/tests/test_migrations_fresh.py @@ -6,13 +6,21 @@ Tests that fresh installations work correctly with the current schema. """ import shutil -import sqlite3 import tempfile import unittest from pathlib import Path +import pytest +from django.db.migrations.recorder import MigrationRecorder + +from archivebox.core.models import ArchiveResult, Snapshot, Tag +from archivebox.crawls.models import Crawl +from archivebox.tests.orm_helpers import use_archivebox_db + from .migrations_helpers import run_archivebox +pytestmark = pytest.mark.django_db(transaction=True) + class TestFreshInstall(unittest.TestCase): """Test that fresh installs work correctly.""" @@ -59,20 +67,9 @@ class TestFreshInstall(unittest.TestCase): result = run_archivebox(work_dir, ["add", "--index-only", "https://example.com"]) self.assertEqual(result.returncode, 0, f"Add command failed: {result.stderr}") - conn = sqlite3.connect(str(work_dir / "index.sqlite3")) - cursor = conn.cursor() - - # Verify a Crawl was created - cursor.execute("SELECT COUNT(*) FROM crawls_crawl") - crawl_count = cursor.fetchone()[0] - self.assertGreaterEqual(crawl_count, 1, "No Crawl was created") - - # Verify at least one snapshot was created - cursor.execute("SELECT COUNT(*) FROM core_snapshot") - snapshot_count = cursor.fetchone()[0] - self.assertGreaterEqual(snapshot_count, 1, "No Snapshot was created") - - conn.close() + with use_archivebox_db(work_dir): + self.assertGreaterEqual(Crawl.objects.count(), 1, "No Crawl was created") + self.assertGreaterEqual(Snapshot.objects.count(), 1, "No Snapshot was created") finally: shutil.rmtree(work_dir, ignore_errors=True) @@ -106,11 +103,8 @@ class TestFreshInstall(unittest.TestCase): result = run_archivebox(work_dir, ["init"]) self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") - conn = sqlite3.connect(str(work_dir / "index.sqlite3")) - cursor = conn.cursor() - cursor.execute("SELECT COUNT(*) FROM django_migrations") - count = cursor.fetchone()[0] - conn.close() + with use_archivebox_db(work_dir): + count = MigrationRecorder.Migration.objects.count() # Should have many migrations applied self.assertGreater(count, 10, f"Expected >10 migrations, got {count}") @@ -126,11 +120,10 @@ class TestFreshInstall(unittest.TestCase): result = run_archivebox(work_dir, ["init"]) self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") - conn = sqlite3.connect(str(work_dir / "index.sqlite3")) - cursor = conn.cursor() - cursor.execute("SELECT name FROM django_migrations WHERE app='core' ORDER BY name") - migrations = [row[0] for row in cursor.fetchall()] - conn.close() + with use_archivebox_db(work_dir): + migrations = list( + MigrationRecorder.Migration.objects.filter(app="core").order_by("name").values_list("name", flat=True), + ) self.assertIn("0001_initial", migrations) @@ -149,11 +142,7 @@ class TestSchemaIntegrity(unittest.TestCase): result = run_archivebox(work_dir, ["init"]) self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") - conn = sqlite3.connect(str(work_dir / "index.sqlite3")) - cursor = conn.cursor() - cursor.execute("PRAGMA table_info(core_snapshot)") - columns = {row[1] for row in cursor.fetchall()} - conn.close() + columns = {field.column for field in Snapshot._meta.local_fields} required = {"id", "url", "timestamp", "title", "status", "created_at", "modified_at"} for col in required: @@ -170,11 +159,7 @@ class TestSchemaIntegrity(unittest.TestCase): result = run_archivebox(work_dir, ["init"]) self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") - conn = sqlite3.connect(str(work_dir / "index.sqlite3")) - cursor = conn.cursor() - cursor.execute("PRAGMA table_info(core_archiveresult)") - columns = {row[1] for row in cursor.fetchall()} - conn.close() + columns = {field.column for field in ArchiveResult._meta.local_fields} required = {"id", "snapshot_id", "plugin", "status", "created_at", "modified_at"} for col in required: @@ -191,11 +176,7 @@ class TestSchemaIntegrity(unittest.TestCase): result = run_archivebox(work_dir, ["init"]) self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") - conn = sqlite3.connect(str(work_dir / "index.sqlite3")) - cursor = conn.cursor() - cursor.execute("PRAGMA table_info(core_tag)") - columns = {row[1] for row in cursor.fetchall()} - conn.close() + columns = {field.column for field in Tag._meta.local_fields} required = {"id", "name"} for col in required: @@ -212,11 +193,7 @@ class TestSchemaIntegrity(unittest.TestCase): result = run_archivebox(work_dir, ["init"]) self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") - conn = sqlite3.connect(str(work_dir / "index.sqlite3")) - cursor = conn.cursor() - cursor.execute("PRAGMA table_info(crawls_crawl)") - columns = {row[1] for row in cursor.fetchall()} - conn.close() + columns = {field.column for field in Crawl._meta.local_fields} required = {"id", "urls", "status", "created_at", "created_by_id"} for col in required: @@ -247,21 +224,12 @@ class TestMultipleSnapshots(unittest.TestCase): result = run_archivebox(work_dir, ["add", "--index-only", "https://example.org"]) self.assertEqual(result.returncode, 0, f"Add 2 failed: {result.stderr}") - conn = sqlite3.connect(str(work_dir / "index.sqlite3")) - cursor = conn.cursor() - - # Verify snapshots were created - cursor.execute("SELECT COUNT(*) FROM core_snapshot") - snapshot_count = cursor.fetchone()[0] + with use_archivebox_db(work_dir): + snapshot_count = Snapshot.objects.count() + crawl_count = Crawl.objects.count() self.assertEqual(snapshot_count, 2, f"Expected 2 snapshots, got {snapshot_count}") - - # Verify crawls were created (one per add call) - cursor.execute("SELECT COUNT(*) FROM crawls_crawl") - crawl_count = cursor.fetchone()[0] self.assertEqual(crawl_count, 2, f"Expected 2 Crawls, got {crawl_count}") - conn.close() - finally: shutil.rmtree(work_dir, ignore_errors=True) @@ -276,16 +244,10 @@ class TestMultipleSnapshots(unittest.TestCase): result = run_archivebox(work_dir, ["add", "--index-only", "https://example.com"]) self.assertEqual(result.returncode, 0, f"Add failed: {result.stderr}") - conn = sqlite3.connect(str(work_dir / "index.sqlite3")) - cursor = conn.cursor() - - # Check that snapshot has a crawl_id - cursor.execute("SELECT crawl_id FROM core_snapshot WHERE url = 'https://example.com'") - row = cursor.fetchone() + with use_archivebox_db(work_dir): + row = Snapshot.objects.filter(url="https://example.com").values_list("crawl_id", flat=True).first() self.assertIsNotNone(row, "Snapshot not found") - self.assertIsNotNone(row[0], "Snapshot should have a crawl_id") - - conn.close() + self.assertIsNotNone(row, "Snapshot should have a crawl_id") finally: shutil.rmtree(work_dir, ignore_errors=True) diff --git a/archivebox/tests/test_persona_admin.py b/archivebox/tests/test_persona_admin.py index c620b0a7..2bb35b3c 100644 --- a/archivebox/tests/test_persona_admin.py +++ b/archivebox/tests/test_persona_admin.py @@ -170,6 +170,7 @@ def test_persona_admin_add_post_runs_shared_importer(client, admin_user, monkeyp { "name": "ImportedPersona", "created_by": str(admin_user.pk), + "permissions": "public", "config": "{}", "import_mode": "discovered", "import_discovered_profile": source.choice_value, @@ -204,6 +205,7 @@ def test_persona_admin_saves_typed_plugin_config(client, admin_user, monkeypatch { "name": "PluginConfigPersona", "created_by": str(admin_user.pk), + "permissions": "public", "config": "{}", "import_mode": "none", "plugin_config__wget__WGET_TIMEOUT": "77", diff --git a/archivebox/tests/test_persona_runtime.py b/archivebox/tests/test_persona_runtime.py index c1efb740..f087162a 100644 --- a/archivebox/tests/test_persona_runtime.py +++ b/archivebox/tests/test_persona_runtime.py @@ -152,12 +152,11 @@ def test_crawl_runner_respects_chrome_isolation_config(initialized_archive): assert payload["explicit_isolation"] == "snapshot" -def test_crawl_resolve_persona_raises_for_missing_persona_id(initialized_archive): +def test_crawl_resolve_persona_treats_missing_persona_id_as_null(initialized_archive): script = textwrap.dedent( """ import json import os - from uuid import uuid4 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'archivebox.core.settings') import django @@ -166,14 +165,12 @@ def test_crawl_resolve_persona_raises_for_missing_persona_id(initialized_archive from archivebox.crawls.models import Crawl from archivebox.personas.models import Persona - crawl = Crawl.objects.create(urls='https://example.com', persona_id=uuid4()) + persona = Persona.objects.create(name='TemporaryPersona') + crawl = Crawl.objects.create(urls='https://example.com', persona_id=persona.id) + persona.delete() + crawl.refresh_from_db() - try: - crawl.resolve_persona() - except Persona.DoesNotExist as err: - print(json.dumps({'raised': True, 'message': str(err)})) - else: - raise SystemExit('resolve_persona unexpectedly succeeded') + print(json.dumps({'persona': crawl.resolve_persona(), 'persona_id': crawl.persona_id})) """, ) @@ -181,16 +178,15 @@ def test_crawl_resolve_persona_raises_for_missing_persona_id(initialized_archive assert code == 0, stderr payload = json.loads(stdout.strip().splitlines()[-1]) - assert payload["raised"] is True - assert "references missing Persona" in payload["message"] + assert payload["persona"] is None + assert payload["persona_id"] is None -def test_get_config_raises_for_missing_persona_id(initialized_archive): +def test_get_config_treats_missing_persona_id_as_null(initialized_archive): script = textwrap.dedent( """ import json import os - from uuid import uuid4 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'archivebox.core.settings') import django @@ -200,14 +196,17 @@ def test_get_config_raises_for_missing_persona_id(initialized_archive): from archivebox.crawls.models import Crawl from archivebox.personas.models import Persona - crawl = Crawl.objects.create(urls='https://example.com', persona_id=uuid4()) + persona = Persona.objects.create(name='TemporaryPersona') + crawl = Crawl.objects.create( + urls='https://example.com', + persona_id=persona.id, + config={'DEFAULT_PERSONA': 'Default'}, + ) + persona.delete() + crawl.refresh_from_db() - try: - get_config(crawl=crawl) - except Persona.DoesNotExist as err: - print(json.dumps({'raised': True, 'message': str(err)})) - else: - raise SystemExit('get_config unexpectedly succeeded') + config = get_config(crawl=crawl) + print(json.dumps({'timeout': config.TIMEOUT, 'persona_id': str(crawl.persona_id)})) """, ) @@ -215,8 +214,8 @@ def test_get_config_raises_for_missing_persona_id(initialized_archive): assert code == 0, stderr payload = json.loads(stdout.strip().splitlines()[-1]) - assert payload["raised"] is True - assert "references missing Persona" in payload["message"] + assert payload["timeout"] + assert payload["persona_id"] == "None" def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(initialized_archive): diff --git a/archivebox/tests/test_process_runtime_paths.py b/archivebox/tests/test_process_runtime_paths.py index e9bb95f0..77db55b6 100644 --- a/archivebox/tests/test_process_runtime_paths.py +++ b/archivebox/tests/test_process_runtime_paths.py @@ -21,7 +21,6 @@ class TestProcessRuntimePaths(unittest.TestCase): self.assertEqual(process.runtime_dir, expected_dir) self.assertEqual(process.stdout_file, expected_dir / "stdout.log") self.assertEqual(process.stderr_file, expected_dir / "stderr.log") - self.assertEqual(process.pid_file, expected_dir / "process.pid") def test_non_hook_processes_keep_runtime_files_in_pwd(self): process = Process( @@ -34,4 +33,3 @@ class TestProcessRuntimePaths(unittest.TestCase): self.assertEqual(process.runtime_dir, expected_dir) self.assertEqual(process.stdout_file, expected_dir / "stdout.log") self.assertEqual(process.stderr_file, expected_dir / "stderr.log") - self.assertEqual(process.pid_file, expected_dir / "process.pid") diff --git a/archivebox/tests/test_recursive_crawl.py b/archivebox/tests/test_recursive_crawl.py index ac1a223d..6ceeb24e 100644 --- a/archivebox/tests/test_recursive_crawl.py +++ b/archivebox/tests/test_recursive_crawl.py @@ -4,23 +4,26 @@ import json import os import subprocess -import sqlite3 import time from pathlib import Path import pytest +from archivebox.core.models import ArchiveResult, Snapshot +from archivebox.crawls.models import Crawl +from archivebox.machine.models import Process +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + def wait_for_db_condition(timeout, condition, interval=0.5): deadline = time.time() + timeout while time.time() < deadline: if os.path.exists("index.sqlite3"): - conn = sqlite3.connect("index.sqlite3") - try: - if condition(conn.cursor()): + with use_archivebox_db("."): + if condition(): return True - finally: - conn.close() time.sleep(interval) return False @@ -87,11 +90,12 @@ def test_background_hooks_dont_block_parser_extractors(tmp_path, process, recurs assert wait_for_db_condition( timeout=120, - condition=lambda c: ( - c.execute( - "SELECT COUNT(*) FROM core_archiveresult WHERE plugin LIKE 'parse_%_urls' AND status IN ('started', 'succeeded', 'failed')", - ).fetchone()[0] - > 0 + condition=lambda: ( + ArchiveResult.objects.filter( + plugin__startswith="parse_", + plugin__endswith="_urls", + status__in=("started", "succeeded", "failed"), + ).exists() ), ), "Parser extractors never progressed beyond queued status" stdout, stderr = stop_process(proc) @@ -101,21 +105,19 @@ def test_background_hooks_dont_block_parser_extractors(tmp_path, process, recurs if stdout: print(f"\n=== STDOUT (last 2000 chars) ===\n{stdout[-2000:]}\n=== END STDOUT ===\n") - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - snapshots = c.execute("SELECT url, depth, status FROM core_snapshot").fetchall() - bg_hooks = c.execute( - "SELECT plugin, status FROM core_archiveresult WHERE plugin IN ('favicon', 'consolelog', 'ssl', 'responses', 'redirects', 'staticfile') ORDER BY plugin", - ).fetchall() - parser_extractors = c.execute( - "SELECT plugin, status FROM core_archiveresult WHERE plugin LIKE 'parse_%_urls' ORDER BY plugin", - ).fetchall() - all_extractors = c.execute( - "SELECT plugin, status FROM core_archiveresult ORDER BY plugin", - ).fetchall() - - conn.close() + with use_archivebox_db(tmp_path): + snapshots = list(Snapshot.objects.values_list("url", "depth", "status")) + bg_hooks = list( + ArchiveResult.objects.filter(plugin__in=("favicon", "consolelog", "ssl", "responses", "redirects", "staticfile")) + .order_by("plugin") + .values_list("plugin", "status"), + ) + parser_extractors = list( + ArchiveResult.objects.filter(plugin__startswith="parse_", plugin__endswith="_urls") + .order_by("plugin") + .values_list("plugin", "status"), + ) + all_extractors = list(ArchiveResult.objects.order_by("plugin").values_list("plugin", "status")) assert len(snapshots) > 0, ( f"Should have created snapshot after Crawl hooks finished. " @@ -167,14 +169,13 @@ def test_parser_extractors_emit_snapshot_jsonl(tmp_path, process, recursive_test ) assert result.returncode == 0, result.stderr - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - parse_html = c.execute( - "SELECT id, status, output_str FROM core_archiveresult WHERE plugin LIKE '%parse_html_urls' ORDER BY id LIMIT 1", - ).fetchone() - - conn.close() + with use_archivebox_db(tmp_path): + parse_html = ( + ArchiveResult.objects.filter(plugin__endswith="parse_html_urls") + .order_by("id") + .values_list("id", "status", "output_str") + .first() + ) if parse_html: status = parse_html[1] @@ -222,9 +223,9 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, process, recursive_te ["archivebox", "add", "--depth=1", "--plugins=wget,parse_html_urls", recursive_test_site["root_url"]], env=env, timeout=120, - condition=lambda c: ( - c.execute("SELECT COUNT(*) FROM core_snapshot WHERE depth = 0").fetchone()[0] >= 1 - and c.execute("SELECT COUNT(*) FROM core_snapshot WHERE depth = 1").fetchone()[0] >= len(recursive_test_site["child_urls"]) + condition=lambda: ( + Snapshot.objects.filter(depth=0).count() >= 1 + and Snapshot.objects.filter(depth=1).count() >= len(recursive_test_site["child_urls"]) ), ) @@ -233,29 +234,29 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, process, recursive_te if stdout: print(f"\n=== STDOUT (last 2000 chars) ===\n{stdout[-2000:]}\n=== END STDOUT ===\n") - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - all_snapshots = c.execute("SELECT url, depth FROM core_snapshot").fetchall() - root_snapshot = c.execute( - "SELECT id, url, depth, parent_snapshot_id FROM core_snapshot WHERE depth = 0 ORDER BY created_at LIMIT 1", - ).fetchone() - child_snapshots = c.execute( - "SELECT id, url, depth, parent_snapshot_id FROM core_snapshot WHERE depth = 1", - ).fetchall() - crawl = c.execute( - "SELECT id, max_depth FROM crawls_crawl ORDER BY created_at DESC LIMIT 1", - ).fetchone() - parser_status = c.execute( - "SELECT plugin, status FROM core_archiveresult WHERE snapshot_id = ? AND plugin LIKE 'parse_%_urls'", - (root_snapshot[0] if root_snapshot else "",), - ).fetchall() - started_extractors = c.execute( - "SELECT plugin, status FROM core_archiveresult WHERE snapshot_id = ? AND status = 'started'", - (root_snapshot[0] if root_snapshot else "",), - ).fetchall() - - conn.close() + with use_archivebox_db(tmp_path): + all_snapshots = list(Snapshot.objects.values_list("url", "depth")) + root_snapshot = ( + Snapshot.objects.filter(depth=0) + .order_by("created_at") + .values_list("id", "url", "depth", "parent_snapshot_id") + .first() + ) + child_snapshots = list(Snapshot.objects.filter(depth=1).values_list("id", "url", "depth", "parent_snapshot_id")) + crawl = Crawl.objects.order_by("-created_at").values_list("id", "max_depth").first() + parser_status = list( + ArchiveResult.objects.filter( + snapshot_id=root_snapshot[0] if root_snapshot else None, + plugin__startswith="parse_", + plugin__endswith="_urls", + ).values_list("plugin", "status"), + ) + started_extractors = list( + ArchiveResult.objects.filter( + snapshot_id=root_snapshot[0] if root_snapshot else None, + status="started", + ).values_list("plugin", "status"), + ) assert root_snapshot is not None, f"Root snapshot should exist at depth=0. All snapshots: {all_snapshots}" root_id = root_snapshot[0] @@ -283,32 +284,26 @@ def test_recursive_crawl_respects_depth_limit(tmp_path, process, disable_extract ["archivebox", "add", "--depth=1", "--plugins=wget,parse_html_urls", recursive_test_site["root_url"]], env=env, timeout=120, - condition=lambda c: ( - c.execute("SELECT COUNT(*) FROM core_snapshot WHERE depth = 0").fetchone()[0] >= 1 - and c.execute("SELECT COUNT(*) FROM core_snapshot WHERE depth = 1").fetchone()[0] >= len(recursive_test_site["child_urls"]) - and c.execute( - "SELECT COUNT(DISTINCT ar.snapshot_id) " - "FROM core_archiveresult ar " - "JOIN core_snapshot s ON s.id = ar.snapshot_id " - "WHERE s.depth = 1 " - "AND ar.plugin LIKE 'parse_%_urls' " - "AND ar.status IN ('started', 'succeeded', 'failed')", - ).fetchone()[0] + condition=lambda: ( + Snapshot.objects.filter(depth=0).count() >= 1 + and Snapshot.objects.filter(depth=1).count() >= len(recursive_test_site["child_urls"]) + and ArchiveResult.objects.filter( + snapshot__depth=1, + plugin__startswith="parse_", + plugin__endswith="_urls", + status__in=("started", "succeeded", "failed"), + ) + .values("snapshot_id") + .distinct() + .count() >= len(recursive_test_site["child_urls"]) ), ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - max_depth_found = c.execute( - "SELECT MAX(depth) FROM core_snapshot", - ).fetchone()[0] - depth_counts = c.execute( - "SELECT depth, COUNT(*) FROM core_snapshot GROUP BY depth ORDER BY depth", - ).fetchall() - - conn.close() + with use_archivebox_db(tmp_path): + depths = list(Snapshot.objects.values_list("depth", flat=True)) + max_depth_found = max(depths) if depths else None + depth_counts = [(depth, Snapshot.objects.filter(depth=depth).count()) for depth in sorted(set(depths))] assert max_depth_found is not None, "Should have at least one snapshot" assert max_depth_found <= 1, f"Max depth should not exceed 1, got {max_depth_found}. Depth distribution: {depth_counts}" @@ -352,18 +347,13 @@ def test_recursive_crawl_respects_max_urls(tmp_path, process, disable_extractors assert result.returncode == 0, result.stderr - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() + with use_archivebox_db(tmp_path): + crawl_obj = Crawl.objects.order_by("-created_at").first() + crawl = (crawl_obj.max_depth, crawl_obj.config["CRAWL_MAX_URLS"]) if crawl_obj else None + snapshot_rows = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "parent_snapshot_id")) + depth_counts = {depth: Snapshot.objects.filter(depth=depth).count() for depth in set(Snapshot.objects.values_list("depth", flat=True))} - crawl = c.execute( - "SELECT max_depth, max_urls, json_extract(config, '$.CRAWL_MAX_URLS') FROM crawls_crawl ORDER BY created_at DESC LIMIT 1", - ).fetchone() - snapshot_rows = c.execute("SELECT url, depth, parent_snapshot_id FROM core_snapshot ORDER BY depth, url").fetchall() - depth_counts = dict(c.execute("SELECT depth, COUNT(*) FROM core_snapshot GROUP BY depth ORDER BY depth").fetchall()) - - conn.close() - - assert crawl == (2, 4, 4) + assert crawl == (2, 4) assert len(snapshot_rows) == 4 assert depth_counts.get(0, 0) == 1 assert depth_counts.get(1, 0) == 3 @@ -401,18 +391,16 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p ["archivebox", "add", "--depth=2", "--plugins=wget,parse_html_urls", recursive_test_site["root_url"]], env=env, timeout=180, - condition=lambda c: ( - c.execute("SELECT COUNT(*) FROM core_snapshot WHERE depth = 0").fetchone()[0] >= 1 - and c.execute("SELECT COUNT(*) FROM core_snapshot WHERE depth = 1").fetchone()[0] >= len(recursive_test_site["child_urls"]) - and c.execute("SELECT COUNT(*) FROM core_snapshot WHERE depth = 2").fetchone()[0] >= len(recursive_test_site["deep_urls"]) - and c.execute( - "SELECT COUNT(*) " - "FROM core_archiveresult ar " - "JOIN core_snapshot s ON s.id = ar.snapshot_id " - "WHERE ar.plugin LIKE 'parse_%_urls' " - "AND s.depth IN (0, 1) " - "AND ar.status IN ('started', 'succeeded', 'failed', 'skipped', 'noresults')", - ).fetchone()[0] + condition=lambda: ( + Snapshot.objects.filter(depth=0).count() >= 1 + and Snapshot.objects.filter(depth=1).count() >= len(recursive_test_site["child_urls"]) + and Snapshot.objects.filter(depth=2).count() >= len(recursive_test_site["deep_urls"]) + and ArchiveResult.objects.filter( + plugin__startswith="parse_", + plugin__endswith="_urls", + snapshot__depth__in=(0, 1), + status__in=("started", "succeeded", "failed", "skipped", "noresults"), + ).count() >= 2 ), ) @@ -422,41 +410,33 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p if stdout: print(f"\n=== STDOUT (last 2000 chars) ===\n{stdout[-2000:]}\n=== END STDOUT ===\n") - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - depth_counts = dict(c.execute("SELECT depth, COUNT(*) FROM core_snapshot GROUP BY depth ORDER BY depth").fetchall()) - crawl = c.execute("SELECT id, max_depth FROM crawls_crawl ORDER BY created_at DESC LIMIT 1").fetchone() - root_snapshot = c.execute( - "SELECT id, url, depth, parent_snapshot_id FROM core_snapshot WHERE depth = 0 ORDER BY created_at LIMIT 1", - ).fetchone() - child_rows = c.execute( - "SELECT id, url, parent_snapshot_id FROM core_snapshot WHERE depth = 1", - ).fetchall() - deep_rows = c.execute( - "SELECT id, url, parent_snapshot_id FROM core_snapshot WHERE depth = 2", - ).fetchall() - parser_results = c.execute( - "SELECT s.url, s.depth, ar.plugin, ar.status, ar.output_files, ar.output_size " - "FROM core_archiveresult ar " - "JOIN core_snapshot s ON s.id = ar.snapshot_id " - "WHERE ar.plugin LIKE 'parse_%_urls' " - "ORDER BY s.depth, s.url", - ).fetchall() - wget_results = c.execute( - "SELECT s.url, s.depth, ar.status, ar.output_files, ar.output_size " - "FROM core_archiveresult ar " - "JOIN core_snapshot s ON s.id = ar.snapshot_id " - "WHERE ar.plugin = 'wget' " - "ORDER BY s.depth, s.url", - ).fetchall() - process_rows = c.execute( - "SELECT process_type, worker_type, status, exit_code, pwd, cmd " - "FROM machine_process " - "WHERE process_type = 'hook' " - "ORDER BY created_at", - ).fetchall() - conn.close() + with use_archivebox_db(tmp_path): + depths = list(Snapshot.objects.values_list("depth", flat=True)) + depth_counts = {depth: Snapshot.objects.filter(depth=depth).count() for depth in sorted(set(depths))} + crawl = Crawl.objects.order_by("-created_at").values_list("id", "max_depth").first() + root_snapshot = ( + Snapshot.objects.filter(depth=0) + .order_by("created_at") + .values_list("id", "url", "depth", "parent_snapshot_id") + .first() + ) + child_rows = list(Snapshot.objects.filter(depth=1).values_list("id", "url", "parent_snapshot_id")) + deep_rows = list(Snapshot.objects.filter(depth=2).values_list("id", "url", "parent_snapshot_id")) + parser_results = list( + ArchiveResult.objects.filter(plugin__startswith="parse_", plugin__endswith="_urls") + .order_by("snapshot__depth", "snapshot__url") + .values_list("snapshot__url", "snapshot__depth", "plugin", "status", "output_files", "output_size"), + ) + wget_results = list( + ArchiveResult.objects.filter(plugin="wget") + .order_by("snapshot__depth", "snapshot__url") + .values_list("snapshot__url", "snapshot__depth", "status", "output_files", "output_size"), + ) + process_rows = list( + Process.objects.filter(process_type="hook") + .order_by("created_at") + .values_list("process_type", "worker_type", "status", "exit_code", "pwd", "cmd"), + ) assert crawl is not None assert crawl[1] == 2 @@ -509,14 +489,7 @@ def test_crawl_snapshot_has_parent_snapshot_field(tmp_path, process, disable_ext """Test that Snapshot model has parent_snapshot field.""" os.chdir(tmp_path) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - # Check schema for parent_snapshot_id column - schema = c.execute("PRAGMA table_info(core_snapshot)").fetchall() - conn.close() - - column_names = [col[1] for col in schema] + column_names = {field.column for field in Snapshot._meta.local_fields} assert "parent_snapshot_id" in column_names, f"Snapshot table should have parent_snapshot_id column. Columns: {column_names}" @@ -525,14 +498,7 @@ def test_snapshot_depth_field_exists(tmp_path, process, disable_extractors_dict) """Test that Snapshot model has depth field.""" os.chdir(tmp_path) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - # Check schema for depth column - schema = c.execute("PRAGMA table_info(core_snapshot)").fetchall() - conn.close() - - column_names = [col[1] for col in schema] + column_names = {field.column for field in Snapshot._meta.local_fields} assert "depth" in column_names, f"Snapshot table should have depth column. Columns: {column_names}" @@ -548,24 +514,13 @@ def test_root_snapshot_has_depth_zero(tmp_path, process, disable_extractors_dict ["archivebox", "add", "--depth=1", "--plugins=wget,parse_html_urls", recursive_test_site["root_url"]], env=env, timeout=120, - condition=lambda c: ( - c.execute( - "SELECT COUNT(*) FROM core_snapshot WHERE url = ?", - (recursive_test_site["root_url"],), - ).fetchone()[0] - >= 1 + condition=lambda: ( + Snapshot.objects.filter(url=recursive_test_site["root_url"]).count() >= 1 ), ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - snapshot = c.execute( - "SELECT id, depth FROM core_snapshot WHERE url = ? ORDER BY created_at LIMIT 1", - (recursive_test_site["root_url"],), - ).fetchone() - - conn.close() + with use_archivebox_db(tmp_path): + snapshot = Snapshot.objects.filter(url=recursive_test_site["root_url"]).order_by("created_at").values_list("id", "depth").first() assert snapshot is not None, "Root snapshot should be created" assert snapshot[1] == 0, f"Root snapshot should have depth=0, got {snapshot[1]}" @@ -590,25 +545,25 @@ def test_archiveresult_worker_queue_filters_by_foreground_extractors(tmp_path, p ["archivebox", "add", "--plugins=favicon,wget,parse_html_urls", recursive_test_site["root_url"]], env=env, timeout=120, - condition=lambda c: ( - c.execute( - "SELECT COUNT(*) FROM core_archiveresult WHERE plugin LIKE 'parse_%_urls' AND status IN ('started', 'succeeded', 'failed')", - ).fetchone()[0] - > 0 + condition=lambda: ( + ArchiveResult.objects.filter( + plugin__startswith="parse_", + plugin__endswith="_urls", + status__in=("started", "succeeded", "failed"), + ).exists() ), ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - bg_results = c.execute( - "SELECT plugin, status FROM core_archiveresult WHERE plugin IN ('favicon', 'consolelog', 'ssl', 'responses', 'redirects', 'staticfile') AND status IN ('started', 'succeeded', 'failed')", - ).fetchall() - parser_status = c.execute( - "SELECT plugin, status FROM core_archiveresult WHERE plugin LIKE 'parse_%_urls'", - ).fetchall() - - conn.close() + with use_archivebox_db(tmp_path): + bg_results = list( + ArchiveResult.objects.filter( + plugin__in=("favicon", "consolelog", "ssl", "responses", "redirects", "staticfile"), + status__in=("started", "succeeded", "failed"), + ).values_list("plugin", "status"), + ) + parser_status = list( + ArchiveResult.objects.filter(plugin__startswith="parse_", plugin__endswith="_urls").values_list("plugin", "status"), + ) if len(bg_results) > 0: parser_statuses = [status for _, status in parser_status] diff --git a/archivebox/tests/test_runner.py b/archivebox/tests/test_runner.py index ee2feda7..c1ae5c36 100644 --- a/archivebox/tests/test_runner.py +++ b/archivebox/tests/test_runner.py @@ -70,7 +70,7 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path): crawl = Crawl.objects.create( urls="https://example.com", max_depth=0, - max_urls=5, + config={"CRAWL_MAX_URLS": 5}, created_by_id=get_or_create_system_user_pk(), ) snapshot = Snapshot.objects.create( @@ -469,8 +469,8 @@ def test_run_snapshot_seals_descendant_when_crawl_max_size_is_reached(tmp_path): "LIB_DIR": str(tmp_path / "lib"), "PLUGINS": "__archivebox_test_no_plugins__", "CHROME_BINARY": "", + "CRAWL_MAX_SIZE": 16, }, - crawl_max_size=16, created_by_id=get_or_create_system_user_pk(), ) root = Snapshot.objects.create( @@ -583,7 +583,7 @@ def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size(): crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), - crawl_max_size=16, + config={"CRAWL_MAX_SIZE": 16}, ) root = Snapshot.objects.create( url="https://example.com", diff --git a/archivebox/tests/test_schedule.py b/archivebox/tests/test_schedule.py index c891f2de..46577af2 100644 --- a/archivebox/tests/test_schedule.py +++ b/archivebox/tests/test_schedule.py @@ -2,18 +2,14 @@ """Integration tests for the database-backed archivebox schedule command.""" import os -import sqlite3 import subprocess import pytest +from archivebox.crawls.models import Crawl, CrawlSchedule +from archivebox.tests.orm_helpers import use_archivebox_db -def _fetchone(tmp_path, query): - conn = sqlite3.connect(tmp_path / "index.sqlite3") - try: - return conn.execute(query).fetchone() - finally: - conn.close() +pytestmark = pytest.mark.django_db(transaction=True) def test_schedule_creates_enabled_db_schedule(tmp_path, process): @@ -27,17 +23,15 @@ def test_schedule_creates_enabled_db_schedule(tmp_path, process): assert result.returncode == 0 - schedule_row = _fetchone( - tmp_path, - "SELECT schedule, is_enabled, label FROM crawls_crawlschedule ORDER BY created_at DESC LIMIT 1", - ) - crawl_row = _fetchone( - tmp_path, - "SELECT urls, status, max_depth FROM crawls_crawl ORDER BY created_at DESC LIMIT 1", - ) + with use_archivebox_db(tmp_path): + schedule_row = CrawlSchedule.objects.order_by("-created_at").values_list("schedule", "is_enabled", "label").first() + crawl = Crawl.objects.order_by("-created_at").first() - assert schedule_row == ("daily", 1, "Scheduled import: https://example.com/feed.xml") - assert crawl_row == ("https://example.com/feed.xml", "sealed", 1) + assert schedule_row == ("daily", True, "Scheduled import: https://example.com/feed.xml") + assert crawl is not None + assert crawl.urls == "https://example.com/feed.xml" + assert crawl.status == "sealed" + assert crawl.max_depth == 1 def test_schedule_show_lists_enabled_schedules(tmp_path, process): @@ -81,14 +75,9 @@ def test_schedule_clear_disables_existing_schedules(tmp_path, process): assert result.returncode == 0 assert "Disabled 1 scheduled crawl" in result.stdout - disabled_count = _fetchone( - tmp_path, - "SELECT COUNT(*) FROM crawls_crawlschedule WHERE is_enabled = 0", - )[0] - enabled_count = _fetchone( - tmp_path, - "SELECT COUNT(*) FROM crawls_crawlschedule WHERE is_enabled = 1", - )[0] + with use_archivebox_db(tmp_path): + disabled_count = CrawlSchedule.objects.filter(is_enabled=False).count() + enabled_count = CrawlSchedule.objects.filter(is_enabled=True).count() assert disabled_count == 1 assert enabled_count == 0 diff --git a/archivebox/tests/test_schedule_e2e.py b/archivebox/tests/test_schedule_e2e.py index 5694ec2c..5cbf34e3 100644 --- a/archivebox/tests/test_schedule_e2e.py +++ b/archivebox/tests/test_schedule_e2e.py @@ -4,18 +4,24 @@ import os import re import socket -import sqlite3 import subprocess import sys import textwrap import time +from datetime import timedelta from pathlib import Path import pytest import requests +from django.utils import timezone +from archivebox.core.models import ArchiveResult, Snapshot, Tag +from archivebox.crawls.models import Crawl, CrawlSchedule +from archivebox.tests.orm_helpers import use_archivebox_db from .conftest import run_python_cwd +pytestmark = pytest.mark.django_db(transaction=True) + REPO_ROOT = Path(__file__).resolve().parents[2] @@ -120,24 +126,13 @@ def wait_for_http(port: int, host: str, path: str = "/", timeout: int = 30) -> r def make_latest_schedule_due(cwd: Path) -> None: - conn = sqlite3.connect(cwd / "index.sqlite3") - try: - conn.execute( - """ - UPDATE crawls_crawl - SET created_at = datetime('now', '-2 day'), - modified_at = datetime('now', '-2 day') - WHERE id = ( - SELECT template_id - FROM crawls_crawlschedule - ORDER BY created_at DESC - LIMIT 1 - ) - """, + with use_archivebox_db(cwd): + schedule = CrawlSchedule.objects.order_by("-created_at").select_related("template").first() + assert schedule is not None + Crawl.objects.filter(pk=schedule.template_id).update( + created_at=timezone.now() - timedelta(days=2), + modified_at=timezone.now() - timedelta(days=2), ) - conn.commit() - finally: - conn.close() def get_snapshot_file_text(cwd: Path, url: str) -> str: @@ -182,7 +177,7 @@ def get_snapshot_file_text(cwd: Path, url: str) -> str: continue if candidate.suffix not in ('.html', '.htm', '.txt'): continue - if candidate.name in ('stdout.log', 'stderr.log', 'cmd.sh'): + if candidate.name in ('stdout.log', 'stderr.log'): continue candidates.append(candidate) @@ -208,36 +203,52 @@ def wait_for_snapshot_capture(cwd: Path, url: str, timeout: int = 180) -> str: def get_counts(cwd: Path, scheduled_url: str, one_shot_url: str) -> tuple[int, int, int]: - conn = sqlite3.connect(cwd / "index.sqlite3") - try: - scheduled_snapshots = conn.execute( - "SELECT COUNT(*) FROM core_snapshot WHERE url = ?", - (scheduled_url,), - ).fetchone()[0] - one_shot_snapshots = conn.execute( - "SELECT COUNT(*) FROM core_snapshot WHERE url = ?", - (one_shot_url,), - ).fetchone()[0] - scheduled_crawls = conn.execute( - """ - SELECT COUNT(*) - FROM crawls_crawl - WHERE schedule_id IS NOT NULL - AND urls = ? - """, - (scheduled_url,), - ).fetchone()[0] - return scheduled_snapshots, one_shot_snapshots, scheduled_crawls - finally: - conn.close() + with use_archivebox_db(cwd): + scheduled_snapshots = Snapshot.objects.filter(url=scheduled_url).count() + one_shot_snapshots = Snapshot.objects.filter(url=one_shot_url).count() + scheduled_crawls = Crawl.objects.filter(schedule__isnull=False, urls=scheduled_url).count() + return scheduled_snapshots, one_shot_snapshots, scheduled_crawls def get_depth_counts(cwd: Path) -> dict[int, int]: - conn = sqlite3.connect(cwd / "index.sqlite3") - try: - return dict(conn.execute("SELECT depth, COUNT(*) FROM core_snapshot GROUP BY depth").fetchall()) - finally: - conn.close() + with use_archivebox_db(cwd): + return {depth: Snapshot.objects.filter(depth=depth).count() for depth in set(Snapshot.objects.values_list("depth", flat=True))} + + +def get_crawl_runtime_state(cwd: Path, crawl_id: str) -> dict[str, object]: + from archivebox.workers.models import RETRY_AT_MAX + + with use_archivebox_db(cwd): + crawl = Crawl.objects.get(id=crawl_id) + snapshots = list( + crawl.snapshot_set.order_by("created_at").values( + "id", + "url", + "status", + "retry_at", + ), + ) + results = list( + ArchiveResult.objects.filter(snapshot__crawl=crawl) + .order_by("snapshot_id", "plugin", "hook_name") + .values( + "snapshot_id", + "plugin", + "hook_name", + "status", + "retry_at", + "output_files", + "output_size", + ), + ) + + return { + "retry_at_max": RETRY_AT_MAX, + "crawl_status": crawl.status, + "crawl_retry_at": crawl.retry_at, + "snapshots": snapshots, + "results": results, + } def create_admin_and_token(cwd: Path) -> str: @@ -415,19 +426,9 @@ def test_schedule_web_ui_post_works_over_running_server(tmp_path, recursive_test assert response.status_code in (302, 303), response.text - conn = sqlite3.connect(tmp_path / "index.sqlite3") - try: - row = conn.execute( - """ - SELECT cs.schedule, c.urls, c.tags_str - FROM crawls_crawlschedule cs - JOIN crawls_crawl c ON c.schedule_id = cs.id - ORDER BY cs.created_at DESC - LIMIT 1 - """, - ).fetchone() - finally: - conn.close() + with use_archivebox_db(tmp_path): + schedule = CrawlSchedule.objects.select_related("template").order_by("-created_at").first() + row = (schedule.schedule, schedule.template.urls, schedule.template.tags_str) if schedule else None assert row == ("daily", recursive_test_site["root_url"], "web-ui") finally: @@ -493,22 +494,15 @@ def test_web_ui_add_depth_two_crawls_and_renders_real_outputs_over_running_serve else: raise AssertionError(f"timed out waiting for depth=2 crawl, got depth counts {get_depth_counts(tmp_path)}") - conn = sqlite3.connect(tmp_path / "index.sqlite3") - try: - depth_counts = dict(conn.execute("SELECT depth, COUNT(*) FROM core_snapshot GROUP BY depth").fetchall()) - crawl = conn.execute( - "SELECT max_depth, max_urls, tags_str, notes FROM crawls_crawl ORDER BY created_at DESC LIMIT 1", - ).fetchone() - snapshot_rows = conn.execute( - "SELECT url, depth, status, parent_snapshot_id FROM core_snapshot ORDER BY depth, url", - ).fetchall() - archive_results = conn.execute( - "SELECT plugin, status, output_files, output_size FROM core_archiveresult ORDER BY plugin, status", - ).fetchall() - finally: - conn.close() + with use_archivebox_db(tmp_path): + depth_counts = get_depth_counts(tmp_path) + crawl_obj = Crawl.objects.order_by("-created_at").first() + crawl = (crawl_obj.max_depth, crawl_obj.tags_str, crawl_obj.notes, crawl_obj.config) if crawl_obj else None + snapshot_rows = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "status", "parent_snapshot_id")) + archive_results = list(ArchiveResult.objects.order_by("plugin", "status").values_list("plugin", "status", "output_files", "output_size")) - assert crawl == (2, 20, "web-depth-two", "created from running-server web ui") + assert crawl[:3] == (2, "web-depth-two", "created from running-server web ui") + assert (crawl[3] or {})["CRAWL_MAX_URLS"] == 20 assert depth_counts.get(0, 0) >= 1 assert depth_counts.get(1, 0) >= len(recursive_test_site["child_urls"]) assert depth_counts.get(2, 0) >= len(recursive_test_site["deep_urls"]) @@ -735,13 +729,202 @@ def test_core_rest_api_crud_uses_token_auth_and_persists_side_effects_over_runni assert delete_crawl.status_code == 200, delete_crawl.text assert delete_crawl.json()["success"] is True - conn = sqlite3.connect(tmp_path / "index.sqlite3") - try: - assert conn.execute("SELECT COUNT(*) FROM crawls_crawl WHERE id = ?", (crawl_id,)).fetchone()[0] == 0 - assert conn.execute("SELECT COUNT(*) FROM core_snapshot WHERE id = ?", (snapshot_id,)).fetchone()[0] == 0 - assert conn.execute("SELECT COUNT(*) FROM core_tag WHERE name = 'api-extra'").fetchone()[0] == 1 - finally: - conn.close() + with use_archivebox_db(tmp_path): + assert Crawl.objects.filter(pk=crawl_id).count() == 0 + assert Snapshot.objects.filter(pk=snapshot_id).count() == 0 + assert Tag.objects.filter(name="api-extra").count() == 1 + finally: + stop_server(tmp_path) + + +@pytest.mark.timeout(240) +def test_pause_resume_crawl_api_survives_server_restart_and_processes_after_resume(tmp_path, recursive_test_site): + os.chdir(tmp_path) + init_archive(tmp_path) + + port = get_free_port() + env = build_test_env(port, PLUGINS="wget", SAVE_WGET="True") + api_token = create_admin_and_token(tmp_path) + api_headers = { + "Host": f"api.archivebox.localhost:{port}", + "X-ArchiveBox-API-Key": api_token, + } + + try: + start_server(tmp_path, env=env, port=port) + wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") + + crawl_response = requests.post( + f"http://127.0.0.1:{port}/api/v1/crawls/crawls", + headers=api_headers, + json={ + "urls": [recursive_test_site["root_url"]], + "max_depth": 0, + "tags": ["pause-resume-e2e"], + "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, + }, + timeout=10, + ) + assert crawl_response.status_code == 200, crawl_response.text + crawl_id = crawl_response.json()["id"] + + pause_response = requests.patch( + f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", + headers=api_headers, + json={"action": "pause"}, + timeout=10, + ) + assert pause_response.status_code == 200, pause_response.text + assert pause_response.json()["status"] == "paused" + + paused_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert paused_state["crawl_status"] == "paused" + assert paused_state["crawl_retry_at"] == paused_state["retry_at_max"] + assert len(paused_state["snapshots"]) == 1 + assert paused_state["snapshots"][0]["status"] == "paused" + assert paused_state["snapshots"][0]["retry_at"] == paused_state["retry_at_max"] + + stop_server(tmp_path) + start_server(tmp_path, env=env, port=port) + wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") + time.sleep(3) + + restarted_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert restarted_state["crawl_status"] == "paused" + assert restarted_state["crawl_retry_at"] == restarted_state["retry_at_max"] + assert restarted_state["snapshots"][0]["status"] == "paused" + assert restarted_state["snapshots"][0]["retry_at"] == restarted_state["retry_at_max"] + assert not any(result["status"] == "succeeded" for result in restarted_state["results"]) + + resume_response = requests.patch( + f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", + headers=api_headers, + json={"action": "resume"}, + timeout=10, + ) + assert resume_response.status_code == 200, resume_response.text + assert resume_response.json()["status"] == "queued" + + captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) + assert "Root" in captured_text + assert "About" in captured_text + + final_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert final_state["snapshots"][0]["status"] == "sealed" + wget_results = [result for result in final_state["results"] if result["plugin"] == "wget"] + assert wget_results + assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results) + finally: + stop_server(tmp_path) + + +@pytest.mark.timeout(180) +def test_index_only_update_processes_paused_snapshot_search_rows_without_resuming_it(tmp_path, recursive_test_site): + os.chdir(tmp_path) + init_archive(tmp_path) + + port = get_free_port() + env = build_test_env(port, PLUGINS="wget", SAVE_WGET="True") + api_token = create_admin_and_token(tmp_path) + api_headers = { + "Host": f"api.archivebox.localhost:{port}", + "X-ArchiveBox-API-Key": api_token, + } + + try: + start_server(tmp_path, env=env, port=port) + wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") + + crawl_response = requests.post( + f"http://127.0.0.1:{port}/api/v1/crawls/crawls", + headers=api_headers, + json={ + "urls": [recursive_test_site["root_url"]], + "max_depth": 0, + "tags": ["paused-index-e2e"], + "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, + }, + timeout=10, + ) + assert crawl_response.status_code == 200, crawl_response.text + crawl_id = crawl_response.json()["id"] + + pause_response = requests.patch( + f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", + headers=api_headers, + json={"action": "pause"}, + timeout=10, + ) + assert pause_response.status_code == 200, pause_response.text + assert pause_response.json()["status"] == "paused" + finally: + stop_server(tmp_path) + + update_env = build_test_env( + port, + PLUGINS="search_backend_sqlite", + SEARCH_BACKEND_ENGINE="sqlite", + USE_INDEXING_BACKEND="True", + USE_SEARCHING_BACKEND="True", + ) + update_process = subprocess.run( + [ + sys.executable, + "-m", + "archivebox", + "update", + "--index-only", + "--crawl-id", + crawl_id, + "--limit", + "1", + "--batch-size", + "1", + ], + cwd=tmp_path, + capture_output=True, + text=True, + env=update_env, + timeout=120, + ) + assert update_process.returncode == 0, update_process.stderr + + indexed_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert indexed_state["crawl_status"] == "paused" + assert indexed_state["crawl_retry_at"] == indexed_state["retry_at_max"] + assert indexed_state["snapshots"][0]["status"] == "paused" + assert indexed_state["snapshots"][0]["retry_at"] == indexed_state["retry_at_max"] + search_results = [result for result in indexed_state["results"] if result["plugin"] == "search_backend_sqlite"] + assert search_results + assert all(result["status"] not in {"queued", "started", "paused"} for result in search_results) + + try: + start_server(tmp_path, env=env, port=port) + wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") + + still_paused_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert still_paused_state["crawl_status"] == "paused" + assert still_paused_state["snapshots"][0]["status"] == "paused" + assert not any(result["plugin"] == "wget" and result["status"] == "succeeded" for result in still_paused_state["results"]) + + resume_response = requests.patch( + f"http://127.0.0.1:{port}/api/v1/crawls/crawl/{crawl_id}", + headers=api_headers, + json={"action": "resume"}, + timeout=10, + ) + assert resume_response.status_code == 200, resume_response.text + assert resume_response.json()["status"] == "queued" + + captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) + assert "Root" in captured_text + assert "About" in captured_text + + resumed_state = get_crawl_runtime_state(tmp_path, crawl_id) + assert resumed_state["snapshots"][0]["status"] == "sealed" + wget_results = [result for result in resumed_state["results"] if result["plugin"] == "wget"] + assert wget_results + assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results) finally: stop_server(tmp_path) @@ -824,19 +1007,14 @@ def test_cli_rest_api_add_search_update_remove_over_running_server(tmp_path, rec assert update_response.status_code == 200, update_response.text assert update_response.json()["success"] is True - conn = sqlite3.connect(tmp_path / "index.sqlite3") - try: - crawl = conn.execute( - "SELECT max_depth, tags_str, config FROM crawls_crawl WHERE id IN (?, ?)", - (crawl_id, crawl_id.replace("-", "")), - ).fetchone() - finally: - conn.close() + with use_archivebox_db(tmp_path): + crawl_obj = Crawl.objects.filter(pk=crawl_id).first() + crawl = (crawl_obj.max_depth, crawl_obj.tags_str, crawl_obj.config) if crawl_obj else None assert crawl is not None assert crawl[0] == 1 assert crawl[1] == "api-cli" - assert '"INDEX_ONLY": true' in crawl[2] or '"INDEX_ONLY":true' in crawl[2] + assert crawl[2]["INDEX_ONLY"] is True remove_response = requests.post( f"http://127.0.0.1:{port}/api/v1/cli/remove", @@ -856,11 +1034,8 @@ def test_cli_rest_api_add_search_update_remove_over_running_server(tmp_path, rec assert remove_payload["result"]["removed_count"] == 1 assert snapshot_id in remove_payload["result"]["removed_snapshot_ids"] - conn = sqlite3.connect(tmp_path / "index.sqlite3") - try: - snapshot_count = conn.execute("SELECT COUNT(*) FROM core_snapshot WHERE id = ?", (snapshot_id,)).fetchone()[0] - finally: - conn.close() + with use_archivebox_db(tmp_path): + snapshot_count = Snapshot.objects.filter(pk=snapshot_id).count() assert snapshot_count == 0 finally: diff --git a/archivebox/tests/test_search_backends_e2e.py b/archivebox/tests/test_search_backends_e2e.py new file mode 100644 index 00000000..f0c1c637 --- /dev/null +++ b/archivebox/tests/test_search_backends_e2e.py @@ -0,0 +1,140 @@ +import os +import signal +import socket +import subprocess +import sys +import time + + +def test_real_fulltext_search_backends_survive_reindex_transition(tmp_path): + data_dir = tmp_path / "archivebox_data" + data_dir.mkdir() + query = "documentation examples" + + def free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + def archivebox(*args: str, env: dict[str, str] | None = None, timeout: int = 120) -> subprocess.CompletedProcess[str]: + merged_env = os.environ.copy() + merged_env.update( + { + "DATA_DIR": str(data_dir), + "USE_COLOR": "False", + "SHOW_PROGRESS": "False", + "SAVE_WARC": "False", + "WGET_WARC_ENABLED": "False", + "WGET_TIMEOUT": "20", + "USE_SEARCHING_BACKEND": "true", + "USE_INDEXING_BACKEND": "true", + }, + ) + if env: + merged_env.update(env) + return subprocess.run( + [sys.executable, "-m", "archivebox", *args], + cwd=data_dir, + env=merged_env, + capture_output=True, + text=True, + timeout=timeout, + ) + + init_result = archivebox("init", "--quick", timeout=90) + assert init_result.returncode == 0, init_result.stderr + + add_result = archivebox("add", "--depth=0", "--plugins=wget", "https://example.com", env={"SEARCH_BACKEND_ENGINE": "ripgrep"}) + assert add_result.returncode == 0, add_result.stderr + + rg_result = archivebox("list", "--search=contents", "--csv=url", query, env={"SEARCH_BACKEND_ENGINE": "ripgrep"}) + assert rg_result.returncode == 0, rg_result.stderr + assert "https://example.com" in rg_result.stdout + + sqlite_update = archivebox("update", "--index-only", "--batch-size=10", env={"SEARCH_BACKEND_ENGINE": "sqlite"}) + assert sqlite_update.returncode == 0, sqlite_update.stderr + sqlite_result = archivebox("list", "--search=contents", "--csv=url", query, env={"SEARCH_BACKEND_ENGINE": "sqlite"}) + assert sqlite_result.returncode == 0, sqlite_result.stderr + assert "https://example.com" in sqlite_result.stdout + + http_port = free_port() + sonic_port = free_port() + sonic_env = os.environ.copy() + sonic_env.update( + { + "DATA_DIR": str(data_dir), + "USE_COLOR": "False", + "SHOW_PROGRESS": "False", + "SEARCH_BACKEND_ENGINE": "sonic", + "USE_SEARCHING_BACKEND": "true", + "USE_INDEXING_BACKEND": "true", + "SEARCH_BACKEND_SONIC_PORT": str(sonic_port), + }, + ) + server_log = data_dir / "server.log" + with server_log.open("w", encoding="utf-8") as log_file: + server = subprocess.Popen( + [sys.executable, "-m", "archivebox", "server", f"127.0.0.1:{http_port}"], + cwd=data_dir, + env=sonic_env, + stdout=log_file, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + try: + for _ in range(80): + if server.poll() is not None: + break + try: + with socket.create_connection(("127.0.0.1", sonic_port), timeout=0.25): + break + except OSError: + time.sleep(0.5) + else: + raise AssertionError(f"Sonic did not start on port {sonic_port}:\n{server_log.read_text(encoding='utf-8', errors='replace')}") + + sonic_update = archivebox( + "update", + "--index-only", + "--batch-size=10", + env={ + "SEARCH_BACKEND_ENGINE": "sonic", + "SEARCH_BACKEND_SONIC_PORT": str(sonic_port), + }, + ) + assert sonic_update.returncode == 0, sonic_update.stderr + sonic_result = archivebox( + "list", + "--search=contents", + "--csv=url", + query, + env={ + "SEARCH_BACKEND_ENGINE": "sonic", + "SEARCH_BACKEND_SONIC_PORT": str(sonic_port), + }, + ) + assert sonic_result.returncode == 0, sonic_result.stderr + assert "https://example.com" in sonic_result.stdout + finally: + if server.poll() is None: + os.killpg(server.pid, signal.SIGTERM) + try: + server.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(server.pid, signal.SIGKILL) + server.wait(timeout=10) + + for _ in range(20): + leftovers = subprocess.run( + ["pgrep", "-af", str(data_dir)], + capture_output=True, + text=True, + timeout=5, + ) + remaining = [line for line in leftovers.stdout.splitlines() if "pgrep -af" not in line] + if not remaining: + break + time.sleep(0.25) + else: + raise AssertionError(f"archivebox server left supervised worker processes running:\n{leftovers.stdout}") diff --git a/archivebox/tests/test_snapshot.py b/archivebox/tests/test_snapshot.py index 4147ec51..9b6b8f92 100644 --- a/archivebox/tests/test_snapshot.py +++ b/archivebox/tests/test_snapshot.py @@ -3,14 +3,17 @@ import os import subprocess -import sqlite3 from archivebox.machine.models import Process -from datetime import datetime from urllib.parse import urlparse import uuid import pytest +from archivebox.core.models import Snapshot, Tag +from archivebox.tests.orm_helpers import use_archivebox_db + +pytestmark = pytest.mark.django_db(transaction=True) + def test_snapshot_creates_snapshot_with_correct_url(tmp_path, process, disable_extractors_dict): """Test that snapshot stores the exact URL in the database.""" @@ -22,29 +25,14 @@ def test_snapshot_creates_snapshot_with_correct_url(tmp_path, process, disable_e env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot_row = c.execute( - "SELECT id, created_at, url, crawl_id FROM core_snapshot WHERE url = ?", - ("https://example.com",), - ).fetchone() - assert snapshot_row is not None - crawl_row = c.execute( - "SELECT id, created_at, urls, created_by_id FROM crawls_crawl WHERE id = ?", - (snapshot_row[3],), - ).fetchone() - assert crawl_row is not None - user_row = c.execute( - "SELECT username FROM auth_user WHERE id = ?", - (crawl_row[3],), - ).fetchone() - assert user_row is not None - conn.close() + with use_archivebox_db(tmp_path): + snapshot = Snapshot.objects.select_related("crawl__created_by").get(url="https://example.com") + snapshot_id_raw = str(snapshot.id) + snapshot_date_str = snapshot.created_at.strftime("%Y%m%d") + snapshot_url = snapshot.url + username = snapshot.crawl.created_by.username - snapshot_id_raw, snapshot_created_at, snapshot_url, crawl_id = snapshot_row snapshot_id = str(uuid.UUID(snapshot_id_raw)) - username = user_row[0] - snapshot_date_str = datetime.fromisoformat(snapshot_created_at).strftime("%Y%m%d") domain = urlparse(snapshot_url).hostname or "unknown" # Verify crawl symlink exists and is relative @@ -75,12 +63,9 @@ def test_snapshot_multiple_urls_creates_multiple_records(tmp_path, process, disa env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - urls = c.execute("SELECT url FROM core_snapshot ORDER BY url").fetchall() - conn.close() + with use_archivebox_db(tmp_path): + urls = list(Snapshot.objects.order_by("url").values_list("url", flat=True)) - urls = [u[0] for u in urls] assert "https://example.com" in urls assert "https://iana.org" in urls assert len(urls) >= 2 @@ -102,33 +87,12 @@ def test_snapshot_tag_creates_tag_and_links_to_snapshot(tmp_path, process, disab env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - # Verify tag was created - tag = c.execute("SELECT id, name FROM core_tag WHERE name = ?", ("mytesttag",)).fetchone() - assert tag is not None, "Tag 'mytesttag' should exist in core_tag" - tag_id = tag[0] - - # Verify snapshot exists - snapshot = c.execute( - "SELECT id FROM core_snapshot WHERE url = ?", - ("https://example.com",), - ).fetchone() - assert snapshot is not None - snapshot_id = snapshot[0] - - # Verify tag is linked to snapshot via join table - link = c.execute( - """ - SELECT * FROM core_snapshot_tags - WHERE snapshot_id = ? AND tag_id = ? - """, - (snapshot_id, tag_id), - ).fetchone() - conn.close() - - assert link is not None, "Tag should be linked to snapshot via core_snapshot_tags" + with use_archivebox_db(tmp_path): + tag = Tag.objects.filter(name="mytesttag").first() + assert tag is not None, "Tag 'mytesttag' should exist in core_tag" + snapshot = Snapshot.objects.filter(url="https://example.com").first() + assert snapshot is not None + assert snapshot.tags.filter(pk=tag.pk).exists(), "Tag should be linked to snapshot via core_snapshot_tags" def test_snapshot_jsonl_output_has_correct_structure(tmp_path, process, disable_extractors_dict): @@ -168,18 +132,11 @@ def test_snapshot_with_tag_stores_tag_name(tmp_path, process, disable_extractors env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - - # Verify tag was created with correct name - tag = c.execute( - "SELECT name FROM core_tag WHERE name = ?", - ("customtag",), - ).fetchone() - conn.close() + with use_archivebox_db(tmp_path): + tag = Tag.objects.filter(name="customtag").first() assert tag is not None - assert tag[0] == "customtag" + assert tag.name == "customtag" def test_snapshot_with_depth_sets_snapshot_depth(tmp_path, process, disable_extractors_dict): @@ -198,13 +155,11 @@ def test_snapshot_with_depth_sets_snapshot_depth(tmp_path, process, disable_extr env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - snapshot = c.execute("SELECT depth FROM core_snapshot ORDER BY created_at DESC LIMIT 1").fetchone() - conn.close() + with use_archivebox_db(tmp_path): + snapshot = Snapshot.objects.order_by("-created_at").first() assert snapshot is not None, "Snapshot should be created when depth is provided" - assert snapshot[0] == 1, "Snapshot depth should match --depth value" + assert snapshot.depth == 1, "Snapshot depth should match --depth value" def test_snapshot_allows_duplicate_urls_across_crawls(tmp_path, process, disable_extractors_dict): @@ -223,13 +178,8 @@ def test_snapshot_allows_duplicate_urls_across_crawls(tmp_path, process, disable env={**disable_extractors_dict, "DATA_DIR": str(tmp_path)}, ) - conn = sqlite3.connect("index.sqlite3") - c = conn.cursor() - count = c.execute( - "SELECT COUNT(*) FROM core_snapshot WHERE url = ?", - ("https://example.com",), - ).fetchone()[0] - conn.close() + with use_archivebox_db(tmp_path): + count = Snapshot.objects.filter(url="https://example.com").count() assert count == 2, "Same URL should create separate snapshots across different crawls" diff --git a/archivebox/tests/test_title.py b/archivebox/tests/test_title.py index 63082a3e..102c041c 100644 --- a/archivebox/tests/test_title.py +++ b/archivebox/tests/test_title.py @@ -1,11 +1,15 @@ -import os -import sqlite3 import subprocess import sys +import pytest + +from archivebox.core.models import Snapshot +from archivebox.tests.orm_helpers import use_archivebox_db from .conftest import _find_system_browser from .fixtures import disable_extractors_dict, process +pytestmark = pytest.mark.django_db(transaction=True) + FIXTURES = (disable_extractors_dict, process) @@ -39,16 +43,11 @@ def test_title_is_extracted(tmp_path, process, disable_extractors_dict): ) assert add_process.returncode == 0, add_process.stderr or add_process.stdout - os.chdir(tmp_path) - conn = sqlite3.connect("index.sqlite3") - conn.row_factory = sqlite3.Row - c = conn.cursor() - c.execute("SELECT title FROM core_snapshot") - snapshot = c.fetchone() - conn.close() + with use_archivebox_db(tmp_path): + title = Snapshot.objects.values_list("title", flat=True).get() - assert snapshot[0] is not None - assert "Example" in snapshot[0] + assert title is not None + assert "Example" in title def test_title_is_htmlencoded_in_index_html(tmp_path, process, disable_extractors_dict): diff --git a/archivebox/tests/test_update.py b/archivebox/tests/test_update.py index 9cd55cff..a0335e8d 100644 --- a/archivebox/tests/test_update.py +++ b/archivebox/tests/test_update.py @@ -1,13 +1,17 @@ import json -import sqlite3 +import os import subprocess from datetime import datetime, timedelta import pytest from django.utils import timezone +from archivebox.core.models import Snapshot +from archivebox.tests.orm_helpers import use_archivebox_db from .fixtures import disable_extractors_dict, process +pytestmark = pytest.mark.django_db(transaction=True) + FIXTURES = (disable_extractors_dict, process) @@ -29,9 +33,9 @@ def test_update_imports_orphaned_snapshots(tmp_path, process, disable_extractors ), ) - # Run update without filters - should import and migrate the legacy directory. + # Run the migration phase only; default update also runs queued crawl work. update_process = subprocess.run( - ["archivebox", "update"], + ["archivebox", "update", "--migrate-only"], capture_output=True, text=True, env=disable_extractors_dict, @@ -39,11 +43,8 @@ def test_update_imports_orphaned_snapshots(tmp_path, process, disable_extractors ) assert update_process.returncode == 0, update_process.stderr - conn = sqlite3.connect(str(tmp_path / "index.sqlite3")) - c = conn.cursor() - row = c.execute("SELECT url, fs_version FROM core_snapshot").fetchone() - conn.commit() - conn.close() + with use_archivebox_db(tmp_path): + row = Snapshot.objects.values_list("url", "fs_version").get() assert row == ("https://example.com", "0.9.0") assert legacy_dir.is_symlink() @@ -54,13 +55,12 @@ def test_update_imports_orphaned_snapshots(tmp_path, process, disable_extractors assert (migrated_dir / "singlefile.html").exists() -@pytest.mark.django_db -def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_plugins(monkeypatch): +@pytest.mark.django_db(transaction=True) +def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_plugins(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.cli.archivebox_update import reindex_snapshots from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl - import archivebox.cli.archivebox_extract as extract_mod crawl = Crawl.objects.create( urls="https://example.com", @@ -74,48 +74,47 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p result = ArchiveResult.objects.create( snapshot=snapshot, plugin="search_backend_sqlite", - hook_name="on_Snapshot__90_index_sqlite.py", + hook_name="on_Snapshot__90_index_sqlite", status=ArchiveResult.StatusChoices.SUCCEEDED, output_str="old index hit", output_json={"indexed": True}, output_files={"search.sqlite3": {"size": 123}}, output_size=123, ) + output_dir = snapshot.output_dir + (output_dir / "title").mkdir(parents=True, exist_ok=True) + (output_dir / "title" / "title.txt").write_text("Example Domain") + (output_dir / "dom").mkdir(parents=True, exist_ok=True) + (output_dir / "dom" / "output.html").write_text("Example searchable text") - captured: dict[str, object] = {} - - def fake_run_plugins(*, args, records, wait, emit_results, plugins=""): - captured["args"] = args - captured["records"] = records - captured["wait"] = wait - captured["emit_results"] = emit_results - captured["plugins"] = plugins - return 0 - - monkeypatch.setattr(extract_mod, "run_plugins", fake_run_plugins) - - stats = reindex_snapshots( - Snapshot.objects.filter(id=snapshot.id), - search_plugins=["search_backend_sqlite"], - batch_size=10, - ) + original_engine = os.environ.get("SEARCH_BACKEND_ENGINE") + original_indexing = os.environ.get("USE_INDEXING_BACKEND") + os.environ["SEARCH_BACKEND_ENGINE"] = "sqlite" + os.environ["USE_INDEXING_BACKEND"] = "true" + try: + stats = reindex_snapshots( + Snapshot.objects.filter(id=snapshot.id), + search_plugins=["search_backend_sqlite"], + batch_size=10, + ) + finally: + if original_engine is None: + os.environ.pop("SEARCH_BACKEND_ENGINE", None) + else: + os.environ["SEARCH_BACKEND_ENGINE"] = original_engine + if original_indexing is None: + os.environ.pop("USE_INDEXING_BACKEND", None) + else: + os.environ["USE_INDEXING_BACKEND"] = original_indexing result.refresh_from_db() assert stats["processed"] == 1 assert stats["queued"] == 1 - assert stats["reindexed"] == 1 + assert stats["reindexed"] == 0 assert result.status == ArchiveResult.StatusChoices.QUEUED assert result.output_str == "" assert result.output_json is None - assert result.output_files == {} - assert captured == { - "args": (), - "records": [{"type": "ArchiveResult", "snapshot_id": str(snapshot.id), "plugin": "search_backend_sqlite"}], - "wait": True, - "emit_results": False, - "plugins": "", - } @pytest.mark.django_db @@ -160,6 +159,47 @@ def test_build_filtered_snapshots_queryset_respects_resume_cutoff(): assert set(map(str, snapshots)) == {str(middle.id), str(older.id)} +@pytest.mark.django_db +def test_build_filtered_snapshots_queryset_accepts_list_style_filters(): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset + from archivebox.core.models import Snapshot, Tag + from archivebox.crawls.models import Crawl + + crawl = Crawl.objects.create( + urls="https://example.com\nhttps://example.org", + created_by_id=get_or_create_system_user_pk(), + ) + tagged = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + title="Example Domain", + status=Snapshot.StatusChoices.SEALED, + ) + Snapshot.objects.create( + url="https://example.org", + crawl=crawl, + title="Other Example", + status=Snapshot.StatusChoices.QUEUED, + ) + tagged.tags.add(Tag.objects.create(name="keep")) + + snapshots = list( + _build_filtered_snapshots_queryset( + filter_patterns=(), + filter_type="exact", + status=Snapshot.StatusChoices.SEALED, + url__icontains="example", + tag="keep", + crawl_id=str(crawl.id), + limit=1, + sort="url", + ).values_list("id", flat=True), + ) + + assert snapshots == [tagged.id] + + @pytest.mark.django_db def test_reconcile_with_index_json_tolerates_null_title(tmp_path): from archivebox.base_models.models import get_or_create_system_user_pk diff --git a/archivebox/workers/management/commands/runner_watch.py b/archivebox/workers/management/commands/runner_watch.py index ab784d50..af5f5e57 100644 --- a/archivebox/workers/management/commands/runner_watch.py +++ b/archivebox/workers/management/commands/runner_watch.py @@ -2,13 +2,13 @@ from django.core.management.base import BaseCommand class Command(BaseCommand): - help = "Watch the runserver autoreload PID file and restart the background runner on reloads." + help = "Watch the debug runserver Process row and restart the background runner on autoreloads." def add_arguments(self, parser): parser.add_argument( - "--pidfile", - default=None, - help="Path to runserver pidfile to watch", + "--bind-url", + default="", + help="Runserver bind URL to watch, e.g. http://127.0.0.1:8000", ) parser.add_argument( "--interval", @@ -21,9 +21,7 @@ class Command(BaseCommand): import os import time - import psutil - - from archivebox.config.common import get_config + from archivebox.config import CONSTANTS from archivebox.machine.models import Machine, Process from archivebox.workers.supervisord_util import ( RUNNER_WORKER, @@ -33,33 +31,31 @@ class Command(BaseCommand): stop_worker, ) - pidfile = kwargs.get("pidfile") or os.environ.get("ARCHIVEBOX_RUNSERVER_PIDFILE") - if not pidfile: - pidfile = str(get_config().TMP_DIR / "runserver.pid") + bind_url = kwargs.get("bind_url") or os.environ.get("ARCHIVEBOX_RUNSERVER_BIND_URL") or "" + current = Process.current() + current.mark_running( + process_type=Process.TypeChoices.WORKER, + worker_type="worker_runner_watch", + pwd=str(CONSTANTS.DATA_DIR), + url=bind_url, + timeout=0, + ) interval = max(0.2, float(kwargs.get("interval", 1.0))) - last_pid = None + last_runserver_id = None def stop_duplicate_watchers() -> None: - current_pid = os.getpid() - for proc in psutil.process_iter(["pid", "cmdline"]): - if proc.info["pid"] == current_pid: - continue - cmdline = proc.info.get("cmdline") or [] - if not cmdline: - continue - if "runner_watch" not in " ".join(cmdline): - continue - if not any(str(arg) == f"--pidfile={pidfile}" or str(arg) == pidfile for arg in cmdline): - continue - try: - proc.terminate() - proc.wait(timeout=2.0) - except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.TimeoutExpired): - try: - proc.kill() - except (psutil.NoSuchProcess, psutil.AccessDenied): - pass + machine = Machine.current() + for proc in Process.objects.filter( + machine=machine, + status=Process.StatusChoices.RUNNING, + process_type=Process.TypeChoices.WORKER, + worker_type="worker_runner_watch", + pwd=str(CONSTANTS.DATA_DIR), + url=bind_url, + ).exclude(id=current.id): + if proc.is_running: + proc.terminate(graceful_timeout=2.0) def get_supervisor(): supervisor = get_existing_supervisord_process() @@ -67,30 +63,40 @@ class Command(BaseCommand): raise RuntimeError("runner_watch requires a running supervisord process") return supervisor + def current_runserver(): + machine = Machine.current() + for proc in Process.objects.filter( + machine=machine, + status=Process.StatusChoices.RUNNING, + process_type=Process.TypeChoices.WORKER, + worker_type="worker_runserver", + pwd=str(CONSTANTS.DATA_DIR), + url=bind_url, + ).order_by("-started_at", "-created_at"): + if proc.is_running: + return proc + return None + stop_duplicate_watchers() start_worker(get_supervisor(), RUNNER_WORKER, lazy=True) def restart_runner() -> None: machine = Machine.current() - running = Process.objects.filter( + for proc in Process.objects.filter( machine=machine, status=Process.StatusChoices.RUNNING, process_type=Process.TypeChoices.ORCHESTRATOR, - ) - for proc in running: - try: + pwd=str(CONSTANTS.DATA_DIR), + ): + if proc.is_running: proc.kill_tree(graceful_timeout=0.5) - except Exception: - continue supervisor = get_supervisor() - try: stop_worker(supervisor, RUNNER_WORKER["name"]) except Exception: pass - start_worker(supervisor, RUNNER_WORKER) def runner_running() -> bool: @@ -99,17 +105,14 @@ class Command(BaseCommand): while True: try: - if os.path.exists(pidfile): - with open(pidfile) as handle: - pid = handle.read().strip() or None - else: - pid = None - - if pid and pid != last_pid: + runserver = current_runserver() + runserver_id = str(runserver.id) if runserver else None + if runserver_id and runserver_id != last_runserver_id: restart_runner() - last_pid = pid + last_runserver_id = runserver_id elif not runner_running(): restart_runner() + current.heartbeat() except Exception: pass diff --git a/archivebox/workers/models.py b/archivebox/workers/models.py index 18254404..8519eb67 100644 --- a/archivebox/workers/models.py +++ b/archivebox/workers/models.py @@ -2,7 +2,7 @@ __package__ = "archivebox.workers" from typing import ClassVar from collections.abc import Iterable -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from statemachine.mixins import MachineMixin from django.db import models @@ -17,6 +17,7 @@ from statemachine import registry, StateMachine, State class DefaultStatusChoices(models.TextChoices): QUEUED = "queued", "Queued" STARTED = "started", "Started" + PAUSED = "paused", "Paused" SEALED = "sealed", "Sealed" @@ -29,6 +30,7 @@ default_status_field: models.CharField = models.CharField( db_index=True, ) default_retry_at_field: models.DateTimeField = models.DateTimeField(default=timezone.now, null=True, blank=True, db_index=True) +RETRY_AT_MAX = datetime.max.replace(tzinfo=UTC) ObjectState = State | str ObjectStateList = Iterable[ObjectState] @@ -193,6 +195,70 @@ class BaseModelWithStateMachine(models.Model, MachineMixin): def bump_retry_at(self, seconds: int = 10): self.RETRY_AT = timezone.now() + timedelta(seconds=seconds) + @property + def is_paused(self) -> bool: + paused_state = getattr(self.StatusChoices, "PAUSED", None) + return paused_state is not None and self.STATE == paused_state + + def pause(self, *, save: bool = True) -> bool: + paused_state = getattr(self.StatusChoices, "PAUSED", None) + if paused_state is None: + return False + if self.STATE in self.FINAL_STATES or self.is_paused: + return False + if save: + transition = getattr(getattr(self, self.state_machine_attr, None), "pause_requested", None) + if callable(transition): + transition() + self.refresh_from_db() + return self.is_paused + self.STATE = paused_state + self.RETRY_AT = RETRY_AT_MAX + updated = type(self).objects.filter(pk=self.pk).exclude( + **{self.state_field_name + "__in": [*self.FINAL_STATES, paused_state]}, + ).update( + **{ + self.state_field_name: paused_state, + self.retry_at_field_name: RETRY_AT_MAX, + "modified_at": timezone.now(), + }, + ) + self.refresh_from_db() + return updated == 1 + self.STATE = paused_state + self.RETRY_AT = RETRY_AT_MAX + return True + + def resume(self, *, when: datetime | None = None, save: bool = True) -> bool: + paused_state = getattr(self.StatusChoices, "PAUSED", None) + if paused_state is None: + return False + if not self.is_paused: + return False + if save: + transition = getattr(getattr(self, self.state_machine_attr, None), "resume_requested", None) + if callable(transition) and when is None: + transition() + self.refresh_from_db() + return self.STATE == self.StatusChoices.QUEUED + self.STATE = self.StatusChoices.QUEUED + self.RETRY_AT = when or timezone.now() + updated = type(self).objects.filter( + pk=self.pk, + **{self.state_field_name: paused_state}, + ).update( + **{ + self.state_field_name: self.StatusChoices.QUEUED, + self.retry_at_field_name: self.RETRY_AT, + "modified_at": timezone.now(), + }, + ) + self.refresh_from_db() + return updated == 1 + self.STATE = self.StatusChoices.QUEUED + self.RETRY_AT = when or timezone.now() + return True + def update_and_requeue(self, **kwargs) -> bool: """ Atomically update fields and schedule retry_at for next worker tick. diff --git a/archivebox/workers/supervisord_util.py b/archivebox/workers/supervisord_util.py index dcea2ade..a7744e5c 100644 --- a/archivebox/workers/supervisord_util.py +++ b/archivebox/workers/supervisord_util.py @@ -3,23 +3,29 @@ __package__ = "archivebox.workers" import sys import time import socket +import os import psutil import shutil import subprocess import shlex from typing import cast -from collections.abc import Iterator from pathlib import Path from functools import cache from rich import print from supervisor.xmlrpc import SupervisorTransport -from xmlrpc.client import ServerProxy +from xmlrpc.client import Fault, ServerProxy from archivebox.config import CONSTANTS from archivebox.config.paths import get_or_create_working_tmp_dir from archivebox.config.permissions import ARCHIVEBOX_USER +from archivebox.core.shutdown_util import ( + configured_stopwaitsecs, + foreground_shutdown_signals, + wait_popen_and_kill_children, + wait_psutil_and_kill_children, +) from archivebox.misc.logging import STDERR from archivebox.misc.logging_util import pretty_path @@ -30,12 +36,58 @@ WORKERS_DIR_NAME = "workers" # Global reference to supervisord process for cleanup _supervisord_proc = None +_desired_supervisord_workers: dict[str, dict[str, str]] = {} def _shell_join(args: list[str]) -> str: return shlex.join(args) +def _record_supervisord_process(proc: subprocess.Popen, config_file: Path) -> None: + try: + from datetime import datetime + from django.utils import timezone + from archivebox.machine.models import Machine, Process + + try: + started_at = datetime.fromtimestamp(psutil.Process(proc.pid).create_time(), tz=timezone.get_current_timezone()) + except (psutil.NoSuchProcess, psutil.AccessDenied): + started_at = timezone.now() + + Process.objects.create( + machine=Machine.current(), + process_type=Process.TypeChoices.SUPERVISORD, + worker_type="supervisord", + pwd=str(CONSTANTS.DATA_DIR), + cmd=["supervisord", f"--configuration={config_file}"], + pid=proc.pid, + started_at=started_at, + status=Process.StatusChoices.RUNNING, + timeout=0, + ) + except Exception: + pass + + +def _fallback_supervisord_process_from_db(): + try: + from archivebox.machine.models import Machine, Process + + for process in Process.objects.filter( + machine=Machine.current(), + process_type=Process.TypeChoices.SUPERVISORD, + status=Process.StatusChoices.RUNNING, + pwd=str(CONSTANTS.DATA_DIR), + ).order_by("-started_at", "-created_at"): + proc = process.proc + if proc is not None: + return proc + process.mark_exited(exit_code=0) + except Exception: + return None + return None + + RUNNER_WORKER = { "name": "worker_runner", "command": _shell_join([sys.executable, "-m", "archivebox", "run", "--daemon"]), @@ -48,9 +100,9 @@ RUNNER_WORKER = { "redirect_stderr": "true", } -RUNNER_WATCH_WORKER = lambda pidfile: { +RUNNER_WATCH_WORKER = lambda bind_url: { "name": "worker_runner_watch", - "command": _shell_join([sys.executable, "-m", "archivebox", "manage", "runner_watch", f"--pidfile={pidfile}"]), + "command": _shell_join([sys.executable, "-m", "archivebox", "manage", "runner_watch", f"--bind-url={bind_url}"]), "autostart": "false", "autorestart": "true", "stdout_logfile": "logs/worker_runner_watch.log", @@ -80,7 +132,7 @@ SERVER_WORKER = lambda host, port: { } -def RUNSERVER_WORKER(host: str, port: str, *, reload: bool, pidfile: str | None = None, nothreading: bool = False): +def RUNSERVER_WORKER(host: str, port: str, *, reload: bool, nothreading: bool = False): command = [sys.executable, "-m", "archivebox", "manage", "runserver", f"{host}:{port}"] if not reload: command.append("--noreload") @@ -89,11 +141,10 @@ def RUNSERVER_WORKER(host: str, port: str, *, reload: bool, pidfile: str | None environment = ['ARCHIVEBOX_RUNSERVER="1"'] if reload: - assert pidfile, "RUNSERVER_WORKER requires a pidfile when reload=True" environment.extend( [ 'ARCHIVEBOX_AUTORELOAD="1"', - f'ARCHIVEBOX_RUNSERVER_PIDFILE="{pidfile}"', + f'ARCHIVEBOX_RUNSERVER_BIND_URL="http://{host}:{port}"', ], ) @@ -129,21 +180,6 @@ def get_sock_file(): return socket_file -def follow(file, sleep_sec=0.1) -> Iterator[str]: - """Yield each line from a file as they are written. - `sleep_sec` is the time to sleep after empty reads.""" - line = "" - while True: - tmp = file.readline() - if tmp is not None and tmp != "": - line += tmp - if line.endswith("\n"): - yield line - line = "" - elif sleep_sec: - time.sleep(sleep_sec) - - def create_supervisord_config(): SOCK_FILE = get_sock_file() WORKERS_DIR = SOCK_FILE.parent / WORKERS_DIR_NAME @@ -179,6 +215,8 @@ files = {WORKERS_DIR}/*.conf """ CONFIG_FILE.write_text(config_content) Path.mkdir(WORKERS_DIR, exist_ok=True, parents=True) + for worker_conf in WORKERS_DIR.glob("*.conf"): + worker_conf.unlink(missing_ok=True) (WORKERS_DIR / "initial_startup.conf").write_text("") # hides error about "no files found to include" when supervisord starts @@ -194,6 +232,8 @@ def create_worker_config(daemon): worker_conf = WORKERS_DIR / f"{name}.conf" worker_str = f"[program:{name}]\n" + if "startsecs" not in daemon: + worker_str += "startsecs=0\n" for key, value in daemon.items(): if key == "name": continue @@ -203,7 +243,77 @@ def create_worker_config(daemon): worker_conf.write_text(worker_str) -def get_existing_supervisord_process(): +def sync_supervisord_workers(supervisor, workers: list[tuple[dict[str, str], bool]], *, prune: bool = True): + """Project desired workers into supervisord from ArchiveBox-owned state. + + The worker conf files are generated supervisor input only. They are never + treated as durable ArchiveBox state; callers either pass a complete worker + set with prune=True or add one explicit worker with prune=False. + """ + assert supervisor.getPID() + + SOCK_FILE = get_sock_file() + WORKERS_DIR = SOCK_FILE.parent / WORKERS_DIR_NAME + Path.mkdir(WORKERS_DIR, exist_ok=True, parents=True) + + global _desired_supervisord_workers + + desired = {worker["name"]: (worker, lazy) for worker, lazy in workers} + if prune: + _desired_supervisord_workers = {name: worker for name, (worker, _lazy) in desired.items()} + else: + _desired_supervisord_workers.update({name: worker for name, (worker, _lazy) in desired.items()}) + if prune: + for worker_conf in WORKERS_DIR.glob("*.conf"): + worker_conf.unlink(missing_ok=True) + + for worker, _lazy in desired.values(): + create_worker_config(worker) + + added, changed, removed = supervisor.reloadConfig()[0] + for group in removed: + try: + supervisor.stopProcessGroup(group) + except Exception: + pass + supervisor.removeProcessGroup(group) + for group in changed: + try: + supervisor.stopProcessGroup(group) + except Exception: + pass + supervisor.removeProcessGroup(group) + supervisor.addProcessGroup(group) + for group in added: + supervisor.addProcessGroup(group) + + procs_by_name = {} + for worker_name, (_worker, lazy) in desired.items(): + print(f"[πŸ¦Έβ€β™‚οΈ] Supervisord syncing subprocess worker: {worker_name}...") + for _ in range(25): + proc = get_worker(supervisor, worker_name) + if proc is None: + time.sleep(0.2) + continue + if proc["statename"] == "RUNNING": + print(f" - Worker {worker_name}: already {proc['statename']} ({proc['description']})") + procs_by_name[worker_name] = proc + break + if not lazy: + supervisor.startProcessGroup(worker_name, True) + proc = supervisor.getProcessInfo(worker_name) + print(f" - Worker {worker_name}: started {proc['statename']} ({proc['description']})") + else: + print(f" - Worker {worker_name}: configured {proc['statename']} ({proc['description']})") + procs_by_name[worker_name] = proc + break + else: + raise Exception(f"Failed to sync worker {worker_name}! Only found: {supervisor.getAllProcessInfo()}") + + return procs_by_name + + +def get_existing_supervisord_process(*, quiet: bool = False): SOCK_FILE = get_sock_file() try: transport = SupervisorTransport(None, None, f"unix://{SOCK_FILE}") @@ -214,12 +324,22 @@ def get_existing_supervisord_process(): current_state = cast(dict[str, int | str], server.supervisor.getState()) if current_state["statename"] == "RUNNING": pid = server.supervisor.getPID() - print(f"[πŸ¦Έβ€β™‚οΈ] Supervisord connected (pid={pid}) via unix://{pretty_path(SOCK_FILE)}.") + if not quiet: + print(f"[πŸ¦Έβ€β™‚οΈ] Supervisord connected (pid={pid}) via unix://{pretty_path(SOCK_FILE)}.") return server.supervisor except FileNotFoundError: return None + except Fault as err: + if err.faultCode == 6 and "SHUTDOWN_STATE" in str(err): + if not quiet: + print(f"[πŸ¦Έβ€β™‚οΈ] Supervisord is already shutting down via unix://{pretty_path(SOCK_FILE)}.") + return None + if not quiet: + print(f"Error connecting to existing supervisord: {str(err)}") + return None except Exception as e: - print(f"Error connecting to existing supervisord: {str(e)}") + if not quiet: + print(f"Error connecting to existing supervisord: {str(e)}") return None @@ -227,53 +347,77 @@ def stop_existing_supervisord_process(): global _supervisord_proc SOCK_FILE = get_sock_file() PID_FILE = SOCK_FILE.parent / PID_FILE_NAME + stop_grace_seconds = configured_stopwaitsecs(tuple(_desired_supervisord_workers.values())) + + supervisor = get_existing_supervisord_process() + supervisor_shutdown_requested = False + if supervisor is not None: + # Ask supervisord to stop each worker first so child shutdown follows + # each worker's own stopasgroup/killasgroup/stopwaitsecs settings. The + # direct psutil kill path below is only the final cleanup bound. + try: + final_states = {"STOPPED", "EXITED", "FATAL", "UNKNOWN"} + if any(proc["statename"] not in final_states for proc in supervisor.getAllProcessInfo()): + supervisor.stopAllProcesses(False) + deadline = time.monotonic() + stop_grace_seconds + while time.monotonic() < deadline: + if all(proc["statename"] in final_states for proc in supervisor.getAllProcessInfo()): + break + time.sleep(0.2) + except Fault as err: + if err.faultCode != 6 or "SHUTDOWN_STATE" not in str(err): + print(f"Error stopping supervisord workers: {str(err)}") + except Exception as err: + print(f"Error stopping supervisord workers: {str(err)}") + + try: + supervisor.shutdown() + supervisor_shutdown_requested = True + except Fault as err: + if err.faultCode == 6 and "SHUTDOWN_STATE" in str(err): + supervisor_shutdown_requested = True + else: + print(f"Error shutting down supervisord: {str(err)}") + except Exception: + supervisor_shutdown_requested = True try: # First try to stop via the global proc reference if _supervisord_proc and _supervisord_proc.poll() is None: try: print(f"[πŸ¦Έβ€β™‚οΈ] Stopping supervisord process (pid={_supervisord_proc.pid})...") - _supervisord_proc.terminate() try: - _supervisord_proc.wait(timeout=5) - except subprocess.TimeoutExpired: - _supervisord_proc.kill() - _supervisord_proc.wait(timeout=2) + psutil_proc = psutil.Process(_supervisord_proc.pid) + children = psutil_proc.children(recursive=True) + except psutil.NoSuchProcess: + children = [] + if not supervisor_shutdown_requested: + _supervisord_proc.terminate() + wait_popen_and_kill_children(_supervisord_proc, children, timeout=stop_grace_seconds) except (BrokenPipeError, OSError): pass finally: _supervisord_proc = None return - # Fallback: if pid file exists, load PID int and kill that process - try: - pid = int(PID_FILE.read_text()) - except (FileNotFoundError, ValueError): + proc = _fallback_supervisord_process_from_db() + if proc is None: return try: - print(f"[πŸ¦Έβ€β™‚οΈ] Stopping supervisord process (pid={pid})...") - proc = psutil.Process(pid) - # Kill the entire process group to ensure all children are stopped + print(f"[πŸ¦Έβ€β™‚οΈ] Stopping supervisord process (pid={proc.pid})...") children = proc.children(recursive=True) - proc.terminate() - # Also terminate all children - for child in children: - try: - child.terminate() - except psutil.NoSuchProcess: - pass - proc.wait(timeout=5) - # Kill any remaining children - for child in children: - try: - if child.is_running(): - child.kill() - except psutil.NoSuchProcess: - pass + if not supervisor_shutdown_requested: + proc.terminate() + for child in children: + try: + child.terminate() + except psutil.NoSuchProcess: + pass + wait_psutil_and_kill_children(proc, children, timeout=stop_grace_seconds) except psutil.NoSuchProcess: pass - except (BrokenPipeError, OSError): + except (BrokenPipeError, OSError, psutil.TimeoutExpired): pass finally: try: @@ -284,6 +428,19 @@ def stop_existing_supervisord_process(): pass +def reap_foreground_supervisord_process() -> None: + """Reap the supervisord child owned by this foreground parent if it exited.""" + + global _supervisord_proc + if _supervisord_proc and _supervisord_proc.poll() is not None: + try: + _supervisord_proc.wait(timeout=0) + except subprocess.TimeoutExpired: + pass + finally: + _supervisord_proc = None + + def start_new_supervisord_process(daemonize=False): SOCK_FILE = get_sock_file() WORKERS_DIR = SOCK_FILE.parent / WORKERS_DIR_NAME @@ -314,29 +471,32 @@ def start_new_supervisord_process(daemonize=False): if daemonize: # Start supervisord in background (daemon mode) - subprocess.Popen( + proc = subprocess.Popen( ["supervisord", f"--configuration={CONFIG_FILE}"], stdin=None, stdout=log_handle, stderr=log_handle, start_new_session=True, ) + _record_supervisord_process(proc, CONFIG_FILE) return wait_for_supervisord_ready() else: - # Start supervisord in FOREGROUND - this will block until supervisord exits - # supervisord with nodaemon=true will run in foreground and handle signals properly - # When supervisord gets SIGINT/SIGTERM, it will stop all child processes before exiting + # Keep supervisord foreground-owned by this process, but isolate it + # from terminal Ctrl+C. The ArchiveBox parent owns user-facing server + # signals and stops supervisord explicitly, so Ctrl+C does not also + # hit crawl workers and trigger the crawl-interactive abort flow. proc = subprocess.Popen( ["supervisord", f"--configuration={CONFIG_FILE}"], stdin=None, stdout=log_handle, stderr=log_handle, - start_new_session=False, # Keep in same process group so signals propagate + start_new_session=True, ) # Store the process so we can wait on it later global _supervisord_proc _supervisord_proc = proc + _record_supervisord_process(proc, CONFIG_FILE) return wait_for_supervisord_ready() @@ -384,44 +544,7 @@ def get_or_create_supervisord_process(daemonize=False): def start_worker(supervisor, daemon, lazy=False): - assert supervisor.getPID() - - print(f"[πŸ¦Έβ€β™‚οΈ] Supervisord starting new subprocess worker: {daemon['name']}...") - create_worker_config(daemon) - - result = supervisor.reloadConfig() - added, changed, removed = result[0] - # print(f"Added: {added}, Changed: {changed}, Removed: {removed}") - for removed in removed: - supervisor.stopProcessGroup(removed) - supervisor.removeProcessGroup(removed) - for changed in changed: - supervisor.stopProcessGroup(changed) - supervisor.removeProcessGroup(changed) - supervisor.addProcessGroup(changed) - for added in added: - supervisor.addProcessGroup(added) - - procs = [] - for _ in range(25): - procs = supervisor.getAllProcessInfo() - for proc in procs: - if proc["name"] == daemon["name"]: - # See process state diagram here: http://supervisord.org/subprocess.html - if proc["statename"] == "RUNNING": - print(f" - Worker {daemon['name']}: already {proc['statename']} ({proc['description']})") - return proc - else: - if not lazy: - supervisor.startProcessGroup(daemon["name"], True) - proc = supervisor.getProcessInfo(daemon["name"]) - print(f" - Worker {daemon['name']}: started {proc['statename']} ({proc['description']})") - return proc - - # retry in a moment in case it's slow to launch - time.sleep(0.2) - - raise Exception(f"Failed to start worker {daemon['name']}! Only found: {procs}") + return sync_supervisord_workers(supervisor, [(daemon, lazy)], prune=False).get(daemon["name"]) def get_worker(supervisor, daemon_name): @@ -456,30 +579,7 @@ def stop_worker(supervisor, daemon_name): raise Exception(f"Failed to stop worker {daemon_name}!") -def tail_worker_logs(log_path: str): - get_or_create_supervisord_process(daemonize=False) - - from rich.live import Live - from rich.table import Table - - table = Table() - table.add_column("TS") - table.add_column("URL") - - try: - with Live(table, refresh_per_second=1) as live: # update 4 times a second to feel fluid - with open(log_path) as f: - for line in follow(f): - if "://" in line: - live.console.print(f"Working on: {line.strip()}") - # table.add_row("123124234", line.strip()) - except (KeyboardInterrupt, BrokenPipeError, OSError): - STDERR.print("\n[πŸ›‘] Got Ctrl+C, stopping gracefully...") - except SystemExit: - pass - - -def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None): +def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep_running=None): """Tail multiple log files simultaneously, interleaving their output. Args: @@ -525,10 +625,14 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None): try: while follow: + if keep_running is not None and not keep_running(): + print("\n[newer ArchiveBox parent is now running the orchestrator and server]") + return "transferred" + # Check if the monitored process has exited if proc is not None and proc.poll() is not None: print(f"\n[server process exited with code {proc.returncode}]") - break + return "exited" had_output = False # Read ALL available lines from all files (not just one per iteration) @@ -548,9 +652,9 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None): time.sleep(0.05) except (KeyboardInterrupt, BrokenPipeError, OSError): - pass # Let the caller handle the cleanup message + return "interrupted" # Let the caller handle the cleanup message except SystemExit: - pass + return "interrupted" finally: # Close all file handles for _, f in file_handles: @@ -558,26 +662,7 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None): f.close() except Exception: pass - - -def watch_worker(supervisor, daemon_name, interval=5): - """loop continuously and monitor worker's health""" - while True: - proc = get_worker(supervisor, daemon_name) - if not proc: - raise Exception("Worker disappeared while running! " + daemon_name) - - if proc["statename"] == "STOPPED": - return proc - - if proc["statename"] == "RUNNING": - time.sleep(1) - continue - - if proc["statename"] in ("STARTING", "BACKOFF", "FATAL", "EXITED", "STOPPING"): - print(f"[πŸ¦Έβ€β™‚οΈ] WARNING: Worker {daemon_name} {proc['statename']} {proc['description']}") - time.sleep(interval) - continue + return "stopped" def get_sonic_supervisord_worker_from_plugin(config) -> dict[str, str] | None: @@ -592,90 +677,135 @@ def get_sonic_supervisord_worker_from_plugin(config) -> dict[str, str] | None: return cast(dict[str, str] | None, worker) -def start_server_workers(host="0.0.0.0", port="8000", daemonize=False, debug=False, reload=False, nothreading=False): +def stop_stale_sonic_processes(sonic_worker: dict[str, str], *, supervisor_pid: int | None) -> None: + command = shlex.split(sonic_worker.get("command") or "") + config_path = Path(command[command.index("-c") + 1]).resolve() if "-c" in command and command.index("-c") + 1 < len(command) else None + if config_path is None: + return + + stale = [] + for proc in psutil.process_iter(["pid", "ppid", "name", "cmdline"]): + try: + cmdline = proc.info.get("cmdline") or [] + if proc.info["pid"] == os.getpid() or proc.info["ppid"] == supervisor_pid: + continue + if Path(cmdline[0]).name != "sonic" or str(config_path) not in cmdline: + continue + stale.append(proc) + except (IndexError, psutil.NoSuchProcess, psutil.AccessDenied): + continue + + if not stale: + return + + print(f"[yellow][*] Taking over stale Sonic daemon(s) using {pretty_path(config_path)}...[/yellow]") + for proc in stale: + try: + proc.terminate() + except psutil.NoSuchProcess: + pass + _gone, alive = psutil.wait_procs(stale, timeout=2.0) + for proc in alive: + try: + proc.kill() + except psutil.NoSuchProcess: + pass + psutil.wait_procs(alive, timeout=2.0) + + +def start_server_workers( + host="0.0.0.0", + port="8000", + daemonize=False, + debug=False, + reload=False, + nothreading=False, + keep_running=None, + should_stop_supervisord=None, +): from archivebox.config.common import get_config config = get_config() - supervisor = get_or_create_supervisord_process(daemonize=daemonize) + shutdown_state = None + tail_result = "stopped" + try: + supervisor = get_or_create_supervisord_process(daemonize=daemonize) + bind_url = f"http://{host}:{port}" - if debug: - pidfile = str(config.TMP_DIR / "runserver.pid") if reload else None - server_worker = RUNSERVER_WORKER(host=host, port=port, reload=reload, pidfile=pidfile, nothreading=nothreading) - bg_workers: list[tuple[dict[str, str], bool]] = ( - [(RUNNER_WORKER, True), (RUNNER_WATCH_WORKER(pidfile), False)] if reload else [(RUNNER_WORKER, False)] - ) - log_files = ["logs/worker_runserver.log", "logs/worker_runner.log"] - if reload: - log_files.insert(1, "logs/worker_runner_watch.log") - else: - server_worker = SERVER_WORKER(host=host, port=port) - bg_workers = [(RUNNER_WORKER, False)] - log_files = ["logs/worker_daphne.log", "logs/worker_runner.log"] - - sonic_worker = get_sonic_supervisord_worker_from_plugin(config) - if sonic_worker is not None: - bg_workers.insert(0, (sonic_worker, False)) - log_files.append(str(sonic_worker["stdout_logfile"])) - - print() - start_worker(supervisor, server_worker) - print() - for worker, lazy in bg_workers: - start_worker(supervisor, worker, lazy=lazy) - print() - - if not daemonize: - try: - # Tail worker logs while supervisord runs - sys.stdout.write("Tailing worker logs (Ctrl+C to stop)...\n\n") - sys.stdout.flush() - tail_multiple_worker_logs( - log_files=log_files, - follow=True, - proc=_supervisord_proc, # Stop tailing when supervisord exits + if debug: + server_worker = RUNSERVER_WORKER(host=host, port=port, reload=reload, nothreading=nothreading) + bg_workers: list[tuple[dict[str, str], bool]] = ( + [(RUNNER_WORKER, True), (RUNNER_WATCH_WORKER(bind_url), False)] if reload else [(RUNNER_WORKER, False)] ) - except (KeyboardInterrupt, BrokenPipeError, OSError): - STDERR.print("\n[πŸ›‘] Got Ctrl+C, stopping gracefully...") - except SystemExit: - pass - except BaseException as e: - STDERR.print(f"\n[πŸ›‘] Got {e.__class__.__name__} exception, stopping gracefully...") - finally: - # Ensure supervisord and all children are stopped - stop_existing_supervisord_process() - time.sleep(1.0) # Give processes time to fully terminate + log_files = ["logs/worker_runserver.log", "logs/worker_runner.log"] + if reload: + log_files.insert(1, "logs/worker_runner_watch.log") + else: + server_worker = SERVER_WORKER(host=host, port=port) + bg_workers = [(RUNNER_WORKER, False)] + log_files = ["logs/worker_daphne.log", "logs/worker_runner.log"] - -def start_cli_workers(watch=False): - from archivebox.config.common import get_config - - supervisor = get_or_create_supervisord_process(daemonize=False) - - sonic_worker = get_sonic_supervisord_worker_from_plugin(get_config()) - if sonic_worker is not None: - start_worker(supervisor, sonic_worker) - - start_worker(supervisor, RUNNER_WORKER) - - if watch: - try: - # Block on supervisord process - it will handle signals and stop children - if _supervisord_proc: - _supervisord_proc.wait() + sonic_worker = get_sonic_supervisord_worker_from_plugin(config) + if sonic_worker is not None: + try: + current_sonic = get_worker(supervisor, sonic_worker["name"]) + supervisor_pid = supervisor.getPID() + except Exception: + current_sonic = None + supervisor_pid = None + if not (isinstance(current_sonic, dict) and current_sonic.get("statename") in ("STARTING", "RUNNING")): + stop_stale_sonic_processes(sonic_worker, supervisor_pid=supervisor_pid) + sonic_host = str(getattr(config, "SEARCH_BACKEND_SONIC_HOST_NAME", "127.0.0.1") or "127.0.0.1") + if sonic_host.strip().lower() == "localhost": + sonic_host = "127.0.0.1" + sonic_port = int(getattr(config, "SEARCH_BACKEND_SONIC_PORT")) + if not (isinstance(current_sonic, dict) and current_sonic.get("statename") in ("STARTING", "RUNNING")) and is_port_in_use( + sonic_host, + sonic_port, + ): + print(f"[yellow][*] Sonic is already listening on {sonic_host}:{sonic_port}; not starting a duplicate worker.[/yellow]") else: - # Fallback to watching worker if no proc reference - watch_worker(supervisor, RUNNER_WORKER["name"]) + bg_workers.insert(0, (sonic_worker, False)) + log_files.append(str(sonic_worker["stdout_logfile"])) + + print() + sync_supervisord_workers(supervisor, [(server_worker, False), *bg_workers], prune=True) + print() + + if daemonize: + return None + + try: + with foreground_shutdown_signals() as shutdown_state: + # Tail worker logs while supervisord runs. + sys.stdout.write("Tailing worker logs (Ctrl+C to stop)...\n\n") + sys.stdout.flush() + tail_result = tail_multiple_worker_logs( + log_files=log_files, + follow=True, + proc=_supervisord_proc, # Stop tailing when supervisord exits + keep_running=keep_running, + ) except (KeyboardInterrupt, BrokenPipeError, OSError): - STDERR.print("\n[πŸ›‘] Got Ctrl+C, stopping gracefully...") + if daemonize: + raise + if not shutdown_state or not shutdown_state.signal_name: + print("\n[πŸ›‘] Got CTRL+C, stopping gracefully...") except SystemExit: + if daemonize: + raise pass except BaseException as e: + if daemonize: + raise STDERR.print(f"\n[πŸ›‘] Got {e.__class__.__name__} exception, stopping gracefully...") - finally: - # Ensure supervisord and all children are stopped + finally: + if not daemonize and (should_stop_supervisord is None or should_stop_supervisord()): + # Ensure supervisord and all children are stopped only while this + # foreground parent is still the active server parent. Standby + # parents must not tear down a newer leader's services. stop_existing_supervisord_process() - time.sleep(1.0) # Give processes time to fully terminate - return [RUNNER_WORKER] + return tail_result # def main(daemons): diff --git a/bin/fuzz_test.sh b/bin/fuzz_test.sh new file mode 100755 index 00000000..f583d76e --- /dev/null +++ b/bin/fuzz_test.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash + +# Chaos-drive ArchiveBox CLI commands and print what happens. +# This is intentionally not a test harness: it does not assert success/failure. + +set -u + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DATA_DIR="${DATA_DIR:-$ROOT_DIR/data}" +LOG_DIR="${FUZZ_LOG_DIR:-$ROOT_DIR/tmp/fuzz-$(date +%Y%m%d-%H%M%S)}" +FUZZ_ROUNDS="${FUZZ_ROUNDS:-8}" +FUZZ_PARALLEL="${FUZZ_PARALLEL:-10}" +COMMAND_TIMEOUT="${FUZZ_COMMAND_TIMEOUT:-180}" +SERVER_HOLD_SECONDS="${FUZZ_SERVER_HOLD_SECONDS:-35}" +SERVER_BASE_PORT="${FUZZ_SERVER_BASE_PORT:-8700}" +SLEEP_BETWEEN_JOBS="${FUZZ_SLEEP_BETWEEN_JOBS:-0.2}" + +if [[ -n "${ARCHIVEBOX_CMD:-}" ]]; then + read -r -a ABX <<< "$ARCHIVEBOX_CMD" +elif command -v archivebox >/dev/null 2>&1; then + ABX=(archivebox) +else + ABX=(uv run archivebox) +fi + +DEFAULT_URLS=( + "https://example.com/" + "https://blog.sweeting.me/" + "https://www.iana.org/domains/reserved" + "https://httpbin.org/html" + "https://www.recurse.com/" +) + +URLS=() +if [[ -n "${FUZZ_URLS:-}" ]]; then + while IFS= read -r url; do + [[ -n "$url" ]] && URLS+=("$url") + done <<< "$FUZZ_URLS" +else + URLS=("${DEFAULT_URLS[@]}") +fi + +ts() { + date "+%Y-%m-%d %H:%M:%S" +} + +pick() { + local arr_name="$1" + local len idx + eval "len=\${#${arr_name}[@]}" + idx=$((RANDOM % len)) + eval "printf '%s\n' \"\${${arr_name}[$idx]}\"" +} + +kill_tree() { + local pid="$1" + local child + if command -v pgrep >/dev/null 2>&1; then + for child in $(pgrep -P "$pid" 2>/dev/null || true); do + kill_tree "$child" + done + fi + kill "$pid" >/dev/null 2>&1 || true +} + +cleanup() { + local pid + echo "[$(ts)] cleanup: stopping active background jobs" + for pid in $(jobs -pr); do + kill_tree "$pid" + done + sleep 2 + for pid in $(jobs -pr); do + kill -9 "$pid" >/dev/null 2>&1 || true + done +} + +trap cleanup INT TERM EXIT + +run_with_timeout() { + local label="$1" + local timeout="$2" + shift 2 + + local slug token + slug="$(echo "$label" | tr ' /:' '____' | tr -cd '[:alnum:]_.-')" + token="$(date +%s).$RANDOM.$RANDOM" + local logfile="$LOG_DIR/${slug}.${token}.log" + + { + echo "[$(ts)] START label=$label shell=$$ data=$DATA_DIR" + echo "[$(ts)] CMD DATA_DIR=$DATA_DIR $*" + } | tee -a "$logfile" + + ( + DATA_DIR="$DATA_DIR" "$@" + ) >> "$logfile" 2>&1 & + local child=$! + + ( + sleep "$timeout" + if kill -0 "$child" >/dev/null 2>&1; then + echo "[$(ts)] TIMEOUT label=$label pid=$child after=${timeout}s" >> "$logfile" + kill "$child" >/dev/null 2>&1 || true + sleep 5 + kill -9 "$child" >/dev/null 2>&1 || true + fi + ) & + local watchdog=$! + + trap 'kill_tree "$child"; kill "$watchdog" >/dev/null 2>&1 || true' INT TERM + + wait "$child" + local code=$? + kill "$watchdog" >/dev/null 2>&1 || true + wait "$watchdog" >/dev/null 2>&1 || true + trap - INT TERM + + echo "[$(ts)] END label=$label pid=$child exit=$code log=$logfile" | tee -a "$logfile" + return 0 +} + +run_server_for_a_bit() { + local label="$1" + local port="$2" + local hold="$3" + local debug_flag="$4" + local token logfile + local server_extra=() + [[ -n "$debug_flag" ]] && read -r -a server_extra <<< "$debug_flag" + token="$(date +%s).$RANDOM.$RANDOM" + logfile="$LOG_DIR/server-${port}.${token}.log" + + { + echo "[$(ts)] START label=$label shell=$$ data=$DATA_DIR" + echo "[$(ts)] CMD DATA_DIR=$DATA_DIR ${ABX[*]} server $debug_flag 127.0.0.1:$port" + } | tee -a "$logfile" + + ( + DATA_DIR="$DATA_DIR" "${ABX[@]}" server "${server_extra[@]}" "127.0.0.1:$port" + ) >> "$logfile" 2>&1 & + local child=$! + + trap 'kill_tree "$child"' INT TERM + + sleep "$hold" + echo "[$(ts)] STOP label=$label pid=$child after=${hold}s" | tee -a "$logfile" + kill_tree "$child" + sleep 5 + kill -9 "$child" >/dev/null 2>&1 || true + wait "$child" >/dev/null 2>&1 + local code=$? + trap - INT TERM + + echo "[$(ts)] END label=$label pid=$child exit=$code log=$logfile" | tee -a "$logfile" + return 0 +} + +job_init() { + run_with_timeout "init" "$COMMAND_TIMEOUT" "${ABX[@]}" init +} + +job_init_install() { + run_with_timeout "init-install" "$((COMMAND_TIMEOUT * 2))" "${ABX[@]}" init --install +} + +job_update_all() { + run_with_timeout "update" "$((COMMAND_TIMEOUT * 2))" "${ABX[@]}" update +} + +job_update_index() { + run_with_timeout "update-index-only" "$((COMMAND_TIMEOUT * 2))" "${ABX[@]}" update --index-only +} + +job_update_migrate() { + run_with_timeout "update-migrate-only" "$((COMMAND_TIMEOUT * 2))" "${ABX[@]}" update --migrate-only +} + +job_update_index_migrate() { + run_with_timeout "update-index-migrate-only" "$((COMMAND_TIMEOUT * 2))" "${ABX[@]}" update --index-only --migrate-only +} + +job_add_depth0() { + local url + url="$(pick URLS)" + run_with_timeout "add-depth0-$url" "$((COMMAND_TIMEOUT * 2))" "${ABX[@]}" add --depth=0 "$url" +} + +job_add_depth1() { + local url max_urls + url="$(pick URLS)" + max_urls=$((5 + RANDOM % 25)) + run_with_timeout "add-depth1-max${max_urls}-$url" "$((COMMAND_TIMEOUT * 3))" "${ABX[@]}" add --depth=1 --max-urls="$max_urls" "$url" +} + +job_server() { + local port hold debug_flag + port=$((SERVER_BASE_PORT + RANDOM % 50)) + hold=$((5 + RANDOM % SERVER_HOLD_SECONDS)) + debug_flag="" + [[ $((RANDOM % 4)) -eq 0 ]] && debug_flag="--debug" + run_server_for_a_bit "server-$port" "$port" "$hold" "$debug_flag" +} + +job_server_reload_debug() { + local port hold + port=$((SERVER_BASE_PORT + 50 + RANDOM % 25)) + hold=$((5 + RANDOM % SERVER_HOLD_SECONDS)) + run_server_for_a_bit "server-reload-debug-$port" "$port" "$hold" "--reload --debug" +} + +job_version() { + run_with_timeout "version" "60" "${ABX[@]}" version +} + +job_list() { + run_with_timeout "list-search" "90" "${ABX[@]}" list --search example +} + +# Used through pick JOBS. +# shellcheck disable=SC2034 +JOBS=( + job_init + job_init_install + job_update_all + job_update_index + job_update_migrate + job_update_index_migrate + job_add_depth0 + job_add_depth1 + job_server + job_server_reload_debug + job_version + job_list +) + +run_random_job() { + local job + job="$(pick JOBS)" + "$job" +} + +run_difficult_sequence() { + local port_a port_b + port_a=$((SERVER_BASE_PORT + 100 + RANDOM % 50)) + port_b=$((SERVER_BASE_PORT + 150 + RANDOM % 50)) + + echo "[$(ts)] SEQUENCE overlap-init-server-update-add ports=$port_a,$port_b" + run_server_for_a_bit "sequence-server-a-$port_a" "$port_a" "$SERVER_HOLD_SECONDS" "" & + sleep 1 + job_init & + sleep 1 + job_update_index & + sleep 1 + job_add_depth0 & + sleep 1 + run_server_for_a_bit "sequence-server-b-$port_b" "$port_b" "$SERVER_HOLD_SECONDS" "--reload --debug" & + wait + + echo "[$(ts)] SEQUENCE update-mode-collision" + job_update_all & + job_update_index & + job_update_migrate & + job_update_index_migrate & + wait +} + +main() { + mkdir -p "$LOG_DIR" + cd "$ROOT_DIR" || exit 1 + + echo "[$(ts)] ArchiveBox fuzz run" + echo " root: $ROOT_DIR" + echo " data: $DATA_DIR" + echo " logs: $LOG_DIR" + echo " command: ${ABX[*]}" + echo " rounds: $FUZZ_ROUNDS" + echo " parallel: $FUZZ_PARALLEL" + echo " urls: ${URLS[*]}" + echo + + for round in $(seq 1 "$FUZZ_ROUNDS"); do + echo "[$(ts)] ROUND $round/$FUZZ_ROUNDS starting random batch" + for slot in $(seq 1 "$FUZZ_PARALLEL"); do + ( + echo "[$(ts)] ROUND $round slot=$slot" + run_random_job + ) & + sleep "$SLEEP_BETWEEN_JOBS" + done + wait + + if [[ $((round % 2)) -eq 0 ]]; then + run_difficult_sequence + fi + + echo "[$(ts)] ROUND $round/$FUZZ_ROUNDS done" + done + + echo "[$(ts)] fuzz run complete; logs in $LOG_DIR" +} + +main "$@" diff --git a/bin/take_screenshot.js b/bin/take_screenshot.js new file mode 100755 index 00000000..9938e773 --- /dev/null +++ b/bin/take_screenshot.js @@ -0,0 +1,154 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const path = require('path'); +const puppeteer = require('puppeteer'); + +const DEFAULT_URL = 'http://web.archivebox.localhost:8000/add/'; +const DEFAULT_OUTPUT = 'tmp/add-page.png'; +const DEFAULT_CHROME = '/Users/squash/Library/Application Support/abx/lib/puppeteer/bin/chromium'; + +function usage() { + console.log(` +Usage: + node bin/take_screenshot.js [url] [output.png] + +Environment: + SESSIONID Django session cookie value for admin.archivebox.localhost + SCREENSHOT_COOKIE_DOMAIN Cookie domain, defaults to admin.archivebox.localhost + CHROME_BINARY Chromium/Chrome executable path + PUPPETEER_EXECUTABLE_PATH Chromium/Chrome executable path + SCREENSHOT_WIDTH Viewport width, defaults to 1600 + SCREENSHOT_HEIGHT Viewport height, defaults to 1400 + SCREENSHOT_FULL_PAGE Set to 1 to capture the full page, defaults to viewport only + SCREENSHOT_SCROLL_SELECTOR Scroll this selector into view before capture + SCREENSHOT_SNAPSHOT_VIEW Set to list or grid before loading the page + SCREENSHOT_RESET_FILTERS Set to 1 to clear the admin filter collapsed preference +`); +} + +function argValue(name) { + const idx = process.argv.indexOf(name); + return idx === -1 ? null : process.argv[idx + 1]; +} + +function firstPositionalArg(index) { + return process.argv.slice(2).filter((arg) => !arg.startsWith('--'))[index] || null; +} + +function chromePath() { + const configured = process.env.CHROME_BINARY || process.env.PUPPETEER_EXECUTABLE_PATH || DEFAULT_CHROME; + return fs.existsSync(configured) ? configured : undefined; +} + +async function main() { + if (process.argv.includes('--help') || process.argv.includes('-h')) { + usage(); + return; + } + + const url = argValue('--url') || firstPositionalArg(0) || DEFAULT_URL; + const output = path.resolve(argValue('--output') || firstPositionalArg(1) || DEFAULT_OUTPUT); + const width = Number(process.env.SCREENSHOT_WIDTH || 1600); + const height = Number(process.env.SCREENSHOT_HEIGHT || 1400); + const fullPage = process.env.SCREENSHOT_FULL_PAGE === '1'; + + fs.mkdirSync(path.dirname(output), { recursive: true }); + + const launchOptions = { + headless: true, + defaultViewport: { width, height }, + }; + const executablePath = chromePath(); + if (executablePath) { + launchOptions.executablePath = executablePath; + } + + const browser = await puppeteer.launch(launchOptions); + try { + const page = await browser.newPage(); + page.setDefaultTimeout(45000); + + if (process.env.SCREENSHOT_SNAPSHOT_VIEW || process.env.SCREENSHOT_RESET_FILTERS === '1') { + await page.evaluateOnNewDocument((snapshotView, resetFilters) => { + if (snapshotView) localStorage.setItem('preferred_snapshot_view_mode', snapshotView); + if (resetFilters) localStorage.removeItem('admin-filters-collapsed'); + }, process.env.SCREENSHOT_SNAPSHOT_VIEW || '', process.env.SCREENSHOT_RESET_FILTERS === '1'); + } + + if (process.env.SESSIONID) { + const cookie = { + name: 'sessionid', + value: process.env.SESSIONID, + path: '/', + }; + if (process.env.SCREENSHOT_COOKIE_DOMAIN) { + cookie.domain = process.env.SCREENSHOT_COOKIE_DOMAIN; + } else { + cookie.url = new URL(url).origin; + } + await page.setCookie(cookie); + } + + await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 }); + + await page.waitForSelector('body'); + await page.waitForSelector('#progress-monitor, #add-form', { timeout: 5000 }).catch(() => {}); + if (process.env.SCREENSHOT_SCROLL_SELECTOR) { + await page.waitForSelector(process.env.SCREENSHOT_SCROLL_SELECTOR, { timeout: 45000 }).catch(() => {}); + await page.evaluate((selector) => { + document.querySelector(selector)?.scrollIntoView({ block: 'start', inline: 'nearest' }); + }, process.env.SCREENSHOT_SCROLL_SELECTOR); + } + + const frameHandle = await page.$('.crawl-snapshots-embed iframe'); + if (frameHandle) { + const frame = await frameHandle.contentFrame(); + if (frame) { + await frame.waitForSelector('#changelist-form', { timeout: 45000 }).catch(() => {}); + await frame.waitForSelector('#result_list', { timeout: 45000 }).catch(() => {}); + } + } + + await new Promise((resolve) => setTimeout(resolve, 1200)); + + const checks = await page.evaluate(() => ({ + url: location.href, + title: document.title, + progressMonitorDisplay: document.querySelector('#progress-monitor') + ? getComputedStyle(document.querySelector('#progress-monitor')).display + : 'missing', + progressCrawls: document.querySelectorAll('#progress-monitor .crawl-item').length, + progressSnapshots: document.querySelectorAll('#progress-monitor .snapshot-item').length, + snapshotEmbed: Boolean(document.querySelector('.crawl-snapshots-embed iframe')), + addForm: Boolean(document.querySelector('#add-form')), + limitFields: Array.from(document.querySelectorAll('.crawl-limit-field label')).map((el) => el.textContent.trim()), + })); + + let frameChecks = null; + const embeddedFrameHandle = await page.$('.crawl-snapshots-embed iframe'); + if (embeddedFrameHandle) { + const frame = await embeddedFrameHandle.contentFrame(); + if (frame) { + frameChecks = await frame.evaluate(() => ({ + rows: document.querySelectorAll('#result_list tbody tr').length, + actionCheckboxes: document.querySelectorAll('#result_list input.action-select').length, + searchModeRadios: document.querySelectorAll('#changelist-search input[type="radio"][name="search_mode"]').length, + progressMonitorDisplay: document.querySelector('#progress-monitor') + ? getComputedStyle(document.querySelector('#progress-monitor')).display + : 'missing', + })); + } + } + + await page.screenshot({ path: output, fullPage }); + console.log(JSON.stringify({ screenshotPath: output, checks, frameChecks }, null, 2)); + } finally { + await browser.close(); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/pyproject.toml b/pyproject.toml index 69fb90ee..57b8bc57 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.32rc36" +version = "0.9.33rc1" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -195,7 +195,8 @@ package-dir = {"archivebox" = "archivebox"} line-length = 140 target-version = "py313" src = ["archivebox"] -exclude = ["*.pyi", "typings/", "migrations/", "archivebox/tests/data/"] +force-exclude = true +exclude = ["*.pyi", "*.html", "**/*.html", "typings/", "migrations/", "archivebox/tests/data/"] # https://docs.astral.sh/ruff/rules/ [tool.ruff.lint]