diff --git a/archivebox/api/v1_crawls.py b/archivebox/api/v1_crawls.py index 5f31d3b5..e69ae287 100644 --- a/archivebox/api/v1_crawls.py +++ b/archivebox/api/v1_crawls.py @@ -1,6 +1,5 @@ __package__ = "archivebox.api" -import json from pathlib import Path from uuid import UUID from datetime import datetime @@ -123,7 +122,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema): config = dict(data.config or {}) config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS)) crawl = Crawl.objects.create( - urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls), + urls="\n".join(urls), max_depth=data.max_depth, tags_str=",".join(tags), label=data.label, diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 51be7758..58fc00ae 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -6,7 +6,6 @@ __command__ = "archivebox add" import sys import os -import json from pathlib import Path from typing import Any, TYPE_CHECKING @@ -71,17 +70,7 @@ def add( created_by_id: int | None = None, config: dict[str, Any] | None = None, ) -> tuple[Crawl, QuerySet[Snapshot]]: - """Add a new URL or list of URLs to your archive. - - The flow is: - 1. Save URLs to sources file - 2. Create Crawl with URLs and max_depth - 3. Crawl runner creates Snapshots from Crawl URLs (depth=0) - 4. Crawl runner runs parser extractors on root snapshots - 5. Parser extractors output to urls.jsonl - 6. URLs are added to Crawl.urls and child Snapshots are created - 7. Repeat until max_depth is reached - """ + """Add a URL list or imported URL document to a new Crawl.""" from rich import print @@ -125,12 +114,7 @@ def add( created_by_id = created_by_id or get_or_create_system_user_pk() started_at = timezone.now() - use_internal_input_root = isinstance(urls, str) - source_text = ( - urls - if use_internal_input_root - else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls) - ) + source_text = urls if isinstance(urls, str) else "\n".join(str(url) for url in urls) # 2. Create a new Crawl with inline URLs # Foreground add must claim runner ownership before publishing runnable @@ -187,12 +171,7 @@ def add( try: crawl = Crawl.objects.create( urls=source_text, - # Stdin/import text gets an extra hop because the synthetic - # archivebox://internal root lives at depth 0 and parser-discovered - # URLs land at depth 1; direct URL args become the depth=0 input - # snapshots themselves so --depth=N matches the deepest hop the user - # asked for. - max_depth=depth + 1 if use_internal_input_root else depth, + max_depth=depth, tags_str=tag, persona_id=persona_obj.id, label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]", @@ -213,9 +192,8 @@ def add( print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]") print(f" [dim]First URL: {first_url}[/dim]") - # 3. The runner will create Snapshots from all URLs after claiming the Crawl - # Parser extractors run on snapshots and discover more URLs - # Discovered URLs become child Snapshots (depth+1) + # The runner parses Crawl.urls after claiming the Crawl, then persists the + # resulting depth-0 Snapshot facts. Recursive discoveries start at depth 1. if index_only: print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]") diff --git a/archivebox/cli/archivebox_archiveresult.py b/archivebox/cli/archivebox_archiveresult.py index dc7cf81a..5f387fad 100644 --- a/archivebox/cli/archivebox_archiveresult.py +++ b/archivebox/cli/archivebox_archiveresult.py @@ -6,13 +6,13 @@ archivebox archiveresult [args...] [--filters] Manage ArchiveResult records (plugin extraction results). Actions: - create - Create ArchiveResults for Snapshots (queue extractions) + create - Emit plugin extraction request records for Snapshots list - List ArchiveResults as JSONL (with optional filters) update - Update ArchiveResults from stdin JSONL delete - Delete ArchiveResults from stdin JSONL Examples: - # Create ArchiveResults for snapshots (queue for extraction) + # Emit extraction requests; `archivebox run` schedules their parent snapshots archivebox snapshot list --status=queued | archivebox archiveresult create archivebox archiveresult create --plugin=screenshot --snapshot-id= @@ -20,9 +20,6 @@ Examples: archivebox archiveresult list --status=failed archivebox archiveresult list --plugin=screenshot --status=succeeded - # Update (reset failed extractions to queued) - archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued - # Delete archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes @@ -157,7 +154,7 @@ def create_archiveresults( write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status)) created_count += 1 - rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr) + rprint(f"[green]Created {created_count} extraction request records[/green]", file=sys.stderr) return 0 @@ -341,7 +338,7 @@ def main(): @click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)") @click.option("--status", "-s", default="queued", help="Initial status (default: queued)") def create_cmd(snapshot_id: str | None, plugin: str | None, status: str): - """Create ArchiveResults for Snapshots from stdin JSONL.""" + """Emit Snapshot plugin extraction requests as JSONL.""" sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status)) diff --git a/archivebox/cli/archivebox_extract.py b/archivebox/cli/archivebox_extract.py index b117df18..9e63851f 100644 --- a/archivebox/cli/archivebox_extract.py +++ b/archivebox/cli/archivebox_extract.py @@ -1,100 +1,126 @@ #!/usr/bin/env python3 -""" -archivebox extract [snapshot_ids...] [--plugins=NAMES] - -Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL. - -Input formats: - - Snapshot UUIDs (one per line) - - JSONL: {"type": "Snapshot", "id": "...", "url": "..."} - - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."} - -Output (JSONL): - {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."} - -Examples: - # Extract specific snapshot - archivebox extract 01234567-89ab-cdef-0123-456789abcdef - - # Pipe from snapshot command - archivebox snapshot https://example.com | archivebox extract - - # Run specific plugins only - archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef - - # Chain commands - archivebox crawl https://example.com | archivebox snapshot | archivebox extract -""" +"""Run abx-dl snapshot hooks for existing ArchiveBox snapshots.""" __package__ = "archivebox.cli" __command__ = "archivebox extract" import sys from collections import defaultdict -from itertools import product import rich_click as click -def process_archiveresult_by_id(archiveresult_id: str) -> int: - """ - Re-run extraction for a single ArchiveResult by ID. +def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[str]], set[str]]: + """Resolve CLI records to snapshot-level execution requests. - ArchiveResults are projected status rows, not queued work items. Re-running - a single result means resetting that row and queueing its parent snapshot - through the shared crawl runner with the corresponding plugin selected. + ArchiveResult input is accepted as a convenient reference to its parent + Snapshot and plugin. It is never reset or converted into queued work. """ - from rich import print as rprint - from archivebox.core.models import ArchiveResult from archivebox.api.v1_core import _uuid_ref_query + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.misc.jsonl import TYPE_ARCHIVERESULT + + explicit_plugins = {name.strip() for name in plugins.split(",") if name.strip()} + requested: dict[str, set[str]] = defaultdict(set) + missing: set[str] = set() + + for record in records: + record_type = record.get("type") + record_id = str(record.get("id") or "") + + if record_type == TYPE_ARCHIVERESULT and record.get("snapshot_id"): + requested[str(record["snapshot_id"])].update(explicit_plugins or {str(record.get("plugin") or "")}) + requested[str(record["snapshot_id"])].discard("") + continue + + if record_type == TYPE_ARCHIVERESULT: + result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first() + if result is not None: + requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin}) + continue + + snapshot_id = str(record.get("snapshot_id") or record_id or "") + snapshot = Snapshot.objects.filter(id=snapshot_id).only("id").first() if snapshot_id else None + if snapshot is None and record.get("url"): + snapshot = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").only("id").first() + # Bare UUID CLI arguments are parsed as Snapshot records because the + # input layer cannot know which model owns them. Preserve the public + # convenience of passing an ArchiveResult ID by trying that reference + # only after the Snapshot lookup misses. + if snapshot is None and record_id and not record.get("url"): + result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first() + if result is not None: + requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin}) + continue + if snapshot is None: + missing.add(snapshot_id or str(record.get("url") or "")) + continue + requested[str(snapshot.id)].update(explicit_plugins) + + return requested, missing + + +def _run_snapshot_requests(requested: dict[str, set[str]], *, wait: bool, show_progress: bool) -> int: + from django.utils import timezone + from rich import print as rprint + + from archivebox.core.models import Snapshot from archivebox.services.runner import run_crawl - try: - archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id)) - except ArchiveResult.DoesNotExist: - rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr) + snapshots = {str(snapshot.id): snapshot for snapshot in Snapshot.objects.filter(id__in=requested).select_related("crawl")} + if not snapshots: + rprint("[red]No snapshots to process[/red]", file=sys.stderr) return 1 - rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr) + from archivebox.config.common import get_config + from archivebox.plugins.discovery import get_enabled_plugins - try: - archiveresult.reset_for_retry() - snapshot = archiveresult.snapshot - snapshot.queue_for_extraction() - crawl = snapshot.crawl - if not crawl.claim_processing_lock(lock_seconds=10): - rprint( - f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]", - file=sys.stderr, + for snapshot_id, plugin_names in requested.items(): + snapshot = snapshots.get(snapshot_id) + if snapshot is not None and not plugin_names: + plugin_names.update(get_enabled_plugins(config=get_config(crawl=snapshot.crawl, snapshot=snapshot))) + + # Explicit extraction resumes open/paused snapshots at the snapshot level. + # Sealed snapshots stay sealed during targeted maintenance backfills. + if wait: + for snapshot in snapshots.values(): + if snapshot.status != Snapshot.StatusChoices.SEALED: + snapshot.update_and_requeue( + status=Snapshot.StatusChoices.QUEUED, + retry_at=timezone.now(), + ) + + if not wait: + for snapshot_id, plugin_names in requested.items(): + snapshot = snapshots.get(snapshot_id) + if snapshot is None: + continue + if plugin_names: + snapshot.config = {**(snapshot.config or {}), "PLUGINS": ",".join(sorted(plugin_names))} + snapshot.save(update_fields=["config", "modified_at"]) + snapshot.update_and_requeue( + status=Snapshot.StatusChoices.QUEUED, + retry_at=timezone.now(), ) - return 1 + if show_progress: + rprint(f"[blue]Queued {len(snapshots)} snapshots for extraction[/blue]", file=sys.stderr) + return 0 + grouped: dict[tuple[str, tuple[str, ...]], list[str]] = defaultdict(list) + for snapshot_id, plugin_names in requested.items(): + snapshot = snapshots.get(snapshot_id) + if snapshot is not None: + grouped[(str(snapshot.crawl_id), tuple(sorted(plugin_names)))].append(snapshot_id) + + for (crawl_id, plugin_names), snapshot_ids in grouped.items(): run_crawl( - str(snapshot.crawl_id), - snapshot_ids=[str(snapshot.id)], - selected_plugins=[archiveresult.plugin], - selected_plugins_are_explicit=False, + crawl_id, + snapshot_ids=sorted(snapshot_ids), + selected_plugins=list(plugin_names) or None, + show_progress=show_progress, ) - archiveresult.refresh_from_db() - - if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED: - print(f"[green]Extraction succeeded: {archiveresult.output_str}[/green]") - return 0 - elif archiveresult.status == ArchiveResult.StatusChoices.NORESULTS: - print(f"[dim]Extraction completed with no results: {archiveresult.output_str}[/dim]") - return 0 - elif archiveresult.status == ArchiveResult.StatusChoices.FAILED: - print(f"[red]Extraction failed: {archiveresult.output_str}[/red]", file=sys.stderr) - return 1 - else: - # Still in progress or backoff - not a failure - print(f"[yellow]Extraction status: {archiveresult.status}[/yellow]") - return 0 - - except Exception as e: - print(f"[red]Extraction error: {type(e).__name__}: {e}[/red]", file=sys.stderr) - return 1 + return 0 def run_plugins( @@ -104,377 +130,64 @@ def run_plugins( wait: bool = True, emit_results: bool = True, show_progress: bool = True, - preserve_queued: bool = False, ) -> int: - """ - Run plugins on Snapshots from input. - - Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL. - - Exit codes: - 0: Success - 1: Failure - """ + """Execute selected plugins through the snapshot-level runner.""" from rich import print as rprint - from django.utils import timezone - from archivebox.misc.jsonl import ( - read_args_or_stdin, - write_record, - TYPE_SNAPSHOT, - TYPE_ARCHIVERESULT, - ) from archivebox.core.models import Snapshot - from archivebox.core.models import ArchiveResult - from archivebox.services.runner import run_crawl - from archivebox.plugins.discovery import get_plugin_catalog + from archivebox.misc.jsonl import read_args_or_stdin, write_record - is_tty = sys.stdout.isatty() - - # Parse comma-separated plugins list once (reused in creation and filtering) - plugins_list = [p.strip() for p in plugins.split(",") if p.strip()] if plugins else [] - - # Parse stdin/args exactly once per CLI invocation. - # `main()` may already have consumed stdin to distinguish Snapshot input from - # ArchiveResult IDs; if so, it must pass the parsed records through here - # instead of asking this helper to reread an already-drained pipe. if records is None: records = list(read_args_or_stdin(args)) - if not records: rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr) return 1 - # Gather snapshot IDs and optional plugin constraints to process - snapshot_ids = set() - requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set) - requested_hooks_by_snapshot: dict[str, set[tuple[str, str]]] = defaultdict(set) - plugin_level_requests_by_snapshot: dict[str, set[str]] = defaultdict(set) - for record in records: - record_type = record.get("type") - - if record_type == TYPE_SNAPSHOT: - snapshot_id = record.get("id") - if snapshot_id: - snapshot_ids.add(str(snapshot_id)) - elif record.get("url"): - # Look up by URL (get most recent if multiple exist) - snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first() - if snap: - snapshot_ids.add(str(snap.id)) - else: - rprint(f"[yellow]Snapshot not found for URL: {record['url']}[/yellow]", file=sys.stderr) - - elif record_type == TYPE_ARCHIVERESULT: - snapshot_id = record.get("snapshot_id") - if snapshot_id: - snapshot_ids.add(str(snapshot_id)) - plugin_name = record.get("plugin") - if plugin_name and not plugins_list: - snapshot_key = str(snapshot_id) - plugin_key = str(plugin_name) - requested_plugins_by_snapshot[snapshot_key].add(plugin_key) - hook_name = str(record.get("hook_name") or "") - if hook_name: - requested_hooks_by_snapshot[snapshot_key].add((plugin_key, hook_name)) - else: - plugin_level_requests_by_snapshot[snapshot_key].add(plugin_key) - - elif "id" in record: - # Assume it's a snapshot ID - snapshot_ids.add(str(record["id"])) - - if not snapshot_ids: - rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr) + requested, missing = _resolve_requests(records, plugins) + for value in sorted(missing): + rprint(f"[yellow]Snapshot or ArchiveResult not found: {value}[/yellow]", file=sys.stderr) + if not requested: return 1 - existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id")) - existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots} - existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots} - missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids) - for snapshot_id in missing_snapshot_ids: - rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr) - - # Queue only the target plugin rows. Bulk updates keep large reindex runs - # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start. - requested_pairs: set[tuple[str, str]] = set() - if plugins_list: - requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list)) - else: - requested_pairs.update( - (snapshot_id, plugin_name) - for snapshot_id, plugin_names in requested_plugins_by_snapshot.items() - if snapshot_id in existing_snapshot_ids - for plugin_name in plugin_names - ) - plugins_by_name = get_plugin_catalog() - requested_rows: set[tuple[str, str, str]] = set() - for snapshot_id, plugin_name in requested_pairs: - exact_hook_names = { - hook_name - for requested_plugin, hook_name in requested_hooks_by_snapshot.get(snapshot_id, set()) - if requested_plugin == plugin_name - } - if exact_hook_names and plugin_name not in plugin_level_requests_by_snapshot.get(snapshot_id, set()): - requested_rows.update((snapshot_id, plugin_name, hook_name) for hook_name in exact_hook_names) - continue - plugin = plugins_by_name.get(plugin_name) - hooks = plugin.filter_hooks("Snapshot") if plugin is not None else [] - if hooks: - requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks) - else: - requested_rows.add((snapshot_id, plugin_name, "")) - - queued_rows: set[tuple[str, str, str]] = set() - if preserve_queued and requested_rows: - queued_rows = { - (str(snapshot_id), plugin_name, hook_name) - for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter( - snapshot_id__in=existing_snapshot_ids, - plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows}, - status=ArchiveResult.StatusChoices.QUEUED, - ).values_list("snapshot_id", "plugin", "hook_name") - } - rows_to_queue = requested_rows - queued_rows - - reset_fields = { - "status": ArchiveResult.StatusChoices.QUEUED, - "output_str": "", - "output_json": None, - "output_files": {}, - "output_size": 0, - "output_mimetypes": "", - "start_ts": None, - "end_ts": None, - "modified_at": timezone.now(), - } - if rows_to_queue and plugins_list: - rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set) - for snapshot_id, plugin_name, hook_name in rows_to_queue: - rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id) - for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items(): - ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update( - **reset_fields, - ) - elif rows_to_queue and requested_plugins_by_snapshot: - snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set) - for snapshot_id, plugin_name, hook_name in rows_to_queue: - snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id) - for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items(): - ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update( - **reset_fields, - ) - existing_rows = ( - { - (str(snapshot_id), plugin_name, hook_name) - for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter( - snapshot_id__in=existing_snapshot_ids, - plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue}, - ).values_list("snapshot_id", "plugin", "hook_name") - } - if rows_to_queue - else set() - ) - missing_rows = rows_to_queue - existing_rows - if missing_rows: - ArchiveResult.objects.bulk_create( - [ - ArchiveResult( - snapshot_id=snapshot_id, - plugin=plugin_name, - hook_name=hook_name, - status=ArchiveResult.StatusChoices.QUEUED, - ) - for snapshot_id, plugin_name, hook_name in sorted(missing_rows) - ], - batch_size=500, - ) - - processed_count = len(existing_snapshot_ids) - queue_at = timezone.now() - if existing_snapshot_ids: - if requested_rows: - # Explicit plugin retries are maintenance on the existing snapshot; - # preserve a sealed lifecycle while making its queued rows due. - affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue} - if preserve_queued and queued_rows: - queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows} - affected_snapshot_ids.update( - str(snapshot_id) - for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids) - .filter(retry_at__gt=queue_at) - .values_list("id", flat=True) - ) - affected_snapshot_ids.update( - str(snapshot_id) - for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list( - "id", - flat=True, - ) - ) - requested_plugins_by_id: dict[str, set[str]] = defaultdict(set) - for snapshot_id, plugin_name, _hook_name in requested_rows: - requested_plugins_by_id[snapshot_id].add(plugin_name) - for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"): - # Guard the read-time status so we never bump retry_at on a - # row that's been re-queued / started by a concurrent runner. - if snapshot.status == Snapshot.StatusChoices.SEALED and requested_plugins_by_id.get(str(snapshot.id)): - snapshot.safe_update( - {"retry_at": queue_at, "modified_at": queue_at}, - refresh=False, - extra_filter={"status": snapshot.status}, - ) - else: - snapshot.update_and_requeue( - status=Snapshot.StatusChoices.QUEUED, - retry_at=queue_at, - current_step=0, - ) - else: - # No plugin rows were requested, so this is a full snapshot retry. - for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"): - snapshot.safe_update( - { - "status": Snapshot.StatusChoices.QUEUED, - "retry_at": queue_at, - "current_step": 0, - "modified_at": queue_at, - }, - refresh=False, - extra_filter={"status": snapshot.status}, - ) - if existing_crawl_ids and not requested_rows: - from archivebox.crawls.models import Crawl - - for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"): - update_fields = { - "retry_at": queue_at, - "modified_at": queue_at, - } - if crawl.status != Crawl.StatusChoices.STARTED: - update_fields["status"] = Crawl.StatusChoices.QUEUED - crawl.safe_update( - update_fields, - refresh=False, - extra_filter={"status": crawl.status}, - ) - - if processed_count == 0: - rprint("[red]No snapshots to process[/red]", file=sys.stderr) - return 1 - - if show_progress: - rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr) - - # Run orchestrator if --wait (default) - if wait: - if show_progress: - rprint("[blue]Running plugins...[/blue]", file=sys.stderr) - snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set) - for snapshot_id, crawl_id in existing_snapshots: - snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id)) - - for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items(): - from archivebox.crawls.models import Crawl - - crawl = Crawl.objects.get(id=crawl_id) - if not crawl.claim_processing_lock(lock_seconds=10): - rprint( - f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]", - file=sys.stderr, - ) - return 1 - selected_plugins = ( - plugins_list - or sorted( - {plugin for snapshot_id, plugin, _hook_name in requested_rows if snapshot_id in crawl_snapshot_ids}, - ) - or None - ) - run_crawl( - crawl_id, - snapshot_ids=sorted(crawl_snapshot_ids), - selected_plugins=selected_plugins, - show_progress=show_progress, - selected_plugins_are_explicit=bool(plugins_list), - ) - - if not emit_results: - return 0 - - # Output results as JSONL (when piped) or human-readable (when TTY) - for snapshot_id in snapshot_ids: - try: - snapshot = Snapshot.objects.get(id=snapshot_id) - results = snapshot.archiveresult_set.all() - if plugins_list: - results = results.filter(plugin__in=plugins_list) - - for result in results: - if is_tty: - status_color = { - "succeeded": "green", - "failed": "red", - "skipped": "yellow", - }.get(result.status, "dim") - rprint( - f" [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ''}", - file=sys.stderr, - ) - else: - write_record(result.to_json()) - except Snapshot.DoesNotExist: - continue + exit_code = _run_snapshot_requests(requested, wait=wait, show_progress=show_progress) + if exit_code or not emit_results: + return exit_code + is_tty = sys.stdout.isatty() + for snapshot in Snapshot.objects.filter(id__in=requested): + results = snapshot.archiveresult_set.all() + requested_plugins = requested[str(snapshot.id)] + if requested_plugins: + results = results.filter(plugin__in=requested_plugins) + for result in results: + if is_tty: + color = {"succeeded": "green", "failed": "red", "skipped": "yellow"}.get(result.status, "dim") + rprint(f" [{color}]{result.status}[/{color}] {result.plugin} → {result.output_str or ''}", file=sys.stderr) + else: + write_record(result.to_json()) return 0 -def is_archiveresult_id(value: str) -> bool: - """Check if value resolves to an ArchiveResult ID.""" - from archivebox.core.models import ArchiveResult - from archivebox.api.v1_core import _uuid_ref_query - - return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists() +def process_archiveresult_by_id(archiveresult_id: str) -> int: + """Re-run the parent Snapshot plugin referenced by an ArchiveResult.""" + return run_plugins((), records=[{"id": archiveresult_id}], wait=True) @click.command() -@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run (e.g., screenshot,singlefile)") +@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run") @click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)") @click.argument("args", nargs=-1) def main(plugins: str, wait: bool, args: tuple): - """Run plugins on Snapshots, or process existing ArchiveResults by ID""" + """Run plugins on Snapshots; ArchiveResult IDs select their parent plugin.""" from archivebox.misc.jsonl import read_args_or_stdin - # Read all input records = list(read_args_or_stdin(args)) - if not records: from rich import print as rprint rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr) sys.exit(1) - - # Check if input looks like existing ArchiveResult IDs to process - all_are_archiveresult_ids = all(is_archiveresult_id(r.get("id") or r.get("url", "")) for r in records) - - if all_are_archiveresult_ids: - # Process existing ArchiveResults by ID - from rich import print as rprint - - exit_code = 0 - for record in records: - archiveresult_id = record.get("id") or record.get("url") - if not isinstance(archiveresult_id, str): - rprint(f"[red]Invalid ArchiveResult input: {record}[/red]", file=sys.stderr) - exit_code = 1 - continue - result = process_archiveresult_by_id(archiveresult_id) - if result != 0: - exit_code = result - sys.exit(exit_code) - else: - # Default behavior: run plugins on Snapshots from input - sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait)) + sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait)) if __name__ == "__main__": diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py index 1275714a..48f10705 100644 --- a/archivebox/cli/archivebox_run.py +++ b/archivebox/cli/archivebox_run.py @@ -72,8 +72,8 @@ def process_stdin_records() -> int: Outputs JSONL of all processed records (for chaining). - Handles any record type: Crawl, Snapshot, ArchiveResult. - Auto-cascades: Crawl → Snapshots → ArchiveResults. + Handles Crawl and Snapshot work records. ArchiveResult records are accepted + as references to their parent Snapshot and plugin. Returns exit code (0 = success, 1 = error). """ @@ -90,6 +90,7 @@ def process_stdin_records() -> int: ) from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.core.models import Snapshot, ArchiveResult + from archivebox.api.v1_core import _uuid_ref_query from archivebox.crawls.models import Crawl from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals from archivebox.machine.models import Binary @@ -156,20 +157,15 @@ def process_stdin_records() -> int: queued_count += 1 elif record_type == TYPE_ARCHIVERESULT: - archiveresult = ArchiveResult.from_json(record) - if archiveresult: - if archiveresult.status in [ - ArchiveResult.StatusChoices.FAILED, - ArchiveResult.StatusChoices.SKIPPED, - ArchiveResult.StatusChoices.NORESULTS, - ArchiveResult.StatusChoices.BACKOFF, - ]: - archiveresult.reset_for_retry() - snapshot = archiveresult.snapshot - plugin_name = archiveresult.plugin - else: - snapshot = None - plugin_name = None + snapshot_id = str(record.get("snapshot_id") or "") + plugin_name = str(record.get("plugin") or "") + archiveresult = None + if not snapshot_id and record_id: + archiveresult = ArchiveResult.objects.filter(_uuid_ref_query("id", str(record_id))).select_related("snapshot").first() + if archiveresult: + snapshot_id = str(archiveresult.snapshot_id) + plugin_name = plugin_name or archiveresult.plugin + snapshot = Snapshot.objects.filter(id=snapshot_id).first() if snapshot_id else None if snapshot: snapshot.queue_for_extraction() @@ -177,7 +173,7 @@ def process_stdin_records() -> int: snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id)) if plugin_name: plugin_names_by_crawl[crawl_id].add(str(plugin_name)) - output_records.append(record if not archiveresult else archiveresult.to_json()) + output_records.append(archiveresult.to_json() if archiveresult else record) queued_count += 1 elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}: @@ -235,7 +231,6 @@ def process_stdin_records() -> int: crawl_id, snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]), selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]), - selected_plugins_are_explicit=False, ) return 0 diff --git a/archivebox/cli/archivebox_schedule.py b/archivebox/cli/archivebox_schedule.py index df74748f..5b5670b9 100644 --- a/archivebox/cli/archivebox_schedule.py +++ b/archivebox/cli/archivebox_schedule.py @@ -57,12 +57,12 @@ def schedule( created_by_id = get_or_create_system_user_pk() is_update_schedule = not import_path - template_urls = import_path or "archivebox://update" + template_urls = import_path or "" template_label = (f"Scheduled import: {template_urls}" if import_path else "Scheduled ArchiveBox update")[:64] template_notes = ( f"Created by archivebox schedule for {template_urls}" if import_path - else "Created by archivebox schedule to queue recurring archivebox://update maintenance crawls." + else "Created by archivebox schedule to run recurring ArchiveBox maintenance." ) template = Crawl.objects.create( @@ -76,7 +76,6 @@ def schedule( retry_at=None, config={ "DEPTH": 0 if is_update_schedule else depth, - "SCHEDULE_KIND": "update" if is_update_schedule else "crawl", # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the # template defaults. Anything left unset falls through to the # standard config stack at crawl-resolution time. @@ -86,6 +85,7 @@ def schedule( crawl_schedule = CrawlSchedule.objects.create( template=template, schedule=schedule_str, + config={**template.config, "SCHEDULE_KIND": "update" if is_update_schedule else "crawl"}, is_enabled=True, label=template_label, notes=template_notes, @@ -119,12 +119,19 @@ def schedule( if run_all: enqueued = 0 + maintained = 0 now = timezone.now() for scheduled_crawl in schedules: - scheduled_crawl.enqueue(queued_at=now) - enqueued += 1 + if scheduled_crawl.dispatch(queued_at=now) is None: + maintained += 1 + else: + enqueued += 1 result["run_all_enqueued"] = enqueued + result["run_all_maintained"] = maintained print(f"[green]\\[*] Enqueued {enqueued} scheduled crawl(s) immediately.[/green]") + if maintained: + run_pending_crawls(maintenance_only=True) + print(f"[green]\\[*] Ran {maintained} scheduled maintenance update(s).[/green]") if enqueued: print( "[yellow]\\[*] Start `archivebox server`, `archivebox run --daemon`, or `archivebox schedule --foreground` to process the queued crawls.[/yellow]", diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index 83f9e81e..70c5aa12 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -77,104 +77,50 @@ def reindex_snapshots( ) -> dict[str, Any]: from archivebox.cli.archivebox_extract import run_plugins from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.plugins.discovery import get_plugin_catalog + from django.db.models import Exists, OuterRef # Search backfill is the one maintenance hook allowed to execute without # reopening a Snapshot. Restrict that exception to already-sealed rows; # every open lifecycle state remains owned by the normal runner lifecycle. snapshots = snapshots.filter(status=Snapshot.StatusChoices.SEALED) - stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []} - records: list[dict[str, str]] = [] - plugins_by_name = get_plugin_catalog() - required_hooks_by_plugin = { - plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot")) - for plugin_name in search_plugins - if plugin_name in plugins_by_name - } + stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []} + print(f"[*] Backfilling missing search indexes with: {', '.join(search_plugins)}") - total = snapshots.count() - print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}") - - def run_batch() -> None: - if not records: - return - if wait_for_turn: - wait_for_turn() - batch_records = list(records) - snapshot_ids = {record["snapshot_id"] for record in batch_records} - plugin_names = {record["plugin"] for record in batch_records} - queued_rows = { - (str(snapshot_id), plugin_name, hook_name) - for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter( - snapshot_id__in=snapshot_ids, - plugin__in=plugin_names, - status=ArchiveResult.StatusChoices.QUEUED, - ).values_list("snapshot_id", "plugin", "hook_name") - } - records_to_queue = [] - for record in batch_records: - snapshot_id = record["snapshot_id"] - plugin_name = record["plugin"] - required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset()) - if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks): - stats["skipped_queued"] += 1 - continue - records_to_queue.append(record) - if not records_to_queue: - print( - f" [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs", - ) - records.clear() - return - # `archivebox update --index-only` intentionally breaks the usual - # "runner discovers work" rule by inserting synthetic queued - # ArchiveResult rows for search backends. run_plugins() keeps this as - # statement-sized UPDATE/bulk_create work, then bumps Snapshot.retry_at - # so the orchestrator owns actual hook execution. Paused snapshots stay - # PAUSED; run_due_snapshot restores retry_at=MAX after targeted rows - # finish. - exit_code = run_plugins( - args=(), - records=records_to_queue, - wait=False, - emit_results=False, - show_progress=False, - preserve_queued=True, + completed_statuses = [ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS] + for plugin_name in search_plugins: + completed_result = ArchiveResult.objects.filter( + snapshot_id=OuterRef("pk"), + plugin=plugin_name, + status__in=completed_statuses, ) - if exit_code != 0: - raise SystemExit(exit_code) - stats["queued"] += len(records_to_queue) - print( - f" [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator", - ) - records.clear() - - for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size): - try: - stats["processed"] += 1 - - if _get_snapshot_crawl(snapshot) is None: - continue - + candidates = snapshots.annotate(has_completed_index=Exists(completed_result)).filter(has_completed_index=False).order_by("id") + after_id = None + while True: + if wait_for_turn: + wait_for_turn() + page = candidates.filter(id__gt=after_id) if after_id is not None else candidates + batch = list(page.only("id", "timestamp")[:batch_size]) + if not batch: + break + after_id = batch[-1].id + records = [{"type": "Snapshot", "id": str(snapshot.id)} for snapshot in batch] + stats["processed"] += len(batch) + stats["requested"] += len(batch) if collect_ids: - stats["snapshot_ids"].append(str(snapshot.id)) - for plugin_name in search_plugins: - records.append( - { - "type": "ArchiveResult", - "snapshot_id": str(snapshot.id), - "plugin": plugin_name, - }, - ) - stats["requested"] += 1 - if len(records) >= batch_size: - run_batch() - except KeyboardInterrupt as err: - err.archivebox_resume = snapshot.timestamp - raise - - run_batch() + stats["snapshot_ids"].extend(str(snapshot.id) for snapshot in batch) + exit_code = run_plugins( + args=(), + records=records, + plugins=plugin_name, + wait=True, + emit_results=False, + show_progress=False, + ) + if exit_code != 0: + raise SystemExit(exit_code) + stats["reindexed"] += len(batch) + print(f" [{plugin_name}] indexed {stats['reindexed']} missing snapshots") return stats @@ -203,9 +149,9 @@ def update( Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving. Three-phase operation (without filters): - - Phase 1: Drain old archive/ dirs by moving to new fs location (0.8.x → 0.9.x) - - Phase 2: O(n) scan over entire DB from most recent to least recent - - No orphan scans needed (trust 1:1 mapping between DB and filesystem after phase 1) + - Phase 1: Drain legacy archive/ directories into the current layout + - Phase 2: Select only stale fs_version rows through the indexed column + - Phase 3: Run queued snapshot-level filesystem maintenance until idle With filters: Only phase 2 (DB query), no filesystem operations. Without filters: All phases (full update). @@ -232,7 +178,6 @@ def update( from archivebox.core.takeover_util import ( command_owns_foreground_runner, current_command, - foreground_runner_owner, standby_until_foreground_runner_needed, ) from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process @@ -291,11 +236,6 @@ def update( while True: do_migrate = migrate_only or not index_only do_index = index_only or not migrate_only - do_run_until_idle = do_migrate or do_index - ran_post_migrate_runner = False - full_update_empty = False - maintenance_work_queued = False - runner_work_queued = False if do_migrate: if ( @@ -327,13 +267,11 @@ def update( after=after, resume=resume, batch_size=batch_size, - queue_for_archiving=do_run_until_idle, + queue_for_archiving=True, wait_for_turn=wait_for_turn, ) print_stats(stats) touched_snapshot_ids.update(stats.get("snapshot_ids", [])) - maintenance_work_queued = stats.get("queued", 0) > 0 - runner_work_queued = runner_work_queued or maintenance_work_queued else: stats_combined = {"phase1": {}, "phase2": {}} @@ -343,162 +281,52 @@ def update( batch_size=batch_size, ) - print("[*] Phase 2: Processing all database snapshots (most recent first)...") + print("[*] Phase 2: Selecting database snapshots with stale filesystem versions...") stats_combined["phase2"] = process_all_db_snapshots( batch_size=batch_size, resume=resume, wait_for_turn=wait_for_turn, ) print_combined_stats(stats_combined) - full_update_empty = ( - stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0 - ) - maintenance_work_queued = any( - ( - stats_combined["phase1"].get("queued", 0), - stats_combined["phase2"].get("queued", 0), - stats_combined["phase2"].get("crawls_sealed", 0), - ), - ) - runner_work_queued = runner_work_queued or maintenance_work_queued - - if do_run_until_idle: - # Filesystem migration is maintenance on existing - # Snapshot rows: Snapshot.save() moves archive/ to - # the current output_dir and preserves the lifecycle - # status. Drain those retry_at ticks before queuing - # search backfill below. Otherwise the sealed search - # runner branch correctly sees queued ArchiveResult - # rows first, runs the targeted plugins, and may leave - # the fs_version maintenance tick hidden behind that - # plugin work until another update pass. - if full_update_empty: - print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.") - elif not maintenance_work_queued: - print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.") - else: - print("[*] Phase 3: Running filesystem maintenance until idle...") - if full_update_empty: - pass - elif not maintenance_work_queued: - pass - elif is_filtered_update: - if not touched_snapshot_ids: - print("[*] No matching snapshots queued work for the runner.") - for snapshot_id in sorted(touched_snapshot_ids): - run_scoped_runner("--snapshot-id", snapshot_id) - else: - run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size)) - ran_post_migrate_runner = True - - if do_index: - if full_update_empty: - print("[*] No snapshots found; skipping search indexing backfill.") - else: - search_plugins = _get_search_indexing_plugins() - if not search_plugins: - print("[*] No search indexing plugins are available, nothing to backfill.") - else: - snapshots = _build_filtered_snapshots_queryset( - filter_patterns=filter_patterns, - filter_type=filter_type, - status=status, - url__icontains=url__icontains, - url__istartswith=url__istartswith, - tag=tag, - crawl_id=crawl_id, - limit=limit, - sort=sort, - search=search, - before=before, - after=after, - resume=resume, - ) - from django.db.models import Exists, OuterRef, Q - from django.utils import timezone - from archivebox.core.models import ArchiveResult, Snapshot - - scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None - queued_index_results = ArchiveResult.objects.filter( - status=ArchiveResult.StatusChoices.QUEUED, - plugin__in=search_plugins, - ) - if scoped_snapshot_ids is not None: - queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids) - - if queued_index_results.exists(): - runner_work_queued = True - now = timezone.now() - queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk")) - snapshots_to_wake = ( - Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED) - .annotate( - has_queued_index_result=Exists(queued_result_for_snapshot), - ) - .filter( - has_queued_index_result=True, - ) - .filter( - Q(retry_at__isnull=True) | Q(retry_at__gt=now), - ) - ) - if scoped_snapshot_ids is not None: - snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids) - woken_count = snapshots_to_wake.update( - retry_at=now, - modified_at=now, - ) - print( - "[*] Existing queued search index jobs found; " - f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.", - ) - else: - collect_index_ids = ( - is_filtered_update - or foreground_runner_owner( - data_dir=CONSTANTS.DATA_DIR, - exclude_id=command.id, - ) - is not None - ) - stats = reindex_snapshots( - snapshots, - search_plugins=search_plugins, - batch_size=batch_size, - collect_ids=collect_index_ids, - wait_for_turn=wait_for_turn, - ) - print_index_stats(stats) - touched_snapshot_ids.update(stats.get("snapshot_ids", [])) - runner_work_queued = runner_work_queued or stats["queued"] > 0 - - if do_run_until_idle and (do_index or not ran_post_migrate_runner): - # Search/index backfill intentionally queues targeted - # ArchiveResult rows without reopening sealed snapshots. - # This second runner pass drains those plugin - # rows after filesystem maintenance has had its own turn. - # For a normal unfiltered `archivebox update`, keep the - # historical final pass broad enough to resume genuinely - # queued/interrupted crawl work after maintenance is done. - if full_update_empty: - print("[*] No snapshots found; skipping queued/interrupted crawl runner.") - elif not runner_work_queued: - print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.") - else: - print("[*] Phase 3: Running queued/interrupted crawl work until idle...") - if full_update_empty: - pass - elif not runner_work_queued: - pass - elif touched_snapshot_ids and is_filtered_update: - if not touched_snapshot_ids: - print("[*] No matching snapshots queued work for the runner.") + # The due selectors are indexed and cheap when empty, so + # always drain them instead of preceding the runner with + # whole-table counts merely to decide whether to call it. + print("[*] Phase 3: Running filesystem maintenance until idle...") + if is_filtered_update: for snapshot_id in sorted(touched_snapshot_ids): run_scoped_runner("--snapshot-id", snapshot_id) else: - run_scoped_runner( - *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []), + run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size)) + + if do_index: + search_plugins = _get_search_indexing_plugins() + if not search_plugins: + print("[*] No search indexing plugins are available, nothing to backfill.") + else: + snapshots = _build_filtered_snapshots_queryset( + filter_patterns=filter_patterns, + filter_type=filter_type, + status=status, + url__icontains=url__icontains, + url__istartswith=url__istartswith, + tag=tag, + crawl_id=crawl_id, + limit=limit, + sort=sort, + search=search, + before=before, + after=after, + resume=resume, ) + stats = reindex_snapshots( + snapshots, + search_plugins=search_plugins, + batch_size=batch_size, + collect_ids=is_filtered_update, + wait_for_turn=wait_for_turn, + ) + print_index_stats(stats) + touched_snapshot_ids.update(stats.get("snapshot_ids", [])) if not continuous: break @@ -737,18 +565,8 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500 def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, wait_for_turn=None) -> dict[str, int]: - """ - O(n) scan over entire DB from most recent to least recent. - - For each snapshot: - 1. Reconcile index.json with DB (merge titles, tags, archive results) - 2. Mark migrated snapshots sealed unless explicitly re-queued elsewhere - - No orphan detection needed - we trust 1:1 mapping between DB and filesystem - after Phase 1 has drained all old archive/ directories. - """ + """Queue only snapshots whose indexed filesystem version is stale.""" from archivebox.core.models import Snapshot - from archivebox.crawls.models import Crawl from django.db.models import Q from django.utils import timezone @@ -761,128 +579,35 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w "sealed": 0, "crawls_sealed": 0, } - current_fs_version = Snapshot._fs_current_version() - - queryset = Snapshot.objects.all() + queryset = Snapshot.objects.filter(fs_version__in=Snapshot._FS_VERSION_MIGRATION_PATHS) if resume: queryset = queryset.filter(timestamp__lte=resume) - total = queryset.count() - stats["snapshots"] = total - print(f"[*] Processing {total} snapshots from database (most recent first)...") - - def update_in_batches(rows, *, label: str, **updates) -> int: - updated = 0 - checked = 0 - while True: - if wait_for_turn: - wait_for_turn() - batch = list(rows.only("id", "modified_at").order_by("-timestamp")[:batch_size]) - if not batch: - if updated: - print(f" [{label}] complete: {updated} rows updated") - return updated - checked += len(batch) - print(f" [{label}] updating next {len(batch)} rows (seen {checked})...") - for snapshot in batch: - # This maintenance scan intentionally bypasses save(); it is - # only normalizing scheduler fields, and Snapshot.save() may - # do filesystem migration work that belongs in the runner. - # Guard each single-row UPDATE with modified_at so stale scan - # pages cannot overwrite newer runner/admin writes. - updated += int( - snapshot.safe_update( - updates, - refresh=False, - extra_filter={"modified_at": snapshot.modified_at}, - ), - ) - print(f" [{label}] updated {updated} rows so far") - - now = timezone.now() - updated_rows = update_in_batches( - queryset.exclude( - status__in=[ - Snapshot.StatusChoices.QUEUED, - Snapshot.StatusChoices.STARTED, - Snapshot.StatusChoices.PAUSED, - Snapshot.StatusChoices.SEALED, - ], - ), - label="snapshot status normalization", - status=Snapshot.StatusChoices.SEALED, - retry_at=None, - modified_at=now, - ) - stats["sealed"] += updated_rows - stats["updated_db"] += updated_rows - - fs_version_rows = queryset.exclude(fs_version=current_fs_version).filter(Q(retry_at__isnull=True) | Q(retry_at__gt=now)) - stale_batch = [] - - def queue_stale_fs_batch() -> None: - if not stale_batch: - return + initial_now = timezone.now() + rows_to_wake = queryset.filter(Q(retry_at__isnull=True) | Q(retry_at__gt=initial_now)) + after_id = None + while True: if wait_for_turn: wait_for_turn() + page = rows_to_wake.filter(id__gt=after_id) if after_id is not None else rows_to_wake + batch = list(page.only("id", "fs_version", "modified_at").order_by("id")[:batch_size]) + if not batch: + break + after_id = batch[-1].id now = timezone.now() - # Do not bump fs_version here. The orchestrator calls Snapshot.save(), - # which performs the idempotent filesystem migration and commits the new - # fs_version in the same serialized worker path as normal crawls. updated = 0 - for snapshot in stale_batch: - # Each row gets its own short autocommit UPDATE because this scan - # can touch millions of snapshots while a server is also alive. - # The modified_at predicate is the CAS guard: if the runner or - # admin changed the snapshot after paged_iterator read it, skip it - # and let the newer state decide whether migration is still due. + for snapshot in batch: updated += int( snapshot.safe_update( - { - "retry_at": now, - "modified_at": now, - }, + {"retry_at": now, "modified_at": now}, refresh=False, extra_filter={"fs_version": snapshot.fs_version}, ), ) - stats["processed"] += len(stale_batch) + stats["processed"] += len(batch) stats["updated_db"] += updated stats["queued"] += updated - print(f" [{stats['processed']}/{total}] Queued {updated} filesystem migrations for orchestrator...") - stale_batch.clear() - - for snapshot in ( - fs_version_rows.only("id", "crawl_id", "timestamp", "fs_version", "modified_at") - .order_by("-timestamp") - .paged_iterator(chunk_size=batch_size) - ): - try: - stale_batch.append(snapshot) - if len(stale_batch) >= batch_size: - queue_stale_fs_batch() - except KeyboardInterrupt as err: - err.archivebox_resume = snapshot.timestamp - raise - queue_stale_fs_batch() - - now = timezone.now() - # Crawls with no open child snapshots are already finished. Seal them here - # instead of waking the foreground runner; otherwise migration/update can - # accidentally re-enter full crawl execution for historical rows. - stats["crawls_sealed"] = ( - Crawl.objects.filter( - status__in=Crawl.RUNNABLE_STATES, - ) - .exclude( - snapshot_set__status__in=Snapshot.OPEN_STATES, - ) - .update( - status=Crawl.StatusChoices.SEALED, - retry_at=None, - modified_at=now, - ) - ) - stats["updated_db"] += stats["crawls_sealed"] + print(f" Queued {stats['queued']} stale filesystem snapshots so far...") + stats["snapshots"] = stats["processed"] return stats @@ -942,8 +667,6 @@ def process_filtered_snapshots( stats["snapshot_ids"].append(str(snapshot.id)) update_values = {} updated = 0 - if not isinstance(snapshot.current_step, int): - update_values["current_step"] = 0 if queue_for_archiving: update_values.update( { @@ -1029,10 +752,9 @@ def print_index_stats(stats: dict[str, Any]) -> None: print(f""" [green]Search Reindex Complete[/green] - Scanned rows: {stats["processed"]} - Requested jobs: {stats.get("requested", stats["queued"])} - Queued index jobs: {stats["queued"]} - Already queued: {stats.get("skipped_queued", 0)} + Missing rows: {stats["processed"]} + Requested runs: {stats.get("requested", 0)} + Indexed snapshots: {stats.get("reindexed", 0)} """) diff --git a/archivebox/config/collection.py b/archivebox/config/collection.py index ba9c8c71..17e57026 100644 --- a/archivebox/config/collection.py +++ b/archivebox/config/collection.py @@ -160,7 +160,7 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]: so they round-trip through INI's string-only storage. When reading the file back into ``Machine.config`` (a JSONField that holds native types) those strings have to be decoded — otherwise downstream consumers like - ``ExecutionPlan.seed_config`` → ``MachineEvent`` → abx-dl see a JSON string + ``MachineEvent`` → abx-dl see a JSON string where they expect a dict and raise ``TypeError``. Declared fields go through pydantic-settings' own ``field_is_complex`` / ``prepare_field_value`` so they're decoded per annotation. Undeclared diff --git a/archivebox/core/migrations/0054_snapshot_fs_version_index.py b/archivebox/core/migrations/0054_snapshot_fs_version_index.py new file mode 100644 index 00000000..56763ad6 --- /dev/null +++ b/archivebox/core/migrations/0054_snapshot_fs_version_index.py @@ -0,0 +1,26 @@ +# Generated by Django 6.1 on 2026-09-02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0053_alter_archiveresult_options"), + ] + + operations = [ + migrations.RemoveField( + model_name="snapshot", + name="current_step", + ), + migrations.AlterField( + model_name="snapshot", + name="fs_version", + field=models.CharField( + db_index=True, + default="0.9.0", + help_text='Filesystem version of this snapshot (e.g., "0.7.0", "0.8.0", "0.9.4").', + max_length=10, + ), + ), + ] diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 8d3141bf..b3aefd5f 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -13,9 +13,9 @@ from django.conf import settings from django.contrib import admin from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError from django.db import IntegrityError, models, transaction -from django.db.models import Case, F, Q, QuerySet, Sum, Value, When +from django.db.models import F, Q, QuerySet, Sum, Value from django.db.models.fields.json import KT -from django.db.models.functions import Coalesce, Concat +from django.db.models.functions import Coalesce from django.urls import reverse_lazy from django.utils import timezone from django.utils.functional import cached_property @@ -538,8 +538,6 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)): # ty: ig class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithQueue): BROWSER_EXTENSION_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload" - INTERNAL_INPUT_URL = "archivebox://internal" - id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True) created_at = models.DateTimeField(default=timezone.now, db_index=True) modified_at = models.DateTimeField(auto_now=True) @@ -567,14 +565,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW fs_version = models.CharField( max_length=10, default="0.9.0", - help_text='Filesystem version of this snapshot (e.g., "0.7.0", "0.8.0", "0.9.0"). Used to trigger lazy migration on save().', - ) - current_step = models.PositiveSmallIntegerField( - default=0, db_index=True, - help_text="Current hook step being executed (0-9). Used for sequential hook execution.", + help_text='Filesystem version of this snapshot (e.g., "0.7.0", "0.8.0", "0.9.4").', ) - retry_at = ModelWithQueue.RetryAtField(default=timezone.now) status = ModelWithQueue.StatusField( choices=ModelWithQueue.StatusChoices, @@ -730,20 +723,13 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW return self.update_and_requeue( status=self.StatusChoices.QUEUED, retry_at=when or timezone.now(), - current_step=0, ) def pause(self, *, save: bool = True) -> bool: - paused = super().pause(save=save) - if paused and self.pk: - ArchiveResult.pause_queryset(self.archiveresult_set.all()) - return paused + return super().pause(save=save) def resume(self, *, when: datetime | None = None, save: bool = True) -> bool: - resumed = super().resume(when=when, save=save) - if resumed and self.pk: - ArchiveResult.resume_queryset(self.archiveresult_set.all(), when=when) - return resumed + return super().resume(when=when, save=save) def restore_paused_scheduler_marker(self) -> None: """ @@ -817,20 +803,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW modified_at=now, ) - def reset_abandoned_results(self) -> tuple[int, int]: - reset_count = 0 - running_count = 0 - for result in self.archiveresult_set.filter( - status__in=[ArchiveResult.StatusChoices.STARTED, ArchiveResult.StatusChoices.BACKOFF], - ).select_related("process"): - process = result.process - if process is not None and process.is_running: - running_count += 1 - continue - result.reset_for_retry() - reset_count += 1 - return reset_count, running_count - def start_processing(self) -> bool: """Atomically move a claimed queued Snapshot into its active lease.""" owned_retry_at = self.retry_at @@ -880,12 +852,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW if self.status == self.StatusChoices.PAUSED: return False if self.status == self.StatusChoices.QUEUED: - results = self.archiveresult_set.all() - if results.exists() and not results.exclude(status__in=ArchiveResult.FINAL_STATES).exists(): - return self.seal() return bool(self.url) and self.start_processing() - if self.status == self.StatusChoices.STARTED and self.is_finished_processing(): - return self.seal() + # abx-dl emits SnapshotCompletedEvent after the complete hook sequence; + # ArchiveResult projection state never drives Snapshot completion. return False def cancel(self) -> None: @@ -1021,9 +990,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW return False def validate_url_for_archiving(self, *, config: Mapping[str, Any] | Any | None = None) -> None: - if self.is_internal_input_url(): - return - try: validate_url(self.url or "") except ValueError as err: @@ -1032,9 +998,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW if self.is_archivebox_internal_url(self.url, config=config): raise ValidationError({"url": "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs."}) - def is_internal_input_url(self) -> bool: - return (self.url or "").strip() == self.INTERNAL_INPUT_URL and self.depth == 0 and bool(self.crawl_id) - def save(self, *args, **kwargs): update_fields = kwargs.get("update_fields") validate_url_field = self._state.adding or update_fields is None or "url" in update_fields @@ -1062,18 +1025,13 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW if self._state.adding or update_fields is None or "notes" in update_fields: self.notes = sanitize_html_text(self.notes) - # Migrate filesystem if needed (happens automatically on save) + # The maintenance runner currently signals its explicit filesystem pass + # with this narrow update. Ordinary model saves must never move archive + # directories as an unrelated metadata side effect. existing_snapshot = self.pk and not self._state.adding - if existing_snapshot and self.fs_migration_needed: + maintenance_update = update_fields is not None and set(update_fields) == {"retry_at", "modified_at"} + if existing_snapshot and maintenance_update and self.fs_migration_needed: self.migrate_filesystem_to_current_version() - update_fields = kwargs.get("update_fields") - if update_fields is not None: - kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "fs_version", "modified_at"])) - elif existing_snapshot: - current_dir = self.get_storage_path_for_version(self._fs_current_version()) - source_dir = Path(self.output_dir) - if source_dir.exists() and source_dir != current_dir and not source_dir.is_symlink(): - self.migrate_filesystem_to_current_version(source_dir=source_dir) super().save(*args, **kwargs) @@ -1098,50 +1056,13 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW Tag.objects.bulk_create(missing_tags, ignore_conflicts=True) tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=crawl_tag_names)} self.add_tag_ids([tag.pk for name in crawl_tag_names if (tag := tags_by_name.get(name))]) - # Snapshot.save() normally appends newly created URLs to Crawl.urls - # so legacy/direct crawls can keep their queue text in sync. For - # internal-input crawls that would corrupt the original submitted - # import text; parsed URLs are represented by child Snapshot rows. - if crawl.has_internal_input_root(): - return - if not crawl.url_passes_filters(self.url, snapshot=self, use_effective_config=False): - return - # Best-effort skip if our URL is already recorded on the crawl; - # the atomic UPDATE below is what actually prevents clobbering. - crawl.refresh_from_db(fields=["urls"]) - if self.url in {url for _raw_line, url in crawl._iter_url_lines() if url}: - return - now = timezone.now() - # Atomic append: SQLite reads `urls` inside the UPDATE statement, - # so concurrent appends never clobber each other (no read-then-write - # window, no CAS retry needed). A rare duplicate URL on a race is - # harmless — downstream consumers dedupe via Snapshot uniqueness. - text = models.TextField() - type(crawl).objects.filter(pk=crawl.pk).update( - urls=Case( - When(Q(urls="") | Q(urls__isnull=True), then=Value(self.url, output_field=text)), - default=Concat( - "urls", - Value("\n", output_field=text), - Value(self.url, output_field=text), - output_field=text, - ), - output_field=text, - ), - modified_at=now, - ) - crawl.modified_at = now + # Crawl.urls remains the original submitted source. Snapshot rows + # are the normalized work queue and discovery projection. # get_or_create/update_or_create wrap save() in atomic(); defer filesystem # work and crawl maintenance so SQLite commits before touching the disk. transaction.on_commit(finish_snapshot_save) - migration_cleanup = self.__dict__.get("_pending_fs_migration_cleanup") - if migration_cleanup: - old_dir, new_dir = migration_cleanup - transaction.on_commit(lambda: self._cleanup_old_migration_dir(old_dir, new_dir)) - delattr(self, "_pending_fs_migration_cleanup") - # ========================================================================= # Filesystem Migration Methods # ========================================================================= @@ -1183,30 +1104,38 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW def migrate_filesystem_to_current_version(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None) -> None: """ - Copy legacy snapshot output into the current layout and defer old-dir cleanup. + Copy legacy snapshot output into the current layout and safely remove the old tree. The ordering is intentionally crash-safe: 1. Copy from the legacy directory into the new directory idempotently. 2. Verify the new directory has every old file. 3. Convert metadata in the new directory. - 4. Update fs_version in memory for the caller to save. - 5. Cleanup is scheduled only after the DB commit succeeds. + 4. Remove the verified legacy source. + 5. Persist fs_version last, so an interruption remains selected by the + indexed stale-version query and resumes naturally. + + Re-running this method also reconciles a legacy timestamp directory + left behind when a machine stopped after the database commit but before + the on-commit cleanup callback ran. """ current = self.fs_version target = self._fs_current_version() cleanup: tuple[Path, Path] | None = None runtime_config = config or get_config() - if source_dir and current == target: + if current == target: current_dir = self.get_storage_path_for_version(target) - cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, target_dir=current_dir) + legacy_dir = Path(source_dir) if source_dir else CONSTANTS.ARCHIVE_DIR / self.timestamp + cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=legacy_dir, target_dir=current_dir) crawl_dir = self.crawl.output_dir old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex))) if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink(): crawl_dir.parent.mkdir(parents=True, exist_ok=True) old_crawl_dir.rename(crawl_dir) if cleanup: - self._pending_fs_migration_cleanup = cleanup + old_dir, new_dir = cleanup + if not self._cleanup_old_migration_dir(old_dir, new_dir): + raise SnapshotMigrationError(f"Could not clean up verified migration directory: {old_dir}") return while current != target: @@ -1231,7 +1160,14 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW source_dir = None if cleanup: - self._pending_fs_migration_cleanup = cleanup + old_dir, new_dir = cleanup + if not self._cleanup_old_migration_dir(old_dir, new_dir): + raise SnapshotMigrationError(f"Could not clean up verified migration directory: {old_dir}") + + if self.pk: + now = timezone.now() + type(self).objects.filter(pk=self.pk).update(fs_version=target, modified_at=now) + self.modified_at = now def _fs_migrate_from_0_7_0_to_0_9_0(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None): return self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, config=config) @@ -1277,7 +1213,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW if old_dir == new_dir: self.convert_index_json_to_jsonl(output_dir=new_dir) - self.hydrate_archiveresult_output_metadata(snapshot_dir=new_dir) return None if old_dir.is_symlink(): @@ -1288,13 +1223,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW if not points_to_target: raise SnapshotMigrationError(f"Legacy output symlink does not point to the expected target: {old_dir}") self.convert_index_json_to_jsonl(output_dir=new_dir) - self.hydrate_archiveresult_output_metadata(snapshot_dir=new_dir) - return None + return (old_dir, new_dir) if not old_dir.exists(): if new_dir.exists(): self.convert_index_json_to_jsonl(output_dir=new_dir) - self.hydrate_archiveresult_output_metadata(snapshot_dir=new_dir) return None return None @@ -1306,7 +1239,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW pass else: self.convert_index_json_to_jsonl(output_dir=new_dir) - self.hydrate_archiveresult_output_metadata(snapshot_dir=new_dir) return (old_dir, new_dir) def copy_file_without_overwriting(source: str, destination: str): @@ -1352,17 +1284,51 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW # Convert index.json to index.jsonl in the new directory. self.convert_index_json_to_jsonl(output_dir=new_dir) - self.hydrate_archiveresult_output_metadata(snapshot_dir=new_dir) return (old_dir, new_dir) - def _cleanup_old_migration_dir(self, old_dir: Path, new_dir: Path): + @staticmethod + def _migration_trees_match(old_dir: Path, new_dir: Path) -> bool: + """Verify every legacy entry exists unchanged at the destination.""" + import filecmp + + if old_dir.is_symlink(): + try: + return new_dir.exists() and old_dir.resolve() == new_dir.resolve() + except OSError: + return False + if not old_dir.exists(): + return True + if not new_dir.is_dir() or new_dir.is_symlink(): + return False + + for source in old_dir.rglob("*"): + destination = new_dir / source.relative_to(old_dir) + if source.is_symlink(): + copied = destination.is_symlink() and destination.readlink() == source.readlink() + elif source.is_dir(): + copied = destination.is_dir() and not destination.is_symlink() + elif source.is_file(): + copied = destination.is_file() and not destination.is_symlink() and filecmp.cmp(source, destination, shallow=False) + else: + copied = False + if not copied: + return False + return True + + def _cleanup_old_migration_dir(self, old_dir: Path, new_dir: Path) -> bool: """Delete the old directory after its contents are verified at the new path.""" import logging import shutil from archivebox.config.permissions import SudoPermission + if not self._migration_trees_match(old_dir, new_dir): + logging.getLogger("archivebox.migration").warning( + f"Refusing to remove unverified migration directory {old_dir}", + ) + return False + # Delete old directory if old_dir.exists() and not old_dir.is_symlink(): try: @@ -1374,11 +1340,12 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW logging.getLogger("archivebox.migration").warning( f"Could not remove old migration directory {old_dir}: {e}", ) - return + return False # Older migration runs may already have left a timestamp projection. if old_dir.is_symlink(): old_dir.unlink(missing_ok=True) + return True # ========================================================================= # Path Calculation and Migration Helpers @@ -2796,18 +2763,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW self.remove_tag_ids(existing_tag_ids - tag_ids) self.add_tag_ids(tag_ids - existing_tag_ids) - def pending_archiveresults(self) -> QuerySet["ArchiveResult"]: - return self.archiveresult_set.exclude(status__in=ArchiveResult.FINAL_OR_ACTIVE_STATES) - - def run(self) -> list["ArchiveResult"]: - """ - Execute snapshot by creating pending ArchiveResults for all enabled hooks. - - Returns: - list[ArchiveResult]: Newly created pending results - """ - return self.create_pending_archiveresults() - def finalize_output_metadata(self) -> None: """ Clean up background ArchiveResult hooks and empty results. @@ -3054,63 +3009,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW return snapshot - def create_pending_archiveresults(self, hooks: Iterable[tuple[str, str]] | None = None) -> list["ArchiveResult"]: - """ - Create ArchiveResult records for all enabled hooks. - - Uses the hooks system to discover available hooks from: - - abx_plugins/plugins/*/on_Snapshot__*.{py,sh,js} - - data/custom_plugins/*/on_Snapshot__*.{py,sh,js} - - Creates one ArchiveResult per hook (not per plugin), with hook_name set. - This enables step-based execution where all hooks in a step can run in parallel. - """ - try: - self.validate_url_for_archiving() - except ValidationError as err: - rprint(f"[yellow][!] Skipping blocked snapshot URL: {(self.url or '')[:120]}... ({err})[/yellow]") - return [] - - if hooks is None: - from archivebox.config.common import get_config - from archivebox.plugins.hooks import discover_hooks - - # Compatibility path for direct model callers. The runner passes its - # abx-dl hook inventory explicitly so queued rows match execution. - config = get_config(crawl=self.crawl, snapshot=self) - hooks = ((hook_path.parent.name, hook_path.stem) for hook_path in discover_hooks("Snapshot", config=config)) - archiveresults = [] - - for plugin, hook_name in hooks: - # Hooks in one plugin share a filesystem directory, but each hook has - # its own durable result row and retries update that exact row. - archiveresult, _created = ArchiveResult.get_or_create_by_hook( - self, - plugin, - hook_name, - defaults={ - "status": ArchiveResult.INITIAL_STATE, - }, - ) - if archiveresult.status == ArchiveResult.INITIAL_STATE: - archiveresults.append(archiveresult) - - return archiveresults - - def is_finished_processing(self) -> bool: - """ - Check if all ArchiveResults are finished. - - Note: This is only called for observability/progress tracking. - The shared runner owns execution and does not poll this. - """ - # Check if any ARs are still pending/started - pending = self.archiveresult_set.exclude( - status__in=ArchiveResult.FINAL_STATES, - ).exists() - - return not pending - def get_progress_stats(self) -> dict: """ Get progress statistics for this snapshot's archiving process. @@ -3171,34 +3069,20 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW } def retry_failed_archiveresults(self) -> int: - """ - Reset failed ArchiveResults to queued for retry. - - Returns count of ArchiveResults reset. - """ - retryable_results = ArchiveResult.objects.filter( - snapshot=self, - status=ArchiveResult.StatusChoices.FAILED, + """Queue the parent Snapshot to rerun plugins with failed facts.""" + plugins = list( + self.archiveresult_set.filter(status=ArchiveResult.StatusChoices.FAILED) + .exclude(plugin="") + .order_by("plugin") + .values_list("plugin", flat=True) + .distinct(), ) - legacy_result_count = retryable_results.filter(hook_name="").count() - now = timezone.now() - count = retryable_results.exclude(hook_name="").update( - status=ArchiveResult.StatusChoices.QUEUED, - output_str="", - output_json=None, - output_files={}, - output_size=0, - output_mimetypes="", - start_ts=None, - end_ts=None, - modified_at=now, - ) - - if count + legacy_result_count > 0: - self.refresh_from_db(fields=["modified_at", "retry_at", "status"]) - self.queue_for_extraction(when=now) - - return count + legacy_result_count + if not plugins: + return 0 + self.config = {**(self.config or {}), "PLUGINS": ",".join(plugins)} + self.save(update_fields=["config", "modified_at"]) + self.queue_for_extraction() + return len(plugins) # ========================================================================= # URL Helper Properties (migrated from Link schema) @@ -4276,74 +4160,10 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes): def get_absolute_url(self): return f"/{self.snapshot.archive_path}/{self.plugin}" - def reset_for_retry(self, *, save: bool = True) -> None: - self.status = self.StatusChoices.QUEUED - self.retry_at = None - self.output_str = "" - self.output_json = None - self.output_files = {} - self.output_size = 0 - self.output_mimetypes = "" - self.start_ts = None - self.end_ts = None - if save: - self.save( - update_fields=[ - "status", - "retry_at", - "output_str", - "output_json", - "output_files", - "output_size", - "output_mimetypes", - "start_ts", - "end_ts", - "modified_at", - ], - ) - @property def is_paused(self) -> bool: return self.status == self.StatusChoices.PAUSED - @classmethod - def pause_queryset(cls, queryset) -> int: - return queryset.exclude(status__in=[*cls.FINAL_STATES, cls.StatusChoices.PAUSED]).update( - status=cls.StatusChoices.PAUSED, - retry_at=RETRY_AT_MAX, - modified_at=timezone.now(), - ) - - @classmethod - def resume_queryset(cls, queryset, *, when: datetime | None = None) -> int: - return queryset.filter(status=cls.StatusChoices.PAUSED).update( - status=cls.StatusChoices.QUEUED, - retry_at=when or timezone.now(), - modified_at=timezone.now(), - ) - - def pause(self, *, save: bool = True) -> bool: - if self.status in self.FINAL_STATES: - return False - if self.is_paused: - return False - self.status = self.StatusChoices.PAUSED - self.retry_at = RETRY_AT_MAX - if save: - self.pause_queryset(type(self).objects.filter(pk=self.pk)) - self.refresh_from_db() - return True - - def resume(self, *, when: datetime | None = None, save: bool = True) -> bool: - if not self.is_paused: - return False - self.status = self.StatusChoices.QUEUED - self.retry_at = when or timezone.now() - if save: - self.resume_queryset(type(self).objects.filter(pk=self.pk), when=self.retry_at) - self.refresh_from_db() - return True - @staticmethod def _normalize_output_files(raw_output_files: Any) -> dict[str, dict[str, Any]]: from abx_dl.output_files import OutputManifest diff --git a/archivebox/core/recovery_util.py b/archivebox/core/recovery_util.py index 5a3f1266..e068c76a 100644 --- a/archivebox/core/recovery_util.py +++ b/archivebox/core/recovery_util.py @@ -36,9 +36,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | "crawls_queued_without_retry_at": 0, "snapshots_queued_without_retry_at": 0, "snapshots_sealed_with_extension_uploads_only": 0, - "archiveresults_backoff": 0, - "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0, - "archiveresults_started_without_running_process": 0, + "archiveresults_interrupted_without_running_process": 0, "archiveresults_missing_for_orphaned_hook_processes": 0, "snapshots_started_without_running_results": 0, "crawls_started_with_due_snapshots": 0, @@ -46,10 +44,10 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | "crawls_started_without_active_snapshots": 0, } - running_archiveresults = ArchiveResult.objects.filter( - snapshot_id=OuterRef("pk"), - status=ArchiveResult.StatusChoices.STARTED, - process__status=Process.StatusChoices.RUNNING, + running_hook_processes = Process.objects.filter( + archiveresult__snapshot_id=OuterRef("pk"), + process_type=Process.TypeChoices.HOOK, + status=Process.StatusChoices.RUNNING, ) active_child_snapshots = Snapshot.objects.filter( crawl_id=OuterRef("pk"), @@ -109,49 +107,18 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | retry_at=now, modified_at=now, ) - backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF, **result_filter) orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED, **result_filter).exclude( process__status=Process.StatusChoices.RUNNING, ) - # ArchiveResult has no retry_at scheduler. Wake only the parent Snapshots - # for result rows we are about to repair, then requeue those exact rows. - # Using subqueries keeps million-row recovery in SQLite instead of building - # Python ID lists or scanning all sealed snapshots. - Snapshot.objects.filter( - id__in=backoff_results.values("snapshot_id"), - status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED], - retry_at__isnull=True, - ).update(retry_at=now, modified_at=now) + # ArchiveResult rows are projections, never work items. Close interrupted + # projections as failed and wake their parent Snapshot so abx-dl can replay + # the snapshot-level sequence. Indexed subqueries keep this bounded. Snapshot.objects.filter( id__in=orphaned_results.values("snapshot_id"), - status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED], - retry_at__isnull=True, + status=Snapshot.StatusChoices.STARTED, ).update(retry_at=now, modified_at=now) - cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now) - # Targeted plugin rows on final/paused Snapshots are scheduled through the - # parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a - # sealed Snapshot and must not be interpreted as queued work just because - # old/synthetic ArchiveResult rows exist. If takeover kills the runner - # after it leases the Snapshot but before queued ArchiveResult rows finish, - # the rows remain QUEUED while retry_at sits in the future. Recovery runs - # only after this runner has won the single-runner gate, so it can safely - # unlock those stale plugin leases for immediate processing instead of - # waiting out the previous owner's full lock timeout. - queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED, **result_filter) - cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = ( - Snapshot.objects.filter( - id__in=queued_plugin_results.values("snapshot_id"), - status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED], - ) - .filter(retry_at__gt=now) - .update(retry_at=now, modified_at=now) - ) - # Impossible state repair: STARTED ArchiveResults without a live Process - # have no owner left to emit completion. Requeue only the result row; the - # snapshot/crawl schedulers will pick up normal retry processing. - cleaned["archiveresults_started_without_running_process"] = orphaned_results.update( - status=ArchiveResult.StatusChoices.QUEUED, - process=None, + cleaned["archiveresults_interrupted_without_running_process"] = orphaned_results.update( + status=ArchiveResult.StatusChoices.FAILED, modified_at=now, ) orphaned_hook_processes = Process.objects.filter( @@ -206,11 +173,11 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | plugin_dir.name, hook_name, defaults={ - "status": ArchiveResult.StatusChoices.QUEUED, + "status": ArchiveResult.StatusChoices.FAILED, }, ) process_is_newer = bool(process.started_at and (result.start_ts is None or process.started_at >= result.start_ts)) - if result.status == ArchiveResult.StatusChoices.QUEUED or process_is_newer: + if created or process_is_newer: requeue_snapshot = False # A runner can die after the hook Process exits but before the # ProcessCompletedEvent projector links/finalizes ArchiveResult. @@ -231,16 +198,14 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | result.start_ts = process.started_at result.end_ts = process.ended_at if _is_signal_interrupted_exit(process.exit_code): - # The owning runner died or was asked to stop while the hook was - # still active. Keep the work item queued so takeover retries the - # same hook; treating an unknown signal exit as success would - # silently skip unfinished side effects. + # The Process is a durable fact; record the interruption as a + # failure and wake the parent Snapshot for replay. result.output_files = {} result.output_size = 0 result.output_mimetypes = "" result.output_str = "" result.output_json = None - result.status = ArchiveResult.StatusChoices.QUEUED + result.status = ArchiveResult.StatusChoices.FAILED requeue_snapshot = True else: result.output_files = output_files @@ -280,7 +245,6 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now) if created: cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1 - Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now) started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter( Q(retry_at__isnull=True) | Q(retry_at__gt=now), **snapshot_filter, @@ -294,8 +258,8 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | # We only unlock scheduling; normal Snapshot runner code owns the next # transition and side effects. cleaned["snapshots_started_without_running_results"] = ( - started_snapshots.annotate(has_running_results=Exists(running_archiveresults)) - .filter(has_running_results=False) + started_snapshots.annotate(has_running_process=Exists(running_hook_processes)) + .filter(has_running_process=False) .update( retry_at=now, modified_at=now, @@ -352,13 +316,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | "Finishing {count} browser-extension Snapshot(s) that received uploaded files before server extractors started " "(uploaded and server-created results will remain together on the same Snapshot)." ), - "archiveresults_backoff": ( - "Retrying {count} extractor result(s) that were waiting to retry " - "(ArchiveBox may have been interrupted before it was able to try them again; affected outputs will be retried)." - ), - "archiveresults_started_without_running_process": ( - "Retrying {count} extractor result(s) that were interrupted before finishing " - "(ArchiveBox may have been interrupted before it was able to save their final status; partial files will be overwritten with fresh results upon retry)." + "archiveresults_interrupted_without_running_process": ( + "Closing {count} interrupted extractor result projection(s) " + "(their parent Snapshots are resumed through the normal snapshot-level runner)." ), "snapshots_started_without_running_results": ( "Resuming {count} Snapshot(s) that were interrupted before finishing " diff --git a/archivebox/crawls/models.py b/archivebox/crawls/models.py index c8a46841..7eda30f2 100755 --- a/archivebox/crawls/models.py +++ b/archivebox/crawls/models.py @@ -90,6 +90,8 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes): @property def last_run_at(self): + if self.kind == "update": + return self.modified_at latest_crawl = self.crawl_set.order_by("-created_at").first() if latest_crawl: return latest_crawl.created_at @@ -105,6 +107,22 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes): now = now or timezone.now() return self.is_enabled and self.next_run_at <= now + @property + def kind(self) -> str: + return str((self.config or {}).get("SCHEDULE_KIND") or "crawl") + + def dispatch(self, queued_at=None) -> "Crawl | None": + """Run maintenance directly or enqueue one ordinary Crawl.""" + queued_at = queued_at or timezone.now() + if self.kind == "update": + from archivebox.cli.archivebox_update import process_all_db_snapshots + + process_all_db_snapshots() + type(self).objects.filter(pk=self.pk).update(modified_at=queued_at) + self.modified_at = queued_at + return None + return self.enqueue(queued_at=queued_at) + def enqueue(self, queued_at=None) -> "Crawl": from archivebox.config.common import build_crawl_config_snapshot @@ -112,10 +130,11 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes): template = self.template label = template.label or self.label persona = template.persona if template.persona_id else None + crawl_config = {key: value for key, value in (self.config or {}).items() if key != "SCHEDULE_KIND"} return Crawl.objects.create( urls=template.urls, - config=build_crawl_config_snapshot(persona=persona, overrides=self.config or {}), + config=build_crawl_config_snapshot(persona=persona, overrides=crawl_config), max_depth=template.max_depth, tags_str=template.tags_str, persona_id=template.persona_id, @@ -217,7 +236,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith def resume(self, *, when=None, save: bool = True) -> bool: resumed = super().resume(when=when, save=save) if resumed and self.pk: - from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.core.models import Snapshot resume_at = when or timezone.now() active_snapshots = self.snapshot_set.filter( @@ -228,7 +247,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith retry_at=resume_at, modified_at=timezone.now(), ) - ArchiveResult.resume_queryset(ArchiveResult.objects.filter(snapshot__crawl=self), when=resume_at) return resumed def cancel(self) -> None: @@ -269,8 +287,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith now = timezone.now() # Parent pause is a scheduler command. Wake child rows only; each - # Snapshot runner claim performs the real pause transition and cascades - # its own ArchiveResults, keeping request/admin transactions tiny. + # Snapshot runner claim performs the real pause transition, keeping + # request/admin transactions tiny. active_children = self.snapshot_set.filter( status__in=Snapshot.RUNNABLE_STATES, ) @@ -515,18 +533,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith return [] return [url for _raw_line, url in self._iter_url_lines() if url] - def has_internal_input_root(self) -> bool: - """Return True when Crawl.urls is preserved source text, not the work queue. - - The runner creates a synthetic root snapshot for raw import text so - parser hooks use the same Snapshot lifecycle as every other extractor. - In that mode the submitted text must remain in Crawl.urls forever; - parsed URLs live as child Snapshot rows and should not be appended back. - """ - from archivebox.core.models import Snapshot - - return self.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL, depth=0).exists() - @staticmethod def normalize_domain(value: str) -> str: candidate = (value or "").strip().lower() @@ -796,15 +802,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith "deleted_snapshots": filter_result["deleted_snapshots"], } - def get_system_task(self) -> str | None: - urls = self.get_urls_list() - if len(urls) != 1: - return None - system_url = urls[0].strip().lower() - if system_url.startswith("archivebox://"): - return system_url - return None - def resolve_persona(self): from archivebox.personas.models import Persona @@ -968,14 +965,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith if self.status == self.StatusChoices.SEALED: return [] - # Internal-input crawls preserve the submitted text verbatim in - # Crawl.urls. The root snapshot's parser hooks are the only supported - # path for turning that text into child snapshots, otherwise a later - # runner pass could reinterpret plain URL-looking lines as direct - # depth-0 work and bypass format-specific metadata parsing. - if self.has_internal_input_root(): - return [] - created_snapshots = [] crawl_tag_names = self.current_tag_names() tags_by_name: dict[str, Tag] = {} @@ -1248,16 +1237,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith if not created_snapshots: return [] - crawl_urls = {url for _raw_line, url in self._iter_url_lines() if url} - new_url_lines = [snapshot.url for snapshot in created_snapshots if snapshot.url not in crawl_urls] - # For internal-input crawls, Crawl.urls is the immutable source text. - # Child snapshots are the parsed/indexed representation, so appending - # discovered URLs here would both duplicate state and destroy the exact - # import artifact users submitted through CLI/API/UI. - if new_url_lines and not self.has_internal_input_root(): - self.urls = (self.urls.rstrip() + "\n" + "\n".join(new_url_lines)).lstrip("\n") - self.save(update_fields=["urls", "modified_at"]) - tag_names_by_url: dict[str, set[str]] = {} for snapshot in created_snapshots: tag_names = { diff --git a/archivebox/services/archive_result_service.py b/archivebox/services/archive_result_service.py index afa157dc..ad309f31 100644 --- a/archivebox/services/archive_result_service.py +++ b/archivebox/services/archive_result_service.py @@ -2,9 +2,7 @@ from __future__ import annotations import asyncio import inspect -import json import os -import signal import sys import time from contextlib import contextmanager @@ -15,7 +13,7 @@ from typing import Any from asgiref.sync import sync_to_async from django.utils import timezone -from abx_dl.events import PROCESS_EXIT_SKIPPED, ArchiveResultEvent, ProcessCompletedEvent, ProcessStartedEvent, SnapshotEvent +from abx_dl.events import ArchiveResultEvent, ProcessStartedEvent from abx_dl.output_files import OutputManifest from abx_dl.services.base import BaseService @@ -121,37 +119,6 @@ def _should_update_snapshot_title(current_title: str, next_title: str, *, snapsh return len(next_title) > len(current) -def _status_for_process_without_archive_result(event: ProcessCompletedEvent) -> str: - if event.exit_code == PROCESS_EXIT_SKIPPED: - return "skipped" - if event.exit_code in {128 + signal.SIGHUP, 128 + signal.SIGINT, 128 + signal.SIGTERM}: - # This fallback only runs when a snapshot hook exited before emitting a - # structured ArchiveResult. A polite shutdown signal means the runner - # was interrupted during ownership transfer, not that the extractor - # produced a durable negative result. Keep the hook queued so the next - # runner can retry the exact work item instead of sealing in a transient - # process-lifecycle failure. - return "queued" - if event.exit_code != 0: - return "failed" - return "noresults" - - -def _iter_archiveresult_records(stdout: str) -> list[dict]: - records: list[dict] = [] - for raw_line in stdout.splitlines(): - line = raw_line.strip() - if not line.startswith("{"): - continue - try: - record = json.loads(line) - except json.JSONDecodeError: - continue - if record.get("type") == "ArchiveResult": - records.append(record) - return records - - @_perf_trace("archivebox.ArchiveResultService._save_archiveresult_event_sync") def _save_archiveresult_event_to_db( event: ArchiveResultEvent, @@ -234,13 +201,6 @@ def _save_archiveresult_event_to_db( with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_update"): result.save(update_fields=[*update_fields, "modified_at"]) - if result.status == ArchiveResult.StatusChoices.QUEUED: - # ArchiveResult has no retry_at column. If a shutdown/takeover projects - # a killed hook back to QUEUED, wake the parent Snapshot/Crawl so the - # next runner retries that exact hook instead of waiting on a stale - # active-state lease. - snapshot.update_and_requeue(retry_at=timezone.now()) - if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS): with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.title_update"): title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else "" @@ -264,36 +224,43 @@ def _save_archiveresult_event_to_db( def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str, process_id: str) -> None: - """Advance an existing queued hook row after its OS process is persisted.""" - from archivebox.core.models import ArchiveResult + """Project a running abx-dl hook after its OS process is persisted.""" + from archivebox.core.models import ArchiveResult, Snapshot started_at = parse_event_datetime(event.start_ts) if started_at is None: raise ValueError("ProcessStartedEvent.start_ts is required") - ArchiveResult.objects.filter( - snapshot_id=snapshot_id, - plugin=event.plugin_name, - hook_name=event.hook_name, - status=ArchiveResult.StatusChoices.QUEUED, - ).update( - status=ArchiveResult.StatusChoices.STARTED, - start_ts=started_at, - end_ts=None, - process_id=process_id, - modified_at=timezone.now(), + snapshot = Snapshot.objects.filter(id=snapshot_id).first() + if snapshot is None: + return + result, _created = ArchiveResult.get_or_create_by_hook( + snapshot, + event.plugin_name, + event.hook_name, + defaults={ + "status": ArchiveResult.StatusChoices.STARTED, + "start_ts": started_at, + "end_ts": None, + "process_id": process_id, + }, ) + result.status = ArchiveResult.StatusChoices.STARTED + result.start_ts = started_at + result.end_ts = None + result.process_id = process_id + result.save(update_fields=["status", "start_ts", "end_ts", "process_id", "modified_at"]) class ArchiveResultService(BaseService): - LISTENS_TO = [ArchiveResultEvent, ProcessCompletedEvent] + """Project abx-dl ArchiveResult facts into Django models.""" + + LISTENS_TO = [ArchiveResultEvent] EMITS = [] def __init__(self, bus): - self._completed_process_event_ids: set[str] = set() self._save_locks: dict[tuple[str, str, str], asyncio.Lock] = {} super().__init__(bus) self.bus.on(ArchiveResultEvent, self.on_ArchiveResultEvent__save_to_db) - self.bus.on(ProcessCompletedEvent, self.on_ProcessCompletedEvent__save_to_db) @_perf_trace("archivebox.ArchiveResultService.on_ArchiveResultEvent__save_to_db") async def on_ArchiveResultEvent__save_to_db(self, event: ArchiveResultEvent) -> None: @@ -309,64 +276,3 @@ class ArchiveResultService(BaseService): lock = self._save_locks.setdefault(key, asyncio.Lock()) async with lock: await sync_to_async(_save_archiveresult_event_to_db, thread_sensitive=True)(event, process_started) - - @_perf_trace("archivebox.ArchiveResultService.on_ProcessCompletedEvent__save_to_db") - async def on_ProcessCompletedEvent__save_to_db(self, event: ProcessCompletedEvent) -> None: - if event.event_id in self._completed_process_event_ids: - return - self._completed_process_event_ids.add(event.event_id) - - if not event.hook_name.startswith("on_Snapshot"): - return - with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.find_snapshot_event"): - snapshot_event = await self.bus.find( - SnapshotEvent, - past=True, - future=False, - where=lambda candidate: self.bus.event_is_child_of(event, candidate), - ) - if snapshot_event is None: - return - - with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.parse_stdout_records"): - records = _iter_archiveresult_records(event.stdout) - if records: - if len(records) > 1: - raise RuntimeError( - f"Hook {event.plugin_name}:{event.hook_name} emitted {len(records)} ArchiveResult records; expected exactly one", - ) - for record in records: - record_status = _normalize_status(record.get("status") or "") - record_failed = record_status == "failed" or (not record_status and event.exit_code not in (0, PROCESS_EXIT_SKIPPED)) - with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_record"): - await event.emit( - ArchiveResultEvent( - snapshot_id=record.get("snapshot_id") or snapshot_event.snapshot_id, - plugin=record.get("plugin") or event.plugin_name, - hook_name=record.get("hook_name") or event.hook_name, - status=record_status, - output_str=record.get("output_str") or "", - output_json=record.get("output_json") if isinstance(record.get("output_json"), dict) else None, - output_files=event.output_files, - start_ts=event.start_ts, - end_ts=event.end_ts, - error=record.get("error") or (event.stderr if record_failed else ""), - ), - ).now() - return - - process_failed = _status_for_process_without_archive_result(event) == "failed" - with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"): - await event.emit( - ArchiveResultEvent( - snapshot_id=snapshot_event.snapshot_id, - plugin=event.plugin_name, - hook_name=event.hook_name, - status=_status_for_process_without_archive_result(event), - output_str=event.stderr if process_failed else "", - output_files=event.output_files, - start_ts=event.start_ts, - end_ts=event.end_ts, - error=event.stderr if process_failed else "", - ), - ).now() diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index f03cc033..39d425e3 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio import contextvars -import json import os import signal import shutil @@ -16,6 +15,7 @@ from tempfile import TemporaryDirectory from typing import Any from asgiref.sync import sync_to_async +from django.core.exceptions import ValidationError from django.utils import timezone from rich.console import Console from rich.text import Text @@ -28,6 +28,8 @@ from abx_dl.events import ( CrawlEvent, CrawlSetupEvent, CrawlStartEvent, + InstallEvent, + MachineEvent, ProcessCompletedEvent, ProcessEvent, SnapshotCompletedEvent, @@ -36,12 +38,19 @@ from abx_dl.events import ( ) from abx_dl.limits import CrawlLimitState from abx_dl.catalog import PluginCatalog +from abx_dl.config import GlobalConfig, RuntimeConfig from abx_dl.models import Snapshot as AbxSnapshot from abx_dl.orchestrator import ( - ExecutionPlan, + compute_install_phase_timeout, + compute_phase_timeout, create_bus, - install_plugins as abx_install_plugins, + get_install_plugins, + parse_input, ) +from abx_dl.services.binary_service import PluginBinaryEnvService +from abx_dl.services.archive_result_service import ArchiveResultService as HookArchiveResultService +from abx_dl.services.crawl_service import CrawlService as HookCrawlService +from abx_dl.services import PluginBinariesService from abx_dl.services.process_service import ProcessService as HookProcessService from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService from abx_dl.cli import LiveBusUI @@ -70,7 +79,7 @@ from .snapshot_service import SnapshotService, finalize_completed_snapshot, proj from .tag_service import TagService -QUEUED_PLUGIN_RESULT_BATCH_SIZE = 100 +MAINTENANCE_BATCH_SIZE = 100 def _bus_name(prefix: str, identifier: str) -> str: @@ -180,7 +189,6 @@ class CrawlRunner: show_progress: bool = True, interactive_interrupts: bool = False, config_overrides: dict[str, Any] | None = None, - selected_plugins_are_explicit: bool = True, ): self.crawl = crawl self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0) @@ -202,9 +210,10 @@ class CrawlRunner: self.bus, crawl_id=str(crawl.id), ) + HookArchiveResultService(self.bus, emit_jsonl=False) ArchiveResultService(self.bus) + self.requested_plugins = selected_plugins self.selected_plugins = selected_plugins - self.selected_plugins_from_args = selected_plugins is not None and selected_plugins_are_explicit self.initial_snapshot_ids = snapshot_ids self.snapshot_tasks: dict[str, asyncio.Task[None]] = {} self.snapshot_semaphore = asyncio.Semaphore(1) @@ -541,17 +550,6 @@ class CrawlRunner: self.selected_plugins = sorted(runtime_plugins) or None if self.crawl.is_paused: return [] - system_task = self.crawl.get_system_task() - if system_task == "archivebox://update": - # Scheduled maintenance crawls are control-plane work, not URL - # input. The pre-unified crawl lifecycle delegated this sentinel - # directly to the database maintenance scan. Preserve that contract so - # the unified runner does not turn it into an archivebox://internal - # Snapshot and feed the literal sentinel through parser plugins. - from archivebox.cli.archivebox_update import process_all_db_snapshots - - process_all_db_snapshots() - return [] if self.initial_snapshot_ids: # Explicit ids select normal runnable work, except for the one # targeted maintenance admitted by allow_maintenance_on_inactive_crawl. @@ -570,68 +568,59 @@ class CrawlRunner: return [str(snapshot.id) for snapshot in snapshots] def create_initial_snapshots(self) -> list: - from archivebox.core.models import Snapshot + from archivebox.misc.util import validate_url if self.crawl.snapshot_set.exists(): return [] - # Direct URL crawls (CLI add, REST, crawl create, schedule, ORM) seed - # Crawl.urls with either explicit CrawlSeed JSONL or one plain URL per - # line. Either shape becomes the input-layer (depth=0) snapshots so the - # rest of the system has one parsing convention. Anything else (RSS, - # Netscape HTML, JSONL imports, free-form text) falls through to the - # synthetic root path below for parser hooks to process. - records = [] + # Plain URL lists need no parser subprocess. Every other submitted + # document is parsed by abx-dl and returned as discovery facts; only + # those facts cross the ArchiveBox persistence boundary. + direct_urls: list[str] = [] for line in (self.crawl.urls or "").splitlines(): raw_line = line.strip() if not raw_line or raw_line.startswith("#"): continue try: - record = json.loads(raw_line) - except json.JSONDecodeError: - if raw_line.startswith(("http://", "https://")): - records.append({"type": "CrawlSeed", "url": raw_line, "depth": 0}) - continue - records = [] + direct_urls.append(validate_url(raw_line)) + except ValueError: + direct_urls = [] break - if not isinstance(record, dict) or record.get("type") != "CrawlSeed" or not record.get("url"): - records = [] - break - records.append(record) - if records: - created = [] - for record in records: - try: - record["depth"] = int(record.get("depth") or 0) - except (TypeError, ValueError): - record["depth"] = 0 - for depth in sorted({record["depth"] for record in records}): - depth_records = [record for record in records if record["depth"] == depth] - created.extend(self.crawl.create_discovered_snapshots(None, depth_records, depth=depth)) - return created + if direct_urls: + return self.crawl.create_discovered_snapshots( + None, + ({"url": url} for url in direct_urls), + depth=0, + ) - # Raw stdin/API/UI import text must remain verbatim in Crawl.urls so a - # resumed crawl sees the exact same source bytes and cannot reparse an - # overwritten temp file. Parser hooks still need the normal Snapshot - # lifecycle and SNAP_DIR/staticfile convention, so the runner creates a - # single synthetic root only after it has claimed the crawl. This keeps - # DB/FS side effects out of request/CLI add paths and lets child URLs be - # discovered by the usual parser -> CrawlService -> Snapshot flow. - root_snapshot = Snapshot( - url=Snapshot.INTERNAL_INPUT_URL, + parser_name = str(self.base_config.get("PARSER") or "auto").strip().lower() + parser_catalog = self.catalog + if parser_name != "auto": + requested = parser_name if parser_name.startswith("parse_") else f"parse_{parser_name}_urls" + parser_catalog = self.catalog.select([requested]) + runtime_config = self.base_config.for_crawl_runtime( crawl=self.crawl, - depth=0, - title="stdin.txt", - status=Snapshot.StatusChoices.QUEUED, - retry_at=timezone.now(), + persona=self.persona, + crawl_output_dir=self.crawl.output_dir, ) - root_snapshot.set_delete_at_from_config(self.base_config.get("DELETE_AFTER", "0")) - root_snapshot.save() - staticfile_dir = root_snapshot.output_dir / "staticfile" - staticfile_dir.mkdir(parents=True, exist_ok=True) - (staticfile_dir / "stdin.txt").write_text(self.crawl.urls, encoding="utf-8") - return [root_snapshot] + discovered = asyncio.run( + parse_input( + self.crawl.urls, + parser_catalog, + self.crawl.output_dir / "input", + config=normalize_runtime_config(runtime_config), + derived_config=normalize_runtime_config(self.derived_config), + runtime="archivebox", + auto_install=True, + emit_jsonl=False, + ), + ) + records = [snapshot.model_dump(mode="json") for snapshot in discovered] + created = self.crawl.create_discovered_snapshots(None, records, depth=0) + if created: + self.primary_url = created[0].url + return created def finalize_run_state(self) -> None: from archivebox.crawls.models import Crawl @@ -778,6 +767,7 @@ class CrawlRunner: "status": snapshot.status, "output_dir": snapshot_output_dir, "config": normalized_config, + "selected_plugins": [name.strip() for name in str((snapshot.config or {}).get("PLUGINS") or "").split(",") if name.strip()], "_snapshot": snapshot, } @@ -791,23 +781,18 @@ class CrawlRunner: config = normalize_runtime_config(snapshot["config"]) derived_config = normalize_runtime_config(self.derived_config) output_dir = Path(self.crawl_output_dir) - plan = ExecutionPlan.build( - self.catalog, - selected_plugins=self.selected_plugins, - config=config, - derived_config=derived_config, - runtime="archivebox", - ) - setup_hooks = [(plugin, hook) for plugin in plan.catalog.values() for hook in plugin.filter_hooks("CrawlSetup")] + plugins = self.catalog.select(self.selected_plugins) + config["ABX_RUNTIME"] = "archivebox" + setup_hooks = plugins.hooks("CrawlSetup") abx_snapshot = AbxSnapshot( id=snapshot["id"], url=snapshot["url"], depth=int(snapshot["depth"]), crawl_id=str(self.crawl.id), ) - crawl_setup_phase_timeout = plan.crawl_setup_timeout + crawl_setup_phase_timeout = compute_phase_timeout(setup_hooks, config) max_snapshot_count = max(1, int(config.get("CRAWL_MAX_URLS") or len(snapshot_ids) or 1)) - snapshot_phase_timeout = plan.snapshot_timeout + 120.0 + snapshot_phase_timeout = compute_phase_timeout(plugins.hooks("Snapshot"), config) + 120.0 all_snapshots_phase_timeout = snapshot_phase_timeout * max_snapshot_count crawl_cleanup_phase_timeout = crawl_setup_phase_timeout crawl_lifecycle_timeout = ( @@ -817,28 +802,17 @@ class CrawlRunner: + CrawlCompletedEvent.model_fields["event_timeout"].default + 30.0 ) - await plan.seed_config(self.bus) - plan.attach_services( + await self.bus.emit(MachineEvent(config=config, config_type="user")).now() + if derived_config: + await self.bus.emit(MachineEvent(config=derived_config, config_type="derived")).now() + PluginBinaryEnvService(self.bus, catalog=plugins) + HookCrawlService( self.bus, url=snapshot["url"], snapshot=abx_snapshot, output_dir=output_dir, - install_enabled=False, - crawl_setup_enabled=True, - crawl_event_enabled=False, - crawl_start_enabled=False, - snapshot_cleanup_enabled=False, - crawl_cleanup_enabled=True, - crawl_completed_enabled=False, - auto_install=True, - emit_jsonl=False, + catalog=plugins, abort_requested=self.crawl_is_cancelled, - PluginBinariesService=None, - BinaryService=None, - ProcessService=None, - ArchiveResultService=None, - TagService=None, - SnapshotService=None, ) async def on_archivebox_CrawlStartEvent(event: CrawlStartEvent) -> None: @@ -980,76 +954,18 @@ class CrawlRunner: if not isinstance(crawl_start_event, CrawlStartEvent): raise RuntimeError("Snapshot events must be emitted from a CrawlStartEvent handler") snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(snapshot_id) + try: + snapshot["_snapshot"].validate_url_for_archiving(config=snapshot["config"]) + except ValidationError as err: + if snapshot["status"] != "sealed": + await sync_to_async(snapshot["_snapshot"].seal, thread_sensitive=True)() + print(f"[X] Refusing to archive invalid Snapshot URL {snapshot['url']!r}: {err}", file=sys.stderr) + return if snapshot["status"] == "sealed" and not self.selected_plugins: await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id) return config = normalize_runtime_config(snapshot["config"]) - snapshot_config_plugins = [name.strip() for name in str(config.get("PLUGINS") or "").split(",") if name.strip()] - snapshot_selected_plugins = ( - self.selected_plugins if self.selected_plugins_from_args else (snapshot_config_plugins or self.selected_plugins) - ) - - def queued_plugins_selected_by_config(queued_plugins: list[str]) -> list[str]: - if not snapshot_selected_plugins: - return queued_plugins - expanded_selected_plugins = set(self.catalog.select(snapshot_selected_plugins)) - return [plugin for plugin in queued_plugins if plugin in expanded_selected_plugins] - - selected_hooks_by_plugin = None - if snapshot["status"] == "started": - _reset_count, running_count = await sync_to_async(snapshot["_snapshot"].reset_abandoned_results, thread_sensitive=True)() - if running_count: - await sync_to_async( - lambda: snapshot["_snapshot"].update_and_requeue( - retry_at=timezone.now() + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS), - ), - thread_sensitive=True, - )() - return - if await sync_to_async(snapshot["_snapshot"].is_finished_processing, thread_sensitive=True)(): - await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)( - snapshot["id"], - output_dir=Path(snapshot["output_dir"]), - ) - return - if not self.selected_plugins_from_args: - queued_plugins, selected_hooks_by_plugin = await sync_to_async( - queued_plugins_and_hooks_for_snapshot, - thread_sensitive=True, - )(snapshot["id"]) - if queued_plugins: - if snapshot_selected_plugins: - original_queued_plugins = queued_plugins - queued_plugins = queued_plugins_selected_by_config(queued_plugins) - disabled_queued_plugins = sorted(set(original_queued_plugins) - set(queued_plugins)) - if disabled_queued_plugins: - await sync_to_async(skip_disabled_queued_plugins, thread_sensitive=True)( - snapshot["id"], - disabled_queued_plugins, - ) - selected_hooks_by_plugin = { - plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins - } - snapshot_selected_plugins = queued_plugins - elif not self.selected_plugins_from_args: - queued_plugins, selected_hooks_by_plugin = await sync_to_async( - queued_plugins_and_hooks_for_snapshot, - thread_sensitive=True, - )(snapshot["id"]) - if queued_plugins: - if snapshot_selected_plugins: - original_queued_plugins = queued_plugins - queued_plugins = queued_plugins_selected_by_config(queued_plugins) - disabled_queued_plugins = sorted(set(original_queued_plugins) - set(queued_plugins)) - if disabled_queued_plugins: - await sync_to_async(skip_disabled_queued_plugins, thread_sensitive=True)( - snapshot["id"], - disabled_queued_plugins, - ) - selected_hooks_by_plugin = { - plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins - } - snapshot_selected_plugins = queued_plugins + snapshot_selected_plugins = self.requested_plugins or snapshot["selected_plugins"] or self.selected_plugins if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() in ( "crawl_max_size", "crawl_timeout", @@ -1059,56 +975,31 @@ class CrawlRunner: derived_config = normalize_runtime_config(self.derived_config) output_dir = Path(snapshot["output_dir"]) plugins = self.catalog.select(snapshot_selected_plugins) if snapshot_selected_plugins else self.catalog - if selected_hooks_by_plugin is not None: - await sync_to_async(fail_unavailable_queued_hooks, thread_sensitive=True)( - snapshot["id"], - selected_hooks_by_plugin, - plugins, - ) - remaining_queued_plugins = await sync_to_async( - queued_plugins_for_snapshot, - thread_sensitive=True, - )(snapshot["id"]) - if snapshot_selected_plugins and remaining_queued_plugins: - remaining_queued_plugins = queued_plugins_selected_by_config(remaining_queued_plugins) - if not remaining_queued_plugins: - if snapshot["status"] in ("queued", "started"): - await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)( - snapshot_id, - output_dir=output_dir, - ) - else: - await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir) - return - snapshot_selected_plugins = remaining_queued_plugins - plugins = self.catalog.select(snapshot_selected_plugins) - selected_hooks_by_plugin = include_background_prerequisite_hooks(selected_hooks_by_plugin, plugins) abx_snapshot = AbxSnapshot( id=snapshot["id"], url=snapshot["url"], depth=int(snapshot["depth"]), crawl_id=str(self.crawl.id), ) - plan = ExecutionPlan.build( - self.catalog, - selected_plugins=snapshot_selected_plugins, - config=config, - derived_config=derived_config, - runtime="archivebox", - ) - plugins = plan.catalog - snapshot_phase_timeout = plan.snapshot_timeout + 120.0 - await plan.seed_config(self.bus, parent_event=crawl_start_event) - snapshot_service = plan.attach_snapshot_service( + config["ABX_RUNTIME"] = "archivebox" + snapshot_phase_timeout = compute_phase_timeout(plugins.hooks("Snapshot"), config) + 120.0 + user_config_event = MachineEvent(config=config, config_type="user") + user_config_event.event_parent_id = crawl_start_event.event_id + await self.bus.emit(user_config_event).now() + if derived_config: + derived_config_event = MachineEvent(config=derived_config, config_type="derived") + derived_config_event.event_parent_id = crawl_start_event.event_id + await self.bus.emit(derived_config_event).now() + snapshot_service = HookSnapshotService( self.bus, url=snapshot["url"], snapshot=abx_snapshot, output_dir=output_dir, - snapshot_service=HookSnapshotService, - timeout_padding=120.0, + catalog=plugins, + config=RuntimeConfig(user=GlobalConfig(**config), derived=derived_config), + snapshot_phase_timeout=snapshot_phase_timeout, + snapshot_cleanup_phase_timeout=snapshot_phase_timeout, abort_requested=self.crawl_is_cancelled, - selected_hooks_by_plugin=selected_hooks_by_plugin, - emit_discovered_snapshot_events=False, ) try: snapshot_event = SnapshotEvent( @@ -1191,7 +1082,6 @@ def run_crawl( show_progress: bool = True, interactive_interrupts: bool = False, config_overrides: dict[str, Any] | None = None, - selected_plugins_are_explicit: bool = True, ) -> None: with crawl_lifecycle_lock(crawl_id): _run_crawl_locked( @@ -1202,7 +1092,6 @@ def run_crawl( show_progress=show_progress, interactive_interrupts=interactive_interrupts, config_overrides=config_overrides, - selected_plugins_are_explicit=selected_plugins_are_explicit, ) @@ -1215,7 +1104,6 @@ def _run_crawl_locked( show_progress: bool = True, interactive_interrupts: bool = False, config_overrides: dict[str, Any] | None = None, - selected_plugins_are_explicit: bool = True, ) -> None: from archivebox.crawls.models import Crawl from django.db import close_old_connections @@ -1233,7 +1121,6 @@ def _run_crawl_locked( show_progress=show_progress, interactive_interrupts=interactive_interrupts, config_overrides=config_overrides, - selected_plugins_are_explicit=selected_plugins_are_explicit, ).run(), ) finally: @@ -1273,28 +1160,14 @@ async def _run_binary(binary_id: str) -> None: binary_process_service = ArchiveBoxBinaryService(bus) BinaryService(bus, lib_dir=Path(config["ABXPKG_LIB_DIR"])) TagService(bus) - ArchiveResultService(bus) MachineService(bus) catalog = get_plugin_catalog() - plan = ExecutionPlan.build( - catalog, - selected_plugins=list(catalog), - config=config, - derived_config=derived_config, - runtime="archivebox", - ) - plan.attach_services( - bus, - install_enabled=False, - crawl_setup_enabled=False, - crawl_start_enabled=False, - snapshot_cleanup_enabled=False, - crawl_cleanup_enabled=False, - auto_install=True, - emit_jsonl=False, - BinaryService=None, - ) - await plan.seed_config(bus) + config["ABX_RUNTIME"] = "archivebox" + PluginBinaryEnvService(bus, catalog=catalog) + HookProcessService(bus, emit_jsonl=False, interactive_tty=False) + await bus.emit(MachineEvent(config=config, config_type="user")).now() + if derived_config: + await bus.emit(MachineEvent(config=derived_config, config_type="derived")).now() try: await bus.emit( @@ -1321,170 +1194,18 @@ def run_binary(binary_id: str) -> None: asyncio.run(_run_binary(binary_id)) -def queued_plugins_and_hooks_for_snapshot(snapshot_id: str) -> tuple[list[str] | None, dict[str, set[str] | None] | None]: - from archivebox.core.models import ArchiveResult - - queued_results = list( - ArchiveResult.objects.filter( - snapshot_id=snapshot_id, - status=ArchiveResult.StatusChoices.QUEUED, - ) - .exclude(plugin="") - .only("id", "plugin", "hook_name"), - ) - - selected_hooks_by_plugin: dict[str, set[str] | None] = {} - queued_plugins = sorted({result.plugin for result in queued_results}) - for result in queued_results: - # hook_name is the modern scheduler identity. Empty hook_name rows are - # legacy plugin-level work and must keep running the whole plugin. - if not result.hook_name: - selected_hooks_by_plugin[result.plugin] = None - elif result.plugin not in selected_hooks_by_plugin: - selected_hooks_by_plugin[result.plugin] = {result.hook_name} - elif selected_hooks_by_plugin[result.plugin] is not None: - selected_hooks_by_plugin[result.plugin].add(result.hook_name) - if queued_plugins: - return queued_plugins, selected_hooks_by_plugin - return None, None - - -def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None: - queued_plugins, _selected_hooks_by_plugin = queued_plugins_and_hooks_for_snapshot(snapshot_id) - return queued_plugins - - -def config_overrides_for_queued_plugins(selected_plugins: list[str], **overrides: Any) -> dict[str, Any]: - config_overrides = dict(overrides) - config_overrides["PLUGINS"] = ",".join(selected_plugins) - selected_plugin_names = set(get_plugin_catalog().select(plugin_name.lower() for plugin_name in selected_plugins)) - for plugin_name, enabled_key in _plugin_enabled_config_keys().items(): - config_overrides[enabled_key] = plugin_name in selected_plugin_names - return config_overrides - - -def fail_unavailable_queued_hooks( - snapshot_id: str, - selected_hooks_by_plugin: dict[str, set[str] | None], - plugins: PluginCatalog, -) -> None: - from archivebox.core.models import ArchiveResult - - now = timezone.now() - for plugin_name, selected_hook_names in selected_hooks_by_plugin.items(): - if selected_hook_names is None: - continue - if plugin_name in plugins: - available_hook_names = { - name for hook in plugins[plugin_name].filter_hooks("Snapshot") for name in (hook.name, Path(hook.name).stem) - } - else: - available_hook_names = set() - missing_hook_names = [hook_name for hook_name in selected_hook_names if hook_name not in available_hook_names] - if not missing_hook_names: - continue - # Hook-level resume rows are durable scheduler state. If a plugin is - # installed but no longer exposes a queued hook, mark that row failed so - # the snapshot is not retried forever with no hook left to execute. - ArchiveResult.objects.filter( - snapshot_id=snapshot_id, - plugin=plugin_name, - hook_name__in=missing_hook_names, - status=ArchiveResult.StatusChoices.QUEUED, - ).update( - status=ArchiveResult.StatusChoices.FAILED, - start_ts=now, - end_ts=now, - output_str="Queued hook is no longer available in the installed plugin", - ) - - -def skip_disabled_queued_plugins(snapshot_id: str, plugin_names: list[str]) -> None: - from archivebox.core.models import ArchiveResult - - if not plugin_names: - return - now = timezone.now() - # Queued ArchiveResult rows are durable scheduler state and can outlive a - # config change or deploy. If a plugin is no longer selected for this - # Snapshot/Crawl, leaving its old row queued keeps the Snapshot STARTED - # forever even though there is no runnable work left. - ArchiveResult.objects.filter( - snapshot_id=snapshot_id, - plugin__in=plugin_names, - status=ArchiveResult.StatusChoices.QUEUED, - ).update( - status=ArchiveResult.StatusChoices.SKIPPED, - start_ts=now, - end_ts=now, - output_str="Queued plugin is disabled by this Snapshot/Crawl config", - ) - - -def include_background_prerequisite_hooks( - selected_hooks_by_plugin: dict[str, set[str] | None], - plugins: PluginCatalog, -) -> dict[str, set[str] | None]: - expanded: dict[str, set[str] | None] = {} - for plugin_name, selected_hook_names in selected_hooks_by_plugin.items(): - if selected_hook_names is None or plugin_name not in plugins: - expanded[plugin_name] = selected_hook_names - continue - plugin_hooks = sorted(plugins[plugin_name].filter_hooks("Snapshot"), key=lambda hook: hook.sort_key) - selected_sort_keys = [ - hook.sort_key for hook in plugin_hooks if hook.name in selected_hook_names or Path(hook.name).stem in selected_hook_names - ] - if not selected_sort_keys: - expanded[plugin_name] = set(selected_hook_names) - continue - first_selected_sort_key = min(selected_sort_keys) - expanded_hook_names = set(selected_hook_names) - # Earlier background hooks publish live resources (e.g. Chrome tabs) - # needed by later foreground hooks, but completed foreground hooks stay - # final and are not rerun during hook-level resume. - for hook in plugin_hooks: - if hook.is_background and hook.sort_key < first_selected_sort_key: - expanded_hook_names.add(hook.name) - expanded_hook_names.add(Path(hook.name).stem) - expanded[plugin_name] = expanded_hook_names - return expanded - - -def snapshot_hooks_for_pending_archiveresults(snapshot) -> list[tuple[str, str]]: - from archivebox.config.common import get_config - from archivebox.core.models import Snapshot - from archivebox.plugins.discovery import get_enabled_plugins - - config = get_config(crawl=snapshot.crawl, snapshot=snapshot) - snapshot_plugin_names = [name.strip() for name in str((snapshot.config or {}).get("PLUGINS") or "").split(",") if name.strip()] - crawl_plugin_names = [name.strip() for name in str((snapshot.crawl.config or {}).get("PLUGINS") or "").split(",") if name.strip()] - config_plugin_names = [name.strip() for name in str(config.PLUGINS or "").split(",") if name.strip()] - plugin_names = snapshot_plugin_names or crawl_plugin_names or config_plugin_names or get_enabled_plugins(config=config) - catalog = get_plugin_catalog() - plugins = catalog.select(plugin_names) if plugin_names else catalog - if snapshot.url == Snapshot.INTERNAL_INPUT_URL: - plugins = {name: plugin for name, plugin in plugins.items() if getattr(plugin.config, "x_accepts_internal_input", False)} - return sorted((plugin.name, hook.name) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")) - - def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None) -> bool: - from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.core.models import Snapshot snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first() if snapshot is None: return False - has_queued_results = snapshot.archiveresult_set.filter(status=ArchiveResult.StatusChoices.QUEUED).exists() - # retry_at is the scheduler signal for both lifecycle work and targeted - # maintenance. Filesystem migration/json rewriting is independent from - # queued ArchiveResult rows, so run it whenever this helper is called. - # The only thing queued work changes is the next scheduler value: - # - open lifecycle rows: leave them due for extraction after maintenance - # - no queued work left on a sealed row: clear retry_at - # - queued rows remain on a sealed Snapshot: leave it due so targeted - # plugin maintenance can process them on the next tick + # ArchiveBox owns filesystem and metadata maintenance at Snapshot + # granularity. ArchiveResult rows are projections and never influence this + # scheduler decision. current_retry_at = snapshot.retry_at - next_retry_at = timezone.now() if has_queued_results or snapshot.status in Snapshot.OPEN_STATES else None + next_retry_at = timezone.now() if snapshot.status in Snapshot.OPEN_STATES else None snapshot.retry_at = next_retry_at if snapshot.fs_migration_needed: snapshot.save(update_fields=["retry_at", "modified_at"]) @@ -1630,11 +1351,8 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru return parent_reconciled if snapshot.is_paused: - # Paused work never executes out of band. Preserve the lifecycle marker - # until an explicit resume moves it through the normal lifecycle. - from archivebox.core.models import ArchiveResult - - ArchiveResult.pause_queryset(snapshot.archiveresult_set.all()) + # Paused work never executes out of band. ArchiveResult rows are + # historical projections and are not rewritten as scheduler state. snapshot.restore_paused_scheduler_marker() return True if snapshot.status == Snapshot.StatusChoices.SEALED: @@ -1642,121 +1360,36 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru return False snapshot.refresh_from_db() snapshot.finalize_completed_upload_results() - maintenance_ran = False - if snapshot.fs_migration_needed: - # Final snapshots can still need filesystem/index maintenance after - # a data-dir migration, but queued ArchiveResult rows are the actual - # runnable work. Do the metadata rewrite first, then continue into - # the targeted plugin path in the same tick so large migrations do - # not starve search/index backfills behind a full maintenance pass. - maintenance_ran = run_snapshot_maintenance(str(snapshot.id)) - snapshot.refresh_from_db() - selected_plugins = queued_plugins_for_snapshot(str(snapshot.id)) - if selected_plugins: - _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot) - run_crawl( - str(snapshot.crawl_id), - snapshot_ids=[str(snapshot.id)], - selected_plugins=selected_plugins, - process_discovered_snapshots_inline=True, - interactive_interrupts=interactive_interrupts, - config_overrides=config_overrides_for_queued_plugins(selected_plugins), - selected_plugins_are_explicit=False, - ) - from archivebox.core.models import ArchiveResult - - has_queued_results = ArchiveResult.objects.filter( - snapshot_id=snapshot.id, - status=ArchiveResult.StatusChoices.QUEUED, - ).exists() - type(snapshot).objects.filter( - pk=snapshot.pk, - status=snapshot.StatusChoices.SEALED, - ).update( - retry_at=timezone.now() if has_queued_results else None, - modified_at=timezone.now(), - ) - return True - if snapshot.status == Snapshot.StatusChoices.SEALED: - if maintenance_ran: - return True - return run_snapshot_maintenance(str(snapshot.id)) - - if snapshot.status == Snapshot.StatusChoices.STARTED: - _reset_count, running_count = snapshot.reset_abandoned_results() - if running_count: - snapshot.update_and_requeue(retry_at=timezone.now() + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS)) - return True + return run_snapshot_maintenance(str(snapshot.id)) if not snapshot.claim_processing_lock(lock_seconds=lock_seconds): return False snapshot.refresh_from_db() + if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()): + # The Snapshot lease may have expired while an abx-dl hook process is + # still alive. Preserve the snapshot-level ownership boundary and do + # not launch a second sequence; ArchiveResult status is irrelevant. + snapshot.update_and_requeue(retry_at=timezone.now() + timedelta(seconds=lock_seconds)) + return True + if snapshot.fs_migration_needed: + # Migrate before abx-dl writes new hook outputs. The claimed Snapshot + # lease remains in place and the idempotent migration persists its + # indexed fs_version marker only after copy/verification/cleanup. + snapshot.save(update_fields=["retry_at", "modified_at"]) + snapshot.refresh_from_db() if snapshot.status == Snapshot.StatusChoices.QUEUED: - has_server_archiveresults = snapshot.archiveresult_set.exclude( - hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME, - ).exists() - if has_server_archiveresults and snapshot.is_finished_processing(): - finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir)) - snapshot.refresh_from_db() - if snapshot.status == Snapshot.StatusChoices.SEALED: - if snapshot.fs_migration_needed: - run_snapshot_maintenance(str(snapshot.id)) - snapshot.refresh_from_db() - _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED") - return True - # A Snapshot with no server hook rows is fresh lifecycle work; materialize - # its configured hook set. Browser-extension uploads are completed external - # outputs, not the durable server workset, and must coexist with these rows. - if not has_server_archiveresults: - snapshot.create_pending_archiveresults(hooks=snapshot_hooks_for_pending_archiveresults(snapshot)) - snapshot.advance_lifecycle() + snapshot.start_processing() snapshot.refresh_from_db() - if snapshot.status == Snapshot.StatusChoices.SEALED: - _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED") - return True - if snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.archiveresult_set.exists() and snapshot.is_finished_processing(): - finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir)) - snapshot.refresh_from_db() - if snapshot.status == Snapshot.StatusChoices.SEALED: - if snapshot.fs_migration_needed: - run_snapshot_maintenance(str(snapshot.id)) - snapshot.refresh_from_db() - _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED") - return True - if snapshot.status == Snapshot.StatusChoices.STARTED: - queued_plugins, selected_hooks_by_plugin = queued_plugins_and_hooks_for_snapshot(str(snapshot.id)) - if queued_plugins and selected_hooks_by_plugin: - fail_unavailable_queued_hooks( - str(snapshot.id), - selected_hooks_by_plugin, - get_plugin_catalog(), - ) - if not queued_plugins_for_snapshot(str(snapshot.id)): - finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir)) - return True + if snapshot.status != Snapshot.StatusChoices.STARTED: + return True _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot) run_crawl( str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], - # Do not pass this snapshot's queued plugin set as a crawl-wide runner - # filter. Internal import roots intentionally queue only parser hooks, - # but any child snapshots discovered from that root must still run the - # normal crawl plugin surface. run_snapshot() narrows the current - # snapshot from its queued ArchiveResult rows immediately before - # execution; leaving the runner unconstrained keeps that narrowing - # local to the one snapshot that owns those rows. selected_plugins=None, process_discovered_snapshots_inline=True, interactive_interrupts=interactive_interrupts, - selected_plugins_are_explicit=False, ) - snapshot.refresh_from_db() - if queued_plugins_for_snapshot(str(snapshot.id)): - # Hook-level resume work is tracked by queued ArchiveResult rows, not by - # the Snapshot lease. If a partial pass returns with rows still queued, - # wake the Snapshot immediately so takeover does not wait out a stale - # active-state lock before running the remaining hooks. - snapshot.update_and_requeue(retry_at=timezone.now()) return True @@ -1789,7 +1422,6 @@ async def _run_install(plugin_names: list[str] | None = None) -> None: ArchiveBoxBinaryService(bus) BinaryService(bus) TagService(bus) - ArchiveResultService(bus) MachineService(bus) live_stream = None bus_destroyed = False @@ -1806,13 +1438,8 @@ async def _run_install(plugin_names: list[str] | None = None) -> None: for plugin in selected_plugins.values(): if plugin.enabled_key in plugin.config.properties: install_config[plugin.enabled_key] = True - plan = ExecutionPlan.build( - catalog, - selected_plugins=list(selected_plugins), - config=install_config, - derived_config=derived_config, - runtime="archivebox", - ) + install_config["ABX_RUNTIME"] = "archivebox" + install_timeout = compute_install_phase_timeout(get_install_plugins(selected_plugins), install_config) timeout_seconds = config["TIMEOUT"] stdout_is_tty = sys.stdout.isatty() stderr_is_tty = sys.stderr.isatty() @@ -1864,13 +1491,32 @@ async def _run_install(plugin_names: list[str] | None = None) -> None: ) with live_ui if live_ui is not None else nullcontext(): try: - await abx_install_plugins( - plan, + HookProcessService(bus, emit_jsonl=False, interactive_tty=interactive_tty) + PluginBinaryEnvService(bus, catalog=selected_plugins) + install_snapshot = AbxSnapshot(url="") + PluginBinariesService( + bus, + catalog=selected_plugins, + auto_install=True, + install_plugins=get_install_plugins(selected_plugins), output_dir=output_dir, - emit_jsonl=False, - bus=bus, - BinaryService=None, + snapshot=install_snapshot, ) + await bus.emit(MachineEvent(config=install_config, config_type="user")).now() + if derived_config: + await bus.emit(MachineEvent(config=derived_config, config_type="derived")).now() + install_event = bus.emit( + InstallEvent( + url="", + snapshot_id=install_snapshot.id, + output_dir=str(output_dir), + event_timeout=install_timeout, + event_handler_slow_timeout=slow_warning_timeout(install_timeout), + ), + ) + await install_event.now(timeout=install_timeout) + await install_event.wait(timeout=install_timeout) + await install_event.event_results_list() finally: try: await bus.wait_until_idle() @@ -1946,116 +1592,6 @@ def _run_due_snapshot_id(snapshot_id, *, lock_seconds: int, interactive_interrup return True -def _run_due_queued_plugin_result( - plugin_names: frozenset[str], - *, - crawl_id: str | None, - lock_seconds: int, - interactive_interrupts: bool, - runtime_config, - batch_size: int = QUEUED_PLUGIN_RESULT_BATCH_SIZE, -) -> bool: - from archivebox.core.models import ArchiveResult, Snapshot - from django.db.models import Exists, OuterRef - - if not plugin_names: - return False - now = timezone.now() - queued_results = ArchiveResult.objects.filter( - snapshot_id=OuterRef("pk"), - status=ArchiveResult.StatusChoices.QUEUED, - plugin__in=plugin_names, - ) - first_due_query = ( - ArchiveResult.objects.filter( - status=ArchiveResult.StatusChoices.QUEUED, - plugin__in=plugin_names, - snapshot__retry_at__lte=now, - snapshot__status=Snapshot.StatusChoices.SEALED, - ) - .filter(**({"snapshot__crawl_id": crawl_id} if crawl_id else {})) - .values("snapshot_id", "snapshot__crawl_id")[:1] - ) - first_due_results = list(first_due_query) - if not first_due_results: - return False - root_crawl_id = str(first_due_results[0]["snapshot__crawl_id"]) - - due_snapshots = Snapshot.objects.filter( - retry_at__lte=now, - status=Snapshot.StatusChoices.SEALED, - ).filter(Exists(queued_results)) - if crawl_id: - due_snapshots = due_snapshots.filter(crawl_id=crawl_id) - batch_candidates = list( - # The crawl picker above starts from enabled queued ArchiveResult rows - # and uses a sliced LIMIT 1. Do not use QuerySet.first() here: it adds - # ordering and can turn this hot scheduler check into a temp-sort over - # hundreds of thousands of plugin rows. Once a crawl is selected, - # sibling order is irrelevant; the crawl_id/status index can fetch this - # small local batch directly while EXISTS proves the enabled queued - # plugin rows via the existing ArchiveResult unique index. - due_snapshots.filter(crawl_id=root_crawl_id).order_by()[:batch_size], - ) - if not batch_candidates: - return False - - selected_plugins: list[str] | None = None - claimed_snapshot_ids: list[str] = [] - for snapshot in batch_candidates: - snapshot_selected_plugins = [ - plugin_name for plugin_name in (queued_plugins_for_snapshot(str(snapshot.id)) or []) if plugin_name in plugin_names - ] - if not snapshot_selected_plugins: - continue - if selected_plugins is None: - selected_plugins = snapshot_selected_plugins - if snapshot_selected_plugins != selected_plugins: - continue - claimed = Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds) - if not claimed: - continue - snapshot.refresh_from_db() - snapshot.finalize_completed_upload_results() - if snapshot.fs_migration_needed: - run_snapshot_maintenance(str(snapshot.id)) - snapshot.refresh_from_db() - if snapshot.status != Snapshot.StatusChoices.SEALED: - continue - claimed_snapshot_ids.append(str(snapshot.id)) - _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot) - - if not claimed_snapshot_ids or selected_plugins is None: - return True - - run_crawl( - root_crawl_id, - snapshot_ids=claimed_snapshot_ids, - selected_plugins=selected_plugins, - process_discovered_snapshots_inline=True, - interactive_interrupts=interactive_interrupts, - config_overrides=config_overrides_for_queued_plugins(selected_plugins, CRAWL_MAX_CONCURRENT_SNAPSHOTS=batch_size), - selected_plugins_are_explicit=False, - ) - queued_results = ArchiveResult.objects.filter( - snapshot_id=OuterRef("pk"), - status=ArchiveResult.StatusChoices.QUEUED, - plugin__in=selected_plugins, - ) - Snapshot.objects.filter( - id__in=claimed_snapshot_ids, - status=Snapshot.StatusChoices.SEALED, - ).annotate( - has_queued_results=Exists(queued_results), - ).filter( - has_queued_results=False, - ).update( - retry_at=None, - modified_at=timezone.now(), - ) - return True - - def _run_due_binary() -> bool: from archivebox.machine.models import Binary @@ -2081,12 +1617,11 @@ def run_pending_crawls( crawl_id: str | None = None, maintenance_only: bool = False, interactive_interrupts: bool = False, - maintenance_batch_size: int = QUEUED_PLUGIN_RESULT_BATCH_SIZE, + maintenance_batch_size: int = MAINTENANCE_BATCH_SIZE, ) -> int: from archivebox.config.common import get_config from archivebox.crawls.models import Crawl, CrawlSchedule from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog from archivebox.machine.models import Process crawl_claim_lock_seconds = 10 @@ -2116,7 +1651,7 @@ def run_pending_crawls( now = timezone.now() for schedule in CrawlSchedule.objects.filter(is_enabled=True).select_related("template", "template__created_by"): if schedule.is_due(now): - schedule.enqueue(queued_at=now) + schedule.dispatch(queued_at=now) if maintenance_only: # Filesystem migration is independent of lifecycle status; do not @@ -2218,42 +1753,14 @@ def run_pending_crawls( ): continue - # Plugin selection is live crawl-execution config, not an installed- - # plugin list. Old queued rows for a plugin that is disabled by the - # current Machine/Crawl/Snapshot config remain queued until the user - # re-enables it. Refresh here to preserve mid-run config edits while - # using the same enabled-hook discovery path that created the rows. - runtime_config = get_config() - catalog = get_plugin_catalog() - enabled_plugins = get_enabled_plugins(config=runtime_config) - queued_plugin_names = frozenset(plugin.name for plugin, _hook in catalog.hooks("Snapshot", names=enabled_plugins)) - if _run_due_queued_plugin_result( - queued_plugin_names, - crawl_id=crawl_id, - lock_seconds=60, - interactive_interrupts=interactive_interrupts, - runtime_config=runtime_config, - batch_size=maintenance_batch_size, - ): - continue - if not maintenance_only: - # Broad final-state maintenance is intentionally a fallback. Specific - # queued plugin work above can use ArchiveResult's scheduler indexes; - # this branch may need to prove that no due sealed snapshot remains, so - # avoid paying that scan while targeted work is already available. + # Final snapshots can still have an explicit filesystem/index-json + # maintenance tick. Search extraction is invoked directly by the + # update path and never enters this scheduler through ArchiveResult. sealed_snapshots = Snapshot.objects.filter( retry_at__lte=timezone.now(), status=Snapshot.StatusChoices.SEALED, ) - if queued_plugin_names: - queued_plugin_snapshot_ids = ArchiveResult.objects.filter( - status=ArchiveResult.StatusChoices.QUEUED, - plugin__in=queued_plugin_names, - ).values("snapshot_id") - sealed_snapshots = sealed_snapshots.exclude( - id__in=queued_plugin_snapshot_ids, - ) if crawl_id: sealed_snapshots = sealed_snapshots.filter(crawl_id=crawl_id) if _run_due_snapshot_query( diff --git a/archivebox/services/snapshot_service.py b/archivebox/services/snapshot_service.py index e1773582..4b9453c5 100644 --- a/archivebox/services/snapshot_service.py +++ b/archivebox/services/snapshot_service.py @@ -1,12 +1,9 @@ from __future__ import annotations -import sys from pathlib import Path from asgiref.sync import sync_to_async from django.utils import timezone -from django.core.exceptions import ValidationError -from rich import print as rprint from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent from abx_dl.limits import CrawlLimitState from abx_dl.services.base import BaseService @@ -76,7 +73,11 @@ def finalize_completed_snapshot( if snapshot.status == Snapshot.StatusChoices.QUEUED: snapshot.advance_lifecycle() snapshot.refresh_from_db() - if snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.is_finished_processing(): + # SnapshotCompletedEvent is abx-dl's authoritative signal that the complete + # snapshot hook sequence (including cleanup) finished. ArchiveResult rows + # are projections of that work, never prerequisites used to decide whether + # the Snapshot may seal. + if snapshot.status == Snapshot.StatusChoices.STARTED: snapshot.seal() snapshot.refresh_from_db() @@ -113,26 +114,7 @@ class SnapshotService(BaseService): if snapshot.is_paused: return if snapshot.status == Snapshot.StatusChoices.QUEUED: - if not await snapshot.archiveresult_set.aexists(): - from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults - - hooks = await sync_to_async(snapshot_hooks_for_pending_archiveresults, thread_sensitive=True)(snapshot) - await sync_to_async(snapshot.create_pending_archiveresults, thread_sensitive=True)(hooks=hooks) - try: - await sync_to_async(snapshot.advance_lifecycle, thread_sensitive=True)() - except ValidationError as err: - if "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs." not in str(err): - raise - await Snapshot.objects.filter(id=snapshot.id).aupdate( - status=Snapshot.StatusChoices.SEALED, - retry_at=None, - modified_at=timezone.now(), - ) - rprint( - f"[red][X] Refusing to archive ArchiveBox internal URL for security: {snapshot.url}[/red]", - file=sys.stderr, - ) - return + await sync_to_async(snapshot.advance_lifecycle, thread_sensitive=True)() await sync_to_async(snapshot.refresh_from_db, thread_sensitive=True)() elif snapshot.status != Snapshot.StatusChoices.STARTED: return diff --git a/archivebox/tests/test_api_v1_cli_add.py b/archivebox/tests/test_api_v1_cli_add.py index 28485a0c..7ffb335d 100644 --- a/archivebox/tests/test_api_v1_cli_add.py +++ b/archivebox/tests/test_api_v1_cli_add.py @@ -264,7 +264,7 @@ def test_basic_success_case_request(client, tmp_path, api_headers): assert response.status_code == 200, response.content assert response.json()["success"] is True crawl = Crawl.objects.get() - assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": submitted_url, "depth": 0} + assert crawl.urls == submitted_url assert Snapshot.objects.count() == 0 @@ -322,10 +322,7 @@ def test_api_cli_add_concurrent_first_time_default_persona_creation(tmp_path): crawls = list(Crawl.objects.order_by("urls").values_list("urls", flat=True)) assert Snapshot.objects.count() == 0 - expected_crawl_sources = sorted( - json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in submitted_urls - ) - assert crawls == expected_crawl_sources + assert crawls == sorted(submitted_urls) @pytest.mark.timeout(360) @@ -366,15 +363,10 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls( stop_archivebox_process(api_server) api_server = None run_queued_crawls(tmp_path, env=env, timeout=240) - with use_archivebox_db(tmp_path): - root_counts = { - str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all() - } - assert root_counts and all(count == 1 for count in root_counts.values()), root_counts with use_archivebox_db(tmp_path): for crawl in Crawl.objects.all(): - root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL) - root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") + assert not crawl.snapshot_set.filter(url__startswith="archivebox://").exists() + root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8") assert root_input == crawl.urls api_server = start_api_server_without_runner(tmp_path, env, port) assert_expected_import_snapshots(tmp_path, expected_urls) diff --git a/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py b/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py index 9b078f7e..2fad621e 100644 --- a/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py +++ b/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py @@ -102,7 +102,7 @@ def test_snapshot_pause_wins_over_concurrent_runner_lease(api_admin_user): assert snapshot.retry_at == RETRY_AT_MAX -def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_finished_rows( +def test_snapshot_pause_resume_api_leaves_archiveresult_facts_unchanged( request, tmp_path, client, @@ -159,10 +159,10 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_ url=blocking_http_server.url, status=Snapshot.StatusChoices.QUEUED, retry_at=now, + config={"PLUGINS": "wget"}, ) Crawl.objects.filter(pk=snapshot.crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now) snapshot.refresh_from_db() - [started_result] = snapshot.create_pending_archiveresults(hooks=[("wget", _snapshot_hook_name("wget"))]) errors = [] def run_snapshot(): @@ -185,10 +185,13 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_ request.addfinalizer(finish_runner) blocking_http_server.request_started.wait() assert errors == [] - started_result.refresh_from_db() + started_result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget") assert started_result.status == ArchiveResult.StatusChoices.STARTED - [queued_result] = snapshot.create_pending_archiveresults( - hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")], + queued_result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="parse_txt_urls", + hook_name="on_Snapshot__71_parse_txt_urls", + status=ArchiveResult.StatusChoices.QUEUED, ) invalid_response = api_client_request( @@ -222,8 +225,8 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_ row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id]) } assert active_rows == { - "parse_txt_urls": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), - "wget": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), + "parse_txt_urls": (ArchiveResult.StatusChoices.QUEUED, None), + "wget": (ArchiveResult.StatusChoices.STARTED, None), } finished_rows = { @@ -262,12 +265,7 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_ resumed_rows = { row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id]) } - assert resumed_rows["parse_txt_urls"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["parse_txt_urls"][1] is not None - assert resumed_rows["parse_txt_urls"][1] != RETRY_AT_MAX - assert resumed_rows["wget"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["wget"][1] is not None - assert resumed_rows["wget"][1] != RETRY_AT_MAX + assert resumed_rows == active_rows assert ArchiveResult.objects.get(id=succeeded_result.id).status == ArchiveResult.StatusChoices.SUCCEEDED assert ArchiveResult.objects.get(id=failed_result.id).status == ArchiveResult.StatusChoices.FAILED @@ -317,8 +315,11 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh assert "wget failed (exit=4)" in wget_result.output_str Snapshot.objects.filter(pk=snapshot.pk).update(url=recursive_test_site["root_url"]) snapshot.refresh_from_db() - [unrelated_result] = snapshot.create_pending_archiveresults( - hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")], + unrelated_result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="parse_txt_urls", + hook_name="on_Snapshot__71_parse_txt_urls", + status=ArchiveResult.StatusChoices.QUEUED, ) snapshot.output_dir.mkdir(parents=True, exist_ok=True) (snapshot.output_dir / "source.txt").write_text("finished row must survive targeted retry", encoding="utf-8") @@ -343,7 +344,7 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh assert snapshot.status == Snapshot.StatusChoices.PAUSED assert snapshot.retry_at == RETRY_AT_MAX assert ArchiveResult.objects.get(id=wget_result.id).status == ArchiveResult.StatusChoices.FAILED - assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.PAUSED + assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.QUEUED finished_row = ArchiveResult.objects.get(id=finished_result.id) finished_output_path = Path(snapshot.output_dir) / finished_row.plugin / next(iter(finished_row.output_files)) assert finished_output_path.is_file() @@ -365,10 +366,9 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh with use_archivebox_db(tmp_path): snapshot = Snapshot.objects.get(id=snapshot_id) - assert snapshot.status == Snapshot.StatusChoices.STARTED - assert snapshot.retry_at is not None - assert snapshot.retry_at != RETRY_AT_MAX - assert snapshot.crawl.status == snapshot.crawl.StatusChoices.STARTED + assert snapshot.status == Snapshot.StatusChoices.SEALED + assert snapshot.retry_at is None + assert snapshot.crawl.status == snapshot.crawl.StatusChoices.SEALED retried_wget = ArchiveResult.objects.get(id=wget_result.id) assert retried_wget.status == ArchiveResult.StatusChoices.SUCCEEDED @@ -376,8 +376,8 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh assert retried_wget.output_files unrelated = ArchiveResult.objects.get(id=unrelated_result.id) - assert unrelated.status == ArchiveResult.StatusChoices.PAUSED - assert unrelated.retry_at == RETRY_AT_MAX + assert unrelated.status == ArchiveResult.StatusChoices.QUEUED + assert unrelated.retry_at is None finished = ArchiveResult.objects.get(id=finished_result.id) assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED diff --git a/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py index 21225797..a2b02362 100644 --- a/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py +++ b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py @@ -12,7 +12,7 @@ from django.utils import timezone from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl from archivebox.tests.test_orm_helpers import use_archivebox_db -from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook, _snapshot_hook_name +from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook from archivebox.workers.models import RETRY_AT_MAX from .conftest import ( @@ -126,7 +126,7 @@ def test_crawl_pause_wins_over_concurrent_runner_lease(api_admin_user): assert crawl.retry_at == RETRY_AT_MAX -def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone( +def test_crawl_pause_resume_api_leaves_archiveresult_facts_unchanged( request, tmp_path, client, @@ -210,9 +210,12 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap ) now = timezone.now() Crawl.objects.filter(pk=crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now) - Snapshot.objects.filter(pk=active_snapshot.pk).update(status=Snapshot.StatusChoices.QUEUED, retry_at=now) + Snapshot.objects.filter(pk=active_snapshot.pk).update( + status=Snapshot.StatusChoices.QUEUED, + retry_at=now, + config={"PLUGINS": "wget"}, + ) active_snapshot.refresh_from_db() - [active_started] = active_snapshot.create_pending_archiveresults(hooks=[("wget", _snapshot_hook_name("wget"))]) errors = [] def run_snapshot(): @@ -235,10 +238,13 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap request.addfinalizer(finish_runner) blocking_http_server.request_started.wait() assert errors == [] - active_started.refresh_from_db() + active_started = ArchiveResult.objects.get(snapshot=active_snapshot, plugin="wget") assert active_started.status == ArchiveResult.StatusChoices.STARTED - [active_queued] = active_snapshot.create_pending_archiveresults( - hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")], + active_queued = ArchiveResult.objects.create( + snapshot=active_snapshot, + plugin="parse_txt_urls", + hook_name="on_Snapshot__71_parse_txt_urls", + status=ArchiveResult.StatusChoices.QUEUED, ) pause_response = api_client_request( client, @@ -260,11 +266,11 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED assert sealed_snapshot.retry_at is None - paused_rows = { + unchanged_rows = { row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id]) } - assert paused_rows["parse_txt_urls"] == (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX) - assert paused_rows["wget"] == (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX) + assert unchanged_rows["parse_txt_urls"] == (ArchiveResult.StatusChoices.QUEUED, None) + assert unchanged_rows["wget"] == (ArchiveResult.StatusChoices.STARTED, None) active_done_row = ArchiveResult.objects.get(id=active_done.id) sealed_done_row = ArchiveResult.objects.get(id=sealed_done.id) @@ -302,12 +308,7 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap resumed_rows = { row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id]) } - assert resumed_rows["parse_txt_urls"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["parse_txt_urls"][1] is not None - assert resumed_rows["parse_txt_urls"][1] != RETRY_AT_MAX - assert resumed_rows["wget"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["wget"][1] is not None - assert resumed_rows["wget"][1] != RETRY_AT_MAX + assert resumed_rows == unchanged_rows assert ArchiveResult.objects.get(id=active_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED assert ArchiveResult.objects.get(id=sealed_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED assert active_done_path.is_file() diff --git a/archivebox/tests/test_archive_result_service.py b/archivebox/tests/test_archive_result_service.py index 71c07311..ab59eb3a 100644 --- a/archivebox/tests/test_archive_result_service.py +++ b/archivebox/tests/test_archive_result_service.py @@ -48,6 +48,7 @@ def _run_shipped_snapshot_hook( import asyncio from abx_dl.services.process_service import ProcessService as HookProcessService + from abx_dl.services.archive_result_service import ArchiveResultService as HookArchiveResultService from abx_plugins.plugins.base.utils import get_hydrated_required_binaries from archivebox.core.models import ArchiveResult from archivebox.machine.models import Process @@ -76,6 +77,7 @@ def _run_shipped_snapshot_hook( output_dir.mkdir(parents=True, exist_ok=True) bus = create_bus(name=f"test_real_{plugin}_{snapshot.id}") HookProcessService(bus, emit_jsonl=False, interactive_tty=False) + HookArchiveResultService(bus, emit_jsonl=False) PersistedProcessService(bus) ArchiveResultService(bus) @@ -291,7 +293,6 @@ def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_p assert result.status == ArchiveResult.StatusChoices.FAILED assert result.process_id == process.id assert "Chrome session" in result.output_str - assert result.output_str in result.notes _cleanup_machine_process_rows() @@ -464,15 +465,12 @@ def test_retry_failed_archiveresults_requeues_snapshot_in_queued_state(): assert reset_count == 1 assert snapshot.status == Snapshot.StatusChoices.QUEUED assert snapshot.retry_at is not None - assert snapshot.current_step == 0 - assert result.status == ArchiveResult.StatusChoices.QUEUED - assert result.output_str == "" - assert result.output_json is None - assert result.output_files == {} - assert result.output_size == 0 - assert result.output_mimetypes == "" - assert result.start_ts is None - assert result.end_ts is None + assert snapshot.config["PLUGINS"] == "chrome" + assert result.status == ArchiveResult.StatusChoices.FAILED + assert result.output_str == "timed out" + assert result.output_files == {"stderr.log": {}} + assert result.output_size == 123 + assert result.output_mimetypes == "text/plain" assert ArchiveResult.objects.get(snapshot=snapshot, plugin="ublock").status == ArchiveResult.StatusChoices.SKIPPED assert ArchiveResult.objects.get(snapshot=snapshot, plugin="forumdl").status == ArchiveResult.StatusChoices.NORESULTS snapshot.refresh_from_db() diff --git a/archivebox/tests/test_cli_add.py b/archivebox/tests/test_cli_add.py index d0cc8c7b..08da08ff 100644 --- a/archivebox/tests/test_cli_add.py +++ b/archivebox/tests/test_cli_add.py @@ -225,7 +225,7 @@ def test_add_single_url_records_url_in_crawl(initialized_archive): crawl = Crawl.objects.get() snapshots = list(Snapshot.objects.all()) - assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0} + assert crawl.urls == "https://example.com" assert crawl.get_urls_list() == ["https://example.com"] assert snapshots == [] @@ -295,13 +295,10 @@ def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initial assert crawl.urls == source_text run_queued_crawls(initialized_archive, env=env, timeout=240) - with use_archivebox_db(initialized_archive): - root_counts = {str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()} - assert root_counts and all(count == 1 for count in root_counts.values()), root_counts with use_archivebox_db(initialized_archive): for crawl in Crawl.objects.all(): - root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL) - root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") + assert not crawl.snapshot_set.filter(url__startswith="archivebox://").exists() + root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8") assert root_input == crawl.urls assert_expected_import_snapshots(initialized_archive, expected_urls) @@ -387,7 +384,7 @@ def test_run_rejects_file_url_injected_directly_into_crawl_urls_with_db_update(i status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now(), ) - bad_jsonl = json.dumps({"type": "CrawlSeed", "url": file_url, "depth": 0, "tags": "sql-file-url"}) + bad_jsonl = json.dumps({"type": "Snapshot", "url": file_url, "depth": 0, "tags": "sql-file-url"}) Crawl.objects.filter(pk=crawl.pk).update(urls=bad_jsonl) result = run_archivebox_cmd( @@ -480,7 +477,7 @@ def test_add_bg_queues_direct_url_snapshot(initialized_archive): assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is not None - assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0} + assert crawl.urls == "https://example.com" assert snapshots == [] @@ -622,7 +619,7 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is None - assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0} + assert crawl.urls == "https://example.com" assert snapshot_urls == set() run_queued_crawls(initialized_archive, env) @@ -633,7 +630,7 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi assert crawl.status == Crawl.StatusChoices.SEALED assert crawl.retry_at is None - assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0} + assert crawl.urls == "https://example.com" assert snapshot_urls == set() @@ -658,9 +655,7 @@ def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive): crawl = Crawl.objects.get() snapshot_urls = set(Snapshot.objects.values_list("url", flat=True)) - assert [json.loads(line) for line in crawl.urls.splitlines()] == [ - {"type": "CrawlSeed", "url": url, "depth": 0} for url in internal_urls - ] + assert crawl.urls.splitlines() == internal_urls assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is None assert snapshot_urls == set() @@ -713,10 +708,7 @@ def test_add_multiple_urls_single_command(initialized_archive): crawl = Crawl.objects.get() snapshots = list(Snapshot.objects.order_by("url").values_list("url", "depth")) - assert [json.loads(line) for line in crawl.urls.splitlines()] == [ - {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}, - {"type": "CrawlSeed", "url": "https://example.org", "depth": 0}, - ] + assert crawl.urls.splitlines() == ["https://example.com", "https://example.org"] assert snapshots == [] @@ -994,7 +986,7 @@ def test_add_index_only_queues_crawl_without_starting_runner(initialized_archive assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is None - assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0} + assert crawl.urls == "https://example.com" assert snapshots == [] @@ -1012,7 +1004,7 @@ def test_add_index_only_creates_direct_url_snapshot(initialized_archive): crawl = Crawl.objects.get() snapshot = Snapshot.objects.get() - assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0} + assert crawl.urls == "https://example.com" assert snapshot.url == "https://example.com" assert snapshot.depth == 0 diff --git a/archivebox/tests/test_cli_extract_input.py b/archivebox/tests/test_cli_extract_input.py index 3f2655b7..eaf7312a 100644 --- a/archivebox/tests/test_cli_extract_input.py +++ b/archivebox/tests/test_cli_extract_input.py @@ -22,7 +22,7 @@ def create_extract_snapshot(initialized_archive, env, url="https://example.com") ) -def test_extract_archiveresult_record_queues_only_exact_hook(initialized_archive): +def test_extract_archiveresult_record_queues_parent_snapshot_plugin(initialized_archive): env = cli_env(PLUGINS="archivewebpage") create_extract_snapshot(initialized_archive, env) @@ -49,10 +49,11 @@ def test_extract_archiveresult_record_queues_only_exact_hook(initialized_archive assert result.returncode == 0, result.stderr or result.stdout with use_archivebox_db(initialized_archive): - rows = list( - ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin="archivewebpage").values_list("hook_name", "status"), - ) - assert rows == [(hook_name, ArchiveResult.StatusChoices.QUEUED)] + snapshot = Snapshot.objects.get(id=snapshot_id) + rows = list(ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin="archivewebpage")) + assert rows == [] + assert snapshot.status == Snapshot.StatusChoices.QUEUED + assert snapshot.config["PLUGINS"] == "archivewebpage" def test_extract_runs_on_snapshot_id(initialized_archive): diff --git a/archivebox/tests/test_cli_list.py b/archivebox/tests/test_cli_list.py index 8af08bf0..bb1a3338 100644 --- a/archivebox/tests/test_cli_list.py +++ b/archivebox/tests/test_cli_list.py @@ -232,7 +232,6 @@ def test_list_limit_zero_streams_one_million_snapshots_without_materializing(ini fs_version, crawl_id, config, - current_step, depth, notes, num_uses_failed, @@ -256,7 +255,6 @@ def test_list_limit_zero_streams_one_million_snapshots_without_materializing(ini %s, '{}', 0, - 0, '', 0, 0, diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index 0ba02e86..1c5df3ab 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -240,7 +240,7 @@ class TestRunWithArchiveResult: """Tests for `archivebox run` with ArchiveResult input.""" @pytest.mark.django_db(transaction=True) - def test_run_creates_and_runs_exact_no_id_archiveresult_request(self, initialized_archive): + def test_run_treats_no_id_archiveresult_as_parent_snapshot_plugin_request(self, initialized_archive): import json from archivebox.core.models import ArchiveResult @@ -284,16 +284,13 @@ class TestRunWithArchiveResult: "output_str", ), ) - assert rows == [ - ( - missing_hook, - ArchiveResult.StatusChoices.FAILED, - "Queued hook is no longer available in the installed plugin", - ), - ] + assert len(rows) == 1 + assert rows[0][0] != missing_hook + assert rows[0][0].startswith("on_Snapshot__") + assert rows[0][1] in ArchiveResult.FINAL_STATES def test_run_requeues_failed_archiveresult(self, initialized_archive): - """Run re-queues a failed ArchiveResult.""" + """Run uses a failed ArchiveResult as a parent Snapshot/plugin reference.""" url = create_test_url() # Create snapshot and archive result @@ -905,43 +902,6 @@ class TestRecoverOrchestratorState: assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED assert sealed_snapshot.retry_at is not None - def test_recover_orchestrator_state_requeues_backoff_archiveresults(self): - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.core.recovery_util import recover_orchestrator_state - from archivebox.crawls.models import Crawl - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by_id=get_or_create_system_user_pk(), - status=Crawl.StatusChoices.SEALED, - retry_at=None, - ) - snapshot = Snapshot.objects.create( - url="https://example.com", - crawl=crawl, - status=Snapshot.StatusChoices.SEALED, - retry_at=None, - ) - result = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="search_backend_sqlite", - hook_name="on_Snapshot__90_index_sqlite", - status=ArchiveResult.StatusChoices.BACKOFF, - ) - - recovered = recover_orchestrator_state() - - result.refresh_from_db() - snapshot.refresh_from_db() - crawl.refresh_from_db() - - assert recovered["archiveresults_backoff"] == 1 - assert result.status == ArchiveResult.StatusChoices.QUEUED - assert snapshot.status == Snapshot.StatusChoices.SEALED - assert snapshot.retry_at is not None - assert crawl.status == Crawl.StatusChoices.SEALED - def test_recover_orchestrator_state_leaves_due_queued_snapshot_for_runner_even_with_final_results(self): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.core.models import ArchiveResult, Snapshot @@ -980,6 +940,7 @@ class TestRecoverOrchestratorState: assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is not None + @pytest.mark.django_db(transaction=True) def test_recover_orchestrator_state_leaves_stale_queued_final_rows_for_runner(self): from datetime import timedelta @@ -989,12 +950,13 @@ class TestRecoverOrchestratorState: from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.recovery_util import recover_orchestrator_state from archivebox.crawls.models import Crawl - from archivebox.services.runner import run_due_crawl, run_due_snapshot + from archivebox.services.runner import run_due_snapshot old = timezone.now() - timedelta(hours=13) crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), + config={"PLUGINS": "__archivebox_test_no_plugins__"}, status=Crawl.StatusChoices.QUEUED, retry_at=old, ) @@ -1033,16 +995,11 @@ class TestRecoverOrchestratorState: assert snapshot.status == Snapshot.StatusChoices.SEALED assert snapshot.retry_at is None - assert crawl.status == Crawl.StatusChoices.QUEUED - assert crawl.retry_at == old - - assert run_due_crawl(crawl, lock_seconds=60) is True - crawl.refresh_from_db() - assert crawl.status == Crawl.StatusChoices.SEALED assert crawl.retry_at is None - def test_run_due_snapshot_seals_queued_snapshot_with_final_results(self): + @pytest.mark.django_db(transaction=True) + def test_run_due_snapshot_runs_snapshot_without_consulting_final_result_rows(self): from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk @@ -1053,6 +1010,7 @@ class TestRecoverOrchestratorState: crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), + config={"PLUGINS": "__archivebox_test_no_plugins__"}, status=Crawl.StatusChoices.STARTED, retry_at=timezone.now(), ) @@ -1076,68 +1034,6 @@ class TestRecoverOrchestratorState: assert snapshot.retry_at is None assert snapshot.downloaded_at is not None - def test_create_pending_archiveresults_uses_canonical_hook_names(self): - from django.utils import timezone - - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.crawls.models import Crawl - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by_id=get_or_create_system_user_pk(), - status=Crawl.StatusChoices.STARTED, - retry_at=timezone.now(), - ) - snapshot = Snapshot.objects.create( - url="https://example.com", - crawl=crawl, - status=Snapshot.StatusChoices.QUEUED, - retry_at=timezone.now(), - ) - - snapshot.create_pending_archiveresults() - - hook_names = list(ArchiveResult.objects.filter(snapshot=snapshot).values_list("hook_name", flat=True)) - assert hook_names - assert all(not hook_name.endswith((".py", ".js", ".sh")) for hook_name in hook_names) - - def test_snapshot_hooks_for_pending_archiveresults_respects_disabled_plugins_when_plugins_empty(self): - from django.utils import timezone - - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.core.models import Snapshot - from archivebox.crawls.models import Crawl - from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by_id=get_or_create_system_user_pk(), - config={ - "CLAUDECHROME_ENABLED": False, - "CLAUDECODEEXTRACT_ENABLED": False, - "CLAUDECODECLEANUP_ENABLED": False, - "SEARCH_BACKEND_SQLITE_ENABLED": False, - }, - status=Crawl.StatusChoices.STARTED, - retry_at=timezone.now(), - ) - snapshot = Snapshot.objects.create( - url="https://example.com", - crawl=crawl, - status=Snapshot.StatusChoices.QUEUED, - retry_at=timezone.now(), - ) - - hooks = snapshot_hooks_for_pending_archiveresults(snapshot) - queued_plugins = {plugin for plugin, _hook_name in hooks} - - assert "title" in queued_plugins - assert "claudechrome" not in queued_plugins - assert "claudecodeextract" not in queued_plugins - assert "claudecodecleanup" not in queued_plugins - assert "search_backend_sqlite" not in queued_plugins - def test_run_due_snapshot_pauses_child_when_parent_is_paused(self): from django.utils import timezone @@ -1172,7 +1068,7 @@ class TestRecoverOrchestratorState: result.refresh_from_db() assert snapshot.status == Snapshot.StatusChoices.PAUSED assert snapshot.retry_at == RETRY_AT_MAX - assert result.status == ArchiveResult.StatusChoices.PAUSED + assert result.status == ArchiveResult.StatusChoices.QUEUED assert snapshot.archiveresult_set.count() == 1 def test_parent_status_transitions_schedule_children_to_follow_parent_status(self): @@ -1230,7 +1126,7 @@ class TestRecoverOrchestratorState: sealed_started_child.refresh_from_db() assert paused_child.status == Snapshot.StatusChoices.PAUSED assert paused_child.retry_at == RETRY_AT_MAX - assert paused_result.status == ArchiveResult.StatusChoices.PAUSED + assert paused_result.status == ArchiveResult.StatusChoices.QUEUED assert sealed_child.status == Snapshot.StatusChoices.PAUSED assert sealed_child.retry_at is not None assert sealed_child.retry_at <= timezone.now() @@ -1671,151 +1567,6 @@ class TestRecoverOrchestratorState: assert snapshot.output_dir.joinpath("index.html").read_text(encoding="utf-8") == "legacy archive" assert not legacy_dir.exists() - @pytest.mark.django_db(transaction=True) - def test_run_due_snapshot_runs_queued_plugin_after_fs_migration(self): - from django.utils import timezone - - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.crawls.models import Crawl - from archivebox.services.runner import run_due_snapshot - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by_id=get_or_create_system_user_pk(), - status=Crawl.StatusChoices.SEALED, - retry_at=None, - ) - snapshot = Snapshot.objects.create( - url="https://example.com", - crawl=crawl, - status=Snapshot.StatusChoices.SEALED, - retry_at=timezone.now(), - ) - Snapshot.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0") - snapshot.refresh_from_db() - snapshot.output_dir.mkdir(parents=True, exist_ok=True) - (snapshot.output_dir / "source.txt").write_text("real targeted maintenance input\n", encoding="utf-8") - result = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="hashes", - hook_name="on_Snapshot__93_hashes.py", - status=ArchiveResult.StatusChoices.QUEUED, - ) - - assert run_due_snapshot(snapshot, lock_seconds=60) is True - - snapshot.refresh_from_db() - result.refresh_from_db() - assert snapshot.fs_version == Snapshot._fs_current_version() - assert result.status in ArchiveResult.FINAL_STATES - assert result.status != ArchiveResult.StatusChoices.QUEUED - assert result.start_ts is not None - assert result.end_ts is not None - - @pytest.mark.django_db(transaction=True) - def test_run_due_snapshot_fails_obsolete_queued_hook_name(self): - from django.utils import timezone - - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.crawls.models import Crawl - from archivebox.services.runner import run_due_snapshot - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by_id=get_or_create_system_user_pk(), - status=Crawl.StatusChoices.SEALED, - retry_at=None, - ) - snapshot = Snapshot.objects.create( - url="https://example.com", - crawl=crawl, - status=Snapshot.StatusChoices.SEALED, - retry_at=timezone.now(), - ) - result = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="singlefile", - hook_name="on_Snapshot__50_singlefile.py", - status=ArchiveResult.StatusChoices.QUEUED, - ) - - assert run_due_snapshot(snapshot, lock_seconds=60) is True - - result.refresh_from_db() - snapshot.refresh_from_db() - assert result.status == ArchiveResult.StatusChoices.FAILED - assert result.output_str == "Queued hook is no longer available in the installed plugin" - assert snapshot.retry_at is None - - @pytest.mark.django_db(transaction=True) - def test_run_due_snapshot_skips_disabled_queued_plugins_and_seals_started_snapshot(self): - from django.utils import timezone - - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.crawls.models import Crawl - from archivebox.services.runner import run_due_snapshot - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by_id=get_or_create_system_user_pk(), - config={ - "CLAUDECHROME_ENABLED": False, - "CLAUDECODEEXTRACT_ENABLED": False, - "CLAUDECODECLEANUP_ENABLED": False, - "SEARCH_BACKEND_SQLITE_ENABLED": False, - }, - status=Crawl.StatusChoices.STARTED, - retry_at=timezone.now(), - ) - snapshot = Snapshot.objects.create( - url="https://example.com", - crawl=crawl, - status=Snapshot.StatusChoices.STARTED, - retry_at=timezone.now(), - downloaded_at=timezone.now(), - ) - ArchiveResult.objects.create( - snapshot=snapshot, - plugin="title", - hook_name="on_Snapshot__01_title", - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_str="Example Domain", - ) - stale_results = [ - ArchiveResult.objects.create( - snapshot=snapshot, - plugin=plugin, - hook_name=hook_name, - status=ArchiveResult.StatusChoices.QUEUED, - ) - for plugin, hook_name in ( - ("claudechrome", "on_Snapshot__47_claudechrome"), - ("claudecodeextract", "on_Snapshot__58_claudecodeextract"), - ("claudecodecleanup", "on_Snapshot__92_claudecodecleanup"), - ("search_backend_sqlite", "on_Snapshot__90_index_sqlite"), - ) - ] - stale_result_ids = [result.id for result in stale_results] - stale_plugins = [result.plugin for result in stale_results] - - assert run_due_snapshot(snapshot, lock_seconds=60) is True - - snapshot.refresh_from_db() - assert snapshot.status == Snapshot.StatusChoices.SEALED - assert snapshot.retry_at is None - assert not snapshot.archiveresult_set.filter( - plugin__in=stale_plugins, - status=ArchiveResult.StatusChoices.QUEUED, - ).exists() - for result in ArchiveResult.objects.filter(id__in=stale_result_ids): - assert result.status == ArchiveResult.StatusChoices.SKIPPED - assert result.start_ts is not None - assert result.end_ts is not None - assert "disabled by this Snapshot/Crawl config" in result.output_str - @pytest.mark.django_db(transaction=True) @pytest.mark.timeout(300) @pytest.mark.parametrize("chrome_isolation", ["crawl", "snapshot"]) @@ -1949,6 +1700,7 @@ class TestRecoverOrchestratorState: assert crawl.status == Crawl.StatusChoices.SEALED assert crawl.retry_at is None + @pytest.mark.django_db(transaction=True) def test_recover_orchestrator_state_unlocks_started_snapshot_with_final_results_for_runner(self): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.core.models import ArchiveResult, Snapshot @@ -1959,6 +1711,7 @@ class TestRecoverOrchestratorState: crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), + config={"PLUGINS": "__archivebox_test_no_plugins__"}, status=Crawl.StatusChoices.STARTED, retry_at=None, ) @@ -2049,7 +1802,7 @@ class TestRunDueCrawlState: assert crawl.retry_at == now assert crawl.snapshot_set.count() == 0 - def test_maintenance_only_runner_ignores_disabled_queued_results_on_sealed_snapshots(self): + def test_maintenance_only_runner_clears_snapshot_tick_without_scheduling_archive_results(self): from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk @@ -2083,7 +1836,7 @@ class TestRunDueCrawlState: result.refresh_from_db() assert snapshot.status == Snapshot.StatusChoices.SEALED assert snapshot.fs_version == Snapshot._fs_current_version() - assert snapshot.retry_at is not None + assert snapshot.retry_at is None assert result.status == ArchiveResult.StatusChoices.QUEUED def test_snapshot_start_writes_short_future_lease(self): @@ -2113,132 +1866,6 @@ class TestRunDueCrawlState: assert snapshot.retry_at is not None assert snapshot.retry_at > timezone.now() - def test_abandoned_started_snapshot_results_are_reset_locally_for_resume(self): - from django.utils import timezone - - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.crawls.models import Crawl - - crawl = Crawl.objects.create( - urls="https://example.com", - created_by_id=get_or_create_system_user_pk(), - status=Crawl.StatusChoices.STARTED, - retry_at=timezone.now(), - ) - snapshot = Snapshot.objects.create( - url="https://example.com", - crawl=crawl, - status=Snapshot.StatusChoices.STARTED, - retry_at=timezone.now(), - ) - abandoned = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="title", - hook_name="on_Snapshot__01_title", - status=ArchiveResult.StatusChoices.STARTED, - output_str="partial output should be cleared", - output_files={"partial.txt": {"size": 12}}, - output_size=12, - start_ts=timezone.now(), - ) - queued = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="wget", - hook_name="on_Snapshot__40_wget", - status=ArchiveResult.StatusChoices.QUEUED, - ) - finished = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="favicon", - hook_name="on_Snapshot__01_favicon", - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_str="keep me", - output_files={"favicon.ico": {"size": 1}}, - output_size=1, - ) - - snapshot.reset_abandoned_results() - - abandoned.refresh_from_db() - queued.refresh_from_db() - finished.refresh_from_db() - - assert abandoned.status == ArchiveResult.StatusChoices.QUEUED - assert abandoned.output_str == "" - assert abandoned.output_files == {} - assert abandoned.output_size == 0 - assert queued.status == ArchiveResult.StatusChoices.QUEUED - assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED - assert finished.output_str == "keep me" - assert finished.output_files == {"favicon.ico": {"size": 1}} - - @pytest.mark.django_db(transaction=True) - def test_finished_parser_result_projects_children_before_resume_seals_snapshot(self): - from importlib.resources import files - from pathlib import Path - - from django.utils import timezone - - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.crawls.models import Crawl - from archivebox.tests.conftest import run_test_hook - from archivebox.services.runner import run_due_snapshot - - crawl = Crawl.objects.create( - urls="Plain text import containing https://example.org/\n", - max_depth=1, - created_by_id=get_or_create_system_user_pk(), - status=Crawl.StatusChoices.STARTED, - retry_at=timezone.now(), - ) - root = Snapshot.objects.create( - url=Snapshot.INTERNAL_INPUT_URL, - crawl=crawl, - depth=0, - status=Snapshot.StatusChoices.STARTED, - retry_at=timezone.now(), - ) - staticfile_dir = root.output_dir / "staticfile" - parser_dir = root.output_dir / "parse_txt_urls" - staticfile_dir.mkdir(parents=True, exist_ok=True) - parser_dir.mkdir(parents=True, exist_ok=True) - (staticfile_dir / "input.txt").write_text( - "Plain text import containing https://example.org/\n", - encoding="utf-8", - ) - hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py"))) - process = run_test_hook( - hook_path, - parser_dir, - config={"ABXPKG_LIB_DIR": str(root.output_dir.parent.parent / "lib"), "SNAP_DIR": str(root.output_dir)}, - timeout=30, - url=root.url, - depth=root.depth, - snapshot_id=str(root.id), - ) - process.refresh_from_db() - assert process.exit_code == 0, process.stderr - result_record = next(record for record in process.get_records() if record.get("type") == "ArchiveResult") - ArchiveResult.objects.create( - snapshot=root, - process=process, - plugin="parse_txt_urls", - hook_name=hook_path.name, - status=result_record["status"], - output_str=result_record.get("output_str", ""), - output_files={"urls.jsonl": {"size": (parser_dir / "urls.jsonl").stat().st_size}}, - ) - - assert run_due_snapshot(root, lock_seconds=60) - - root.refresh_from_db() - child = Snapshot.objects.get(crawl=crawl, url="https://example.org/") - assert root.status == Snapshot.StatusChoices.SEALED - assert child.parent_snapshot_id == root.id - assert child.status == Snapshot.StatusChoices.QUEUED - def test_due_started_snapshot_with_live_child_extends_lease_without_reset(self): import os from datetime import datetime @@ -2577,7 +2204,7 @@ class TestRecoverOrchestratorStateRedFailureModes: assert crawl.status == Crawl.StatusChoices.STARTED assert crawl.retry_at == future - def test_recovery_requeues_started_archiveresult_without_process(self): + def test_recovery_closes_interrupted_result_and_requeues_parent_snapshot(self): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.recovery_util import recover_orchestrator_state @@ -2605,9 +2232,12 @@ class TestRecoverOrchestratorStateRedFailureModes: recover_orchestrator_state() result.refresh_from_db() - assert result.status == ArchiveResult.StatusChoices.QUEUED + snapshot.refresh_from_db() + assert result.status == ArchiveResult.StatusChoices.FAILED + assert snapshot.status == Snapshot.StatusChoices.STARTED + assert snapshot.retry_at is not None - def test_recovery_requeues_started_archiveresult_with_exited_process(self): + def test_recovery_closes_started_archiveresult_with_exited_process(self): from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk @@ -2646,9 +2276,11 @@ class TestRecoverOrchestratorStateRedFailureModes: recover_orchestrator_state() result.refresh_from_db() - assert result.status == ArchiveResult.StatusChoices.QUEUED + snapshot.refresh_from_db() + assert result.status == ArchiveResult.StatusChoices.FAILED + assert snapshot.retry_at is not None - def test_recovery_requeues_sealed_snapshot_started_result_with_exited_process_result_too(self): + def test_recovery_does_not_reopen_sealed_snapshot_for_interrupted_result_projection(self): from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk @@ -2694,10 +2326,10 @@ class TestRecoverOrchestratorStateRedFailureModes: snapshot.refresh_from_db() result.refresh_from_db() assert snapshot.status == Snapshot.StatusChoices.SEALED - assert snapshot.retry_at is not None - assert result.status == ArchiveResult.StatusChoices.QUEUED + assert snapshot.retry_at is None + assert result.status == ArchiveResult.StatusChoices.FAILED - def test_recovery_requeues_started_snapshot_result_before_unlocking_snapshot(self): + def test_recovery_closes_result_projection_before_unlocking_snapshot(self): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.recovery_util import recover_orchestrator_state @@ -2726,7 +2358,7 @@ class TestRecoverOrchestratorStateRedFailureModes: snapshot.refresh_from_db() result.refresh_from_db() - assert result.status == ArchiveResult.StatusChoices.QUEUED + assert result.status == ArchiveResult.StatusChoices.FAILED assert snapshot.retry_at is not None def test_crawl_runner_load_run_state_does_not_return_future_retry_snapshots(self): diff --git a/archivebox/tests/test_cli_schedule.py b/archivebox/tests/test_cli_schedule.py index ff970f35..4290d73f 100644 --- a/archivebox/tests/test_cli_schedule.py +++ b/archivebox/tests/test_cli_schedule.py @@ -56,9 +56,11 @@ def test_schedule_without_import_path_creates_maintenance_schedule(initialized_a assert "Created scheduled maintenance update" in result.stdout with use_archivebox_db(initialized_archive): - row = Crawl.objects.order_by("-created_at").values_list("urls", "status").first() + schedule_row = CrawlSchedule.objects.select_related("template").get() - assert row == ("archivebox://update", "sealed") + assert schedule_row.kind == "update" + assert schedule_row.template.urls == "" + assert schedule_row.template.status == "sealed" def test_schedule_creates_enabled_db_schedule(initialized_archive): diff --git a/archivebox/tests/test_cli_update_reindex_snapshots.py b/archivebox/tests/test_cli_update_reindex_snapshots.py index b45db278..3823ee87 100644 --- a/archivebox/tests/test_cli_update_reindex_snapshots.py +++ b/archivebox/tests/test_cli_update_reindex_snapshots.py @@ -15,84 +15,6 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) -def test_targeted_plugin_retries_preserve_sealed_snapshot_lifecycle(): - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.cli.archivebox_extract import run_plugins - from archivebox.core.models import ArchiveResult - from archivebox.crawls.models import Crawl - - search_crawl = Crawl.objects.create( - urls="https://example.com/search", - created_by_id=get_or_create_system_user_pk(), - status=Crawl.StatusChoices.SEALED, - ) - search_snapshot = Snapshot.objects.create( - url="https://example.com/search", - crawl=search_crawl, - status=Snapshot.StatusChoices.SEALED, - ) - extract_crawl = Crawl.objects.create( - urls="https://example.com/extract", - created_by_id=get_or_create_system_user_pk(), - status=Crawl.StatusChoices.SEALED, - ) - extract_snapshot = Snapshot.objects.create( - url="https://example.com/extract", - crawl=extract_crawl, - status=Snapshot.StatusChoices.SEALED, - ) - - assert ( - run_plugins( - args=(), - records=[ - { - "type": "ArchiveResult", - "snapshot_id": str(search_snapshot.id), - "plugin": "search_backend_sqlite", - }, - ], - wait=False, - emit_results=False, - show_progress=False, - ) - == 0 - ) - search_crawl.refresh_from_db() - search_snapshot.refresh_from_db() - assert search_crawl.status == Crawl.StatusChoices.SEALED - assert search_snapshot.status == Snapshot.StatusChoices.SEALED - assert search_snapshot.archiveresult_set.filter( - plugin="search_backend_sqlite", - status=ArchiveResult.StatusChoices.QUEUED, - ).exists() - - assert ( - run_plugins( - args=(), - records=[ - { - "type": "ArchiveResult", - "snapshot_id": str(extract_snapshot.id), - "plugin": "wget", - }, - ], - wait=False, - emit_results=False, - show_progress=False, - ) - == 0 - ) - extract_crawl.refresh_from_db() - extract_snapshot.refresh_from_db() - assert extract_crawl.status == Crawl.StatusChoices.SEALED - assert extract_snapshot.status == Snapshot.StatusChoices.SEALED - assert extract_snapshot.archiveresult_set.filter( - plugin="wget", - status=ArchiveResult.StatusChoices.QUEUED, - ).exists() - - def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive): """Test that archivebox update imports real legacy archive directories.""" env = cli_env(disable_extractors=True) @@ -196,8 +118,9 @@ def test_update_migrates_every_declared_filesystem_version(tmp_path, initialized assert snapshot.status == Snapshot.StatusChoices.QUEUED assert snapshot.retry_at is not None result.refresh_from_db() - assert result.output_files - assert result.output_size > 0 + # Filesystem layout migration does not infer or mutate ArchiveResult facts. + assert result.output_files == {} + assert result.output_size == 0 if legacy_layout: assert (migrated_dir / "existing-user-output.bin").read_bytes() == b"preserve interrupted migration output" @@ -215,15 +138,16 @@ def test_update_migrates_every_declared_filesystem_version(tmp_path, initialized @pytest.mark.django_db(transaction=True) -def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_plugins(): +def test_reindex_snapshots_runs_only_missing_sealed_search_indexes(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.cli.archivebox_update import reindex_snapshots from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl crawl = Crawl.objects.create( - urls="https://example.com", + urls="https://example.com\nhttps://example.org", created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.SEALED, ) snapshot = Snapshot.objects.create( url="https://example.com", @@ -250,12 +174,20 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p (output_dir / "title" / "title.txt").write_text("Example Domain") (output_dir / "dom").mkdir(parents=True, exist_ok=True) (output_dir / "dom" / "output.html").write_text("Example searchable text") + missing_snapshot = Snapshot.objects.create( + url="https://example.org", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + ) + missing_output_dir = missing_snapshot.output_dir + (missing_output_dir / "title").mkdir(parents=True, exist_ok=True) + (missing_output_dir / "title" / "title.txt").write_text("Missing Search Index") original_engine = os.environ.get("SEARCH_BACKEND_ENGINE") os.environ["SEARCH_BACKEND_ENGINE"] = "sqlite" try: stats = reindex_snapshots( - Snapshot.objects.filter(id__in=(snapshot.id, paused_snapshot.id)), + Snapshot.objects.filter(id__in=(snapshot.id, missing_snapshot.id, paused_snapshot.id)), search_plugins=["search_backend_sqlite"], batch_size=10, ) @@ -267,12 +199,21 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p result.refresh_from_db() + snapshot.refresh_from_db() + missing_snapshot.refresh_from_db() assert stats["processed"] == 1 - assert stats["queued"] == 1 - assert stats["reindexed"] == 0 - assert result.status == ArchiveResult.StatusChoices.QUEUED - assert result.output_str == "" - assert result.output_json is None + assert stats["requested"] == 1 + assert stats["queued"] == 0 + assert stats["reindexed"] == 1 + assert result.status == ArchiveResult.StatusChoices.SUCCEEDED + assert result.output_str == "old index hit" + assert result.output_json == {"indexed": True} + assert snapshot.status == Snapshot.StatusChoices.SEALED + assert missing_snapshot.status == Snapshot.StatusChoices.SEALED + assert missing_snapshot.archiveresult_set.filter( + plugin="search_backend_sqlite", + status__in=[ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS], + ).exists() assert not paused_snapshot.archiveresult_set.exists() diff --git a/archivebox/tests/test_crawl_runner.py b/archivebox/tests/test_crawl_runner.py index fa2a5ee8..8d541c3c 100644 --- a/archivebox/tests/test_crawl_runner.py +++ b/archivebox/tests/test_crawl_runner.py @@ -271,45 +271,6 @@ def test_snapshot_started_state_keeps_retry_at_lease(): assert snapshot.retry_at > before -@pytest.mark.django_db(transaction=True) -def test_system_update_crawl_runs_database_maintenance_without_snapshot_work(): - from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl - from archivebox.core.models import Snapshot - from archivebox.services.runner import CrawlRunner - from django.utils import timezone - - owner_id = get_or_create_system_user_pk() - archived_crawl = Crawl.objects.create( - urls="https://example.com", - created_by_id=owner_id, - status=Crawl.StatusChoices.SEALED, - retry_at=None, - ) - archived_snapshot = Snapshot.objects.create( - url="https://example.com", - crawl=archived_crawl, - status=Snapshot.StatusChoices.SEALED, - retry_at=None, - ) - Snapshot.objects.filter(pk=archived_snapshot.pk).update(fs_version=0, retry_at=None) - maintenance_crawl = Crawl.objects.create( - urls="archivebox://update", - created_by_id=owner_id, - status=Crawl.StatusChoices.QUEUED, - retry_at=timezone.now(), - ) - - snapshot_ids = CrawlRunner(maintenance_crawl, show_progress=False).load_run_state() - - maintenance_crawl.refresh_from_db() - archived_snapshot.refresh_from_db() - assert snapshot_ids == [] - assert maintenance_crawl.snapshot_set.count() == 0 - assert archived_snapshot.status == Snapshot.StatusChoices.SEALED - assert archived_snapshot.retry_at is not None - - @pytest.mark.django_db(transaction=True) def test_crawl_start_event_keeps_retry_at_lease(): from abx_dl.events import CrawlStartEvent diff --git a/archivebox/tests/test_crawl_service.py b/archivebox/tests/test_crawl_service.py index fd2ccf76..bd1ef151 100644 --- a/archivebox/tests/test_crawl_service.py +++ b/archivebox/tests/test_crawl_service.py @@ -80,7 +80,7 @@ def test_crawl_service_run_processes_queued_crawl_and_applies_crawl_config(tmp_p assert queued_state["retry_at"] is not None assert queued_state["config"]["PLUGINS"] == "wget,parse_html_urls" assert queued_state["config"]["URL_DENYLIST"] == "/contact$" - # add --bg seeds Crawl.urls as CrawlSeed JSONL and returns; the runner + # add --bg stores the submitted URL list in Crawl.urls and returns; the runner # materializes Snapshot rows + applies URL_DENYLIST when it claims the # crawl, not at add time. The post-run assertions below verify those. assert queued_state["snapshots"] == [] diff --git a/archivebox/tests/test_migrations_08_to_09.py b/archivebox/tests/test_migrations_08_to_09.py index f9265962..ea2111a2 100644 --- a/archivebox/tests/test_migrations_08_to_09.py +++ b/archivebox/tests/test_migrations_08_to_09.py @@ -92,7 +92,7 @@ def convert_legacy_tags_to_uuid(conn: sqlite3.Connection) -> None: @pytest.mark.parametrize("uuid_tags", (False, True), ids=("integer-tags", "uuid-tags")) -def test_migration_preserves_extended_08_metadata(migration_08_data, uuid_tags): +def test_migration_preserves_supported_extended_08_metadata(migration_08_data, uuid_tags): work_dir, db_path, original_data = migration_08_data snapshot = original_data["snapshots"][1] parent = original_data["snapshots"][0] @@ -152,11 +152,12 @@ def test_migration_preserves_extended_08_metadata(migration_08_data, uuid_tags): snapshot_row = conn.execute( """ SELECT depth, config, notes, num_uses_failed, num_uses_succeeded, - parent_snapshot_id, current_step, fs_version + parent_snapshot_id, fs_version FROM core_snapshot WHERE id = ? """, (snapshot["id"],), ).fetchone() + snapshot_columns = {row[1] for row in conn.execute("PRAGMA table_info(core_snapshot)")} result_notes = conn.execute( "SELECT notes FROM core_archiveresult WHERE id = REPLACE(?, '-', '')", (archiveresult["uuid"],), @@ -174,9 +175,9 @@ def test_migration_preserves_extended_08_metadata(migration_08_data, uuid_tags): metadata["num_uses_failed"], metadata["num_uses_succeeded"], parent["id"], - metadata["current_step"], metadata["fs_version"], ) + assert "current_step" not in snapshot_columns assert result_notes == ("legacy archive result notes",) assert migrated_tag == tag_metadata assert migrated_counts == expected_counts diff --git a/archivebox/tests/test_snapshot_filesystem_migration.py b/archivebox/tests/test_snapshot_filesystem_migration.py new file mode 100644 index 00000000..00a32796 --- /dev/null +++ b/archivebox/tests/test_snapshot_filesystem_migration.py @@ -0,0 +1,109 @@ +from pathlib import Path + +import pytest +from django.db import connection + +from archivebox.config import CONSTANTS +from archivebox.core.models import Snapshot, SnapshotMigrationError + + +pytestmark = pytest.mark.django_db(transaction=True) + + +def _make_legacy_snapshot(snapshot: Snapshot) -> tuple[Path, Path]: + current_dir = snapshot.get_storage_path_for_version(snapshot._fs_current_version()) + legacy_dir = CONSTANTS.ARCHIVE_DIR / snapshot.timestamp + Snapshot.objects.filter(pk=snapshot.pk).update(fs_version="0.8.0") + snapshot.refresh_from_db() + legacy_dir.mkdir(parents=True, exist_ok=True) + (legacy_dir / "unknown" / "nested").mkdir(parents=True) + (legacy_dir / "unknown" / "payload.bin").write_bytes(b"filesystem migration payload\x00\xff") + return legacy_dir, current_dir + + +def test_ordinary_snapshot_save_does_not_migrate_directories(snapshot): + legacy_dir, current_dir = _make_legacy_snapshot(snapshot) + + snapshot.title = "Metadata-only update" + snapshot.save() + snapshot.refresh_from_db() + + assert snapshot.fs_version == "0.8.0" + assert legacy_dir.exists() + assert not (current_dir / "unknown" / "payload.bin").exists() + + +def test_maintenance_save_keeps_existing_runner_entrypoint(snapshot): + legacy_dir, current_dir = _make_legacy_snapshot(snapshot) + + snapshot.save(update_fields=["retry_at", "modified_at"]) + snapshot.refresh_from_db() + + assert snapshot.fs_version == snapshot._fs_current_version() + assert not legacy_dir.exists() + assert (current_dir / "unknown" / "payload.bin").read_bytes() == b"filesystem migration payload\x00\xff" + + +def test_filesystem_migration_resumes_after_shutdown_before_cleanup(snapshot, monkeypatch): + legacy_dir, current_dir = _make_legacy_snapshot(snapshot) + real_cleanup = Snapshot._cleanup_old_migration_dir + + def simulate_shutdown_before_cleanup(self, old_dir, new_dir): + raise RuntimeError("simulated shutdown") + + monkeypatch.setattr(Snapshot, "_cleanup_old_migration_dir", simulate_shutdown_before_cleanup) + with pytest.raises(RuntimeError, match="simulated shutdown"): + snapshot.migrate_filesystem_to_current_version() + snapshot.refresh_from_db() + + assert snapshot.fs_version == "0.8.0" + assert legacy_dir.exists() + assert (current_dir / "unknown" / "payload.bin").read_bytes() == b"filesystem migration payload\x00\xff" + + monkeypatch.setattr(Snapshot, "_cleanup_old_migration_dir", real_cleanup) + snapshot.migrate_filesystem_to_current_version() + snapshot.refresh_from_db() + + assert snapshot.fs_version == snapshot._fs_current_version() + assert not legacy_dir.exists() + assert (current_dir / "unknown" / "payload.bin").read_bytes() == b"filesystem migration payload\x00\xff" + + +def test_filesystem_migration_cleans_legacy_source_when_version_is_current(snapshot, monkeypatch): + legacy_dir, current_dir = _make_legacy_snapshot(snapshot) + monkeypatch.setattr(Snapshot, "_cleanup_old_migration_dir", lambda *_args: True) + snapshot.migrate_filesystem_to_current_version() + snapshot.refresh_from_db() + assert snapshot.fs_version == snapshot._fs_current_version() + assert legacy_dir.exists() + + monkeypatch.undo() + snapshot.migrate_filesystem_to_current_version() + + assert not legacy_dir.exists() + assert (current_dir / "unknown" / "payload.bin").read_bytes() == b"filesystem migration payload\x00\xff" + + +def test_fs_version_has_database_index(): + assert Snapshot._meta.get_field("fs_version").db_index is True + constraints = connection.introspection.get_constraints(connection.cursor(), Snapshot._meta.db_table) + assert any(index["index"] and index["columns"] == ["fs_version"] for index in constraints.values()) + + +def test_stale_filesystem_selector_uses_fs_version_index(): + queryset = Snapshot.objects.filter(fs_version__in=Snapshot._FS_VERSION_MIGRATION_PATHS) + + assert "fs_version" in queryset.explain().lower() + + +def test_resume_refuses_to_overwrite_changed_legacy_output(snapshot, monkeypatch): + legacy_dir, current_dir = _make_legacy_snapshot(snapshot) + monkeypatch.setattr(Snapshot, "_cleanup_old_migration_dir", lambda *_args: True) + snapshot.migrate_filesystem_to_current_version() + (legacy_dir / "unknown" / "payload.bin").write_bytes(b"changed after database commit") + + with pytest.raises(SnapshotMigrationError, match="overwrite a different output"): + snapshot.migrate_filesystem_to_current_version() + + assert legacy_dir.exists() + assert (current_dir / "unknown" / "payload.bin").read_bytes() == b"filesystem migration payload\x00\xff" diff --git a/archivebox/tests/test_ui_add_view.py b/archivebox/tests/test_ui_add_view.py index 11c377db..48fe99ad 100644 --- a/archivebox/tests/test_ui_add_view.py +++ b/archivebox/tests/test_ui_add_view.py @@ -565,7 +565,7 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user): assert crawl is not None assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is not None - assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0} + assert crawl.urls == "https://example.com" assert crawl.snapshot_set.count() == 0 @@ -599,7 +599,7 @@ def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, ad assert crawl is not None assert crawl.status == Crawl.StatusChoices.PAUSED assert crawl.retry_at == RETRY_AT_MAX - assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com/paused", "depth": 0} + assert crawl.urls == "https://example.com/paused" assert crawl.snapshot_set.count() == 0 assert crawl.config.get("INDEX_ONLY") is not True diff --git a/archivebox/tests/test_ui_add_view_runtime.py b/archivebox/tests/test_ui_add_view_runtime.py index be94cc2a..5efc16ca 100644 --- a/archivebox/tests/test_ui_add_view_runtime.py +++ b/archivebox/tests/test_ui_add_view_runtime.py @@ -241,11 +241,7 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s crawl = Crawl.objects.order_by("-created_at").first() assert crawl is not None assert crawl.tags_str == "restart-supervised-runner" - assert json.loads(crawl.urls) == { - "type": "CrawlSeed", - "url": recursive_test_site["root_url"], - "depth": 0, - } + assert crawl.urls == recursive_test_site["root_url"] finally: stop_server(tmp_path) @@ -482,8 +478,8 @@ def test_public_add_view_import_text_formats_preserve_metadata_and_resume_withou with use_archivebox_db(tmp_path): for crawl in Crawl.objects.order_by("created_at"): - root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL) - root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") + assert not crawl.snapshot_set.filter(url__startswith="archivebox://").exists() + root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8") assert root_input == crawl.urls start_archivebox_server(tmp_path, env=env, port=port) @@ -659,7 +655,7 @@ def test_add_view_post_creates_schedule_over_server(tmp_path, recursive_test_sit schedule = CrawlSchedule.objects.select_related("template").order_by("-created_at").first() row = None if schedule: - template_url = json.loads(schedule.template.urls)["url"] + template_url = schedule.template.urls.strip() row = (schedule.schedule, template_url, schedule.template.tags_str) assert row == ("daily", recursive_test_site["root_url"], "web-ui") diff --git a/archivebox/tests/test_ui_admin_snapshot.py b/archivebox/tests/test_ui_admin_snapshot.py index 807ef203..ab7172c9 100644 --- a/archivebox/tests/test_ui_admin_snapshot.py +++ b/archivebox/tests/test_ui_admin_snapshot.py @@ -18,7 +18,7 @@ from django.urls import reverse from archivebox.core.middleware import ADMIN_LOGIN_HINT_COOKIE from archivebox.tests.conftest import ADMIN_TEST_HOST -from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook, _snapshot_hook_name +from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook pytestmark = pytest.mark.django_db(transaction=True) REPO_ROOT = Path(__file__).resolve().parents[2] @@ -98,9 +98,9 @@ def running_wget_projection(snapshot, blocking_http_server): retry_at=now, downloaded_at=None, url=blocking_http_server.url, + config={"PLUGINS": "wget"}, ) snapshot.refresh_from_db() - [result] = snapshot.create_pending_archiveresults(hooks=[("wget", _snapshot_hook_name("wget"))]) errors = [] def run_snapshot(): @@ -115,7 +115,7 @@ def running_wget_projection(snapshot, blocking_http_server): runner.start() blocking_http_server.request_started.wait() assert errors == [] - result.refresh_from_db() + result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget") assert result.status == ArchiveResult.StatusChoices.STARTED yield result blocking_http_server.release_response.set() @@ -1353,7 +1353,10 @@ class TestAdminSnapshotListView: assert response.status_code == 302 assert response["Location"].endswith(f"/admin/core/snapshot/{snapshot.pk}/change/") failed.refresh_from_db() - assert failed.status == ArchiveResult.StatusChoices.QUEUED + snapshot.refresh_from_db() + assert failed.status == ArchiveResult.StatusChoices.FAILED + assert snapshot.status == snapshot.StatusChoices.QUEUED + assert snapshot.config["PLUGINS"] == "title" def test_list_redo_failed_action_requeues_failed_archiveresults_only( self, @@ -1391,14 +1394,12 @@ class TestAdminSnapshotListView: failed.refresh_from_db() succeeded.refresh_from_db() snapshot.refresh_from_db() - assert failed.status == ArchiveResult.StatusChoices.QUEUED - assert failed.output_str == "" - assert failed.output_files == {} - assert failed.output_size == 0 - assert failed.output_mimetypes == "" + assert failed.status == ArchiveResult.StatusChoices.FAILED + assert failed.output_str assert succeeded.status == ArchiveResult.StatusChoices.SUCCEEDED assert succeeded.output_str == succeeded_output assert snapshot.status == snapshot.StatusChoices.QUEUED + assert snapshot.config["PLUGINS"] == "title" def test_archive_now_action_uses_original_snapshot_url_without_timestamp_suffix(self, client, admin_user, snapshot): from archivebox.crawls.models import Crawl diff --git a/archivebox/tests/test_ui_live_progress.py b/archivebox/tests/test_ui_live_progress.py index 3af12cd8..e4de8859 100644 --- a/archivebox/tests/test_ui_live_progress.py +++ b/archivebox/tests/test_ui_live_progress.py @@ -14,7 +14,7 @@ from django.utils import timezone from archivebox.tests.conftest import ADMIN_TEST_HOST from archivebox.tests.conftest import cli_env, resolve_abxpkg_binary_env, run_archivebox_cmd -from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook, _snapshot_hook_name +from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook pytestmark = pytest.mark.django_db(transaction=True) @@ -179,7 +179,12 @@ class TestLiveProgressView: ended_at=now - timedelta(hours=1), modified_at=now - timedelta(hours=1), ) - snapshot.create_pending_archiveresults(hooks=[("chrome", "on_Snapshot__11_chrome_wait")]) + ArchiveResult.objects.create( + snapshot=snapshot, + plugin="chrome", + hook_name="on_Snapshot__11_chrome_wait", + status=ArchiveResult.StatusChoices.QUEUED, + ) response = client.get("/progress.json", HTTP_HOST=ADMIN_TEST_HOST) @@ -218,9 +223,9 @@ class TestLiveProgressView: created_at=now - timedelta(hours=2), downloaded_at=None, url=blocking_http_server.url, + config={"PLUGINS": "wget"}, ) snapshot.refresh_from_db() - [result] = snapshot.create_pending_archiveresults(hooks=[("wget", _snapshot_hook_name("wget"))]) errors = [] def run_snapshot(): @@ -236,7 +241,7 @@ class TestLiveProgressView: try: blocking_http_server.request_started.wait() assert errors == [] - result.refresh_from_db() + result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget") assert result.status == ArchiveResult.StatusChoices.STARTED Snapshot.objects.filter(pk=snapshot.pk).update(modified_at=timezone.now()) @@ -257,7 +262,7 @@ class TestLiveProgressView: assert result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS) def test_live_progress_hides_finished_cancelled_crawl(self, client, admin_user, crawl, snapshot): - from archivebox.core.models import Snapshot + from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl now = timezone.now() @@ -272,7 +277,12 @@ class TestLiveProgressView: downloaded_at=None, modified_at=now, ) - snapshot.create_pending_archiveresults(hooks=[("singlefile", "on_Snapshot__50_singlefile")]) + ArchiveResult.objects.create( + snapshot=snapshot, + plugin="singlefile", + hook_name="on_Snapshot__50_singlefile", + status=ArchiveResult.StatusChoices.QUEUED, + ) client.force_login(admin_user) response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST) diff --git a/docs/ArchiveBox-Architecture-Diagrams.md b/docs/ArchiveBox-Architecture-Diagrams.md index daad4aa7..231784d0 100644 --- a/docs/ArchiveBox-Architecture-Diagrams.md +++ b/docs/ArchiveBox-Architecture-Diagrams.md @@ -82,7 +82,7 @@ stateDiagram-v2 SEALED --> [*] ``` -A crawl owns a set of snapshots. The runner creates or discovers those snapshots and projects crawl events while the row is `STARTED`; sealing waits for their normal lifecycle to finish. Pausing also schedules child snapshots to pause, and resuming returns the crawl to the runnable queue. The `archivebox://update` sentinel remains control-plane work: the runner invokes database maintenance directly and never turns it into a Snapshot. +A crawl owns a set of snapshots. The runner creates or discovers those snapshots and projects crawl events while the row is `STARTED`; sealing waits for their normal lifecycle to finish. Pausing also schedules child snapshots to pause, and resuming returns the crawl to the runnable queue. Scheduled maintenance is dispatched directly by `CrawlSchedule`; it does not create a synthetic crawl or snapshot. ## `Snapshot` Queue Lifecycle diff --git a/docs/Scheduled-Archiving.md b/docs/Scheduled-Archiving.md index 917955c8..db7eb0f1 100644 --- a/docs/Scheduled-Archiving.md +++ b/docs/Scheduled-Archiving.md @@ -33,7 +33,7 @@ Accepted schedule formats: `archivebox schedule --foreground` runs the global orchestrator in the foreground, which is useful outside `archivebox server` if you want a dedicated long-running scheduler/worker process without the web UI. -Running `archivebox schedule --every=day` with no `import_path` creates a recurring maintenance schedule that queues `archivebox://update` crawls. +Running `archivebox schedule --every=day` with no `import_path` creates a recurring maintenance schedule. The scheduler dispatches its bounded database/filesystem maintenance directly instead of creating a synthetic Crawl or Snapshot. ## Docker Compose