The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

refactor: simplify snapshot maintenance orchestration

Browse Source
This commit is contained in:
Nick Sweeting 2026-09-02 12:05:07 -07:00
parent 0a8f15e130
commit 2bba758314
No known key found for this signature in database
35 changed files with 873 additions and 2647 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
archivebox/api/v1_crawls.py
View File

@ -1,6 +1,5 @@
__package__ = "archivebox.api"
import json
from pathlib import Path
from uuid import UUID
from datetime import datetime
@ -123,7 +122,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
config = dict(data.config or {})
config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
crawl = Crawl.objects.create(
urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
urls="\n".join(urls),
max_depth=data.max_depth,
tags_str=",".join(tags),
label=data.label,

32
archivebox/cli/archivebox_add.py
View File

@ -6,7 +6,6 @@ __command__ = "archivebox add"
import sys
import os
import json
from pathlib import Path
from typing import Any, TYPE_CHECKING
@ -71,17 +70,7 @@ def add(
created_by_id: int | None = None,
config: dict[str, Any] | None = None,
) -> tuple[Crawl, QuerySet[Snapshot]]:
"""Add a new URL or list of URLs to your archive.
The flow is:
1. Save URLs to sources file
2. Create Crawl with URLs and max_depth
3. Crawl runner creates Snapshots from Crawl URLs (depth=0)
4. Crawl runner runs parser extractors on root snapshots
5. Parser extractors output to urls.jsonl
6. URLs are added to Crawl.urls and child Snapshots are created
7. Repeat until max_depth is reached
"""
"""Add a URL list or imported URL document to a new Crawl."""
from rich import print
@ -125,12 +114,7 @@ def add(
created_by_id = created_by_id or get_or_create_system_user_pk()
started_at = timezone.now()
use_internal_input_root = isinstance(urls, str)
source_text = (
urls
if use_internal_input_root
else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls)
)
source_text = urls if isinstance(urls, str) else "\n".join(str(url) for url in urls)
# 2. Create a new Crawl with inline URLs
# Foreground add must claim runner ownership before publishing runnable
@ -187,12 +171,7 @@ def add(
try:
crawl = Crawl.objects.create(
urls=source_text,
# Stdin/import text gets an extra hop because the synthetic
# archivebox://internal root lives at depth 0 and parser-discovered
# URLs land at depth 1; direct URL args become the depth=0 input
# snapshots themselves so --depth=N matches the deepest hop the user
# asked for.
max_depth=depth + 1 if use_internal_input_root else depth,
max_depth=depth,
tags_str=tag,
persona_id=persona_obj.id,
label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
@ -213,9 +192,8 @@ def add(
print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
print(f" [dim]First URL: {first_url}[/dim]")
# 3. The runner will create Snapshots from all URLs after claiming the Crawl
# Parser extractors run on snapshots and discover more URLs
# Discovered URLs become child Snapshots (depth+1)
# The runner parses Crawl.urls after claiming the Crawl, then persists the
# resulting depth-0 Snapshot facts. Recursive discoveries start at depth 1.
if index_only:
print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")

11
archivebox/cli/archivebox_archiveresult.py
View File

@ -6,13 +6,13 @@ archivebox archiveresult <action> [args...] [--filters]
Manage ArchiveResult records (plugin extraction results).
Actions:
create - Create ArchiveResults for Snapshots (queue extractions)
create - Emit plugin extraction request records for Snapshots
list - List ArchiveResults as JSONL (with optional filters)
update - Update ArchiveResults from stdin JSONL
delete - Delete ArchiveResults from stdin JSONL
Examples:
# Create ArchiveResults for snapshots (queue for extraction)
# Emit extraction requests; `archivebox run` schedules their parent snapshots
archivebox snapshot list --status=queued | archivebox archiveresult create
archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
@ -20,9 +20,6 @@ Examples:
archivebox archiveresult list --status=failed
archivebox archiveresult list --plugin=screenshot --status=succeeded
# Update (reset failed extractions to queued)
archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
# Delete
archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
@ -157,7 +154,7 @@ def create_archiveresults(
write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
created_count += 1
rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
rprint(f"[green]Created {created_count} extraction request records[/green]", file=sys.stderr)
return 0
@ -341,7 +338,7 @@ def main():
@click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
"""Create ArchiveResults for Snapshots from stdin JSONL."""
"""Emit Snapshot plugin extraction requests as JSONL."""
sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))

539
archivebox/cli/archivebox_extract.py
View File

@ -1,100 +1,126 @@
#!/usr/bin/env python3
"""
archivebox extract [snapshot_ids...] [--plugins=NAMES]
Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
Input formats:
- Snapshot UUIDs (one per line)
- JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
- JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
Output (JSONL):
{"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
Examples:
# Extract specific snapshot
archivebox extract 01234567-89ab-cdef-0123-456789abcdef
# Pipe from snapshot command
archivebox snapshot https://example.com | archivebox extract
# Run specific plugins only
archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
# Chain commands
archivebox crawl https://example.com | archivebox snapshot | archivebox extract
"""
"""Run abx-dl snapshot hooks for existing ArchiveBox snapshots."""
__package__ = "archivebox.cli"
__command__ = "archivebox extract"
import sys
from collections import defaultdict
from itertools import product
import rich_click as click
def process_archiveresult_by_id(archiveresult_id: str) -> int:
"""
Re-run extraction for a single ArchiveResult by ID.
def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[str]], set[str]]:
"""Resolve CLI records to snapshot-level execution requests.
ArchiveResults are projected status rows, not queued work items. Re-running
a single result means resetting that row and queueing its parent snapshot
through the shared crawl runner with the corresponding plugin selected.
ArchiveResult input is accepted as a convenient reference to its parent
Snapshot and plugin. It is never reset or converted into queued work.
"""
from rich import print as rprint
from archivebox.core.models import ArchiveResult
from archivebox.api.v1_core import _uuid_ref_query
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
explicit_plugins = {name.strip() for name in plugins.split(",") if name.strip()}
requested: dict[str, set[str]] = defaultdict(set)
missing: set[str] = set()
for record in records:
record_type = record.get("type")
record_id = str(record.get("id") or "")
if record_type == TYPE_ARCHIVERESULT and record.get("snapshot_id"):
requested[str(record["snapshot_id"])].update(explicit_plugins or {str(record.get("plugin") or "")})
requested[str(record["snapshot_id"])].discard("")
continue
if record_type == TYPE_ARCHIVERESULT:
result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
if result is not None:
requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
continue
snapshot_id = str(record.get("snapshot_id") or record_id or "")
snapshot = Snapshot.objects.filter(id=snapshot_id).only("id").first() if snapshot_id else None
if snapshot is None and record.get("url"):
snapshot = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").only("id").first()
# Bare UUID CLI arguments are parsed as Snapshot records because the
# input layer cannot know which model owns them. Preserve the public
# convenience of passing an ArchiveResult ID by trying that reference
# only after the Snapshot lookup misses.
if snapshot is None and record_id and not record.get("url"):
result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
if result is not None:
requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
continue
if snapshot is None:
missing.add(snapshot_id or str(record.get("url") or ""))
continue
requested[str(snapshot.id)].update(explicit_plugins)
return requested, missing
def _run_snapshot_requests(requested: dict[str, set[str]], *, wait: bool, show_progress: bool) -> int:
from django.utils import timezone
from rich import print as rprint
from archivebox.core.models import Snapshot
from archivebox.services.runner import run_crawl
try:
archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
except ArchiveResult.DoesNotExist:
rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
snapshots = {str(snapshot.id): snapshot for snapshot in Snapshot.objects.filter(id__in=requested).select_related("crawl")}
if not snapshots:
rprint("[red]No snapshots to process[/red]", file=sys.stderr)
return 1
rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
from archivebox.config.common import get_config
from archivebox.plugins.discovery import get_enabled_plugins
try:
archiveresult.reset_for_retry()
snapshot = archiveresult.snapshot
snapshot.queue_for_extraction()
crawl = snapshot.crawl
if not crawl.claim_processing_lock(lock_seconds=10):
rprint(
f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]",
file=sys.stderr,
for snapshot_id, plugin_names in requested.items():
snapshot = snapshots.get(snapshot_id)
if snapshot is not None and not plugin_names:
plugin_names.update(get_enabled_plugins(config=get_config(crawl=snapshot.crawl, snapshot=snapshot)))
# Explicit extraction resumes open/paused snapshots at the snapshot level.
# Sealed snapshots stay sealed during targeted maintenance backfills.
if wait:
for snapshot in snapshots.values():
if snapshot.status != Snapshot.StatusChoices.SEALED:
snapshot.update_and_requeue(
status=Snapshot.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
if not wait:
for snapshot_id, plugin_names in requested.items():
snapshot = snapshots.get(snapshot_id)
if snapshot is None:
continue
if plugin_names:
snapshot.config = {**(snapshot.config or {}), "PLUGINS": ",".join(sorted(plugin_names))}
snapshot.save(update_fields=["config", "modified_at"])
snapshot.update_and_requeue(
status=Snapshot.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
return 1
if show_progress:
rprint(f"[blue]Queued {len(snapshots)} snapshots for extraction[/blue]", file=sys.stderr)
return 0
grouped: dict[tuple[str, tuple[str, ...]], list[str]] = defaultdict(list)
for snapshot_id, plugin_names in requested.items():
snapshot = snapshots.get(snapshot_id)
if snapshot is not None:
grouped[(str(snapshot.crawl_id), tuple(sorted(plugin_names)))].append(snapshot_id)
for (crawl_id, plugin_names), snapshot_ids in grouped.items():
run_crawl(
str(snapshot.crawl_id),
snapshot_ids=[str(snapshot.id)],
selected_plugins=[archiveresult.plugin],
selected_plugins_are_explicit=False,
crawl_id,
snapshot_ids=sorted(snapshot_ids),
selected_plugins=list(plugin_names) or None,
show_progress=show_progress,
)
archiveresult.refresh_from_db()
if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
print(f"[green]Extraction succeeded: {archiveresult.output_str}[/green]")
return 0
elif archiveresult.status == ArchiveResult.StatusChoices.NORESULTS:
print(f"[dim]Extraction completed with no results: {archiveresult.output_str}[/dim]")
return 0
elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
print(f"[red]Extraction failed: {archiveresult.output_str}[/red]", file=sys.stderr)
return 1
else:
# Still in progress or backoff - not a failure
print(f"[yellow]Extraction status: {archiveresult.status}[/yellow]")
return 0
except Exception as e:
print(f"[red]Extraction error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
return 1
return 0
def run_plugins(
@ -104,377 +130,64 @@ def run_plugins(
wait: bool = True,
emit_results: bool = True,
show_progress: bool = True,
preserve_queued: bool = False,
) -> int:
"""
Run plugins on Snapshots from input.
Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
Exit codes:
0: Success
1: Failure
"""
"""Execute selected plugins through the snapshot-level runner."""
from rich import print as rprint
from django.utils import timezone
from archivebox.misc.jsonl import (
read_args_or_stdin,
write_record,
TYPE_SNAPSHOT,
TYPE_ARCHIVERESULT,
)
from archivebox.core.models import Snapshot
from archivebox.core.models import ArchiveResult
from archivebox.services.runner import run_crawl
from archivebox.plugins.discovery import get_plugin_catalog
from archivebox.misc.jsonl import read_args_or_stdin, write_record
is_tty = sys.stdout.isatty()
# Parse comma-separated plugins list once (reused in creation and filtering)
plugins_list = [p.strip() for p in plugins.split(",") if p.strip()] if plugins else []
# Parse stdin/args exactly once per CLI invocation.
# `main()` may already have consumed stdin to distinguish Snapshot input from
# ArchiveResult IDs; if so, it must pass the parsed records through here
# instead of asking this helper to reread an already-drained pipe.
if records is None:
records = list(read_args_or_stdin(args))
if not records:
rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
return 1
# Gather snapshot IDs and optional plugin constraints to process
snapshot_ids = set()
requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
requested_hooks_by_snapshot: dict[str, set[tuple[str, str]]] = defaultdict(set)
plugin_level_requests_by_snapshot: dict[str, set[str]] = defaultdict(set)
for record in records:
record_type = record.get("type")
if record_type == TYPE_SNAPSHOT:
snapshot_id = record.get("id")
if snapshot_id:
snapshot_ids.add(str(snapshot_id))
elif record.get("url"):
# Look up by URL (get most recent if multiple exist)
snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
if snap:
snapshot_ids.add(str(snap.id))
else:
rprint(f"[yellow]Snapshot not found for URL: {record['url']}[/yellow]", file=sys.stderr)
elif record_type == TYPE_ARCHIVERESULT:
snapshot_id = record.get("snapshot_id")
if snapshot_id:
snapshot_ids.add(str(snapshot_id))
plugin_name = record.get("plugin")
if plugin_name and not plugins_list:
snapshot_key = str(snapshot_id)
plugin_key = str(plugin_name)
requested_plugins_by_snapshot[snapshot_key].add(plugin_key)
hook_name = str(record.get("hook_name") or "")
if hook_name:
requested_hooks_by_snapshot[snapshot_key].add((plugin_key, hook_name))
else:
plugin_level_requests_by_snapshot[snapshot_key].add(plugin_key)
elif "id" in record:
# Assume it's a snapshot ID
snapshot_ids.add(str(record["id"]))
if not snapshot_ids:
rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
requested, missing = _resolve_requests(records, plugins)
for value in sorted(missing):
rprint(f"[yellow]Snapshot or ArchiveResult not found: {value}[/yellow]", file=sys.stderr)
if not requested:
return 1
existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id"))
existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots}
existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots}
missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids)
for snapshot_id in missing_snapshot_ids:
rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
# Queue only the target plugin rows. Bulk updates keep large reindex runs
# from doing one SELECT+UPDATE per snapshot/plugin before hooks even start.
requested_pairs: set[tuple[str, str]] = set()
if plugins_list:
requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list))
else:
requested_pairs.update(
(snapshot_id, plugin_name)
for snapshot_id, plugin_names in requested_plugins_by_snapshot.items()
if snapshot_id in existing_snapshot_ids
for plugin_name in plugin_names
)
plugins_by_name = get_plugin_catalog()
requested_rows: set[tuple[str, str, str]] = set()
for snapshot_id, plugin_name in requested_pairs:
exact_hook_names = {
hook_name
for requested_plugin, hook_name in requested_hooks_by_snapshot.get(snapshot_id, set())
if requested_plugin == plugin_name
}
if exact_hook_names and plugin_name not in plugin_level_requests_by_snapshot.get(snapshot_id, set()):
requested_rows.update((snapshot_id, plugin_name, hook_name) for hook_name in exact_hook_names)
continue
plugin = plugins_by_name.get(plugin_name)
hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
if hooks:
requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
else:
requested_rows.add((snapshot_id, plugin_name, ""))
queued_rows: set[tuple[str, str, str]] = set()
if preserve_queued and requested_rows:
queued_rows = {
(str(snapshot_id), plugin_name, hook_name)
for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
snapshot_id__in=existing_snapshot_ids,
plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
status=ArchiveResult.StatusChoices.QUEUED,
).values_list("snapshot_id", "plugin", "hook_name")
}
rows_to_queue = requested_rows - queued_rows
reset_fields = {
"status": ArchiveResult.StatusChoices.QUEUED,
"output_str": "",
"output_json": None,
"output_files": {},
"output_size": 0,
"output_mimetypes": "",
"start_ts": None,
"end_ts": None,
"modified_at": timezone.now(),
}
if rows_to_queue and plugins_list:
rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
for snapshot_id, plugin_name, hook_name in rows_to_queue:
rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
**reset_fields,
)
elif rows_to_queue and requested_plugins_by_snapshot:
snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
for snapshot_id, plugin_name, hook_name in rows_to_queue:
snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
**reset_fields,
)
existing_rows = (
{
(str(snapshot_id), plugin_name, hook_name)
for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
snapshot_id__in=existing_snapshot_ids,
plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
).values_list("snapshot_id", "plugin", "hook_name")
}
if rows_to_queue
else set()
)
missing_rows = rows_to_queue - existing_rows
if missing_rows:
ArchiveResult.objects.bulk_create(
[
ArchiveResult(
snapshot_id=snapshot_id,
plugin=plugin_name,
hook_name=hook_name,
status=ArchiveResult.StatusChoices.QUEUED,
)
for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
],
batch_size=500,
)
processed_count = len(existing_snapshot_ids)
queue_at = timezone.now()
if existing_snapshot_ids:
if requested_rows:
# Explicit plugin retries are maintenance on the existing snapshot;
# preserve a sealed lifecycle while making its queued rows due.
affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
if preserve_queued and queued_rows:
queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
affected_snapshot_ids.update(
str(snapshot_id)
for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
.filter(retry_at__gt=queue_at)
.values_list("id", flat=True)
)
affected_snapshot_ids.update(
str(snapshot_id)
for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
"id",
flat=True,
)
)
requested_plugins_by_id: dict[str, set[str]] = defaultdict(set)
for snapshot_id, plugin_name, _hook_name in requested_rows:
requested_plugins_by_id[snapshot_id].add(plugin_name)
for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
# Guard the read-time status so we never bump retry_at on a
# row that's been re-queued / started by a concurrent runner.
if snapshot.status == Snapshot.StatusChoices.SEALED and requested_plugins_by_id.get(str(snapshot.id)):
snapshot.safe_update(
{"retry_at": queue_at, "modified_at": queue_at},
refresh=False,
extra_filter={"status": snapshot.status},
)
else:
snapshot.update_and_requeue(
status=Snapshot.StatusChoices.QUEUED,
retry_at=queue_at,
current_step=0,
)
else:
# No plugin rows were requested, so this is a full snapshot retry.
for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
snapshot.safe_update(
{
"status": Snapshot.StatusChoices.QUEUED,
"retry_at": queue_at,
"current_step": 0,
"modified_at": queue_at,
},
refresh=False,
extra_filter={"status": snapshot.status},
)
if existing_crawl_ids and not requested_rows:
from archivebox.crawls.models import Crawl
for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
update_fields = {
"retry_at": queue_at,
"modified_at": queue_at,
}
if crawl.status != Crawl.StatusChoices.STARTED:
update_fields["status"] = Crawl.StatusChoices.QUEUED
crawl.safe_update(
update_fields,
refresh=False,
extra_filter={"status": crawl.status},
)
if processed_count == 0:
rprint("[red]No snapshots to process[/red]", file=sys.stderr)
return 1
if show_progress:
rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
# Run orchestrator if --wait (default)
if wait:
if show_progress:
rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
for snapshot_id, crawl_id in existing_snapshots:
snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))
for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.get(id=crawl_id)
if not crawl.claim_processing_lock(lock_seconds=10):
rprint(
f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]",
file=sys.stderr,
)
return 1
selected_plugins = (
plugins_list
or sorted(
{plugin for snapshot_id, plugin, _hook_name in requested_rows if snapshot_id in crawl_snapshot_ids},
)
or None
)
run_crawl(
crawl_id,
snapshot_ids=sorted(crawl_snapshot_ids),
selected_plugins=selected_plugins,
show_progress=show_progress,
selected_plugins_are_explicit=bool(plugins_list),
)
if not emit_results:
return 0
# Output results as JSONL (when piped) or human-readable (when TTY)
for snapshot_id in snapshot_ids:
try:
snapshot = Snapshot.objects.get(id=snapshot_id)
results = snapshot.archiveresult_set.all()
if plugins_list:
results = results.filter(plugin__in=plugins_list)
for result in results:
if is_tty:
status_color = {
"succeeded": "green",
"failed": "red",
"skipped": "yellow",
}.get(result.status, "dim")
rprint(
f" [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ''}",
file=sys.stderr,
)
else:
write_record(result.to_json())
except Snapshot.DoesNotExist:
continue
exit_code = _run_snapshot_requests(requested, wait=wait, show_progress=show_progress)
if exit_code or not emit_results:
return exit_code
is_tty = sys.stdout.isatty()
for snapshot in Snapshot.objects.filter(id__in=requested):
results = snapshot.archiveresult_set.all()
requested_plugins = requested[str(snapshot.id)]
if requested_plugins:
results = results.filter(plugin__in=requested_plugins)
for result in results:
if is_tty:
color = {"succeeded": "green", "failed": "red", "skipped": "yellow"}.get(result.status, "dim")
rprint(f" [{color}]{result.status}[/{color}] {result.plugin} → {result.output_str or ''}", file=sys.stderr)
else:
write_record(result.to_json())
return 0
def is_archiveresult_id(value: str) -> bool:
"""Check if value resolves to an ArchiveResult ID."""
from archivebox.core.models import ArchiveResult
from archivebox.api.v1_core import _uuid_ref_query
return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()
def process_archiveresult_by_id(archiveresult_id: str) -> int:
"""Re-run the parent Snapshot plugin referenced by an ArchiveResult."""
return run_plugins((), records=[{"id": archiveresult_id}], wait=True)
@click.command()
@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run (e.g., screenshot,singlefile)")
@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
@click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
@click.argument("args", nargs=-1)
def main(plugins: str, wait: bool, args: tuple):
"""Run plugins on Snapshots, or process existing ArchiveResults by ID"""
"""Run plugins on Snapshots; ArchiveResult IDs select their parent plugin."""
from archivebox.misc.jsonl import read_args_or_stdin
# Read all input
records = list(read_args_or_stdin(args))
if not records:
from rich import print as rprint
rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
sys.exit(1)
# Check if input looks like existing ArchiveResult IDs to process
all_are_archiveresult_ids = all(is_archiveresult_id(r.get("id") or r.get("url", "")) for r in records)
if all_are_archiveresult_ids:
# Process existing ArchiveResults by ID
from rich import print as rprint
exit_code = 0
for record in records:
archiveresult_id = record.get("id") or record.get("url")
if not isinstance(archiveresult_id, str):
rprint(f"[red]Invalid ArchiveResult input: {record}[/red]", file=sys.stderr)
exit_code = 1
continue
result = process_archiveresult_by_id(archiveresult_id)
if result != 0:
exit_code = result
sys.exit(exit_code)
else:
# Default behavior: run plugins on Snapshots from input
sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
if __name__ == "__main__":

31
archivebox/cli/archivebox_run.py
View File

@ -72,8 +72,8 @@ def process_stdin_records() -> int:
Outputs JSONL of all processed records (for chaining).
Handles any record type: Crawl, Snapshot, ArchiveResult.
Auto-cascades: Crawl → Snapshots → ArchiveResults.
Handles Crawl and Snapshot work records. ArchiveResult records are accepted
as references to their parent Snapshot and plugin.
Returns exit code (0 = success, 1 = error).
"""
@ -90,6 +90,7 @@ def process_stdin_records() -> int:
)
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import Snapshot, ArchiveResult
from archivebox.api.v1_core import _uuid_ref_query
from archivebox.crawls.models import Crawl
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
from archivebox.machine.models import Binary
@ -156,20 +157,15 @@ def process_stdin_records() -> int:
queued_count += 1
elif record_type == TYPE_ARCHIVERESULT:
archiveresult = ArchiveResult.from_json(record)
if archiveresult:
if archiveresult.status in [
ArchiveResult.StatusChoices.FAILED,
ArchiveResult.StatusChoices.SKIPPED,
ArchiveResult.StatusChoices.NORESULTS,
ArchiveResult.StatusChoices.BACKOFF,
]:
archiveresult.reset_for_retry()
snapshot = archiveresult.snapshot
plugin_name = archiveresult.plugin
else:
snapshot = None
plugin_name = None
snapshot_id = str(record.get("snapshot_id") or "")
plugin_name = str(record.get("plugin") or "")
archiveresult = None
if not snapshot_id and record_id:
archiveresult = ArchiveResult.objects.filter(_uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
if archiveresult:
snapshot_id = str(archiveresult.snapshot_id)
plugin_name = plugin_name or archiveresult.plugin
snapshot = Snapshot.objects.filter(id=snapshot_id).first() if snapshot_id else None
if snapshot:
snapshot.queue_for_extraction()
@ -177,7 +173,7 @@ def process_stdin_records() -> int:
snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
if plugin_name:
plugin_names_by_crawl[crawl_id].add(str(plugin_name))
output_records.append(record if not archiveresult else archiveresult.to_json())
output_records.append(archiveresult.to_json() if archiveresult else record)
queued_count += 1
elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}:
@ -235,7 +231,6 @@ def process_stdin_records() -> int:
crawl_id,
snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
selected_plugins_are_explicit=False,
)
return 0

17
archivebox/cli/archivebox_schedule.py
View File

@ -57,12 +57,12 @@ def schedule(
created_by_id = get_or_create_system_user_pk()
is_update_schedule = not import_path
template_urls = import_path or "archivebox://update"
template_urls = import_path or ""
template_label = (f"Scheduled import: {template_urls}" if import_path else "Scheduled ArchiveBox update")[:64]
template_notes = (
f"Created by archivebox schedule for {template_urls}"
if import_path
else "Created by archivebox schedule to queue recurring archivebox://update maintenance crawls."
else "Created by archivebox schedule to run recurring ArchiveBox maintenance."
)
template = Crawl.objects.create(
@ -76,7 +76,6 @@ def schedule(
retry_at=None,
config={
"DEPTH": 0 if is_update_schedule else depth,
"SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
# Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
# template defaults. Anything left unset falls through to the
# standard config stack at crawl-resolution time.
@ -86,6 +85,7 @@ def schedule(
crawl_schedule = CrawlSchedule.objects.create(
template=template,
schedule=schedule_str,
config={**template.config, "SCHEDULE_KIND": "update" if is_update_schedule else "crawl"},
is_enabled=True,
label=template_label,
notes=template_notes,
@ -119,12 +119,19 @@ def schedule(
if run_all:
enqueued = 0
maintained = 0
now = timezone.now()
for scheduled_crawl in schedules:
scheduled_crawl.enqueue(queued_at=now)
enqueued += 1
if scheduled_crawl.dispatch(queued_at=now) is None:
maintained += 1
else:
enqueued += 1
result["run_all_enqueued"] = enqueued
result["run_all_maintained"] = maintained
print(f"[green]\\[*] Enqueued {enqueued} scheduled crawl(s) immediately.[/green]")
if maintained:
run_pending_crawls(maintenance_only=True)
print(f"[green]\\[*] Ran {maintained} scheduled maintenance update(s).[/green]")
if enqueued:
print(
"[yellow]\\[*] Start `archivebox server`, `archivebox run --daemon`, or `archivebox schedule --foreground` to process the queued crawls.[/yellow]",

466
archivebox/cli/archivebox_update.py
View File

@ -77,104 +77,50 @@ def reindex_snapshots(
) -> dict[str, Any]:
from archivebox.cli.archivebox_extract import run_plugins
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.plugins.discovery import get_plugin_catalog
from django.db.models import Exists, OuterRef
# Search backfill is the one maintenance hook allowed to execute without
# reopening a Snapshot. Restrict that exception to already-sealed rows;
# every open lifecycle state remains owned by the normal runner lifecycle.
snapshots = snapshots.filter(status=Snapshot.StatusChoices.SEALED)
stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
records: list[dict[str, str]] = []
plugins_by_name = get_plugin_catalog()
required_hooks_by_plugin = {
plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
for plugin_name in search_plugins
if plugin_name in plugins_by_name
}
stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
print(f"[*] Backfilling missing search indexes with: {', '.join(search_plugins)}")
total = snapshots.count()
print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
def run_batch() -> None:
if not records:
return
if wait_for_turn:
wait_for_turn()
batch_records = list(records)
snapshot_ids = {record["snapshot_id"] for record in batch_records}
plugin_names = {record["plugin"] for record in batch_records}
queued_rows = {
(str(snapshot_id), plugin_name, hook_name)
for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
snapshot_id__in=snapshot_ids,
plugin__in=plugin_names,
status=ArchiveResult.StatusChoices.QUEUED,
).values_list("snapshot_id", "plugin", "hook_name")
}
records_to_queue = []
for record in batch_records:
snapshot_id = record["snapshot_id"]
plugin_name = record["plugin"]
required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset())
if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks):
stats["skipped_queued"] += 1
continue
records_to_queue.append(record)
if not records_to_queue:
print(
f" [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs",
)
records.clear()
return
# `archivebox update --index-only` intentionally breaks the usual
# "runner discovers work" rule by inserting synthetic queued
# ArchiveResult rows for search backends. run_plugins() keeps this as
# statement-sized UPDATE/bulk_create work, then bumps Snapshot.retry_at
# so the orchestrator owns actual hook execution. Paused snapshots stay
# PAUSED; run_due_snapshot restores retry_at=MAX after targeted rows
# finish.
exit_code = run_plugins(
args=(),
records=records_to_queue,
wait=False,
emit_results=False,
show_progress=False,
preserve_queued=True,
completed_statuses = [ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS]
for plugin_name in search_plugins:
completed_result = ArchiveResult.objects.filter(
snapshot_id=OuterRef("pk"),
plugin=plugin_name,
status__in=completed_statuses,
)
if exit_code != 0:
raise SystemExit(exit_code)
stats["queued"] += len(records_to_queue)
print(
f" [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator",
)
records.clear()
for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size):
try:
stats["processed"] += 1
if _get_snapshot_crawl(snapshot) is None:
continue
candidates = snapshots.annotate(has_completed_index=Exists(completed_result)).filter(has_completed_index=False).order_by("id")
after_id = None
while True:
if wait_for_turn:
wait_for_turn()
page = candidates.filter(id__gt=after_id) if after_id is not None else candidates
batch = list(page.only("id", "timestamp")[:batch_size])
if not batch:
break
after_id = batch[-1].id
records = [{"type": "Snapshot", "id": str(snapshot.id)} for snapshot in batch]
stats["processed"] += len(batch)
stats["requested"] += len(batch)
if collect_ids:
stats["snapshot_ids"].append(str(snapshot.id))
for plugin_name in search_plugins:
records.append(
{
"type": "ArchiveResult",
"snapshot_id": str(snapshot.id),
"plugin": plugin_name,
},
)
stats["requested"] += 1
if len(records) >= batch_size:
run_batch()
except KeyboardInterrupt as err:
err.archivebox_resume = snapshot.timestamp
raise
run_batch()
stats["snapshot_ids"].extend(str(snapshot.id) for snapshot in batch)
exit_code = run_plugins(
args=(),
records=records,
plugins=plugin_name,
wait=True,
emit_results=False,
show_progress=False,
)
if exit_code != 0:
raise SystemExit(exit_code)
stats["reindexed"] += len(batch)
print(f" [{plugin_name}] indexed {stats['reindexed']} missing snapshots")
return stats
@ -203,9 +149,9 @@ def update(
Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving.
Three-phase operation (without filters):
- Phase 1: Drain old archive/ dirs by moving to new fs location (0.8.x → 0.9.x)
- Phase 2: O(n) scan over entire DB from most recent to least recent
- No orphan scans needed (trust 1:1 mapping between DB and filesystem after phase 1)
- Phase 1: Drain legacy archive/ directories into the current layout
- Phase 2: Select only stale fs_version rows through the indexed column
- Phase 3: Run queued snapshot-level filesystem maintenance until idle
With filters: Only phase 2 (DB query), no filesystem operations.
Without filters: All phases (full update).
@ -232,7 +178,6 @@ def update(
from archivebox.core.takeover_util import (
command_owns_foreground_runner,
current_command,
foreground_runner_owner,
standby_until_foreground_runner_needed,
)
from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
@ -291,11 +236,6 @@ def update(
while True:
do_migrate = migrate_only or not index_only
do_index = index_only or not migrate_only
do_run_until_idle = do_migrate or do_index
ran_post_migrate_runner = False
full_update_empty = False
maintenance_work_queued = False
runner_work_queued = False
if do_migrate:
if (
@ -327,13 +267,11 @@ def update(
after=after,
resume=resume,
batch_size=batch_size,
queue_for_archiving=do_run_until_idle,
queue_for_archiving=True,
wait_for_turn=wait_for_turn,
)
print_stats(stats)
touched_snapshot_ids.update(stats.get("snapshot_ids", []))
maintenance_work_queued = stats.get("queued", 0) > 0
runner_work_queued = runner_work_queued or maintenance_work_queued
else:
stats_combined = {"phase1": {}, "phase2": {}}
@ -343,162 +281,52 @@ def update(
batch_size=batch_size,
)
print("[*] Phase 2: Processing all database snapshots (most recent first)...")
print("[*] Phase 2: Selecting database snapshots with stale filesystem versions...")
stats_combined["phase2"] = process_all_db_snapshots(
batch_size=batch_size,
resume=resume,
wait_for_turn=wait_for_turn,
)
print_combined_stats(stats_combined)
full_update_empty = (
stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0
)
maintenance_work_queued = any(
(
stats_combined["phase1"].get("queued", 0),
stats_combined["phase2"].get("queued", 0),
stats_combined["phase2"].get("crawls_sealed", 0),
),
)
runner_work_queued = runner_work_queued or maintenance_work_queued
if do_run_until_idle:
# Filesystem migration is maintenance on existing
# Snapshot rows: Snapshot.save() moves archive/<ts> to
# the current output_dir and preserves the lifecycle
# status. Drain those retry_at ticks before queuing
# search backfill below. Otherwise the sealed search
# runner branch correctly sees queued ArchiveResult
# rows first, runs the targeted plugins, and may leave
# the fs_version maintenance tick hidden behind that
# plugin work until another update pass.
if full_update_empty:
print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.")
elif not maintenance_work_queued:
print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.")
else:
print("[*] Phase 3: Running filesystem maintenance until idle...")
if full_update_empty:
pass
elif not maintenance_work_queued:
pass
elif is_filtered_update:
if not touched_snapshot_ids:
print("[*] No matching snapshots queued work for the runner.")
for snapshot_id in sorted(touched_snapshot_ids):
run_scoped_runner("--snapshot-id", snapshot_id)
else:
run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
ran_post_migrate_runner = True
if do_index:
if full_update_empty:
print("[*] No snapshots found; skipping search indexing backfill.")
else:
search_plugins = _get_search_indexing_plugins()
if not search_plugins:
print("[*] No search indexing plugins are available, nothing to backfill.")
else:
snapshots = _build_filtered_snapshots_queryset(
filter_patterns=filter_patterns,
filter_type=filter_type,
status=status,
url__icontains=url__icontains,
url__istartswith=url__istartswith,
tag=tag,
crawl_id=crawl_id,
limit=limit,
sort=sort,
search=search,
before=before,
after=after,
resume=resume,
)
from django.db.models import Exists, OuterRef, Q
from django.utils import timezone
from archivebox.core.models import ArchiveResult, Snapshot
scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
queued_index_results = ArchiveResult.objects.filter(
status=ArchiveResult.StatusChoices.QUEUED,
plugin__in=search_plugins,
)
if scoped_snapshot_ids is not None:
queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
if queued_index_results.exists():
runner_work_queued = True
now = timezone.now()
queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
snapshots_to_wake = (
Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED)
.annotate(
has_queued_index_result=Exists(queued_result_for_snapshot),
)
.filter(
has_queued_index_result=True,
)
.filter(
Q(retry_at__isnull=True) | Q(retry_at__gt=now),
)
)
if scoped_snapshot_ids is not None:
snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
woken_count = snapshots_to_wake.update(
retry_at=now,
modified_at=now,
)
print(
"[*] Existing queued search index jobs found; "
f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
)
else:
collect_index_ids = (
is_filtered_update
or foreground_runner_owner(
data_dir=CONSTANTS.DATA_DIR,
exclude_id=command.id,
)
is not None
)
stats = reindex_snapshots(
snapshots,
search_plugins=search_plugins,
batch_size=batch_size,
collect_ids=collect_index_ids,
wait_for_turn=wait_for_turn,
)
print_index_stats(stats)
touched_snapshot_ids.update(stats.get("snapshot_ids", []))
runner_work_queued = runner_work_queued or stats["queued"] > 0
if do_run_until_idle and (do_index or not ran_post_migrate_runner):
# Search/index backfill intentionally queues targeted
# ArchiveResult rows without reopening sealed snapshots.
# This second runner pass drains those plugin
# rows after filesystem maintenance has had its own turn.
# For a normal unfiltered `archivebox update`, keep the
# historical final pass broad enough to resume genuinely
# queued/interrupted crawl work after maintenance is done.
if full_update_empty:
print("[*] No snapshots found; skipping queued/interrupted crawl runner.")
elif not runner_work_queued:
print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.")
else:
print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
if full_update_empty:
pass
elif not runner_work_queued:
pass
elif touched_snapshot_ids and is_filtered_update:
if not touched_snapshot_ids:
print("[*] No matching snapshots queued work for the runner.")
# The due selectors are indexed and cheap when empty, so
# always drain them instead of preceding the runner with
# whole-table counts merely to decide whether to call it.
print("[*] Phase 3: Running filesystem maintenance until idle...")
if is_filtered_update:
for snapshot_id in sorted(touched_snapshot_ids):
run_scoped_runner("--snapshot-id", snapshot_id)
else:
run_scoped_runner(
*(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
if do_index:
search_plugins = _get_search_indexing_plugins()
if not search_plugins:
print("[*] No search indexing plugins are available, nothing to backfill.")
else:
snapshots = _build_filtered_snapshots_queryset(
filter_patterns=filter_patterns,
filter_type=filter_type,
status=status,
url__icontains=url__icontains,
url__istartswith=url__istartswith,
tag=tag,
crawl_id=crawl_id,
limit=limit,
sort=sort,
search=search,
before=before,
after=after,
resume=resume,
)
stats = reindex_snapshots(
snapshots,
search_plugins=search_plugins,
batch_size=batch_size,
collect_ids=is_filtered_update,
wait_for_turn=wait_for_turn,
)
print_index_stats(stats)
touched_snapshot_ids.update(stats.get("snapshot_ids", []))
if not continuous:
break
@ -737,18 +565,8 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, wait_for_turn=None) -> dict[str, int]:
"""
O(n) scan over entire DB from most recent to least recent.
For each snapshot:
1. Reconcile index.json with DB (merge titles, tags, archive results)
2. Mark migrated snapshots sealed unless explicitly re-queued elsewhere
No orphan detection needed - we trust 1:1 mapping between DB and filesystem
after Phase 1 has drained all old archive/ directories.
"""
"""Queue only snapshots whose indexed filesystem version is stale."""
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from django.db.models import Q
from django.utils import timezone
@ -761,128 +579,35 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
"sealed": 0,
"crawls_sealed": 0,
}
current_fs_version = Snapshot._fs_current_version()
queryset = Snapshot.objects.all()
queryset = Snapshot.objects.filter(fs_version__in=Snapshot._FS_VERSION_MIGRATION_PATHS)
if resume:
queryset = queryset.filter(timestamp__lte=resume)
total = queryset.count()
stats["snapshots"] = total
print(f"[*] Processing {total} snapshots from database (most recent first)...")
def update_in_batches(rows, *, label: str, **updates) -> int:
updated = 0
checked = 0
while True:
if wait_for_turn:
wait_for_turn()
batch = list(rows.only("id", "modified_at").order_by("-timestamp")[:batch_size])
if not batch:
if updated:
print(f" [{label}] complete: {updated} rows updated")
return updated
checked += len(batch)
print(f" [{label}] updating next {len(batch)} rows (seen {checked})...")
for snapshot in batch:
# This maintenance scan intentionally bypasses save(); it is
# only normalizing scheduler fields, and Snapshot.save() may
# do filesystem migration work that belongs in the runner.
# Guard each single-row UPDATE with modified_at so stale scan
# pages cannot overwrite newer runner/admin writes.
updated += int(
snapshot.safe_update(
updates,
refresh=False,
extra_filter={"modified_at": snapshot.modified_at},
),
)
print(f" [{label}] updated {updated} rows so far")
now = timezone.now()
updated_rows = update_in_batches(
queryset.exclude(
status__in=[
Snapshot.StatusChoices.QUEUED,
Snapshot.StatusChoices.STARTED,
Snapshot.StatusChoices.PAUSED,
Snapshot.StatusChoices.SEALED,
],
),
label="snapshot status normalization",
status=Snapshot.StatusChoices.SEALED,
retry_at=None,
modified_at=now,
)
stats["sealed"] += updated_rows
stats["updated_db"] += updated_rows
fs_version_rows = queryset.exclude(fs_version=current_fs_version).filter(Q(retry_at__isnull=True) | Q(retry_at__gt=now))
stale_batch = []
def queue_stale_fs_batch() -> None:
if not stale_batch:
return
initial_now = timezone.now()
rows_to_wake = queryset.filter(Q(retry_at__isnull=True) | Q(retry_at__gt=initial_now))
after_id = None
while True:
if wait_for_turn:
wait_for_turn()
page = rows_to_wake.filter(id__gt=after_id) if after_id is not None else rows_to_wake
batch = list(page.only("id", "fs_version", "modified_at").order_by("id")[:batch_size])
if not batch:
break
after_id = batch[-1].id
now = timezone.now()
# Do not bump fs_version here. The orchestrator calls Snapshot.save(),
# which performs the idempotent filesystem migration and commits the new
# fs_version in the same serialized worker path as normal crawls.
updated = 0
for snapshot in stale_batch:
# Each row gets its own short autocommit UPDATE because this scan
# can touch millions of snapshots while a server is also alive.
# The modified_at predicate is the CAS guard: if the runner or
# admin changed the snapshot after paged_iterator read it, skip it
# and let the newer state decide whether migration is still due.
for snapshot in batch:
updated += int(
snapshot.safe_update(
{
"retry_at": now,
"modified_at": now,
},
{"retry_at": now, "modified_at": now},
refresh=False,
extra_filter={"fs_version": snapshot.fs_version},
),
)
stats["processed"] += len(stale_batch)
stats["processed"] += len(batch)
stats["updated_db"] += updated
stats["queued"] += updated
print(f" [{stats['processed']}/{total}] Queued {updated} filesystem migrations for orchestrator...")
stale_batch.clear()
for snapshot in (
fs_version_rows.only("id", "crawl_id", "timestamp", "fs_version", "modified_at")
.order_by("-timestamp")
.paged_iterator(chunk_size=batch_size)
):
try:
stale_batch.append(snapshot)
if len(stale_batch) >= batch_size:
queue_stale_fs_batch()
except KeyboardInterrupt as err:
err.archivebox_resume = snapshot.timestamp
raise
queue_stale_fs_batch()
now = timezone.now()
# Crawls with no open child snapshots are already finished. Seal them here
# instead of waking the foreground runner; otherwise migration/update can
# accidentally re-enter full crawl execution for historical rows.
stats["crawls_sealed"] = (
Crawl.objects.filter(
status__in=Crawl.RUNNABLE_STATES,
)
.exclude(
snapshot_set__status__in=Snapshot.OPEN_STATES,
)
.update(
status=Crawl.StatusChoices.SEALED,
retry_at=None,
modified_at=now,
)
)
stats["updated_db"] += stats["crawls_sealed"]
print(f" Queued {stats['queued']} stale filesystem snapshots so far...")
stats["snapshots"] = stats["processed"]
return stats
@ -942,8 +667,6 @@ def process_filtered_snapshots(
stats["snapshot_ids"].append(str(snapshot.id))
update_values = {}
updated = 0
if not isinstance(snapshot.current_step, int):
update_values["current_step"] = 0
if queue_for_archiving:
update_values.update(
{
@ -1029,10 +752,9 @@ def print_index_stats(stats: dict[str, Any]) -> None:
print(f"""
[green]Search Reindex Complete[/green]
Scanned rows: {stats["processed"]}
Requested jobs: {stats.get("requested", stats["queued"])}
Queued index jobs: {stats["queued"]}
Already queued: {stats.get("skipped_queued", 0)}
Missing rows: {stats["processed"]}
Requested runs: {stats.get("requested", 0)}
Indexed snapshots: {stats.get("reindexed", 0)}
""")

2
archivebox/config/collection.py
View File

@ -160,7 +160,7 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
so they round-trip through INI's string-only storage. When reading the
file back into ``Machine.config`` (a JSONField that holds native types)
those strings have to be decoded — otherwise downstream consumers like
``ExecutionPlan.seed_config`` → ``MachineEvent`` → abx-dl see a JSON string
``MachineEvent`` → abx-dl see a JSON string
where they expect a dict and raise ``TypeError``.
Declared fields go through pydantic-settings' own ``field_is_complex`` /
``prepare_field_value`` so they're decoded per annotation. Undeclared

26
archivebox/core/migrations/0054_snapshot_fs_version_index.py Normal file
View File

@ -0,0 +1,26 @@
# Generated by Django 6.1 on 2026-09-02
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("core", "0053_alter_archiveresult_options"),
]
operations = [
migrations.RemoveField(
model_name="snapshot",
name="current_step",
),
migrations.AlterField(
model_name="snapshot",
name="fs_version",
field=models.CharField(
db_index=True,
default="0.9.0",
help_text='Filesystem version of this snapshot (e.g., "0.7.0", "0.8.0", "0.9.4").',
max_length=10,
),
),
]

356
archivebox/core/models.py
View File

@ -13,9 +13,9 @@ from django.conf import settings
from django.contrib import admin
from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError
from django.db import IntegrityError, models, transaction
from django.db.models import Case, F, Q, QuerySet, Sum, Value, When
from django.db.models import F, Q, QuerySet, Sum, Value
from django.db.models.fields.json import KT
from django.db.models.functions import Coalesce, Concat
from django.db.models.functions import Coalesce
from django.urls import reverse_lazy
from django.utils import timezone
from django.utils.functional import cached_property
@ -538,8 +538,6 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)): # ty: ig
class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithQueue):
BROWSER_EXTENSION_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
INTERNAL_INPUT_URL = "archivebox://internal"
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
created_at = models.DateTimeField(default=timezone.now, db_index=True)
modified_at = models.DateTimeField(auto_now=True)
@ -567,14 +565,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
fs_version = models.CharField(
max_length=10,
default="0.9.0",
help_text='Filesystem version of this snapshot (e.g., "0.7.0", "0.8.0", "0.9.0"). Used to trigger lazy migration on save().',
)
current_step = models.PositiveSmallIntegerField(
default=0,
db_index=True,
help_text="Current hook step being executed (0-9). Used for sequential hook execution.",
help_text='Filesystem version of this snapshot (e.g., "0.7.0", "0.8.0", "0.9.4").',
)
retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
status = ModelWithQueue.StatusField(
choices=ModelWithQueue.StatusChoices,
@ -730,20 +723,13 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
return self.update_and_requeue(
status=self.StatusChoices.QUEUED,
retry_at=when or timezone.now(),
current_step=0,
)
def pause(self, *, save: bool = True) -> bool:
paused = super().pause(save=save)
if paused and self.pk:
ArchiveResult.pause_queryset(self.archiveresult_set.all())
return paused
return super().pause(save=save)
def resume(self, *, when: datetime | None = None, save: bool = True) -> bool:
resumed = super().resume(when=when, save=save)
if resumed and self.pk:
ArchiveResult.resume_queryset(self.archiveresult_set.all(), when=when)
return resumed
return super().resume(when=when, save=save)
def restore_paused_scheduler_marker(self) -> None:
"""
@ -817,20 +803,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
modified_at=now,
)
def reset_abandoned_results(self) -> tuple[int, int]:
reset_count = 0
running_count = 0
for result in self.archiveresult_set.filter(
status__in=[ArchiveResult.StatusChoices.STARTED, ArchiveResult.StatusChoices.BACKOFF],
).select_related("process"):
process = result.process
if process is not None and process.is_running:
running_count += 1
continue
result.reset_for_retry()
reset_count += 1
return reset_count, running_count
def start_processing(self) -> bool:
"""Atomically move a claimed queued Snapshot into its active lease."""
owned_retry_at = self.retry_at
@ -880,12 +852,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
if self.status == self.StatusChoices.PAUSED:
return False
if self.status == self.StatusChoices.QUEUED:
results = self.archiveresult_set.all()
if results.exists() and not results.exclude(status__in=ArchiveResult.FINAL_STATES).exists():
return self.seal()
return bool(self.url) and self.start_processing()
if self.status == self.StatusChoices.STARTED and self.is_finished_processing():
return self.seal()
# abx-dl emits SnapshotCompletedEvent after the complete hook sequence;
# ArchiveResult projection state never drives Snapshot completion.
return False
def cancel(self) -> None:
@ -1021,9 +990,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
return False
def validate_url_for_archiving(self, *, config: Mapping[str, Any] | Any | None = None) -> None:
if self.is_internal_input_url():
return
try:
validate_url(self.url or "")
except ValueError as err:
@ -1032,9 +998,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
if self.is_archivebox_internal_url(self.url, config=config):
raise ValidationError({"url": "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs."})
def is_internal_input_url(self) -> bool:
return (self.url or "").strip() == self.INTERNAL_INPUT_URL and self.depth == 0 and bool(self.crawl_id)
def save(self, *args, **kwargs):
update_fields = kwargs.get("update_fields")
validate_url_field = self._state.adding or update_fields is None or "url" in update_fields
@ -1062,18 +1025,13 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
if self._state.adding or update_fields is None or "notes" in update_fields:
self.notes = sanitize_html_text(self.notes)
# Migrate filesystem if needed (happens automatically on save)
# The maintenance runner currently signals its explicit filesystem pass
# with this narrow update. Ordinary model saves must never move archive
# directories as an unrelated metadata side effect.
existing_snapshot = self.pk and not self._state.adding
if existing_snapshot and self.fs_migration_needed:
maintenance_update = update_fields is not None and set(update_fields) == {"retry_at", "modified_at"}
if existing_snapshot and maintenance_update and self.fs_migration_needed:
self.migrate_filesystem_to_current_version()
update_fields = kwargs.get("update_fields")
if update_fields is not None:
kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "fs_version", "modified_at"]))
elif existing_snapshot:
current_dir = self.get_storage_path_for_version(self._fs_current_version())
source_dir = Path(self.output_dir)
if source_dir.exists() and source_dir != current_dir and not source_dir.is_symlink():
self.migrate_filesystem_to_current_version(source_dir=source_dir)
super().save(*args, **kwargs)
@ -1098,50 +1056,13 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
Tag.objects.bulk_create(missing_tags, ignore_conflicts=True)
tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=crawl_tag_names)}
self.add_tag_ids([tag.pk for name in crawl_tag_names if (tag := tags_by_name.get(name))])
# Snapshot.save() normally appends newly created URLs to Crawl.urls
# so legacy/direct crawls can keep their queue text in sync. For
# internal-input crawls that would corrupt the original submitted
# import text; parsed URLs are represented by child Snapshot rows.
if crawl.has_internal_input_root():
return
if not crawl.url_passes_filters(self.url, snapshot=self, use_effective_config=False):
return
# Best-effort skip if our URL is already recorded on the crawl;
# the atomic UPDATE below is what actually prevents clobbering.
crawl.refresh_from_db(fields=["urls"])
if self.url in {url for _raw_line, url in crawl._iter_url_lines() if url}:
return
now = timezone.now()
# Atomic append: SQLite reads `urls` inside the UPDATE statement,
# so concurrent appends never clobber each other (no read-then-write
# window, no CAS retry needed). A rare duplicate URL on a race is
# harmless — downstream consumers dedupe via Snapshot uniqueness.
text = models.TextField()
type(crawl).objects.filter(pk=crawl.pk).update(
urls=Case(
When(Q(urls="") | Q(urls__isnull=True), then=Value(self.url, output_field=text)),
default=Concat(
"urls",
Value("\n", output_field=text),
Value(self.url, output_field=text),
output_field=text,
),
output_field=text,
),
modified_at=now,
)
crawl.modified_at = now
# Crawl.urls remains the original submitted source. Snapshot rows
# are the normalized work queue and discovery projection.
# get_or_create/update_or_create wrap save() in atomic(); defer filesystem
# work and crawl maintenance so SQLite commits before touching the disk.
transaction.on_commit(finish_snapshot_save)
migration_cleanup = self.__dict__.get("_pending_fs_migration_cleanup")
if migration_cleanup:
old_dir, new_dir = migration_cleanup
transaction.on_commit(lambda: self._cleanup_old_migration_dir(old_dir, new_dir))
delattr(self, "_pending_fs_migration_cleanup")
# =========================================================================
# Filesystem Migration Methods
# =========================================================================
@ -1183,30 +1104,38 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
def migrate_filesystem_to_current_version(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None) -> None:
"""
Copy legacy snapshot output into the current layout and defer old-dir cleanup.
Copy legacy snapshot output into the current layout and safely remove the old tree.
The ordering is intentionally crash-safe:
1. Copy from the legacy directory into the new directory idempotently.
2. Verify the new directory has every old file.
3. Convert metadata in the new directory.
4. Update fs_version in memory for the caller to save.
5. Cleanup is scheduled only after the DB commit succeeds.
4. Remove the verified legacy source.
5. Persist fs_version last, so an interruption remains selected by the
indexed stale-version query and resumes naturally.
Re-running this method also reconciles a legacy timestamp directory
left behind when a machine stopped after the database commit but before
the on-commit cleanup callback ran.
"""
current = self.fs_version
target = self._fs_current_version()
cleanup: tuple[Path, Path] | None = None
runtime_config = config or get_config()
if source_dir and current == target:
if current == target:
current_dir = self.get_storage_path_for_version(target)
cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, target_dir=current_dir)
legacy_dir = Path(source_dir) if source_dir else CONSTANTS.ARCHIVE_DIR / self.timestamp
cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=legacy_dir, target_dir=current_dir)
crawl_dir = self.crawl.output_dir
old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex)))
if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink():
crawl_dir.parent.mkdir(parents=True, exist_ok=True)
old_crawl_dir.rename(crawl_dir)
if cleanup:
self._pending_fs_migration_cleanup = cleanup
old_dir, new_dir = cleanup
if not self._cleanup_old_migration_dir(old_dir, new_dir):
raise SnapshotMigrationError(f"Could not clean up verified migration directory: {old_dir}")
return
while current != target:
@ -1231,7 +1160,14 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
source_dir = None
if cleanup:
self._pending_fs_migration_cleanup = cleanup
old_dir, new_dir = cleanup
if not self._cleanup_old_migration_dir(old_dir, new_dir):
raise SnapshotMigrationError(f"Could not clean up verified migration directory: {old_dir}")
if self.pk:
now = timezone.now()
type(self).objects.filter(pk=self.pk).update(fs_version=target, modified_at=now)
self.modified_at = now
def _fs_migrate_from_0_7_0_to_0_9_0(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None):
return self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, config=config)
@ -1277,7 +1213,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
if old_dir == new_dir:
self.convert_index_json_to_jsonl(output_dir=new_dir)
self.hydrate_archiveresult_output_metadata(snapshot_dir=new_dir)
return None
if old_dir.is_symlink():
@ -1288,13 +1223,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
if not points_to_target:
raise SnapshotMigrationError(f"Legacy output symlink does not point to the expected target: {old_dir}")
self.convert_index_json_to_jsonl(output_dir=new_dir)
self.hydrate_archiveresult_output_metadata(snapshot_dir=new_dir)
return None
return (old_dir, new_dir)
if not old_dir.exists():
if new_dir.exists():
self.convert_index_json_to_jsonl(output_dir=new_dir)
self.hydrate_archiveresult_output_metadata(snapshot_dir=new_dir)
return None
return None
@ -1306,7 +1239,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
pass
else:
self.convert_index_json_to_jsonl(output_dir=new_dir)
self.hydrate_archiveresult_output_metadata(snapshot_dir=new_dir)
return (old_dir, new_dir)
def copy_file_without_overwriting(source: str, destination: str):
@ -1352,17 +1284,51 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
# Convert index.json to index.jsonl in the new directory.
self.convert_index_json_to_jsonl(output_dir=new_dir)
self.hydrate_archiveresult_output_metadata(snapshot_dir=new_dir)
return (old_dir, new_dir)
def _cleanup_old_migration_dir(self, old_dir: Path, new_dir: Path):
@staticmethod
def _migration_trees_match(old_dir: Path, new_dir: Path) -> bool:
"""Verify every legacy entry exists unchanged at the destination."""
import filecmp
if old_dir.is_symlink():
try:
return new_dir.exists() and old_dir.resolve() == new_dir.resolve()
except OSError:
return False
if not old_dir.exists():
return True
if not new_dir.is_dir() or new_dir.is_symlink():
return False
for source in old_dir.rglob("*"):
destination = new_dir / source.relative_to(old_dir)
if source.is_symlink():
copied = destination.is_symlink() and destination.readlink() == source.readlink()
elif source.is_dir():
copied = destination.is_dir() and not destination.is_symlink()
elif source.is_file():
copied = destination.is_file() and not destination.is_symlink() and filecmp.cmp(source, destination, shallow=False)
else:
copied = False
if not copied:
return False
return True
def _cleanup_old_migration_dir(self, old_dir: Path, new_dir: Path) -> bool:
"""Delete the old directory after its contents are verified at the new path."""
import logging
import shutil
from archivebox.config.permissions import SudoPermission
if not self._migration_trees_match(old_dir, new_dir):
logging.getLogger("archivebox.migration").warning(
f"Refusing to remove unverified migration directory {old_dir}",
)
return False
# Delete old directory
if old_dir.exists() and not old_dir.is_symlink():
try:
@ -1374,11 +1340,12 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
logging.getLogger("archivebox.migration").warning(
f"Could not remove old migration directory {old_dir}: {e}",
)
return
return False
# Older migration runs may already have left a timestamp projection.
if old_dir.is_symlink():
old_dir.unlink(missing_ok=True)
return True
# =========================================================================
# Path Calculation and Migration Helpers
@ -2796,18 +2763,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
self.remove_tag_ids(existing_tag_ids - tag_ids)
self.add_tag_ids(tag_ids - existing_tag_ids)
def pending_archiveresults(self) -> QuerySet["ArchiveResult"]:
return self.archiveresult_set.exclude(status__in=ArchiveResult.FINAL_OR_ACTIVE_STATES)
def run(self) -> list["ArchiveResult"]:
"""
Execute snapshot by creating pending ArchiveResults for all enabled hooks.
Returns:
list[ArchiveResult]: Newly created pending results
"""
return self.create_pending_archiveresults()
def finalize_output_metadata(self) -> None:
"""
Clean up background ArchiveResult hooks and empty results.
@ -3054,63 +3009,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
return snapshot
def create_pending_archiveresults(self, hooks: Iterable[tuple[str, str]] | None = None) -> list["ArchiveResult"]:
"""
Create ArchiveResult records for all enabled hooks.
Uses the hooks system to discover available hooks from:
- abx_plugins/plugins/*/on_Snapshot__*.{py,sh,js}
- data/custom_plugins/*/on_Snapshot__*.{py,sh,js}
Creates one ArchiveResult per hook (not per plugin), with hook_name set.
This enables step-based execution where all hooks in a step can run in parallel.
"""
try:
self.validate_url_for_archiving()
except ValidationError as err:
rprint(f"[yellow][!] Skipping blocked snapshot URL: {(self.url or '')[:120]}... ({err})[/yellow]")
return []
if hooks is None:
from archivebox.config.common import get_config
from archivebox.plugins.hooks import discover_hooks
# Compatibility path for direct model callers. The runner passes its
# abx-dl hook inventory explicitly so queued rows match execution.
config = get_config(crawl=self.crawl, snapshot=self)
hooks = ((hook_path.parent.name, hook_path.stem) for hook_path in discover_hooks("Snapshot", config=config))
archiveresults = []
for plugin, hook_name in hooks:
# Hooks in one plugin share a filesystem directory, but each hook has
# its own durable result row and retries update that exact row.
archiveresult, _created = ArchiveResult.get_or_create_by_hook(
self,
plugin,
hook_name,
defaults={
"status": ArchiveResult.INITIAL_STATE,
},
)
if archiveresult.status == ArchiveResult.INITIAL_STATE:
archiveresults.append(archiveresult)
return archiveresults
def is_finished_processing(self) -> bool:
"""
Check if all ArchiveResults are finished.
Note: This is only called for observability/progress tracking.
The shared runner owns execution and does not poll this.
"""
# Check if any ARs are still pending/started
pending = self.archiveresult_set.exclude(
status__in=ArchiveResult.FINAL_STATES,
).exists()
return not pending
def get_progress_stats(self) -> dict:
"""
Get progress statistics for this snapshot's archiving process.
@ -3171,34 +3069,20 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
}
def retry_failed_archiveresults(self) -> int:
"""
Reset failed ArchiveResults to queued for retry.
Returns count of ArchiveResults reset.
"""
retryable_results = ArchiveResult.objects.filter(
snapshot=self,
status=ArchiveResult.StatusChoices.FAILED,
"""Queue the parent Snapshot to rerun plugins with failed facts."""
plugins = list(
self.archiveresult_set.filter(status=ArchiveResult.StatusChoices.FAILED)
.exclude(plugin="")
.order_by("plugin")
.values_list("plugin", flat=True)
.distinct(),
)
legacy_result_count = retryable_results.filter(hook_name="").count()
now = timezone.now()
count = retryable_results.exclude(hook_name="").update(
status=ArchiveResult.StatusChoices.QUEUED,
output_str="",
output_json=None,
output_files={},
output_size=0,
output_mimetypes="",
start_ts=None,
end_ts=None,
modified_at=now,
)
if count + legacy_result_count > 0:
self.refresh_from_db(fields=["modified_at", "retry_at", "status"])
self.queue_for_extraction(when=now)
return count + legacy_result_count
if not plugins:
return 0
self.config = {**(self.config or {}), "PLUGINS": ",".join(plugins)}
self.save(update_fields=["config", "modified_at"])
self.queue_for_extraction()
return len(plugins)
# =========================================================================
# URL Helper Properties (migrated from Link schema)
@ -4276,74 +4160,10 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
def get_absolute_url(self):
return f"/{self.snapshot.archive_path}/{self.plugin}"
def reset_for_retry(self, *, save: bool = True) -> None:
self.status = self.StatusChoices.QUEUED
self.retry_at = None
self.output_str = ""
self.output_json = None
self.output_files = {}
self.output_size = 0
self.output_mimetypes = ""
self.start_ts = None
self.end_ts = None
if save:
self.save(
update_fields=[
"status",
"retry_at",
"output_str",
"output_json",
"output_files",
"output_size",
"output_mimetypes",
"start_ts",
"end_ts",
"modified_at",
],
)
@property
def is_paused(self) -> bool:
return self.status == self.StatusChoices.PAUSED
@classmethod
def pause_queryset(cls, queryset) -> int:
return queryset.exclude(status__in=[*cls.FINAL_STATES, cls.StatusChoices.PAUSED]).update(
status=cls.StatusChoices.PAUSED,
retry_at=RETRY_AT_MAX,
modified_at=timezone.now(),
)
@classmethod
def resume_queryset(cls, queryset, *, when: datetime | None = None) -> int:
return queryset.filter(status=cls.StatusChoices.PAUSED).update(
status=cls.StatusChoices.QUEUED,
retry_at=when or timezone.now(),
modified_at=timezone.now(),
)
def pause(self, *, save: bool = True) -> bool:
if self.status in self.FINAL_STATES:
return False
if self.is_paused:
return False
self.status = self.StatusChoices.PAUSED
self.retry_at = RETRY_AT_MAX
if save:
self.pause_queryset(type(self).objects.filter(pk=self.pk))
self.refresh_from_db()
return True
def resume(self, *, when: datetime | None = None, save: bool = True) -> bool:
if not self.is_paused:
return False
self.status = self.StatusChoices.QUEUED
self.retry_at = when or timezone.now()
if save:
self.resume_queryset(type(self).objects.filter(pk=self.pk), when=self.retry_at)
self.refresh_from_db()
return True
@staticmethod
def _normalize_output_files(raw_output_files: Any) -> dict[str, dict[str, Any]]:
from abx_dl.output_files import OutputManifest

82
archivebox/core/recovery_util.py
View File

@ -36,9 +36,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
"crawls_queued_without_retry_at": 0,
"snapshots_queued_without_retry_at": 0,
"snapshots_sealed_with_extension_uploads_only": 0,
"archiveresults_backoff": 0,
"snapshots_queued_plugin_rows_waiting_on_stale_lease": 0,
"archiveresults_started_without_running_process": 0,
"archiveresults_interrupted_without_running_process": 0,
"archiveresults_missing_for_orphaned_hook_processes": 0,
"snapshots_started_without_running_results": 0,
"crawls_started_with_due_snapshots": 0,
@ -46,10 +44,10 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
"crawls_started_without_active_snapshots": 0,
}
running_archiveresults = ArchiveResult.objects.filter(
snapshot_id=OuterRef("pk"),
status=ArchiveResult.StatusChoices.STARTED,
process__status=Process.StatusChoices.RUNNING,
running_hook_processes = Process.objects.filter(
archiveresult__snapshot_id=OuterRef("pk"),
process_type=Process.TypeChoices.HOOK,
status=Process.StatusChoices.RUNNING,
)
active_child_snapshots = Snapshot.objects.filter(
crawl_id=OuterRef("pk"),
@ -109,49 +107,18 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
retry_at=now,
modified_at=now,
)
backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF, **result_filter)
orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED, **result_filter).exclude(
process__status=Process.StatusChoices.RUNNING,
)
# ArchiveResult has no retry_at scheduler. Wake only the parent Snapshots
# for result rows we are about to repair, then requeue those exact rows.
# Using subqueries keeps million-row recovery in SQLite instead of building
# Python ID lists or scanning all sealed snapshots.
Snapshot.objects.filter(
id__in=backoff_results.values("snapshot_id"),
status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
retry_at__isnull=True,
).update(retry_at=now, modified_at=now)
# ArchiveResult rows are projections, never work items. Close interrupted
# projections as failed and wake their parent Snapshot so abx-dl can replay
# the snapshot-level sequence. Indexed subqueries keep this bounded.
Snapshot.objects.filter(
id__in=orphaned_results.values("snapshot_id"),
status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
retry_at__isnull=True,
status=Snapshot.StatusChoices.STARTED,
).update(retry_at=now, modified_at=now)
cleaned["archiveresults_backoff"] = backoff_results.update(status=ArchiveResult.StatusChoices.QUEUED, modified_at=now)
# Targeted plugin rows on final/paused Snapshots are scheduled through the
# parent Snapshot.retry_at. retry_at=NULL is the normal idle marker for a
# sealed Snapshot and must not be interpreted as queued work just because
# old/synthetic ArchiveResult rows exist. If takeover kills the runner
# after it leases the Snapshot but before queued ArchiveResult rows finish,
# the rows remain QUEUED while retry_at sits in the future. Recovery runs
# only after this runner has won the single-runner gate, so it can safely
# unlock those stale plugin leases for immediate processing instead of
# waiting out the previous owner's full lock timeout.
queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED, **result_filter)
cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
Snapshot.objects.filter(
id__in=queued_plugin_results.values("snapshot_id"),
status__in=[Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED],
)
.filter(retry_at__gt=now)
.update(retry_at=now, modified_at=now)
)
# Impossible state repair: STARTED ArchiveResults without a live Process
# have no owner left to emit completion. Requeue only the result row; the
# snapshot/crawl schedulers will pick up normal retry processing.
cleaned["archiveresults_started_without_running_process"] = orphaned_results.update(
status=ArchiveResult.StatusChoices.QUEUED,
process=None,
cleaned["archiveresults_interrupted_without_running_process"] = orphaned_results.update(
status=ArchiveResult.StatusChoices.FAILED,
modified_at=now,
)
orphaned_hook_processes = Process.objects.filter(
@ -206,11 +173,11 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
plugin_dir.name,
hook_name,
defaults={
"status": ArchiveResult.StatusChoices.QUEUED,
"status": ArchiveResult.StatusChoices.FAILED,
},
)
process_is_newer = bool(process.started_at and (result.start_ts is None or process.started_at >= result.start_ts))
if result.status == ArchiveResult.StatusChoices.QUEUED or process_is_newer:
if created or process_is_newer:
requeue_snapshot = False
# A runner can die after the hook Process exits but before the
# ProcessCompletedEvent projector links/finalizes ArchiveResult.
@ -231,16 +198,14 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
result.start_ts = process.started_at
result.end_ts = process.ended_at
if _is_signal_interrupted_exit(process.exit_code):
# The owning runner died or was asked to stop while the hook was
# still active. Keep the work item queued so takeover retries the
# same hook; treating an unknown signal exit as success would
# silently skip unfinished side effects.
# The Process is a durable fact; record the interruption as a
# failure and wake the parent Snapshot for replay.
result.output_files = {}
result.output_size = 0
result.output_mimetypes = ""
result.output_str = ""
result.output_json = None
result.status = ArchiveResult.StatusChoices.QUEUED
result.status = ArchiveResult.StatusChoices.FAILED
requeue_snapshot = True
else:
result.output_files = output_files
@ -280,7 +245,6 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
if created:
cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1
Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
Q(retry_at__isnull=True) | Q(retry_at__gt=now),
**snapshot_filter,
@ -294,8 +258,8 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
# We only unlock scheduling; normal Snapshot runner code owns the next
# transition and side effects.
cleaned["snapshots_started_without_running_results"] = (
started_snapshots.annotate(has_running_results=Exists(running_archiveresults))
.filter(has_running_results=False)
started_snapshots.annotate(has_running_process=Exists(running_hook_processes))
.filter(has_running_process=False)
.update(
retry_at=now,
modified_at=now,
@ -352,13 +316,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
"Finishing {count} browser-extension Snapshot(s) that received uploaded files before server extractors started "
"(uploaded and server-created results will remain together on the same Snapshot)."
),
"archiveresults_backoff": (
"Retrying {count} extractor result(s) that were waiting to retry "
"(ArchiveBox may have been interrupted before it was able to try them again; affected outputs will be retried)."
),
"archiveresults_started_without_running_process": (
"Retrying {count} extractor result(s) that were interrupted before finishing "
"(ArchiveBox may have been interrupted before it was able to save their final status; partial files will be overwritten with fresh results upon retry)."
"archiveresults_interrupted_without_running_process": (
"Closing {count} interrupted extractor result projection(s) "
"(their parent Snapshots are resumed through the normal snapshot-level runner)."
),
"snapshots_started_without_running_results": (
"Resuming {count} Snapshot(s) that were interrupted before finishing "

67
archivebox/crawls/models.py
View File

@ -90,6 +90,8 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
@property
def last_run_at(self):
if self.kind == "update":
return self.modified_at
latest_crawl = self.crawl_set.order_by("-created_at").first()
if latest_crawl:
return latest_crawl.created_at
@ -105,6 +107,22 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
now = now or timezone.now()
return self.is_enabled and self.next_run_at <= now
@property
def kind(self) -> str:
return str((self.config or {}).get("SCHEDULE_KIND") or "crawl")
def dispatch(self, queued_at=None) -> "Crawl | None":
"""Run maintenance directly or enqueue one ordinary Crawl."""
queued_at = queued_at or timezone.now()
if self.kind == "update":
from archivebox.cli.archivebox_update import process_all_db_snapshots
process_all_db_snapshots()
type(self).objects.filter(pk=self.pk).update(modified_at=queued_at)
self.modified_at = queued_at
return None
return self.enqueue(queued_at=queued_at)
def enqueue(self, queued_at=None) -> "Crawl":
from archivebox.config.common import build_crawl_config_snapshot
@ -112,10 +130,11 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
template = self.template
label = template.label or self.label
persona = template.persona if template.persona_id else None
crawl_config = {key: value for key, value in (self.config or {}).items() if key != "SCHEDULE_KIND"}
return Crawl.objects.create(
urls=template.urls,
config=build_crawl_config_snapshot(persona=persona, overrides=self.config or {}),
config=build_crawl_config_snapshot(persona=persona, overrides=crawl_config),
max_depth=template.max_depth,
tags_str=template.tags_str,
persona_id=template.persona_id,
@ -217,7 +236,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
def resume(self, *, when=None, save: bool = True) -> bool:
resumed = super().resume(when=when, save=save)
if resumed and self.pk:
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.models import Snapshot
resume_at = when or timezone.now()
active_snapshots = self.snapshot_set.filter(
@ -228,7 +247,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
retry_at=resume_at,
modified_at=timezone.now(),
)
ArchiveResult.resume_queryset(ArchiveResult.objects.filter(snapshot__crawl=self), when=resume_at)
return resumed
def cancel(self) -> None:
@ -269,8 +287,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
now = timezone.now()
# Parent pause is a scheduler command. Wake child rows only; each
# Snapshot runner claim performs the real pause transition and cascades
# its own ArchiveResults, keeping request/admin transactions tiny.
# Snapshot runner claim performs the real pause transition, keeping
# request/admin transactions tiny.
active_children = self.snapshot_set.filter(
status__in=Snapshot.RUNNABLE_STATES,
)
@ -515,18 +533,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
return []
return [url for _raw_line, url in self._iter_url_lines() if url]
def has_internal_input_root(self) -> bool:
"""Return True when Crawl.urls is preserved source text, not the work queue.
The runner creates a synthetic root snapshot for raw import text so
parser hooks use the same Snapshot lifecycle as every other extractor.
In that mode the submitted text must remain in Crawl.urls forever;
parsed URLs live as child Snapshot rows and should not be appended back.
"""
from archivebox.core.models import Snapshot
return self.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL, depth=0).exists()
@staticmethod
def normalize_domain(value: str) -> str:
candidate = (value or "").strip().lower()
@ -796,15 +802,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
"deleted_snapshots": filter_result["deleted_snapshots"],
}
def get_system_task(self) -> str | None:
urls = self.get_urls_list()
if len(urls) != 1:
return None
system_url = urls[0].strip().lower()
if system_url.startswith("archivebox://"):
return system_url
return None
def resolve_persona(self):
from archivebox.personas.models import Persona
@ -968,14 +965,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
if self.status == self.StatusChoices.SEALED:
return []
# Internal-input crawls preserve the submitted text verbatim in
# Crawl.urls. The root snapshot's parser hooks are the only supported
# path for turning that text into child snapshots, otherwise a later
# runner pass could reinterpret plain URL-looking lines as direct
# depth-0 work and bypass format-specific metadata parsing.
if self.has_internal_input_root():
return []
created_snapshots = []
crawl_tag_names = self.current_tag_names()
tags_by_name: dict[str, Tag] = {}
@ -1248,16 +1237,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
if not created_snapshots:
return []
crawl_urls = {url for _raw_line, url in self._iter_url_lines() if url}
new_url_lines = [snapshot.url for snapshot in created_snapshots if snapshot.url not in crawl_urls]
# For internal-input crawls, Crawl.urls is the immutable source text.
# Child snapshots are the parsed/indexed representation, so appending
# discovered URLs here would both duplicate state and destroy the exact
# import artifact users submitted through CLI/API/UI.
if new_url_lines and not self.has_internal_input_root():
self.urls = (self.urls.rstrip() + "\n" + "\n".join(new_url_lines)).lstrip("\n")
self.save(update_fields=["urls", "modified_at"])
tag_names_by_url: dict[str, set[str]] = {}
for snapshot in created_snapshots:
tag_names = {

142
archivebox/services/archive_result_service.py
View File

@ -2,9 +2,7 @@ from __future__ import annotations
import asyncio
import inspect
import json
import os
import signal
import sys
import time
from contextlib import contextmanager
@ -15,7 +13,7 @@ from typing import Any
from asgiref.sync import sync_to_async
from django.utils import timezone
from abx_dl.events import PROCESS_EXIT_SKIPPED, ArchiveResultEvent, ProcessCompletedEvent, ProcessStartedEvent, SnapshotEvent
from abx_dl.events import ArchiveResultEvent, ProcessStartedEvent
from abx_dl.output_files import OutputManifest
from abx_dl.services.base import BaseService
@ -121,37 +119,6 @@ def _should_update_snapshot_title(current_title: str, next_title: str, *, snapsh
return len(next_title) > len(current)
def _status_for_process_without_archive_result(event: ProcessCompletedEvent) -> str:
if event.exit_code == PROCESS_EXIT_SKIPPED:
return "skipped"
if event.exit_code in {128 + signal.SIGHUP, 128 + signal.SIGINT, 128 + signal.SIGTERM}:
# This fallback only runs when a snapshot hook exited before emitting a
# structured ArchiveResult. A polite shutdown signal means the runner
# was interrupted during ownership transfer, not that the extractor
# produced a durable negative result. Keep the hook queued so the next
# runner can retry the exact work item instead of sealing in a transient
# process-lifecycle failure.
return "queued"
if event.exit_code != 0:
return "failed"
return "noresults"
def _iter_archiveresult_records(stdout: str) -> list[dict]:
records: list[dict] = []
for raw_line in stdout.splitlines():
line = raw_line.strip()
if not line.startswith("{"):
continue
try:
record = json.loads(line)
except json.JSONDecodeError:
continue
if record.get("type") == "ArchiveResult":
records.append(record)
return records
@_perf_trace("archivebox.ArchiveResultService._save_archiveresult_event_sync")
def _save_archiveresult_event_to_db(
event: ArchiveResultEvent,
@ -234,13 +201,6 @@ def _save_archiveresult_event_to_db(
with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_update"):
result.save(update_fields=[*update_fields, "modified_at"])
if result.status == ArchiveResult.StatusChoices.QUEUED:
# ArchiveResult has no retry_at column. If a shutdown/takeover projects
# a killed hook back to QUEUED, wake the parent Snapshot/Crawl so the
# next runner retries that exact hook instead of waiting on a stale
# active-state lease.
snapshot.update_and_requeue(retry_at=timezone.now())
if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.title_update"):
title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else ""
@ -264,36 +224,43 @@ def _save_archiveresult_event_to_db(
def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str, process_id: str) -> None:
"""Advance an existing queued hook row after its OS process is persisted."""
from archivebox.core.models import ArchiveResult
"""Project a running abx-dl hook after its OS process is persisted."""
from archivebox.core.models import ArchiveResult, Snapshot
started_at = parse_event_datetime(event.start_ts)
if started_at is None:
raise ValueError("ProcessStartedEvent.start_ts is required")
ArchiveResult.objects.filter(
snapshot_id=snapshot_id,
plugin=event.plugin_name,
hook_name=event.hook_name,
status=ArchiveResult.StatusChoices.QUEUED,
).update(
status=ArchiveResult.StatusChoices.STARTED,
start_ts=started_at,
end_ts=None,
process_id=process_id,
modified_at=timezone.now(),
snapshot = Snapshot.objects.filter(id=snapshot_id).first()
if snapshot is None:
return
result, _created = ArchiveResult.get_or_create_by_hook(
snapshot,
event.plugin_name,
event.hook_name,
defaults={
"status": ArchiveResult.StatusChoices.STARTED,
"start_ts": started_at,
"end_ts": None,
"process_id": process_id,
},
)
result.status = ArchiveResult.StatusChoices.STARTED
result.start_ts = started_at
result.end_ts = None
result.process_id = process_id
result.save(update_fields=["status", "start_ts", "end_ts", "process_id", "modified_at"])
class ArchiveResultService(BaseService):
LISTENS_TO = [ArchiveResultEvent, ProcessCompletedEvent]
"""Project abx-dl ArchiveResult facts into Django models."""
LISTENS_TO = [ArchiveResultEvent]
EMITS = []
def __init__(self, bus):
self._completed_process_event_ids: set[str] = set()
self._save_locks: dict[tuple[str, str, str], asyncio.Lock] = {}
super().__init__(bus)
self.bus.on(ArchiveResultEvent, self.on_ArchiveResultEvent__save_to_db)
self.bus.on(ProcessCompletedEvent, self.on_ProcessCompletedEvent__save_to_db)
@_perf_trace("archivebox.ArchiveResultService.on_ArchiveResultEvent__save_to_db")
async def on_ArchiveResultEvent__save_to_db(self, event: ArchiveResultEvent) -> None:
@ -309,64 +276,3 @@ class ArchiveResultService(BaseService):
lock = self._save_locks.setdefault(key, asyncio.Lock())
async with lock:
await sync_to_async(_save_archiveresult_event_to_db, thread_sensitive=True)(event, process_started)
@_perf_trace("archivebox.ArchiveResultService.on_ProcessCompletedEvent__save_to_db")
async def on_ProcessCompletedEvent__save_to_db(self, event: ProcessCompletedEvent) -> None:
if event.event_id in self._completed_process_event_ids:
return
self._completed_process_event_ids.add(event.event_id)
if not event.hook_name.startswith("on_Snapshot"):
return
with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.find_snapshot_event"):
snapshot_event = await self.bus.find(
SnapshotEvent,
past=True,
future=False,
where=lambda candidate: self.bus.event_is_child_of(event, candidate),
)
if snapshot_event is None:
return
with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.parse_stdout_records"):
records = _iter_archiveresult_records(event.stdout)
if records:
if len(records) > 1:
raise RuntimeError(
f"Hook {event.plugin_name}:{event.hook_name} emitted {len(records)} ArchiveResult records; expected exactly one",
)
for record in records:
record_status = _normalize_status(record.get("status") or "")
record_failed = record_status == "failed" or (not record_status and event.exit_code not in (0, PROCESS_EXIT_SKIPPED))
with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_record"):
await event.emit(
ArchiveResultEvent(
snapshot_id=record.get("snapshot_id") or snapshot_event.snapshot_id,
plugin=record.get("plugin") or event.plugin_name,
hook_name=record.get("hook_name") or event.hook_name,
status=record_status,
output_str=record.get("output_str") or "",
output_json=record.get("output_json") if isinstance(record.get("output_json"), dict) else None,
output_files=event.output_files,
start_ts=event.start_ts,
end_ts=event.end_ts,
error=record.get("error") or (event.stderr if record_failed else ""),
),
).now()
return
process_failed = _status_for_process_without_archive_result(event) == "failed"
with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"):
await event.emit(
ArchiveResultEvent(
snapshot_id=snapshot_event.snapshot_id,
plugin=event.plugin_name,
hook_name=event.hook_name,
status=_status_for_process_without_archive_result(event),
output_str=event.stderr if process_failed else "",
output_files=event.output_files,
start_ts=event.start_ts,
end_ts=event.end_ts,
error=event.stderr if process_failed else "",
),
).now()

789
archivebox/services/runner.py
View File

File diff suppressed because it is too large Load Diff

30
archivebox/services/snapshot_service.py
View File

@ -1,12 +1,9 @@
from __future__ import annotations
import sys
from pathlib import Path
from asgiref.sync import sync_to_async
from django.utils import timezone
from django.core.exceptions import ValidationError
from rich import print as rprint
from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent
from abx_dl.limits import CrawlLimitState
from abx_dl.services.base import BaseService
@ -76,7 +73,11 @@ def finalize_completed_snapshot(
if snapshot.status == Snapshot.StatusChoices.QUEUED:
snapshot.advance_lifecycle()
snapshot.refresh_from_db()
if snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.is_finished_processing():
# SnapshotCompletedEvent is abx-dl's authoritative signal that the complete
# snapshot hook sequence (including cleanup) finished. ArchiveResult rows
# are projections of that work, never prerequisites used to decide whether
# the Snapshot may seal.
if snapshot.status == Snapshot.StatusChoices.STARTED:
snapshot.seal()
snapshot.refresh_from_db()
@ -113,26 +114,7 @@ class SnapshotService(BaseService):
if snapshot.is_paused:
return
if snapshot.status == Snapshot.StatusChoices.QUEUED:
if not await snapshot.archiveresult_set.aexists():
from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults
hooks = await sync_to_async(snapshot_hooks_for_pending_archiveresults, thread_sensitive=True)(snapshot)
await sync_to_async(snapshot.create_pending_archiveresults, thread_sensitive=True)(hooks=hooks)
try:
await sync_to_async(snapshot.advance_lifecycle, thread_sensitive=True)()
except ValidationError as err:
if "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs." not in str(err):
raise
await Snapshot.objects.filter(id=snapshot.id).aupdate(
status=Snapshot.StatusChoices.SEALED,
retry_at=None,
modified_at=timezone.now(),
)
rprint(
f"[red][X] Refusing to archive ArchiveBox internal URL for security: {snapshot.url}[/red]",
file=sys.stderr,
)
return
await sync_to_async(snapshot.advance_lifecycle, thread_sensitive=True)()
await sync_to_async(snapshot.refresh_from_db, thread_sensitive=True)()
elif snapshot.status != Snapshot.StatusChoices.STARTED:
return

16
archivebox/tests/test_api_v1_cli_add.py
View File

@ -264,7 +264,7 @@ def test_basic_success_case_request(client, tmp_path, api_headers):
assert response.status_code == 200, response.content
assert response.json()["success"] is True
crawl = Crawl.objects.get()
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": submitted_url, "depth": 0}
assert crawl.urls == submitted_url
assert Snapshot.objects.count() == 0
@ -322,10 +322,7 @@ def test_api_cli_add_concurrent_first_time_default_persona_creation(tmp_path):
crawls = list(Crawl.objects.order_by("urls").values_list("urls", flat=True))
assert Snapshot.objects.count() == 0
expected_crawl_sources = sorted(
json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in submitted_urls
)
assert crawls == expected_crawl_sources
assert crawls == sorted(submitted_urls)
@pytest.mark.timeout(360)
@ -366,15 +363,10 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls(
stop_archivebox_process(api_server)
api_server = None
run_queued_crawls(tmp_path, env=env, timeout=240)
with use_archivebox_db(tmp_path):
root_counts = {
str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()
}
assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
with use_archivebox_db(tmp_path):
for crawl in Crawl.objects.all():
root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert not crawl.snapshot_set.filter(url__startswith="archivebox://").exists()
root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert root_input == crawl.urls
api_server = start_api_server_without_runner(tmp_path, env, port)
assert_expected_import_snapshots(tmp_path, expected_urls)

44
archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py
View File

@ -102,7 +102,7 @@ def test_snapshot_pause_wins_over_concurrent_runner_lease(api_admin_user):
assert snapshot.retry_at == RETRY_AT_MAX
def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_finished_rows(
def test_snapshot_pause_resume_api_leaves_archiveresult_facts_unchanged(
request,
tmp_path,
client,
@ -159,10 +159,10 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
url=blocking_http_server.url,
status=Snapshot.StatusChoices.QUEUED,
retry_at=now,
config={"PLUGINS": "wget"},
)
Crawl.objects.filter(pk=snapshot.crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now)
snapshot.refresh_from_db()
[started_result] = snapshot.create_pending_archiveresults(hooks=[("wget", _snapshot_hook_name("wget"))])
errors = []
def run_snapshot():
@ -185,10 +185,13 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
request.addfinalizer(finish_runner)
blocking_http_server.request_started.wait()
assert errors == []
started_result.refresh_from_db()
started_result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget")
assert started_result.status == ArchiveResult.StatusChoices.STARTED
[queued_result] = snapshot.create_pending_archiveresults(
hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")],
queued_result = ArchiveResult.objects.create(
snapshot=snapshot,
plugin="parse_txt_urls",
hook_name="on_Snapshot__71_parse_txt_urls",
status=ArchiveResult.StatusChoices.QUEUED,
)
invalid_response = api_client_request(
@ -222,8 +225,8 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id])
}
assert active_rows == {
"parse_txt_urls": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
"wget": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
"parse_txt_urls": (ArchiveResult.StatusChoices.QUEUED, None),
"wget": (ArchiveResult.StatusChoices.STARTED, None),
}
finished_rows = {
@ -262,12 +265,7 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
resumed_rows = {
row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id])
}
assert resumed_rows["parse_txt_urls"][0] == ArchiveResult.StatusChoices.QUEUED
assert resumed_rows["parse_txt_urls"][1] is not None
assert resumed_rows["parse_txt_urls"][1] != RETRY_AT_MAX
assert resumed_rows["wget"][0] == ArchiveResult.StatusChoices.QUEUED
assert resumed_rows["wget"][1] is not None
assert resumed_rows["wget"][1] != RETRY_AT_MAX
assert resumed_rows == active_rows
assert ArchiveResult.objects.get(id=succeeded_result.id).status == ArchiveResult.StatusChoices.SUCCEEDED
assert ArchiveResult.objects.get(id=failed_result.id).status == ArchiveResult.StatusChoices.FAILED
@ -317,8 +315,11 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh
assert "wget failed (exit=4)" in wget_result.output_str
Snapshot.objects.filter(pk=snapshot.pk).update(url=recursive_test_site["root_url"])
snapshot.refresh_from_db()
[unrelated_result] = snapshot.create_pending_archiveresults(
hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")],
unrelated_result = ArchiveResult.objects.create(
snapshot=snapshot,
plugin="parse_txt_urls",
hook_name="on_Snapshot__71_parse_txt_urls",
status=ArchiveResult.StatusChoices.QUEUED,
)
snapshot.output_dir.mkdir(parents=True, exist_ok=True)
(snapshot.output_dir / "source.txt").write_text("finished row must survive targeted retry", encoding="utf-8")
@ -343,7 +344,7 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh
assert snapshot.status == Snapshot.StatusChoices.PAUSED
assert snapshot.retry_at == RETRY_AT_MAX
assert ArchiveResult.objects.get(id=wget_result.id).status == ArchiveResult.StatusChoices.FAILED
assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.PAUSED
assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.QUEUED
finished_row = ArchiveResult.objects.get(id=finished_result.id)
finished_output_path = Path(snapshot.output_dir) / finished_row.plugin / next(iter(finished_row.output_files))
assert finished_output_path.is_file()
@ -365,10 +366,9 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh
with use_archivebox_db(tmp_path):
snapshot = Snapshot.objects.get(id=snapshot_id)
assert snapshot.status == Snapshot.StatusChoices.STARTED
assert snapshot.retry_at is not None
assert snapshot.retry_at != RETRY_AT_MAX
assert snapshot.crawl.status == snapshot.crawl.StatusChoices.STARTED
assert snapshot.status == Snapshot.StatusChoices.SEALED
assert snapshot.retry_at is None
assert snapshot.crawl.status == snapshot.crawl.StatusChoices.SEALED
retried_wget = ArchiveResult.objects.get(id=wget_result.id)
assert retried_wget.status == ArchiveResult.StatusChoices.SUCCEEDED
@ -376,8 +376,8 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh
assert retried_wget.output_files
unrelated = ArchiveResult.objects.get(id=unrelated_result.id)
assert unrelated.status == ArchiveResult.StatusChoices.PAUSED
assert unrelated.retry_at == RETRY_AT_MAX
assert unrelated.status == ArchiveResult.StatusChoices.QUEUED
assert unrelated.retry_at is None
finished = ArchiveResult.objects.get(id=finished_result.id)
assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED

33
archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py
View File

@ -12,7 +12,7 @@ from django.utils import timezone
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.tests.test_orm_helpers import use_archivebox_db
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook, _snapshot_hook_name
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook
from archivebox.workers.models import RETRY_AT_MAX
from .conftest import (
@ -126,7 +126,7 @@ def test_crawl_pause_wins_over_concurrent_runner_lease(api_admin_user):
assert crawl.retry_at == RETRY_AT_MAX
def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone(
def test_crawl_pause_resume_api_leaves_archiveresult_facts_unchanged(
request,
tmp_path,
client,
@ -210,9 +210,12 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
)
now = timezone.now()
Crawl.objects.filter(pk=crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now)
Snapshot.objects.filter(pk=active_snapshot.pk).update(status=Snapshot.StatusChoices.QUEUED, retry_at=now)
Snapshot.objects.filter(pk=active_snapshot.pk).update(
status=Snapshot.StatusChoices.QUEUED,
retry_at=now,
config={"PLUGINS": "wget"},
)
active_snapshot.refresh_from_db()
[active_started] = active_snapshot.create_pending_archiveresults(hooks=[("wget", _snapshot_hook_name("wget"))])
errors = []
def run_snapshot():
@ -235,10 +238,13 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
request.addfinalizer(finish_runner)
blocking_http_server.request_started.wait()
assert errors == []
active_started.refresh_from_db()
active_started = ArchiveResult.objects.get(snapshot=active_snapshot, plugin="wget")
assert active_started.status == ArchiveResult.StatusChoices.STARTED
[active_queued] = active_snapshot.create_pending_archiveresults(
hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")],
active_queued = ArchiveResult.objects.create(
snapshot=active_snapshot,
plugin="parse_txt_urls",
hook_name="on_Snapshot__71_parse_txt_urls",
status=ArchiveResult.StatusChoices.QUEUED,
)
pause_response = api_client_request(
client,
@ -260,11 +266,11 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
assert sealed_snapshot.retry_at is None
paused_rows = {
unchanged_rows = {
row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id])
}
assert paused_rows["parse_txt_urls"] == (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX)
assert paused_rows["wget"] == (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX)
assert unchanged_rows["parse_txt_urls"] == (ArchiveResult.StatusChoices.QUEUED, None)
assert unchanged_rows["wget"] == (ArchiveResult.StatusChoices.STARTED, None)
active_done_row = ArchiveResult.objects.get(id=active_done.id)
sealed_done_row = ArchiveResult.objects.get(id=sealed_done.id)
@ -302,12 +308,7 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
resumed_rows = {
row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id])
}
assert resumed_rows["parse_txt_urls"][0] == ArchiveResult.StatusChoices.QUEUED
assert resumed_rows["parse_txt_urls"][1] is not None
assert resumed_rows["parse_txt_urls"][1] != RETRY_AT_MAX
assert resumed_rows["wget"][0] == ArchiveResult.StatusChoices.QUEUED
assert resumed_rows["wget"][1] is not None
assert resumed_rows["wget"][1] != RETRY_AT_MAX
assert resumed_rows == unchanged_rows
assert ArchiveResult.objects.get(id=active_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
assert ArchiveResult.objects.get(id=sealed_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
assert active_done_path.is_file()

18
archivebox/tests/test_archive_result_service.py
View File

@ -48,6 +48,7 @@ def _run_shipped_snapshot_hook(
import asyncio
from abx_dl.services.process_service import ProcessService as HookProcessService
from abx_dl.services.archive_result_service import ArchiveResultService as HookArchiveResultService
from abx_plugins.plugins.base.utils import get_hydrated_required_binaries
from archivebox.core.models import ArchiveResult
from archivebox.machine.models import Process
@ -76,6 +77,7 @@ def _run_shipped_snapshot_hook(
output_dir.mkdir(parents=True, exist_ok=True)
bus = create_bus(name=f"test_real_{plugin}_{snapshot.id}")
HookProcessService(bus, emit_jsonl=False, interactive_tty=False)
HookArchiveResultService(bus, emit_jsonl=False)
PersistedProcessService(bus)
ArchiveResultService(bus)
@ -291,7 +293,6 @@ def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_p
assert result.status == ArchiveResult.StatusChoices.FAILED
assert result.process_id == process.id
assert "Chrome session" in result.output_str
assert result.output_str in result.notes
_cleanup_machine_process_rows()
@ -464,15 +465,12 @@ def test_retry_failed_archiveresults_requeues_snapshot_in_queued_state():
assert reset_count == 1
assert snapshot.status == Snapshot.StatusChoices.QUEUED
assert snapshot.retry_at is not None
assert snapshot.current_step == 0
assert result.status == ArchiveResult.StatusChoices.QUEUED
assert result.output_str == ""
assert result.output_json is None
assert result.output_files == {}
assert result.output_size == 0
assert result.output_mimetypes == ""
assert result.start_ts is None
assert result.end_ts is None
assert snapshot.config["PLUGINS"] == "chrome"
assert result.status == ArchiveResult.StatusChoices.FAILED
assert result.output_str == "timed out"
assert result.output_files == {"stderr.log": {}}
assert result.output_size == 123
assert result.output_mimetypes == "text/plain"
assert ArchiveResult.objects.get(snapshot=snapshot, plugin="ublock").status == ArchiveResult.StatusChoices.SKIPPED
assert ArchiveResult.objects.get(snapshot=snapshot, plugin="forumdl").status == ArchiveResult.StatusChoices.NORESULTS
snapshot.refresh_from_db()

30
archivebox/tests/test_cli_add.py
View File

@ -225,7 +225,7 @@ def test_add_single_url_records_url_in_crawl(initialized_archive):
crawl = Crawl.objects.get()
snapshots = list(Snapshot.objects.all())
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
assert crawl.urls == "https://example.com"
assert crawl.get_urls_list() == ["https://example.com"]
assert snapshots == []
@ -295,13 +295,10 @@ def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initial
assert crawl.urls == source_text
run_queued_crawls(initialized_archive, env=env, timeout=240)
with use_archivebox_db(initialized_archive):
root_counts = {str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()}
assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
with use_archivebox_db(initialized_archive):
for crawl in Crawl.objects.all():
root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert not crawl.snapshot_set.filter(url__startswith="archivebox://").exists()
root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert root_input == crawl.urls
assert_expected_import_snapshots(initialized_archive, expected_urls)
@ -387,7 +384,7 @@ def test_run_rejects_file_url_injected_directly_into_crawl_urls_with_db_update(i
status=Crawl.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
bad_jsonl = json.dumps({"type": "CrawlSeed", "url": file_url, "depth": 0, "tags": "sql-file-url"})
bad_jsonl = json.dumps({"type": "Snapshot", "url": file_url, "depth": 0, "tags": "sql-file-url"})
Crawl.objects.filter(pk=crawl.pk).update(urls=bad_jsonl)
result = run_archivebox_cmd(
@ -480,7 +477,7 @@ def test_add_bg_queues_direct_url_snapshot(initialized_archive):
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is not None
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
assert crawl.urls == "https://example.com"
assert snapshots == []
@ -622,7 +619,7 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
assert crawl.urls == "https://example.com"
assert snapshot_urls == set()
run_queued_crawls(initialized_archive, env)
@ -633,7 +630,7 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi
assert crawl.status == Crawl.StatusChoices.SEALED
assert crawl.retry_at is None
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
assert crawl.urls == "https://example.com"
assert snapshot_urls == set()
@ -658,9 +655,7 @@ def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive):
crawl = Crawl.objects.get()
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
assert [json.loads(line) for line in crawl.urls.splitlines()] == [
{"type": "CrawlSeed", "url": url, "depth": 0} for url in internal_urls
]
assert crawl.urls.splitlines() == internal_urls
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert snapshot_urls == set()
@ -713,10 +708,7 @@ def test_add_multiple_urls_single_command(initialized_archive):
crawl = Crawl.objects.get()
snapshots = list(Snapshot.objects.order_by("url").values_list("url", "depth"))
assert [json.loads(line) for line in crawl.urls.splitlines()] == [
{"type": "CrawlSeed", "url": "https://example.com", "depth": 0},
{"type": "CrawlSeed", "url": "https://example.org", "depth": 0},
]
assert crawl.urls.splitlines() == ["https://example.com", "https://example.org"]
assert snapshots == []
@ -994,7 +986,7 @@ def test_add_index_only_queues_crawl_without_starting_runner(initialized_archive
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
assert crawl.urls == "https://example.com"
assert snapshots == []
@ -1012,7 +1004,7 @@ def test_add_index_only_creates_direct_url_snapshot(initialized_archive):
crawl = Crawl.objects.get()
snapshot = Snapshot.objects.get()
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
assert crawl.urls == "https://example.com"
assert snapshot.url == "https://example.com"
assert snapshot.depth == 0

11
archivebox/tests/test_cli_extract_input.py
View File

@ -22,7 +22,7 @@ def create_extract_snapshot(initialized_archive, env, url="https://example.com")
)
def test_extract_archiveresult_record_queues_only_exact_hook(initialized_archive):
def test_extract_archiveresult_record_queues_parent_snapshot_plugin(initialized_archive):
env = cli_env(PLUGINS="archivewebpage")
create_extract_snapshot(initialized_archive, env)
@ -49,10 +49,11 @@ def test_extract_archiveresult_record_queues_only_exact_hook(initialized_archive
assert result.returncode == 0, result.stderr or result.stdout
with use_archivebox_db(initialized_archive):
rows = list(
ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin="archivewebpage").values_list("hook_name", "status"),
)
assert rows == [(hook_name, ArchiveResult.StatusChoices.QUEUED)]
snapshot = Snapshot.objects.get(id=snapshot_id)
rows = list(ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin="archivewebpage"))
assert rows == []
assert snapshot.status == Snapshot.StatusChoices.QUEUED
assert snapshot.config["PLUGINS"] == "archivewebpage"
def test_extract_runs_on_snapshot_id(initialized_archive):

2
archivebox/tests/test_cli_list.py
View File

@ -232,7 +232,6 @@ def test_list_limit_zero_streams_one_million_snapshots_without_materializing(ini
fs_version,
crawl_id,
config,
current_step,
depth,
notes,
num_uses_failed,
@ -256,7 +255,6 @@ def test_list_limit_zero_streams_one_million_snapshots_without_materializing(ini
%s,
'{}',
0,
0,
'',
0,
0,

432
archivebox/tests/test_cli_run.py
View File

@ -240,7 +240,7 @@ class TestRunWithArchiveResult:
"""Tests for `archivebox run` with ArchiveResult input."""
@pytest.mark.django_db(transaction=True)
def test_run_creates_and_runs_exact_no_id_archiveresult_request(self, initialized_archive):
def test_run_treats_no_id_archiveresult_as_parent_snapshot_plugin_request(self, initialized_archive):
import json
from archivebox.core.models import ArchiveResult
@ -284,16 +284,13 @@ class TestRunWithArchiveResult:
"output_str",
),
)
assert rows == [
(
missing_hook,
ArchiveResult.StatusChoices.FAILED,
"Queued hook is no longer available in the installed plugin",
),
]
assert len(rows) == 1
assert rows[0][0] != missing_hook
assert rows[0][0].startswith("on_Snapshot__")
assert rows[0][1] in ArchiveResult.FINAL_STATES
def test_run_requeues_failed_archiveresult(self, initialized_archive):
"""Run re-queues a failed ArchiveResult."""
"""Run uses a failed ArchiveResult as a parent Snapshot/plugin reference."""
url = create_test_url()
# Create snapshot and archive result
@ -905,43 +902,6 @@ class TestRecoverOrchestratorState:
assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
assert sealed_snapshot.retry_at is not None
def test_recover_orchestrator_state_requeues_backoff_archiveresults(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.SEALED,
retry_at=None,
)
snapshot = Snapshot.objects.create(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.SEALED,
retry_at=None,
)
result = ArchiveResult.objects.create(
snapshot=snapshot,
plugin="search_backend_sqlite",
hook_name="on_Snapshot__90_index_sqlite",
status=ArchiveResult.StatusChoices.BACKOFF,
)
recovered = recover_orchestrator_state()
result.refresh_from_db()
snapshot.refresh_from_db()
crawl.refresh_from_db()
assert recovered["archiveresults_backoff"] == 1
assert result.status == ArchiveResult.StatusChoices.QUEUED
assert snapshot.status == Snapshot.StatusChoices.SEALED
assert snapshot.retry_at is not None
assert crawl.status == Crawl.StatusChoices.SEALED
def test_recover_orchestrator_state_leaves_due_queued_snapshot_for_runner_even_with_final_results(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import ArchiveResult, Snapshot
@ -980,6 +940,7 @@ class TestRecoverOrchestratorState:
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is not None
@pytest.mark.django_db(transaction=True)
def test_recover_orchestrator_state_leaves_stale_queued_final_rows_for_runner(self):
from datetime import timedelta
@ -989,12 +950,13 @@ class TestRecoverOrchestratorState:
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_crawl, run_due_snapshot
from archivebox.services.runner import run_due_snapshot
old = timezone.now() - timedelta(hours=13)
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
config={"PLUGINS": "__archivebox_test_no_plugins__"},
status=Crawl.StatusChoices.QUEUED,
retry_at=old,
)
@ -1033,16 +995,11 @@ class TestRecoverOrchestratorState:
assert snapshot.status == Snapshot.StatusChoices.SEALED
assert snapshot.retry_at is None
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at == old
assert run_due_crawl(crawl, lock_seconds=60) is True
crawl.refresh_from_db()
assert crawl.status == Crawl.StatusChoices.SEALED
assert crawl.retry_at is None
def test_run_due_snapshot_seals_queued_snapshot_with_final_results(self):
@pytest.mark.django_db(transaction=True)
def test_run_due_snapshot_runs_snapshot_without_consulting_final_result_rows(self):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
@ -1053,6 +1010,7 @@ class TestRecoverOrchestratorState:
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
config={"PLUGINS": "__archivebox_test_no_plugins__"},
status=Crawl.StatusChoices.STARTED,
retry_at=timezone.now(),
)
@ -1076,68 +1034,6 @@ class TestRecoverOrchestratorState:
assert snapshot.retry_at is None
assert snapshot.downloaded_at is not None
def test_create_pending_archiveresults_uses_canonical_hook_names(self):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.STARTED,
retry_at=timezone.now(),
)
snapshot = Snapshot.objects.create(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
snapshot.create_pending_archiveresults()
hook_names = list(ArchiveResult.objects.filter(snapshot=snapshot).values_list("hook_name", flat=True))
assert hook_names
assert all(not hook_name.endswith((".py", ".js", ".sh")) for hook_name in hook_names)
def test_snapshot_hooks_for_pending_archiveresults_respects_disabled_plugins_when_plugins_empty(self):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
config={
"CLAUDECHROME_ENABLED": False,
"CLAUDECODEEXTRACT_ENABLED": False,
"CLAUDECODECLEANUP_ENABLED": False,
"SEARCH_BACKEND_SQLITE_ENABLED": False,
},
status=Crawl.StatusChoices.STARTED,
retry_at=timezone.now(),
)
snapshot = Snapshot.objects.create(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
hooks = snapshot_hooks_for_pending_archiveresults(snapshot)
queued_plugins = {plugin for plugin, _hook_name in hooks}
assert "title" in queued_plugins
assert "claudechrome" not in queued_plugins
assert "claudecodeextract" not in queued_plugins
assert "claudecodecleanup" not in queued_plugins
assert "search_backend_sqlite" not in queued_plugins
def test_run_due_snapshot_pauses_child_when_parent_is_paused(self):
from django.utils import timezone
@ -1172,7 +1068,7 @@ class TestRecoverOrchestratorState:
result.refresh_from_db()
assert snapshot.status == Snapshot.StatusChoices.PAUSED
assert snapshot.retry_at == RETRY_AT_MAX
assert result.status == ArchiveResult.StatusChoices.PAUSED
assert result.status == ArchiveResult.StatusChoices.QUEUED
assert snapshot.archiveresult_set.count() == 1
def test_parent_status_transitions_schedule_children_to_follow_parent_status(self):
@ -1230,7 +1126,7 @@ class TestRecoverOrchestratorState:
sealed_started_child.refresh_from_db()
assert paused_child.status == Snapshot.StatusChoices.PAUSED
assert paused_child.retry_at == RETRY_AT_MAX
assert paused_result.status == ArchiveResult.StatusChoices.PAUSED
assert paused_result.status == ArchiveResult.StatusChoices.QUEUED
assert sealed_child.status == Snapshot.StatusChoices.PAUSED
assert sealed_child.retry_at is not None
assert sealed_child.retry_at <= timezone.now()
@ -1671,151 +1567,6 @@ class TestRecoverOrchestratorState:
assert snapshot.output_dir.joinpath("index.html").read_text(encoding="utf-8") == "legacy archive"
assert not legacy_dir.exists()
@pytest.mark.django_db(transaction=True)
def test_run_due_snapshot_runs_queued_plugin_after_fs_migration(self):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.SEALED,
retry_at=None,
)
snapshot = Snapshot.objects.create(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.SEALED,
retry_at=timezone.now(),
)
Snapshot.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
snapshot.refresh_from_db()
snapshot.output_dir.mkdir(parents=True, exist_ok=True)
(snapshot.output_dir / "source.txt").write_text("real targeted maintenance input\n", encoding="utf-8")
result = ArchiveResult.objects.create(
snapshot=snapshot,
plugin="hashes",
hook_name="on_Snapshot__93_hashes.py",
status=ArchiveResult.StatusChoices.QUEUED,
)
assert run_due_snapshot(snapshot, lock_seconds=60) is True
snapshot.refresh_from_db()
result.refresh_from_db()
assert snapshot.fs_version == Snapshot._fs_current_version()
assert result.status in ArchiveResult.FINAL_STATES
assert result.status != ArchiveResult.StatusChoices.QUEUED
assert result.start_ts is not None
assert result.end_ts is not None
@pytest.mark.django_db(transaction=True)
def test_run_due_snapshot_fails_obsolete_queued_hook_name(self):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.SEALED,
retry_at=None,
)
snapshot = Snapshot.objects.create(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.SEALED,
retry_at=timezone.now(),
)
result = ArchiveResult.objects.create(
snapshot=snapshot,
plugin="singlefile",
hook_name="on_Snapshot__50_singlefile.py",
status=ArchiveResult.StatusChoices.QUEUED,
)
assert run_due_snapshot(snapshot, lock_seconds=60) is True
result.refresh_from_db()
snapshot.refresh_from_db()
assert result.status == ArchiveResult.StatusChoices.FAILED
assert result.output_str == "Queued hook is no longer available in the installed plugin"
assert snapshot.retry_at is None
@pytest.mark.django_db(transaction=True)
def test_run_due_snapshot_skips_disabled_queued_plugins_and_seals_started_snapshot(self):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
config={
"CLAUDECHROME_ENABLED": False,
"CLAUDECODEEXTRACT_ENABLED": False,
"CLAUDECODECLEANUP_ENABLED": False,
"SEARCH_BACKEND_SQLITE_ENABLED": False,
},
status=Crawl.StatusChoices.STARTED,
retry_at=timezone.now(),
)
snapshot = Snapshot.objects.create(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.STARTED,
retry_at=timezone.now(),
downloaded_at=timezone.now(),
)
ArchiveResult.objects.create(
snapshot=snapshot,
plugin="title",
hook_name="on_Snapshot__01_title",
status=ArchiveResult.StatusChoices.SUCCEEDED,
output_str="Example Domain",
)
stale_results = [
ArchiveResult.objects.create(
snapshot=snapshot,
plugin=plugin,
hook_name=hook_name,
status=ArchiveResult.StatusChoices.QUEUED,
)
for plugin, hook_name in (
("claudechrome", "on_Snapshot__47_claudechrome"),
("claudecodeextract", "on_Snapshot__58_claudecodeextract"),
("claudecodecleanup", "on_Snapshot__92_claudecodecleanup"),
("search_backend_sqlite", "on_Snapshot__90_index_sqlite"),
)
]
stale_result_ids = [result.id for result in stale_results]
stale_plugins = [result.plugin for result in stale_results]
assert run_due_snapshot(snapshot, lock_seconds=60) is True
snapshot.refresh_from_db()
assert snapshot.status == Snapshot.StatusChoices.SEALED
assert snapshot.retry_at is None
assert not snapshot.archiveresult_set.filter(
plugin__in=stale_plugins,
status=ArchiveResult.StatusChoices.QUEUED,
).exists()
for result in ArchiveResult.objects.filter(id__in=stale_result_ids):
assert result.status == ArchiveResult.StatusChoices.SKIPPED
assert result.start_ts is not None
assert result.end_ts is not None
assert "disabled by this Snapshot/Crawl config" in result.output_str
@pytest.mark.django_db(transaction=True)
@pytest.mark.timeout(300)
@pytest.mark.parametrize("chrome_isolation", ["crawl", "snapshot"])
@ -1949,6 +1700,7 @@ class TestRecoverOrchestratorState:
assert crawl.status == Crawl.StatusChoices.SEALED
assert crawl.retry_at is None
@pytest.mark.django_db(transaction=True)
def test_recover_orchestrator_state_unlocks_started_snapshot_with_final_results_for_runner(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import ArchiveResult, Snapshot
@ -1959,6 +1711,7 @@ class TestRecoverOrchestratorState:
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
config={"PLUGINS": "__archivebox_test_no_plugins__"},
status=Crawl.StatusChoices.STARTED,
retry_at=None,
)
@ -2049,7 +1802,7 @@ class TestRunDueCrawlState:
assert crawl.retry_at == now
assert crawl.snapshot_set.count() == 0
def test_maintenance_only_runner_ignores_disabled_queued_results_on_sealed_snapshots(self):
def test_maintenance_only_runner_clears_snapshot_tick_without_scheduling_archive_results(self):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
@ -2083,7 +1836,7 @@ class TestRunDueCrawlState:
result.refresh_from_db()
assert snapshot.status == Snapshot.StatusChoices.SEALED
assert snapshot.fs_version == Snapshot._fs_current_version()
assert snapshot.retry_at is not None
assert snapshot.retry_at is None
assert result.status == ArchiveResult.StatusChoices.QUEUED
def test_snapshot_start_writes_short_future_lease(self):
@ -2113,132 +1866,6 @@ class TestRunDueCrawlState:
assert snapshot.retry_at is not None
assert snapshot.retry_at > timezone.now()
def test_abandoned_started_snapshot_results_are_reset_locally_for_resume(self):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.STARTED,
retry_at=timezone.now(),
)
snapshot = Snapshot.objects.create(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.STARTED,
retry_at=timezone.now(),
)
abandoned = ArchiveResult.objects.create(
snapshot=snapshot,
plugin="title",
hook_name="on_Snapshot__01_title",
status=ArchiveResult.StatusChoices.STARTED,
output_str="partial output should be cleared",
output_files={"partial.txt": {"size": 12}},
output_size=12,
start_ts=timezone.now(),
)
queued = ArchiveResult.objects.create(
snapshot=snapshot,
plugin="wget",
hook_name="on_Snapshot__40_wget",
status=ArchiveResult.StatusChoices.QUEUED,
)
finished = ArchiveResult.objects.create(
snapshot=snapshot,
plugin="favicon",
hook_name="on_Snapshot__01_favicon",
status=ArchiveResult.StatusChoices.SUCCEEDED,
output_str="keep me",
output_files={"favicon.ico": {"size": 1}},
output_size=1,
)
snapshot.reset_abandoned_results()
abandoned.refresh_from_db()
queued.refresh_from_db()
finished.refresh_from_db()
assert abandoned.status == ArchiveResult.StatusChoices.QUEUED
assert abandoned.output_str == ""
assert abandoned.output_files == {}
assert abandoned.output_size == 0
assert queued.status == ArchiveResult.StatusChoices.QUEUED
assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED
assert finished.output_str == "keep me"
assert finished.output_files == {"favicon.ico": {"size": 1}}
@pytest.mark.django_db(transaction=True)
def test_finished_parser_result_projects_children_before_resume_seals_snapshot(self):
from importlib.resources import files
from pathlib import Path
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.tests.conftest import run_test_hook
from archivebox.services.runner import run_due_snapshot
crawl = Crawl.objects.create(
urls="Plain text import containing https://example.org/\n",
max_depth=1,
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.STARTED,
retry_at=timezone.now(),
)
root = Snapshot.objects.create(
url=Snapshot.INTERNAL_INPUT_URL,
crawl=crawl,
depth=0,
status=Snapshot.StatusChoices.STARTED,
retry_at=timezone.now(),
)
staticfile_dir = root.output_dir / "staticfile"
parser_dir = root.output_dir / "parse_txt_urls"
staticfile_dir.mkdir(parents=True, exist_ok=True)
parser_dir.mkdir(parents=True, exist_ok=True)
(staticfile_dir / "input.txt").write_text(
"Plain text import containing https://example.org/\n",
encoding="utf-8",
)
hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
process = run_test_hook(
hook_path,
parser_dir,
config={"ABXPKG_LIB_DIR": str(root.output_dir.parent.parent / "lib"), "SNAP_DIR": str(root.output_dir)},
timeout=30,
url=root.url,
depth=root.depth,
snapshot_id=str(root.id),
)
process.refresh_from_db()
assert process.exit_code == 0, process.stderr
result_record = next(record for record in process.get_records() if record.get("type") == "ArchiveResult")
ArchiveResult.objects.create(
snapshot=root,
process=process,
plugin="parse_txt_urls",
hook_name=hook_path.name,
status=result_record["status"],
output_str=result_record.get("output_str", ""),
output_files={"urls.jsonl": {"size": (parser_dir / "urls.jsonl").stat().st_size}},
)
assert run_due_snapshot(root, lock_seconds=60)
root.refresh_from_db()
child = Snapshot.objects.get(crawl=crawl, url="https://example.org/")
assert root.status == Snapshot.StatusChoices.SEALED
assert child.parent_snapshot_id == root.id
assert child.status == Snapshot.StatusChoices.QUEUED
def test_due_started_snapshot_with_live_child_extends_lease_without_reset(self):
import os
from datetime import datetime
@ -2577,7 +2204,7 @@ class TestRecoverOrchestratorStateRedFailureModes:
assert crawl.status == Crawl.StatusChoices.STARTED
assert crawl.retry_at == future
def test_recovery_requeues_started_archiveresult_without_process(self):
def test_recovery_closes_interrupted_result_and_requeues_parent_snapshot(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
@ -2605,9 +2232,12 @@ class TestRecoverOrchestratorStateRedFailureModes:
recover_orchestrator_state()
result.refresh_from_db()
assert result.status == ArchiveResult.StatusChoices.QUEUED
snapshot.refresh_from_db()
assert result.status == ArchiveResult.StatusChoices.FAILED
assert snapshot.status == Snapshot.StatusChoices.STARTED
assert snapshot.retry_at is not None
def test_recovery_requeues_started_archiveresult_with_exited_process(self):
def test_recovery_closes_started_archiveresult_with_exited_process(self):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
@ -2646,9 +2276,11 @@ class TestRecoverOrchestratorStateRedFailureModes:
recover_orchestrator_state()
result.refresh_from_db()
assert result.status == ArchiveResult.StatusChoices.QUEUED
snapshot.refresh_from_db()
assert result.status == ArchiveResult.StatusChoices.FAILED
assert snapshot.retry_at is not None
def test_recovery_requeues_sealed_snapshot_started_result_with_exited_process_result_too(self):
def test_recovery_does_not_reopen_sealed_snapshot_for_interrupted_result_projection(self):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
@ -2694,10 +2326,10 @@ class TestRecoverOrchestratorStateRedFailureModes:
snapshot.refresh_from_db()
result.refresh_from_db()
assert snapshot.status == Snapshot.StatusChoices.SEALED
assert snapshot.retry_at is not None
assert result.status == ArchiveResult.StatusChoices.QUEUED
assert snapshot.retry_at is None
assert result.status == ArchiveResult.StatusChoices.FAILED
def test_recovery_requeues_started_snapshot_result_before_unlocking_snapshot(self):
def test_recovery_closes_result_projection_before_unlocking_snapshot(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
@ -2726,7 +2358,7 @@ class TestRecoverOrchestratorStateRedFailureModes:
snapshot.refresh_from_db()
result.refresh_from_db()
assert result.status == ArchiveResult.StatusChoices.QUEUED
assert result.status == ArchiveResult.StatusChoices.FAILED
assert snapshot.retry_at is not None
def test_crawl_runner_load_run_state_does_not_return_future_retry_snapshots(self):

6
archivebox/tests/test_cli_schedule.py
View File

@ -56,9 +56,11 @@ def test_schedule_without_import_path_creates_maintenance_schedule(initialized_a
assert "Created scheduled maintenance update" in result.stdout
with use_archivebox_db(initialized_archive):
row = Crawl.objects.order_by("-created_at").values_list("urls", "status").first()
schedule_row = CrawlSchedule.objects.select_related("template").get()
assert row == ("archivebox://update", "sealed")
assert schedule_row.kind == "update"
assert schedule_row.template.urls == ""
assert schedule_row.template.status == "sealed"
def test_schedule_creates_enabled_db_schedule(initialized_archive):

117
archivebox/tests/test_cli_update_reindex_snapshots.py
View File

@ -15,84 +15,6 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
pytestmark = pytest.mark.django_db(transaction=True)
def test_targeted_plugin_retries_preserve_sealed_snapshot_lifecycle():
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.cli.archivebox_extract import run_plugins
from archivebox.core.models import ArchiveResult
from archivebox.crawls.models import Crawl
search_crawl = Crawl.objects.create(
urls="https://example.com/search",
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.SEALED,
)
search_snapshot = Snapshot.objects.create(
url="https://example.com/search",
crawl=search_crawl,
status=Snapshot.StatusChoices.SEALED,
)
extract_crawl = Crawl.objects.create(
urls="https://example.com/extract",
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.SEALED,
)
extract_snapshot = Snapshot.objects.create(
url="https://example.com/extract",
crawl=extract_crawl,
status=Snapshot.StatusChoices.SEALED,
)
assert (
run_plugins(
args=(),
records=[
{
"type": "ArchiveResult",
"snapshot_id": str(search_snapshot.id),
"plugin": "search_backend_sqlite",
},
],
wait=False,
emit_results=False,
show_progress=False,
)
== 0
)
search_crawl.refresh_from_db()
search_snapshot.refresh_from_db()
assert search_crawl.status == Crawl.StatusChoices.SEALED
assert search_snapshot.status == Snapshot.StatusChoices.SEALED
assert search_snapshot.archiveresult_set.filter(
plugin="search_backend_sqlite",
status=ArchiveResult.StatusChoices.QUEUED,
).exists()
assert (
run_plugins(
args=(),
records=[
{
"type": "ArchiveResult",
"snapshot_id": str(extract_snapshot.id),
"plugin": "wget",
},
],
wait=False,
emit_results=False,
show_progress=False,
)
== 0
)
extract_crawl.refresh_from_db()
extract_snapshot.refresh_from_db()
assert extract_crawl.status == Crawl.StatusChoices.SEALED
assert extract_snapshot.status == Snapshot.StatusChoices.SEALED
assert extract_snapshot.archiveresult_set.filter(
plugin="wget",
status=ArchiveResult.StatusChoices.QUEUED,
).exists()
def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive):
"""Test that archivebox update imports real legacy archive directories."""
env = cli_env(disable_extractors=True)
@ -196,8 +118,9 @@ def test_update_migrates_every_declared_filesystem_version(tmp_path, initialized
assert snapshot.status == Snapshot.StatusChoices.QUEUED
assert snapshot.retry_at is not None
result.refresh_from_db()
assert result.output_files
assert result.output_size > 0
# Filesystem layout migration does not infer or mutate ArchiveResult facts.
assert result.output_files == {}
assert result.output_size == 0
if legacy_layout:
assert (migrated_dir / "existing-user-output.bin").read_bytes() == b"preserve interrupted migration output"
@ -215,15 +138,16 @@ def test_update_migrates_every_declared_filesystem_version(tmp_path, initialized
@pytest.mark.django_db(transaction=True)
def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_plugins():
def test_reindex_snapshots_runs_only_missing_sealed_search_indexes():
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.cli.archivebox_update import reindex_snapshots
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
urls="https://example.com\nhttps://example.org",
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.SEALED,
)
snapshot = Snapshot.objects.create(
url="https://example.com",
@ -250,12 +174,20 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
(output_dir / "title" / "title.txt").write_text("Example Domain")
(output_dir / "dom").mkdir(parents=True, exist_ok=True)
(output_dir / "dom" / "output.html").write_text("<html><body>Example searchable text</body></html>")
missing_snapshot = Snapshot.objects.create(
url="https://example.org",
crawl=crawl,
status=Snapshot.StatusChoices.SEALED,
)
missing_output_dir = missing_snapshot.output_dir
(missing_output_dir / "title").mkdir(parents=True, exist_ok=True)
(missing_output_dir / "title" / "title.txt").write_text("Missing Search Index")
original_engine = os.environ.get("SEARCH_BACKEND_ENGINE")
os.environ["SEARCH_BACKEND_ENGINE"] = "sqlite"
try:
stats = reindex_snapshots(
Snapshot.objects.filter(id__in=(snapshot.id, paused_snapshot.id)),
Snapshot.objects.filter(id__in=(snapshot.id, missing_snapshot.id, paused_snapshot.id)),
search_plugins=["search_backend_sqlite"],
batch_size=10,
)
@ -267,12 +199,21 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
result.refresh_from_db()
snapshot.refresh_from_db()
missing_snapshot.refresh_from_db()
assert stats["processed"] == 1
assert stats["queued"] == 1
assert stats["reindexed"] == 0
assert result.status == ArchiveResult.StatusChoices.QUEUED
assert result.output_str == ""
assert result.output_json is None
assert stats["requested"] == 1
assert stats["queued"] == 0
assert stats["reindexed"] == 1
assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
assert result.output_str == "old index hit"
assert result.output_json == {"indexed": True}
assert snapshot.status == Snapshot.StatusChoices.SEALED
assert missing_snapshot.status == Snapshot.StatusChoices.SEALED
assert missing_snapshot.archiveresult_set.filter(
plugin="search_backend_sqlite",
status__in=[ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS],
).exists()
assert not paused_snapshot.archiveresult_set.exists()

39
archivebox/tests/test_crawl_runner.py
View File

@ -271,45 +271,6 @@ def test_snapshot_started_state_keeps_retry_at_lease():
assert snapshot.retry_at > before
@pytest.mark.django_db(transaction=True)
def test_system_update_crawl_runs_database_maintenance_without_snapshot_work():
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.services.runner import CrawlRunner
from django.utils import timezone
owner_id = get_or_create_system_user_pk()
archived_crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=owner_id,
status=Crawl.StatusChoices.SEALED,
retry_at=None,
)
archived_snapshot = Snapshot.objects.create(
url="https://example.com",
crawl=archived_crawl,
status=Snapshot.StatusChoices.SEALED,
retry_at=None,
)
Snapshot.objects.filter(pk=archived_snapshot.pk).update(fs_version=0, retry_at=None)
maintenance_crawl = Crawl.objects.create(
urls="archivebox://update",
created_by_id=owner_id,
status=Crawl.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
snapshot_ids = CrawlRunner(maintenance_crawl, show_progress=False).load_run_state()
maintenance_crawl.refresh_from_db()
archived_snapshot.refresh_from_db()
assert snapshot_ids == []
assert maintenance_crawl.snapshot_set.count() == 0
assert archived_snapshot.status == Snapshot.StatusChoices.SEALED
assert archived_snapshot.retry_at is not None
@pytest.mark.django_db(transaction=True)
def test_crawl_start_event_keeps_retry_at_lease():
from abx_dl.events import CrawlStartEvent

2
archivebox/tests/test_crawl_service.py
View File

@ -80,7 +80,7 @@ def test_crawl_service_run_processes_queued_crawl_and_applies_crawl_config(tmp_p
assert queued_state["retry_at"] is not None
assert queued_state["config"]["PLUGINS"] == "wget,parse_html_urls"
assert queued_state["config"]["URL_DENYLIST"] == "/contact$"
# add --bg seeds Crawl.urls as CrawlSeed JSONL and returns; the runner
# add --bg stores the submitted URL list in Crawl.urls and returns; the runner
# materializes Snapshot rows + applies URL_DENYLIST when it claims the
# crawl, not at add time. The post-run assertions below verify those.
assert queued_state["snapshots"] == []

7
archivebox/tests/test_migrations_08_to_09.py
View File

@ -92,7 +92,7 @@ def convert_legacy_tags_to_uuid(conn: sqlite3.Connection) -> None:
@pytest.mark.parametrize("uuid_tags", (False, True), ids=("integer-tags", "uuid-tags"))
def test_migration_preserves_extended_08_metadata(migration_08_data, uuid_tags):
def test_migration_preserves_supported_extended_08_metadata(migration_08_data, uuid_tags):
work_dir, db_path, original_data = migration_08_data
snapshot = original_data["snapshots"][1]
parent = original_data["snapshots"][0]
@ -152,11 +152,12 @@ def test_migration_preserves_extended_08_metadata(migration_08_data, uuid_tags):
snapshot_row = conn.execute(
"""
SELECT depth, config, notes, num_uses_failed, num_uses_succeeded,
parent_snapshot_id, current_step, fs_version
parent_snapshot_id, fs_version
FROM core_snapshot WHERE id = ?
""",
(snapshot["id"],),
).fetchone()
snapshot_columns = {row[1] for row in conn.execute("PRAGMA table_info(core_snapshot)")}
result_notes = conn.execute(
"SELECT notes FROM core_archiveresult WHERE id = REPLACE(?, '-', '')",
(archiveresult["uuid"],),
@ -174,9 +175,9 @@ def test_migration_preserves_extended_08_metadata(migration_08_data, uuid_tags):
metadata["num_uses_failed"],
metadata["num_uses_succeeded"],
parent["id"],
metadata["current_step"],
metadata["fs_version"],
)
assert "current_step" not in snapshot_columns
assert result_notes == ("legacy archive result notes",)
assert migrated_tag == tag_metadata
assert migrated_counts == expected_counts

109
archivebox/tests/test_snapshot_filesystem_migration.py Normal file
View File

@ -0,0 +1,109 @@
from pathlib import Path
import pytest
from django.db import connection
from archivebox.config import CONSTANTS
from archivebox.core.models import Snapshot, SnapshotMigrationError
pytestmark = pytest.mark.django_db(transaction=True)
def _make_legacy_snapshot(snapshot: Snapshot) -> tuple[Path, Path]:
current_dir = snapshot.get_storage_path_for_version(snapshot._fs_current_version())
legacy_dir = CONSTANTS.ARCHIVE_DIR / snapshot.timestamp
Snapshot.objects.filter(pk=snapshot.pk).update(fs_version="0.8.0")
snapshot.refresh_from_db()
legacy_dir.mkdir(parents=True, exist_ok=True)
(legacy_dir / "unknown" / "nested").mkdir(parents=True)
(legacy_dir / "unknown" / "payload.bin").write_bytes(b"filesystem migration payload\x00\xff")
return legacy_dir, current_dir
def test_ordinary_snapshot_save_does_not_migrate_directories(snapshot):
legacy_dir, current_dir = _make_legacy_snapshot(snapshot)
snapshot.title = "Metadata-only update"
snapshot.save()
snapshot.refresh_from_db()
assert snapshot.fs_version == "0.8.0"
assert legacy_dir.exists()
assert not (current_dir / "unknown" / "payload.bin").exists()
def test_maintenance_save_keeps_existing_runner_entrypoint(snapshot):
legacy_dir, current_dir = _make_legacy_snapshot(snapshot)
snapshot.save(update_fields=["retry_at", "modified_at"])
snapshot.refresh_from_db()
assert snapshot.fs_version == snapshot._fs_current_version()
assert not legacy_dir.exists()
assert (current_dir / "unknown" / "payload.bin").read_bytes() == b"filesystem migration payload\x00\xff"
def test_filesystem_migration_resumes_after_shutdown_before_cleanup(snapshot, monkeypatch):
legacy_dir, current_dir = _make_legacy_snapshot(snapshot)
real_cleanup = Snapshot._cleanup_old_migration_dir
def simulate_shutdown_before_cleanup(self, old_dir, new_dir):
raise RuntimeError("simulated shutdown")
monkeypatch.setattr(Snapshot, "_cleanup_old_migration_dir", simulate_shutdown_before_cleanup)
with pytest.raises(RuntimeError, match="simulated shutdown"):
snapshot.migrate_filesystem_to_current_version()
snapshot.refresh_from_db()
assert snapshot.fs_version == "0.8.0"
assert legacy_dir.exists()
assert (current_dir / "unknown" / "payload.bin").read_bytes() == b"filesystem migration payload\x00\xff"
monkeypatch.setattr(Snapshot, "_cleanup_old_migration_dir", real_cleanup)
snapshot.migrate_filesystem_to_current_version()
snapshot.refresh_from_db()
assert snapshot.fs_version == snapshot._fs_current_version()
assert not legacy_dir.exists()
assert (current_dir / "unknown" / "payload.bin").read_bytes() == b"filesystem migration payload\x00\xff"
def test_filesystem_migration_cleans_legacy_source_when_version_is_current(snapshot, monkeypatch):
legacy_dir, current_dir = _make_legacy_snapshot(snapshot)
monkeypatch.setattr(Snapshot, "_cleanup_old_migration_dir", lambda *_args: True)
snapshot.migrate_filesystem_to_current_version()
snapshot.refresh_from_db()
assert snapshot.fs_version == snapshot._fs_current_version()
assert legacy_dir.exists()
monkeypatch.undo()
snapshot.migrate_filesystem_to_current_version()
assert not legacy_dir.exists()
assert (current_dir / "unknown" / "payload.bin").read_bytes() == b"filesystem migration payload\x00\xff"
def test_fs_version_has_database_index():
assert Snapshot._meta.get_field("fs_version").db_index is True
constraints = connection.introspection.get_constraints(connection.cursor(), Snapshot._meta.db_table)
assert any(index["index"] and index["columns"] == ["fs_version"] for index in constraints.values())
def test_stale_filesystem_selector_uses_fs_version_index():
queryset = Snapshot.objects.filter(fs_version__in=Snapshot._FS_VERSION_MIGRATION_PATHS)
assert "fs_version" in queryset.explain().lower()
def test_resume_refuses_to_overwrite_changed_legacy_output(snapshot, monkeypatch):
legacy_dir, current_dir = _make_legacy_snapshot(snapshot)
monkeypatch.setattr(Snapshot, "_cleanup_old_migration_dir", lambda *_args: True)
snapshot.migrate_filesystem_to_current_version()
(legacy_dir / "unknown" / "payload.bin").write_bytes(b"changed after database commit")
with pytest.raises(SnapshotMigrationError, match="overwrite a different output"):
snapshot.migrate_filesystem_to_current_version()
assert legacy_dir.exists()
assert (current_dir / "unknown" / "payload.bin").read_bytes() == b"filesystem migration payload\x00\xff"

4
archivebox/tests/test_ui_add_view.py
View File

@ -565,7 +565,7 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user):
assert crawl is not None
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is not None
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com", "depth": 0}
assert crawl.urls == "https://example.com"
assert crawl.snapshot_set.count() == 0
@ -599,7 +599,7 @@ def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, ad
assert crawl is not None
assert crawl.status == Crawl.StatusChoices.PAUSED
assert crawl.retry_at == RETRY_AT_MAX
assert json.loads(crawl.urls) == {"type": "CrawlSeed", "url": "https://example.com/paused", "depth": 0}
assert crawl.urls == "https://example.com/paused"
assert crawl.snapshot_set.count() == 0
assert crawl.config.get("INDEX_ONLY") is not True

12
archivebox/tests/test_ui_add_view_runtime.py
View File

@ -241,11 +241,7 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s
crawl = Crawl.objects.order_by("-created_at").first()
assert crawl is not None
assert crawl.tags_str == "restart-supervised-runner"
assert json.loads(crawl.urls) == {
"type": "CrawlSeed",
"url": recursive_test_site["root_url"],
"depth": 0,
}
assert crawl.urls == recursive_test_site["root_url"]
finally:
stop_server(tmp_path)
@ -482,8 +478,8 @@ def test_public_add_view_import_text_formats_preserve_metadata_and_resume_withou
with use_archivebox_db(tmp_path):
for crawl in Crawl.objects.order_by("created_at"):
root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert not crawl.snapshot_set.filter(url__startswith="archivebox://").exists()
root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert root_input == crawl.urls
start_archivebox_server(tmp_path, env=env, port=port)
@ -659,7 +655,7 @@ def test_add_view_post_creates_schedule_over_server(tmp_path, recursive_test_sit
schedule = CrawlSchedule.objects.select_related("template").order_by("-created_at").first()
row = None
if schedule:
template_url = json.loads(schedule.template.urls)["url"]
template_url = schedule.template.urls.strip()
row = (schedule.schedule, template_url, schedule.template.tags_str)
assert row == ("daily", recursive_test_site["root_url"], "web-ui")

19
archivebox/tests/test_ui_admin_snapshot.py
View File

@ -18,7 +18,7 @@ from django.urls import reverse
from archivebox.core.middleware import ADMIN_LOGIN_HINT_COOKIE
from archivebox.tests.conftest import ADMIN_TEST_HOST
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook, _snapshot_hook_name
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook
pytestmark = pytest.mark.django_db(transaction=True)
REPO_ROOT = Path(__file__).resolve().parents[2]
@ -98,9 +98,9 @@ def running_wget_projection(snapshot, blocking_http_server):
retry_at=now,
downloaded_at=None,
url=blocking_http_server.url,
config={"PLUGINS": "wget"},
)
snapshot.refresh_from_db()
[result] = snapshot.create_pending_archiveresults(hooks=[("wget", _snapshot_hook_name("wget"))])
errors = []
def run_snapshot():
@ -115,7 +115,7 @@ def running_wget_projection(snapshot, blocking_http_server):
runner.start()
blocking_http_server.request_started.wait()
assert errors == []
result.refresh_from_db()
result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget")
assert result.status == ArchiveResult.StatusChoices.STARTED
yield result
blocking_http_server.release_response.set()
@ -1353,7 +1353,10 @@ class TestAdminSnapshotListView:
assert response.status_code == 302
assert response["Location"].endswith(f"/admin/core/snapshot/{snapshot.pk}/change/")
failed.refresh_from_db()
assert failed.status == ArchiveResult.StatusChoices.QUEUED
snapshot.refresh_from_db()
assert failed.status == ArchiveResult.StatusChoices.FAILED
assert snapshot.status == snapshot.StatusChoices.QUEUED
assert snapshot.config["PLUGINS"] == "title"
def test_list_redo_failed_action_requeues_failed_archiveresults_only(
self,
@ -1391,14 +1394,12 @@ class TestAdminSnapshotListView:
failed.refresh_from_db()
succeeded.refresh_from_db()
snapshot.refresh_from_db()
assert failed.status == ArchiveResult.StatusChoices.QUEUED
assert failed.output_str == ""
assert failed.output_files == {}
assert failed.output_size == 0
assert failed.output_mimetypes == ""
assert failed.status == ArchiveResult.StatusChoices.FAILED
assert failed.output_str
assert succeeded.status == ArchiveResult.StatusChoices.SUCCEEDED
assert succeeded.output_str == succeeded_output
assert snapshot.status == snapshot.StatusChoices.QUEUED
assert snapshot.config["PLUGINS"] == "title"
def test_archive_now_action_uses_original_snapshot_url_without_timestamp_suffix(self, client, admin_user, snapshot):
from archivebox.crawls.models import Crawl

22
archivebox/tests/test_ui_live_progress.py
View File

@ -14,7 +14,7 @@ from django.utils import timezone
from archivebox.tests.conftest import ADMIN_TEST_HOST
from archivebox.tests.conftest import cli_env, resolve_abxpkg_binary_env, run_archivebox_cmd
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook, _snapshot_hook_name
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook
pytestmark = pytest.mark.django_db(transaction=True)
@ -179,7 +179,12 @@ class TestLiveProgressView:
ended_at=now - timedelta(hours=1),
modified_at=now - timedelta(hours=1),
)
snapshot.create_pending_archiveresults(hooks=[("chrome", "on_Snapshot__11_chrome_wait")])
ArchiveResult.objects.create(
snapshot=snapshot,
plugin="chrome",
hook_name="on_Snapshot__11_chrome_wait",
status=ArchiveResult.StatusChoices.QUEUED,
)
response = client.get("/progress.json", HTTP_HOST=ADMIN_TEST_HOST)
@ -218,9 +223,9 @@ class TestLiveProgressView:
created_at=now - timedelta(hours=2),
downloaded_at=None,
url=blocking_http_server.url,
config={"PLUGINS": "wget"},
)
snapshot.refresh_from_db()
[result] = snapshot.create_pending_archiveresults(hooks=[("wget", _snapshot_hook_name("wget"))])
errors = []
def run_snapshot():
@ -236,7 +241,7 @@ class TestLiveProgressView:
try:
blocking_http_server.request_started.wait()
assert errors == []
result.refresh_from_db()
result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget")
assert result.status == ArchiveResult.StatusChoices.STARTED
Snapshot.objects.filter(pk=snapshot.pk).update(modified_at=timezone.now())
@ -257,7 +262,7 @@ class TestLiveProgressView:
assert result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS)
def test_live_progress_hides_finished_cancelled_crawl(self, client, admin_user, crawl, snapshot):
from archivebox.core.models import Snapshot
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
now = timezone.now()
@ -272,7 +277,12 @@ class TestLiveProgressView:
downloaded_at=None,
modified_at=now,
)
snapshot.create_pending_archiveresults(hooks=[("singlefile", "on_Snapshot__50_singlefile")])
ArchiveResult.objects.create(
snapshot=snapshot,
plugin="singlefile",
hook_name="on_Snapshot__50_singlefile",
status=ArchiveResult.StatusChoices.QUEUED,
)
client.force_login(admin_user)
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)

2
docs/ArchiveBox-Architecture-Diagrams.md
View File

@ -82,7 +82,7 @@ stateDiagram-v2
SEALED --> [*]
```
A crawl owns a set of snapshots. The runner creates or discovers those snapshots and projects crawl events while the row is `STARTED`; sealing waits for their normal lifecycle to finish. Pausing also schedules child snapshots to pause, and resuming returns the crawl to the runnable queue. The `archivebox://update` sentinel remains control-plane work: the runner invokes database maintenance directly and never turns it into a Snapshot.
A crawl owns a set of snapshots. The runner creates or discovers those snapshots and projects crawl events while the row is `STARTED`; sealing waits for their normal lifecycle to finish. Pausing also schedules child snapshots to pause, and resuming returns the crawl to the runnable queue. Scheduled maintenance is dispatched directly by `CrawlSchedule`; it does not create a synthetic crawl or snapshot.
## `Snapshot` Queue Lifecycle

2
docs/Scheduled-Archiving.md
View File

@ -33,7 +33,7 @@ Accepted schedule formats:
`archivebox schedule --foreground` runs the global orchestrator in the foreground, which is useful outside `archivebox server` if you want a dedicated long-running scheduler/worker process without the web UI.
Running `archivebox schedule --every=day` with no `import_path` creates a recurring maintenance schedule that queues `archivebox://update` crawls.
Running `archivebox schedule --every=day` with no `import_path` creates a recurring maintenance schedule. The scheduler dispatches its bounded database/filesystem maintenance directly instead of creating a synthetic Crawl or Snapshot.
## Docker Compose