{html.escape(capture['name'])}
" f'{html.escape(route)} · '
- f'View code
{"".join(html_figures)}
mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run --rm -v $PWD:/data -it archivebox/archivebox:dev init
-docker run --rm -v $PWD:/data -it archivebox/archivebox:dev install
+docker run -d --name archivebox -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
/admin/ on the hostname or IP used to reach ArchiveBox (local example: http://admin.archivebox.localhost:8000/admin/) to create the first admin. If BASE_URL is not configured yet, continue through the web setup wizard.
-docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
+- Open
/admin/ on the hostname or IP used to reach ArchiveBox (local example: http://admin.archivebox.localhost:8000/admin/) to create the first admin. If BASE_URL is not configured yet, continue through the web setup wizard.
+
# completely optional, CLI can always be used without running a server
-# docker run -v $PWD:/data -it archivebox/archivebox:dev [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
+# docker exec archivebox archivebox [subcommand] [--help]
+docker exec archivebox archivebox help
For more info, see Install: Docker Compose in the Wiki. ➡️
@@ -497,12 +494,10 @@ archivebox help # get list of archivebox subcommands that can be ru
# make sure you have `docker-compose.yml` from the Quickstart instructions first
-# docker compose run --rm archivebox [subcommand] [--help]
-docker compose run --rm archivebox init
-docker compose run --rm archivebox install
-docker compose run --rm archivebox version
-docker compose run --rm archivebox help
-docker compose run --rm archivebox add 'https://example.com'
+# docker compose exec archivebox archivebox [subcommand] [--help]
+docker compose exec archivebox archivebox version
+docker compose exec archivebox archivebox help
+docker compose exec archivebox archivebox add 'https://example.com'
# to start webserver: docker compose up
For more info, see our Usage: Docker Compose CLI wiki. ➡️
@@ -514,15 +509,12 @@ docker compose run --rm archivebox add 'https://example.com'
CLI Usage Examples: Docker
-# make sure you create and cd into in a new empty directory first
+# make sure the `archivebox` server container from the Quickstart is running first
-# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev init
-docker run -v $PWD:/data -it archivebox/archivebox:dev install
-docker run -v $PWD:/data -it archivebox/archivebox:dev version
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
-docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
-# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
+# docker exec archivebox archivebox [subcommand] [--help]
+docker exec archivebox archivebox version
+docker exec archivebox archivebox help
+docker exec archivebox archivebox add 'https://example.com'
For more info, see our Usage: Docker CLI wiki. ➡️
@@ -1345,14 +1337,11 @@ archivebox server 0.0.0.0:8000
# inside the container will reload and pick up your changes
./bin/build_docker.sh dev
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
-
# Run the development server w/ autoreloading (but no bg workers)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init --debug --reload 0.0.0.0:8000
# Run the production server (with bg workers but no autoreloading)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init
# (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
# When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@@ -1413,7 +1402,7 @@ services:
# or with plain Docker:
docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data archivebox:dev init
+docker run -it -v $PWD:/data -p 8000:8000 archivebox:dev
# or with uv:
uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py
index 432483d8..320e0e62 100644
--- a/archivebox/cli/archivebox_run.py
+++ b/archivebox/cli/archivebox_run.py
@@ -270,7 +270,7 @@ def run_runner(
from archivebox.config import CONSTANTS
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
from archivebox.machine.models import Machine, Process
- from archivebox.core.takeover_util import enter_single_runner_gate, maintain_runner_lease, standby_until_foreground_runner_needed
+ from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.services.runner import run_pending_crawls
@@ -292,40 +292,40 @@ def run_runner(
current.mark_exited()
return 0
- try:
- with maintain_runner_lease(current):
- recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
- if crawl_id:
- from django.utils import timezone
- from archivebox.crawls.models import Crawl
+ recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
+ if crawl_id:
+ from django.utils import timezone
+ from archivebox.crawls.models import Crawl
- crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
- now = timezone.now()
- # Winning the single-runner gate terminates and waits for every older
- # runner. Requeue the explicitly requested crawl so an abandoned
- # future ownership lease cannot hide it from the unified scheduler.
- if crawl is not None:
- crawl.update_and_requeue(retry_at=now, refresh=False)
- # Only a foreground `archivebox add` gets the interactive "abort current
- # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
- # orchestrators should shut down immediately and cleanly on the first signal.
- interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
- if daemon:
- os.environ[RUNNER_DAEMON_ENV] = "1"
- with (
- foreground_shutdown_signals(
- on_signal=_exit_daemon_runner_on_signal if daemon else None,
- raise_on_first_signal=not daemon,
- ),
- foreground_parent_watchdog(enabled=not daemon),
- ):
- run_pending_crawls(
- daemon=daemon,
- crawl_id=crawl_id,
- maintenance_only=maintenance_only,
- interactive_interrupts=interactive_interrupts,
- **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
- )
+ crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
+ now = timezone.now()
+ # Winning the single-runner gate terminates and waits for every older
+ # runner. Requeue the explicitly requested crawl so an abandoned
+ # future ownership lease cannot hide it from the unified scheduler.
+ if crawl is not None:
+ crawl.update_and_requeue(retry_at=now, refresh=False)
+ # Only a foreground `archivebox add` gets the interactive "abort current
+ # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
+ # orchestrators should shut down immediately and cleanly on the first signal.
+ interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+ if daemon:
+ os.environ[RUNNER_DAEMON_ENV] = "1"
+
+ try:
+ with (
+ foreground_shutdown_signals(
+ on_signal=_exit_daemon_runner_on_signal if daemon else None,
+ raise_on_first_signal=not daemon,
+ ),
+ foreground_parent_watchdog(enabled=not daemon),
+ ):
+ run_pending_crawls(
+ daemon=daemon,
+ crawl_id=crawl_id,
+ maintenance_only=maintenance_only,
+ interactive_interrupts=interactive_interrupts,
+ **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
+ )
return 0
except KeyboardInterrupt:
return 0
@@ -432,7 +432,7 @@ def main(
def run_snapshot_worker(snapshot_id: str) -> int:
from archivebox.config import CONSTANTS
- from archivebox.core.takeover_util import enter_single_runner_gate, maintain_runner_lease
+ from archivebox.core.takeover_util import enter_single_runner_gate
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
from archivebox.machine.models import Process
from archivebox.core.models import Snapshot
@@ -446,16 +446,15 @@ def run_snapshot_worker(snapshot_id: str) -> int:
snapshot = None
try:
- with maintain_runner_lease(current):
- with foreground_shutdown_signals(), foreground_parent_watchdog():
- for _ in range(10):
- snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
- if snapshot.retry_at is None:
- snapshot.update_and_requeue(retry_at=timezone.now())
- elif snapshot.retry_at > timezone.now():
- break
- if not run_due_snapshot(snapshot, lock_seconds=60):
- break
+ with foreground_shutdown_signals(), foreground_parent_watchdog():
+ for _ in range(10):
+ snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+ if snapshot.retry_at is None:
+ snapshot.update_and_requeue(retry_at=timezone.now())
+ elif snapshot.retry_at > timezone.now():
+ break
+ if not run_due_snapshot(snapshot, lock_seconds=60):
+ break
return 0
except KeyboardInterrupt:
try:
diff --git a/archivebox/core/takeover_util.py b/archivebox/core/takeover_util.py
index 9862aa33..3872ade1 100644
--- a/archivebox/core/takeover_util.py
+++ b/archivebox/core/takeover_util.py
@@ -1,14 +1,25 @@
+"""Coordinate local foreground processes without pretending to provide a distributed lock.
+
+ArchiveBox wants one orchestrator per collection, but Process rows can only prove
+liveness for PIDs visible on the current machine/PID namespace. We therefore
+enforce one active runner per ``(Machine, DATA_DIR)`` locally, retire stale rows
+from sequential containers on that machine, and only warn about rows owned by a
+different machine. Foreign-machine rows must never block progress or be killed:
+multi-machine coordination belongs in the Crawl/Snapshot CAS claim layer, not in
+process takeover.
+
+These helpers only hand local supervisord/runner ownership between CLI parents.
+They must not hold database transactions or filesystem locks while work runs.
+"""
+
from __future__ import annotations
import time
import sys
-import threading
from collections.abc import Callable
-from contextlib import contextmanager
-from datetime import timedelta
from pathlib import Path
-from django.db import DatabaseError, IntegrityError, close_old_connections
+from django.db import IntegrityError
from django.utils import timezone
from archivebox.config import CONSTANTS
from archivebox.config.common import rprint
@@ -16,27 +27,6 @@ from archivebox.config.common import rprint
RUNNER_ACTIVE_WORKER_TYPE = "worker_runner"
RUNNER_WAITING_WORKER_TYPE = "runner_waiting"
RUNNER_GATE_WORKER_TYPES = (RUNNER_ACTIVE_WORKER_TYPE, RUNNER_WAITING_WORKER_TYPE, "")
-RUNNER_LEASE_HEARTBEAT_SECONDS = 5.0
-RUNNER_LEASE_TIMEOUT = timedelta(seconds=30)
-
-
-def runtime_stack_owner_types():
- from archivebox.machine.models import Process
-
- return (
- Process.TypeChoices.SERVER,
- Process.TypeChoices.ORCHESTRATOR,
- )
-
-
-def foreground_runner_owner_types():
- from archivebox.machine.models import Process
-
- return (
- Process.TypeChoices.SERVER,
- Process.TypeChoices.ADD,
- Process.TypeChoices.UPDATE,
- )
def current_command(process_type: str, *, data_dir: str | Path, url: str | None = None):
@@ -47,31 +37,8 @@ def current_command(process_type: str, *, data_dir: str | Path, url: str | None
return proc
-def live_processes(*, process_type: str, data_dir: str | Path, url: str | None = None):
- from archivebox.machine.models import Machine, Process
-
- qs = Process.objects.filter(
- machine=Machine.current(),
- process_type=process_type,
- status=Process.StatusChoices.RUNNING,
- pwd=str(data_dir),
- )
- if url is not None:
- qs = qs.filter(url=url)
- return [proc for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50) if proc.is_running]
-
-
-def newest_live_process(*, process_type: str, data_dir: str | Path, url: str | None = None):
- processes = live_processes(process_type=process_type, data_dir=data_dir, url=url)
- return processes[0] if processes else None
-
-
-def command_is_newest(command, *, process_type: str, data_dir: str | Path, url: str | None = None) -> bool:
- leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
- return bool(leader and leader.id == command.id)
-
-
def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
+ """Return the live local parent allowed to own the server runtime stack."""
from archivebox.machine.models import Machine, Process
machine = Machine.current()
@@ -79,7 +46,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
machine=machine,
status=Process.StatusChoices.RUNNING,
pwd=str(data_dir),
- process_type__in=runtime_stack_owner_types(),
+ process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ORCHESTRATOR),
)
if exclude_id is not None:
base_qs = base_qs.exclude(id=exclude_id)
@@ -108,6 +75,7 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
+ """Return the newest live local parent allowed to borrow runner/sonic."""
from archivebox.machine.models import Machine, Process
machine = Machine.current()
@@ -115,7 +83,7 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
machine=machine,
status=Process.StatusChoices.RUNNING,
pwd=str(data_dir),
- process_type__in=foreground_runner_owner_types(),
+ process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ADD, Process.TypeChoices.UPDATE),
)
if exclude_id is not None:
qs = qs.exclude(id=exclude_id)
@@ -131,26 +99,6 @@ def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
return bool(owner and owner.id == command.id)
-def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
- try:
- from archivebox.workers.supervisord_util import active_supervisord_runtime_components
-
- components = active_supervisord_runtime_components()
- except Exception:
- components = []
-
- names = list(components)
- if not names and owner is not None:
- from archivebox.machine.models import Process
-
- if owner.process_type == Process.TypeChoices.SERVER:
- names = ["orchestrator", "server"]
- elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
- names = ["orchestrator"]
-
- return ", ".join(dict.fromkeys(names)) or "runtime stack"
-
-
def ensure_daemon_stack(*, reason: str = ""):
from archivebox.config.common import get_config
from archivebox.workers.supervisord_util import (
@@ -186,115 +134,65 @@ def ensure_daemon_stack(*, reason: str = ""):
return start_worker(supervisor, sonic_worker)
-def healthy_orchestrator(*, data_dir: str | Path):
- from archivebox.machine.models import Machine, Process
- from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+def live_runner_processes(*, data_dir: str | Path):
+ """Return locally verifiable runners and warn about unsupported overlap.
- supervisor = get_existing_supervisord_process()
- worker = get_worker(supervisor, "worker_runner") if supervisor else None
- if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
- return worker
-
- for proc in Process.objects.filter(
- machine=Machine.current(),
- process_type=Process.TypeChoices.ORCHESTRATOR,
- status=Process.StatusChoices.RUNNING,
- pwd=str(data_dir),
- ).order_by("-created_at"):
- if proc.is_running:
- return proc
- return None
-
-
-def _runner_sort_key(process):
- return (process.started_at or process.created_at, process.created_at, str(process.id))
-
-
-def live_runner_processes(*, data_dir: str | Path, exclude_id=None):
+ A Process row from another machine is observability only: its PID cannot be
+ checked or signalled here, so it neither joins the local election nor gets
+ mutated. A row for this same Machine from another PID namespace represents
+ a previous sequential container under the supported model; warn, retire the
+ unreachable row, and let the new container continue.
+ """
from archivebox.machine.models import Machine, Process
machine = Machine.current()
Process.cleanup_stale_running(machine=machine)
qs = Process.objects.filter(
- machine=machine,
status=Process.StatusChoices.RUNNING,
process_type=Process.TypeChoices.ORCHESTRATOR,
worker_type__in=RUNNER_GATE_WORKER_TYPES,
pwd=str(data_dir),
)
- if exclude_id is not None:
- qs = qs.exclude(id=exclude_id)
- now = timezone.now()
+ foreign_machine_exists = qs.exclude(machine=machine).exists()
+ qs = qs.filter(machine=machine)
live = []
+ foreign_namespace_ids = []
for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
- if not process.shares_pid_namespace and process.modified_at < now - RUNNER_LEASE_TIMEOUT:
- # A PID from another container cannot be inspected or signaled
- # safely. Its DB heartbeat is the only cross-namespace liveness
- # proof, so release the runner gate only after that lease expires.
- exited = Process.objects.filter(
- pk=process.pk,
- status=Process.StatusChoices.RUNNING,
- modified_at=process.modified_at,
- ).update(
- status=Process.StatusChoices.EXITED,
- exit_code=process.exit_code if process.exit_code is not None else 0,
- ended_at=now,
- retry_at=None,
- modified_at=now,
- )
- if exited:
- continue
- process.refresh_from_db()
+ if not process.shares_pid_namespace:
+ foreign_namespace_ids.append(process.id)
+ continue
if process.is_running:
live.append(process)
+ if foreign_machine_exists or foreign_namespace_ids:
+ rprint(
+ "[bold yellow]WARNING: Multiple orchestrators sharing a single collection is not officially supported! "
+ "Corruption may occur if you run two ArchiveBox workers on the same collection at once.[/bold yellow]",
+ file=sys.stderr,
+ soft_wrap=True,
+ )
+ if foreign_namespace_ids:
+ now = timezone.now()
+ Process.objects.filter(id__in=foreign_namespace_ids, status=Process.StatusChoices.RUNNING).update(
+ status=Process.StatusChoices.EXITED,
+ exit_code=0,
+ ended_at=now,
+ retry_at=None,
+ modified_at=now,
+ )
return live
-@contextmanager
-def maintain_runner_lease(command, *, interval: float = RUNNER_LEASE_HEARTBEAT_SECONDS):
- """Keep the active runner row fresh for takeover across PID namespaces."""
- from archivebox.machine.models import Process
-
- stopped = threading.Event()
-
- def heartbeat() -> None:
- close_old_connections()
- try:
- while not stopped.wait(interval):
- try:
- updated = Process.objects.filter(
- pk=command.pk,
- status=Process.StatusChoices.RUNNING,
- worker_type=RUNNER_ACTIVE_WORKER_TYPE,
- ).update(modified_at=timezone.now())
- if not updated:
- return
- except DatabaseError:
- # A transient SQLite lock must not terminate the runner.
- # The timeout allows several missed heartbeats before a
- # cross-namespace contender may reclaim the gate.
- continue
- finally:
- close_old_connections()
-
- heartbeat_thread = threading.Thread(target=heartbeat, name="archivebox-runner-lease", daemon=True)
- heartbeat_thread.start()
- try:
- yield
- finally:
- stopped.set()
- heartbeat_thread.join(timeout=interval + 1.0)
-
-
def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout: float = 5.0) -> bool:
"""
- Admit exactly one active runner for this DATA_DIR using Process rows.
+ Admit one active runner for this Machine and DATA_DIR using Process rows.
The current process is a real OS process while it waits, so we keep its
Process row RUNNING but mark worker_type=runner_waiting. Only the process
that wins takeover is promoted to worker_type=worker_runner, which is
- protected by a partial unique DB constraint. Older runners are terminated
- and fully waited out before promotion, so the runner work loop never overlaps.
+ protected by a partial unique DB constraint scoped to (Machine, DATA_DIR).
+ Older locally verifiable runners are terminated and fully waited out before
+ promotion, so runner work never overlaps on one machine. Foreign machines
+ are intentionally outside this gate and only produce a warning above.
"""
from archivebox.machine.models import Process
@@ -304,12 +202,7 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
pwd=str(data_dir),
timeout=CONSTANTS.MAX_HOOK_RUNTIME_SECONDS,
)
- last_waiting_heartbeat = time.monotonic()
- announced_cross_namespace_runners = set()
while True:
- if time.monotonic() - last_waiting_heartbeat >= RUNNER_LEASE_HEARTBEAT_SECONDS:
- command.heartbeat()
- last_waiting_heartbeat = time.monotonic()
runners = live_runner_processes(data_dir=data_dir)
if all(process.id != command.id for process in runners):
command.refresh_from_db()
@@ -321,7 +214,7 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
)
runners = live_runner_processes(data_dir=data_dir)
- newest = max(runners, key=_runner_sort_key)
+ newest = max(runners, key=lambda process: (process.started_at or process.created_at, process.created_at, str(process.id)))
if newest.id != command.id:
rprint(
f"[yellow][*] Newer ArchiveBox runner pid={newest.pid} is taking over; exiting this runner.[/yellow]",
@@ -332,16 +225,8 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
older_runners = [process for process in runners if process.id != command.id]
if older_runners:
for process in older_runners:
- if process.shares_pid_namespace:
- rprint(f"[yellow][*] Stopping older ArchiveBox runner process (pid={process.pid})...[/yellow]", file=sys.stderr)
- process.kill_tree(graceful_timeout=graceful_timeout)
- else:
- if process.id not in announced_cross_namespace_runners:
- rprint(
- "[yellow][*] Waiting for older ArchiveBox runner in another PID namespace to stop...[/yellow]",
- file=sys.stderr,
- )
- announced_cross_namespace_runners.add(process.id)
+ rprint(f"[yellow][*] Stopping older ArchiveBox runner process (pid={process.pid})...[/yellow]", file=sys.stderr)
+ process.kill_tree(graceful_timeout=graceful_timeout)
time.sleep(0.1)
continue
@@ -361,24 +246,9 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
time.sleep(0.1)
-def standby_until_leader_needed(command, *, process_type: str, data_dir: str | Path, url: str | None = None, interval: float = 2.0) -> None:
- from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
-
- announced = False
- while not command_is_newest(command, process_type=process_type, data_dir=data_dir, url=url):
- reap_foreground_supervisord_process()
- if not announced:
- leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
- leader_pid = leader.pid if leader else "unknown"
- rprint(f"[yellow][*] Standing by; newer ArchiveBox process pid={leader_pid} is running the orchestrator and server.[/yellow]")
- announced = True
- time.sleep(interval)
- command.modified_at = timezone.now()
- command.save(update_fields=["modified_at"])
-
-
def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
- from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+ from archivebox.machine.models import Process
+ from archivebox.workers.supervisord_util import active_supervisord_runtime_components, reap_foreground_supervisord_process
announced = False
previous_owner_pid = None
@@ -387,7 +257,16 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
if not announced:
owner = runtime_stack_owner(data_dir=data_dir)
owner_pid = owner.pid if owner else "unknown"
- components = runtime_stack_component_label(owner=owner, data_dir=data_dir)
+ try:
+ component_names = list(active_supervisord_runtime_components())
+ except Exception:
+ component_names = []
+ if not component_names and owner is not None:
+ if owner.process_type == Process.TypeChoices.SERVER:
+ component_names = ["orchestrator", "server"]
+ elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
+ component_names = ["orchestrator"]
+ components = ", ".join(dict.fromkeys(component_names)) or "runtime stack"
previous_owner_pid = owner_pid
rprint(
f"[yellow][*] A newer archivebox process took over the {components} "
diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py
index b7565d5b..0b774a08 100755
--- a/archivebox/machine/models.py
+++ b/archivebox/machine/models.py
@@ -1187,6 +1187,11 @@ class Process(ModelWithDeleteAfter, models.Model):
models.Index(fields=["machine", "status", "process_type"], name="mach_proc_running_idx"),
]
constraints = [
+ # This is deliberately machine-scoped. It prevents two locally
+ # verifiable runners from working the same collection, while not
+ # claiming to coordinate independent hosts that share a database.
+ # Cross-machine work ownership belongs to short Crawl/Snapshot CAS
+ # claims so PostgreSQL deployments can support that model later.
models.UniqueConstraint(
fields=["machine", "pwd"],
condition=Q(status="running", process_type="orchestrator", worker_type="worker_runner"),
@@ -1393,7 +1398,7 @@ class Process(ModelWithDeleteAfter, models.Model):
self.save(update_fields=updates)
def heartbeat(self) -> None:
- """Touch modified_at so standby/leader selection can see this parent is alive."""
+ """Keep a long-lived watcher visible in recent-process monitoring."""
self.save(update_fields=["modified_at"])
def mark_exited(self, *, exit_code: int = 0) -> None:
diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py
index aa8b3217..a43bc912 100644
--- a/archivebox/services/runner.py
+++ b/archivebox/services/runner.py
@@ -484,6 +484,11 @@ class CrawlRunner:
await self.enqueue_pending_snapshots_from_projection()
async def heartbeat_active_leases(self) -> None:
+ # These are resumable work-item leases, not orchestrator-election
+ # heartbeats. Each update is a short autocommit statement; network and
+ # filesystem work continues outside a database transaction. A future
+ # PostgreSQL multi-machine runner uses these Crawl/Snapshot claims as
+ # its coordination boundary while SQLite keeps one local orchestrator.
if self._run_task is None:
return
now_monotonic = time.monotonic()
diff --git a/archivebox/tests/test_cli_mcp.py b/archivebox/tests/test_cli_mcp.py
index 4090ae6f..c225b4d4 100644
--- a/archivebox/tests/test_cli_mcp.py
+++ b/archivebox/tests/test_cli_mcp.py
@@ -1,11 +1,18 @@
#!/usr/bin/env python3
-"""
-Tests for archivebox mcp command.
-"""
+"""Tests for the ArchiveBox MCP server and CLI entry point."""
+import json
+import os
+
+import pytest
+
+from archivebox.mcp.server import MCPServer
from archivebox.tests.conftest import run_archivebox_cmd
+pytestmark = pytest.mark.django_db(transaction=True)
+
+
def test_mcp_help_runs_successfully(tmp_path):
"""The mcp command should be registered and expose help."""
@@ -13,3 +20,150 @@ def test_mcp_help_runs_successfully(tmp_path):
assert result.returncode == 0
assert "mcp" in result.stdout.lower()
+
+
+def test_mcp_stdio_handles_handshake_notification_and_ping(initialized_archive):
+ requests = [
+ {
+ "jsonrpc": "2.0",
+ "id": 1,
+ "method": "initialize",
+ "params": {
+ "protocolVersion": "2025-11-25",
+ "capabilities": {},
+ "clientInfo": {"name": "archivebox-test", "version": "1"},
+ },
+ },
+ {"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}},
+ {"jsonrpc": "2.0", "id": 2, "method": "ping", "params": {}},
+ ]
+ result = run_archivebox_cmd(
+ ["mcp"],
+ cwd=initialized_archive,
+ input="".join(json.dumps(request) + "\n" for request in requests),
+ default_cli_env=True,
+ )
+ responses = [json.loads(line) for line in result.stdout.splitlines()]
+
+ assert result.returncode == 0
+ assert [response["id"] for response in responses] == [1, 2]
+ assert responses[0]["result"]["protocolVersion"] == "2025-11-25"
+ assert responses[1]["result"] == {}
+
+
+def test_mcp_exposes_six_focused_tools():
+ tools = MCPServer().handle_tools_list({})["tools"]
+ tools_by_name = {tool["name"]: tool for tool in tools}
+
+ assert set(tools_by_name) == {"add", "search", "crawl", "snapshot", "archiveresult", "shell"}
+ assert len(tools) == 6
+ assert all("outputSchema" in tool for tool in tools)
+
+ crawl_schema = tools_by_name["crawl"]["inputSchema"]
+ assert crawl_schema["properties"]["action"]["enum"] == ["create", "delete", "list", "update"]
+ assert crawl_schema["properties"]["urls"]["type"] == "array"
+ assert crawl_schema["properties"]["records"]["type"] == "array"
+ assert tools_by_name["search"]["annotations"]["readOnlyHint"] is True
+ assert tools_by_name["shell"]["inputSchema"]["required"] == ["code"]
+ assert tools_by_name["shell"]["annotations"]["destructiveHint"] is True
+
+
+def test_mcp_crawl_create_returns_structured_json(initialized_archive):
+ os.chdir(initialized_archive)
+ result = MCPServer().handle_tools_call(
+ {
+ "name": "crawl",
+ "arguments": {
+ "action": "create",
+ "urls": ["https://mcp-test.example.com/"],
+ "depth": 1,
+ "tag": "mcp-test",
+ },
+ },
+ )
+
+ assert result["isError"] is False, result
+ assert result["structuredContent"]["success"] is True
+ assert result["structuredContent"]["error"] is None
+ assert result["structuredContent"]["command"] == "archivebox crawl create"
+ assert result["structuredContent"]["exitCode"] == 0
+ assert result["structuredContent"]["records"][0]["urls"] == "https://mcp-test.example.com/"
+ assert result["structuredContent"]["records"][0]["max_depth"] == 1
+ assert json.loads(result["content"][0]["text"]) == result["structuredContent"]
+
+
+def test_mcp_snapshot_update_accepts_records_without_a_jsonl_pipeline(initialized_archive):
+ os.chdir(initialized_archive)
+ server = MCPServer()
+ created = server.handle_tools_call(
+ {
+ "name": "snapshot",
+ "arguments": {
+ "action": "create",
+ "urls": ["https://mcp-update.example.com/"],
+ },
+ },
+ )
+ assert created["isError"] is False, created
+ snapshot = created["structuredContent"]["records"][0]
+
+ updated = server.handle_tools_call(
+ {
+ "name": "snapshot",
+ "arguments": {
+ "action": "update",
+ "records": [{"id": snapshot["id"]}],
+ "tag": "updated-through-mcp",
+ },
+ },
+ )
+
+ assert updated["isError"] is False
+ assert updated["structuredContent"]["records"][0]["id"] == snapshot["id"]
+ assert "updated-through-mcp" in updated["structuredContent"]["records"][0]["tags"]
+
+
+def test_mcp_cli_errors_are_structured_for_agents(initialized_archive):
+ os.chdir(initialized_archive)
+ result = MCPServer().handle_tools_call(
+ {
+ "name": "crawl",
+ "arguments": {"action": "create"},
+ },
+ )
+
+ assert result["isError"] is True
+ assert result["structuredContent"]["success"] is False
+ assert result["structuredContent"]["exitCode"] == 1
+ assert "No URLs provided" in result["structuredContent"]["error"]
+ assert json.loads(result["content"][0]["text"]) == result["structuredContent"]
+
+
+def test_mcp_invalid_action_is_a_protocol_error():
+ response = MCPServer().handle_request(
+ {
+ "jsonrpc": "2.0",
+ "id": 1,
+ "method": "tools/call",
+ "params": {"name": "crawl", "arguments": {"action": "bogus"}},
+ },
+ )
+
+ assert response["error"]["code"] == -32602
+ assert "Choose one of: create, delete, list, update" in response["error"]["message"]
+
+
+def test_mcp_shell_runs_python_through_archivebox_shell(initialized_archive):
+ os.chdir(initialized_archive)
+ result = MCPServer().handle_tools_call(
+ {
+ "name": "shell",
+ "arguments": {
+ "code": "from archivebox.core.models import Snapshot; print(f'shell_ok={Snapshot.objects.count() >= 0}')",
+ },
+ },
+ )
+
+ assert result["isError"] is False, result
+ assert result["structuredContent"]["command"] == "archivebox shell"
+ assert result["structuredContent"]["stdout"] == "shell_ok=True\n"
diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py
index 21887922..a3f30dd3 100644
--- a/archivebox/tests/test_cli_run.py
+++ b/archivebox/tests/test_cli_run.py
@@ -10,7 +10,6 @@ Tests cover:
import os
import signal
import subprocess
-from datetime import timedelta
import psutil
import pytest
@@ -734,7 +733,7 @@ class TestRunDaemonMode:
cleanup_process_group(proc.pid)
proc.wait(timeout=15)
- def test_run_daemon_recovers_stopped_runner_from_previous_pid_namespace(self, initialized_archive, db):
+ def test_run_daemon_retires_runner_from_previous_pid_namespace(self, initialized_archive, db):
from django.utils import timezone
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE
@@ -753,8 +752,6 @@ class TestRunDaemonMode:
started_at=timezone.now(),
env={PROCESS_PID_NAMESPACE_KEY: f"{get_current_pid_namespace()}-stopped-container"},
)
- Process.objects.filter(pk=stopped_runner.pk).update(modified_at=timezone.now() - timedelta(minutes=5))
-
queued = run_archivebox_cmd(["crawl", "create", create_test_url()], cwd=initialized_archive, env=env, timeout=60)
assert queued.returncode == 0, queued.stderr or queued.stdout
@@ -774,6 +771,7 @@ class TestRunDaemonMode:
try:
wait_for_log(daemon_log, "[Crawl#", timeout=10)
+ assert "Multiple orchestrators sharing a single collection is not officially supported" in daemon_log.read_text()
with use_archivebox_db(initialized_archive):
stopped_runner.refresh_from_db()
assert stopped_runner.status == Process.StatusChoices.EXITED
diff --git a/archivebox/tests/test_takeover_util.py b/archivebox/tests/test_takeover_util.py
index 6246e377..8d075459 100644
--- a/archivebox/tests/test_takeover_util.py
+++ b/archivebox/tests/test_takeover_util.py
@@ -7,7 +7,6 @@ import re
import signal
import subprocess
import sys
-import time
from pathlib import Path
import pytest
@@ -963,27 +962,32 @@ def test_runtime_stack_owner_allows_top_level_runner_when_no_parent_command_exis
_stop_archivebox_shells([proc])
-def test_active_runner_lease_heartbeat_updates_persisted_process(tmp_path, initialized_archive):
- from django.utils import timezone
-
- from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, maintain_runner_lease
+def test_foreign_machine_runner_only_warns(tmp_path, initialized_archive, capsys):
+ from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, live_runner_processes
from archivebox.machine.models import Machine
- runner = Process.objects.create(
- machine=Machine.current(),
+ foreign_machine = Machine.objects.create(
+ guid="foreign-machine",
+ hostname="foreign-host",
+ hw_manufacturer="Test",
+ hw_product="Test",
+ hw_uuid="foreign-hardware",
+ os_arch="x86_64",
+ os_family="linux",
+ os_platform="linux",
+ os_release="test",
+ os_kernel="test",
+ )
+ foreign_runner = Process.objects.create(
+ machine=foreign_machine,
process_type=Process.TypeChoices.ORCHESTRATOR,
worker_type=RUNNER_ACTIVE_WORKER_TYPE,
pwd=str(tmp_path),
- pid=os.getpid(),
- started_at=timezone.now(),
+ pid=1,
status=Process.StatusChoices.RUNNING,
)
- initial_heartbeat = runner.modified_at
- with maintain_runner_lease(runner, interval=0.01):
- deadline = time.monotonic() + 2.0
- while runner.modified_at <= initial_heartbeat and time.monotonic() < deadline:
- time.sleep(0.01)
- runner.refresh_from_db()
-
- assert runner.modified_at > initial_heartbeat
+ assert live_runner_processes(data_dir=tmp_path) == []
+ foreign_runner.refresh_from_db()
+ assert foreign_runner.status == Process.StatusChoices.RUNNING
+ assert "Multiple orchestrators sharing a single collection is not officially supported" in capsys.readouterr().err
diff --git a/bin/collect_ui_screenshots.sh b/bin/collect_ui_screenshots.sh
index 3cc3a411..5d2fb0ae 100755
--- a/bin/collect_ui_screenshots.sh
+++ b/bin/collect_ui_screenshots.sh
@@ -18,6 +18,8 @@ ARCHIVE_PID=""
CREATED_TEMP_USER=0
CREATE_API_TOKEN=0
CREATE_WEBHOOK=0
+ABXPKG_LIB_DIR=""
+SCREENSHOT_CHROME_BINARY=""
CAPTURE_ROOT="$(mktemp -d)"
MANIFEST_FILE="$CAPTURE_ROOT/manifest.jsonl"
PERSONAS_DIR="$CAPTURE_ROOT/personas"
@@ -203,6 +205,13 @@ fi
# the Sweeting.me example below runs through its own real foreground runner.
stop_background_runner
+ABXPKG_LIB_DIR="$(uv run --no-cache --project "$REPO_DIR" abx-dl config --get ABXPKG_LIB_DIR | sed 's/^[^=]*=//; s/^"//; s/"$//')"
+SCREENSHOT_CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
+if [[ ! -x "$SCREENSHOT_CHROME_BINARY" ]]; then
+ echo "[!] abx-dl projected Chromium was not found at $SCREENSHOT_CHROME_BINARY" >&2
+ exit 1
+fi
+
VIEWS=(
"Login|$ADMIN_BASE_URL/admin/login/|/admin/login/|archivebox/templates/admin/login.html"
"Public snapshot list|$PUBLIC_BASE_URL/public/|/public/|archivebox/core/views.py"
@@ -225,9 +234,11 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
SCREENSHOT_HEIGHT=1000 \
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$CAPTURE_ROOT/snapshot-header-state.png" >/dev/null
fi
+ view_timing_report=""
while IFS='|' read -r profile viewport_width viewport_height; do
filename="$(printf '%02d' "$capture_index")-$slug-$profile.png"
capture_dir="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/$profile"
+ timing_report_path="$view_timing_report"
capture_env=(
"RESOLUTION=$viewport_width,$viewport_height"
"CHROME_RESOLUTION=$viewport_width,$viewport_height"
@@ -272,6 +283,8 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
SCREENSHOT_SNAPSHOT_HEADER=expanded \
SCREENSHOT_EXPECT_LIVE_PROGRESS=1 \
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
+ view_timing_report="$capture_dir/report.json"
+ timing_report_path="$view_timing_report"
fi
elif [[ "$profile" == "desktop" || "$capture_mode" == wait-replay:* || -z "${ABXPKG_LIB_DIR:-}" ]]; then
capture_log="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")-$profile-abx-dl.log"
@@ -311,6 +324,8 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
SCREENSHOT_VARIANTS_JSON="$responsive_variants" \
SCREENSHOT_COLLAPSE_FILTERS=1 \
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
+ view_timing_report="$capture_dir/report.json"
+ timing_report_path="$view_timing_report"
uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" validate \
"$capture_dir/report.json" "$expected_path"
fi
@@ -324,6 +339,7 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
cp "$screenshot_path" "$PUBLIC_OUTPUT_DIR/$filename"
UI_SCREENSHOT_NAME="$name" UI_SCREENSHOT_URL="$url" UI_SCREENSHOT_SOURCE="$source" \
UI_SCREENSHOT_FILENAME="$filename" UI_SCREENSHOT_PROFILE="$profile" \
+ UI_SCREENSHOT_TIMING_REPORT="$timing_report_path" \
uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" append \
"$MANIFEST_FILE" "$OUTPUT_DIR/$filename"
done <<<"$CAPTURE_PROFILES"
@@ -331,12 +347,6 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
# Capture the public views before login because the real admin-login hint
# intentionally redirects authenticated personas away from /public/.
if [[ "$capture_index" == "2" ]]; then
- ABXPKG_LIB_DIR="$(uv run --no-cache --project "$REPO_DIR" abx-dl config --get ABXPKG_LIB_DIR | sed 's/^[^=]*=//; s/^"//; s/"$//')"
- SCREENSHOT_CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
- if [[ ! -x "$SCREENSHOT_CHROME_BINARY" ]]; then
- echo "[!] abx-dl projected Chromium was not found at $SCREENSHOT_CHROME_BINARY" >&2
- exit 1
- fi
echo "[*] Logging in through the real $ACTIVE_PERSONA browser persona"
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
diff --git a/bin/generate_ui_screenshot_gallery.py b/bin/generate_ui_screenshot_gallery.py
index 2ef01c27..1c327cfd 100755
--- a/bin/generate_ui_screenshot_gallery.py
+++ b/bin/generate_ui_screenshot_gallery.py
@@ -112,6 +112,12 @@ def append_manifest(manifest_path: Path, screenshot_path: Path) -> None:
"width": dimensions[0],
"height": dimensions[1],
}
+ timing_report_path = os.environ.get("UI_SCREENSHOT_TIMING_REPORT", "").strip()
+ if timing_report_path:
+ timing_report = json.loads(Path(timing_report_path).read_text(encoding="utf-8"))
+ ttfb_ms = timing_report.get("checks", {}).get("ttfbMs")
+ if isinstance(ttfb_ms, int | float):
+ item["ttfb_ms"] = round(ttfb_ms)
with manifest_path.open("a", encoding="utf-8") as manifest:
manifest.write(json.dumps(item) + "\n")
@@ -179,6 +185,8 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
if parsed_url.fragment:
route = f"{route}#{parsed_url.fragment}"
source_url = f"{source_base_url}{capture['source']}"
+ ttfb_values = [variant["ttfb_ms"] for variant in variants.values() if isinstance(variant.get("ttfb_ms"), int | float)]
+ ttfb_text = f" · ~{round(sum(ttfb_values) / len(ttfb_values))}ms TTFB" if ttfb_values else ""
markdown_cells = []
html_figures = []
for profile, (width, height) in CAPTURE_PROFILES.items():
@@ -206,7 +214,7 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
(
f"## {capture['name']}",
"",
- f"View: [`{route}`]({capture['url']}) · [View code]({source_url})",
+ f"View: [`{route}`]({capture['url']}) · [View code]({source_url}){ttfb_text}",
"",
"",
"".join(f"{profile.title()} " for profile in CAPTURE_PROFILES),
@@ -219,7 +227,7 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
html_sections.append(
f"`
- - ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner_types
- :summary:
- ```
-* - {py:obj}`foreground_runner_owner_types `
- - ```{autodoc2-docstring} archivebox.core.takeover_util.foreground_runner_owner_types
- :summary:
- ```
* - {py:obj}`current_command `
- ```{autodoc2-docstring} archivebox.core.takeover_util.current_command
:summary:
```
-* - {py:obj}`live_processes `
- - ```{autodoc2-docstring} archivebox.core.takeover_util.live_processes
- :summary:
- ```
-* - {py:obj}`newest_live_process `
- - ```{autodoc2-docstring} archivebox.core.takeover_util.newest_live_process
- :summary:
- ```
-* - {py:obj}`command_is_newest `
- - ```{autodoc2-docstring} archivebox.core.takeover_util.command_is_newest
- :summary:
- ```
* - {py:obj}`runtime_stack_owner `
- ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner
:summary:
@@ -55,22 +35,10 @@
- ```{autodoc2-docstring} archivebox.core.takeover_util.command_owns_foreground_runner
:summary:
```
-* - {py:obj}`runtime_stack_component_label `
- - ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_component_label
- :summary:
- ```
* - {py:obj}`ensure_daemon_stack `
- ```{autodoc2-docstring} archivebox.core.takeover_util.ensure_daemon_stack
:summary:
```
-* - {py:obj}`healthy_orchestrator `
- - ```{autodoc2-docstring} archivebox.core.takeover_util.healthy_orchestrator
- :summary:
- ```
-* - {py:obj}`_runner_sort_key `
- - ```{autodoc2-docstring} archivebox.core.takeover_util._runner_sort_key
- :summary:
- ```
* - {py:obj}`live_runner_processes `
- ```{autodoc2-docstring} archivebox.core.takeover_util.live_runner_processes
:summary:
@@ -79,10 +47,6 @@
- ```{autodoc2-docstring} archivebox.core.takeover_util.enter_single_runner_gate
:summary:
```
-* - {py:obj}`standby_until_leader_needed `
- - ```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_leader_needed
- :summary:
- ```
* - {py:obj}`standby_until_runtime_stack_needed `
- ```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_runtime_stack_needed
:summary:
@@ -145,20 +109,6 @@
````
-````{py:function} runtime_stack_owner_types()
-:canonical: archivebox.core.takeover_util.runtime_stack_owner_types
-
-```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner_types
-```
-````
-
-````{py:function} foreground_runner_owner_types()
-:canonical: archivebox.core.takeover_util.foreground_runner_owner_types
-
-```{autodoc2-docstring} archivebox.core.takeover_util.foreground_runner_owner_types
-```
-````
-
````{py:function} current_command(process_type: str, *, data_dir: str | pathlib.Path, url: str | None = None)
:canonical: archivebox.core.takeover_util.current_command
@@ -166,27 +116,6 @@
```
````
-````{py:function} live_processes(*, process_type: str, data_dir: str | pathlib.Path, url: str | None = None)
-:canonical: archivebox.core.takeover_util.live_processes
-
-```{autodoc2-docstring} archivebox.core.takeover_util.live_processes
-```
-````
-
-````{py:function} newest_live_process(*, process_type: str, data_dir: str | pathlib.Path, url: str | None = None)
-:canonical: archivebox.core.takeover_util.newest_live_process
-
-```{autodoc2-docstring} archivebox.core.takeover_util.newest_live_process
-```
-````
-
-````{py:function} command_is_newest(command, *, process_type: str, data_dir: str | pathlib.Path, url: str | None = None) -> bool
-:canonical: archivebox.core.takeover_util.command_is_newest
-
-```{autodoc2-docstring} archivebox.core.takeover_util.command_is_newest
-```
-````
-
````{py:function} runtime_stack_owner(*, data_dir: str | pathlib.Path, exclude_id=None)
:canonical: archivebox.core.takeover_util.runtime_stack_owner
@@ -215,13 +144,6 @@
```
````
-````{py:function} runtime_stack_component_label(*, owner=None, data_dir: str | pathlib.Path) -> str
-:canonical: archivebox.core.takeover_util.runtime_stack_component_label
-
-```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_component_label
-```
-````
-
````{py:function} ensure_daemon_stack(*, reason: str = '')
:canonical: archivebox.core.takeover_util.ensure_daemon_stack
@@ -229,21 +151,7 @@
```
````
-````{py:function} healthy_orchestrator(*, data_dir: str | pathlib.Path)
-:canonical: archivebox.core.takeover_util.healthy_orchestrator
-
-```{autodoc2-docstring} archivebox.core.takeover_util.healthy_orchestrator
-```
-````
-
-````{py:function} _runner_sort_key(process)
-:canonical: archivebox.core.takeover_util._runner_sort_key
-
-```{autodoc2-docstring} archivebox.core.takeover_util._runner_sort_key
-```
-````
-
-````{py:function} live_runner_processes(*, data_dir: str | pathlib.Path, exclude_id=None)
+````{py:function} live_runner_processes(*, data_dir: str | pathlib.Path)
:canonical: archivebox.core.takeover_util.live_runner_processes
```{autodoc2-docstring} archivebox.core.takeover_util.live_runner_processes
@@ -257,13 +165,6 @@
```
````
-````{py:function} standby_until_leader_needed(command, *, process_type: str, data_dir: str | pathlib.Path, url: str | None = None, interval: float = 2.0) -> None
-:canonical: archivebox.core.takeover_util.standby_until_leader_needed
-
-```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_leader_needed
-```
-````
-
````{py:function} standby_until_runtime_stack_needed(command, *, data_dir: str | pathlib.Path, interval: float = 2.0) -> dict[str, object]
:canonical: archivebox.core.takeover_util.standby_until_runtime_stack_needed
diff --git a/etc/package.json b/etc/package.json
index ff26beeb..7167e2fe 100644
--- a/etc/package.json
+++ b/etc/package.json
@@ -1,6 +1,6 @@
{
"name": "archivebox",
- "version": "0.9.35rc362",
+ "version": "0.9.35rc365",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {
diff --git a/publicsite/index.html b/publicsite/index.html
index bebf5965..c0978cf6 100644
--- a/publicsite/index.html
+++ b/publicsite/index.html
@@ -99,8 +99,9 @@
# Docker Compose is the recommended setup
$ mkdir -p ~/archivebox/data && cd ~/archivebox
$ curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
-$ docker compose run archivebox init
--> created ./data/index.sqlite3
+$ docker compose up -d --wait
+-> initialized ./data/index.sqlite3
+ok listening on http://127.0.0.1:8000
$
@@ -207,16 +208,15 @@
3
- Initialize and start
- $ docker compose run archivebox init
-$ docker compose up
+ Start ArchiveBox (initializes automatically)
+ $ docker compose up -d --wait
4
Add your first URL
- $ docker compose run archivebox add 'https://example.com'
+ $ docker compose exec archivebox archivebox add 'https://example.com'
diff --git a/pyproject.toml b/pyproject.toml
index f975615d..6a5e7bca 100755
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "archivebox"
-version = "0.9.35rc362"
+version = "0.9.35rc365"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -342,7 +342,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
[tool.bumpver]
-current_version = "v0.9.35rc362"
+current_version = "v0.9.35rc365"
version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
commit_message = "bump version {old_version} -> {new_version}"
tag_message = "{new_version}"
diff --git a/uv.lock b/uv.lock
index 3e7793c8..6c18fe21 100644
--- a/uv.lock
+++ b/uv.lock
@@ -120,7 +120,7 @@ wheels = [
[[package]]
name = "archivebox"
-version = "0.9.35rc362"
+version = "0.9.35rc365"
source = { editable = "." }
dependencies = [
{ name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },