Merge remote-tracking branch 'origin/dev' into audit-pr1850

2026-09-12 19:50:57 +05:00 · 2026-09-01 11:38:24 -07:00 · 2026-09-01 11:38:24 -07:00 · 92a939a6a2
commit 92a939a6a2
parent 3fc67705a5 39c64b21d0
19 changed files with 398 additions and 416 deletions
--- a/README.md
+++ b/README.md
@ -80,8 +80,6 @@ docker compose up -d --wait                                                # ini
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev init
-docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev install
 docker run -d --name archivebox -v "$PWD:/data" -p 8000:8000 archivebox/archivebox:dev
 # open http://admin.archivebox.localhost:8000 to finish setup
 # docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
@ -187,17 +185,16 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
-<li>Create a new empty directory and initialize your collection (can be anywhere).
+<li>Create a new empty directory and start the server, which initializes the collection automatically (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run --rm -v $PWD:/data -it archivebox/archivebox:dev init
-docker run --rm -v $PWD:/data -it archivebox/archivebox:dev install
+docker run -d --name archivebox -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 </li>
-<li>Optional: Start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
-<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
+<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
+<pre lang="bash"><code style="white-space: pre-line">
 # completely optional, CLI can always be used without running a server
-# docker run -v $PWD:/data -it archivebox/archivebox:dev [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
+# docker exec archivebox archivebox [subcommand] [--help]
+docker exec archivebox archivebox help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@ -497,12 +494,10 @@ archivebox help              # get list of archivebox subcommands that can be ru
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
-# docker compose run --rm archivebox [subcommand] [--help]
-docker compose run --rm archivebox init
-docker compose run --rm archivebox install
-docker compose run --rm archivebox version
-docker compose run --rm archivebox help
-docker compose run --rm archivebox add 'https://example.com'
+# docker compose exec archivebox archivebox [subcommand] [--help]
+docker compose exec archivebox archivebox version
+docker compose exec archivebox archivebox help
+docker compose exec archivebox archivebox add 'https://example.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@ -514,15 +509,12 @@ docker compose run --rm archivebox add 'https://example.com'
 <summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
-# make sure you create and cd into in a new empty directory first  
+# make sure the `archivebox` server container from the Quickstart is running first
 <br/>
-# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev init
-docker run -v $PWD:/data -it archivebox/archivebox:dev install
-docker run -v $PWD:/data -it archivebox/archivebox:dev version
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
-docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
-# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
+# docker exec archivebox archivebox [subcommand] [--help]
+docker exec archivebox archivebox version
+docker exec archivebox archivebox help
+docker exec archivebox archivebox add 'https://example.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
 </details>
@ -1345,14 +1337,11 @@ archivebox server 0.0.0.0:8000
 # inside the container will reload and pick up your changes
 ./bin/build_docker.sh dev

-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
-
 # Run the development server w/ autoreloading (but no bg workers)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init --debug --reload 0.0.0.0:8000

 # Run the production server (with bg workers but no autoreloading)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init

 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
 # When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@ -1413,7 +1402,7 @@ services:

 # or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data archivebox:dev init
+docker run -it -v $PWD:/data -p 8000:8000 archivebox:dev

 # or with uv:
 uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
--- a/archivebox/cli/archivebox_run.py
+++ b/archivebox/cli/archivebox_run.py
@ -270,7 +270,7 @@ def run_runner(
    from archivebox.config import CONSTANTS
    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
    from archivebox.machine.models import Machine, Process
-    from archivebox.core.takeover_util import enter_single_runner_gate, maintain_runner_lease, standby_until_foreground_runner_needed
+    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
    from archivebox.core.recovery_util import recover_orchestrator_state
    from archivebox.services.runner import run_pending_crawls

@ -292,40 +292,40 @@ def run_runner(
        current.mark_exited()
        return 0

-    try:
-        with maintain_runner_lease(current):
-            recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
-            if crawl_id:
-                from django.utils import timezone
-                from archivebox.crawls.models import Crawl
+    recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
+    if crawl_id:
+        from django.utils import timezone
+        from archivebox.crawls.models import Crawl

-                crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
-                now = timezone.now()
-                # Winning the single-runner gate terminates and waits for every older
-                # runner. Requeue the explicitly requested crawl so an abandoned
-                # future ownership lease cannot hide it from the unified scheduler.
-                if crawl is not None:
-                    crawl.update_and_requeue(retry_at=now, refresh=False)
-            # Only a foreground `archivebox add` gets the interactive "abort current
-            # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
-            # orchestrators should shut down immediately and cleanly on the first signal.
-            interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
-            if daemon:
-                os.environ[RUNNER_DAEMON_ENV] = "1"
-            with (
-                foreground_shutdown_signals(
-                    on_signal=_exit_daemon_runner_on_signal if daemon else None,
-                    raise_on_first_signal=not daemon,
-                ),
-                foreground_parent_watchdog(enabled=not daemon),
-            ):
-                run_pending_crawls(
-                    daemon=daemon,
-                    crawl_id=crawl_id,
-                    maintenance_only=maintenance_only,
-                    interactive_interrupts=interactive_interrupts,
-                    **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
-                )
+        crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
+        now = timezone.now()
+        # Winning the single-runner gate terminates and waits for every older
+        # runner. Requeue the explicitly requested crawl so an abandoned
+        # future ownership lease cannot hide it from the unified scheduler.
+        if crawl is not None:
+            crawl.update_and_requeue(retry_at=now, refresh=False)
+    # Only a foreground `archivebox add` gets the interactive "abort current
+    # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
+    # orchestrators should shut down immediately and cleanly on the first signal.
+    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    if daemon:
+        os.environ[RUNNER_DAEMON_ENV] = "1"
+
+    try:
+        with (
+            foreground_shutdown_signals(
+                on_signal=_exit_daemon_runner_on_signal if daemon else None,
+                raise_on_first_signal=not daemon,
+            ),
+            foreground_parent_watchdog(enabled=not daemon),
+        ):
+            run_pending_crawls(
+                daemon=daemon,
+                crawl_id=crawl_id,
+                maintenance_only=maintenance_only,
+                interactive_interrupts=interactive_interrupts,
+                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
+            )
        return 0
    except KeyboardInterrupt:
        return 0
@ -432,7 +432,7 @@ def main(

 def run_snapshot_worker(snapshot_id: str) -> int:
    from archivebox.config import CONSTANTS
-    from archivebox.core.takeover_util import enter_single_runner_gate, maintain_runner_lease
+    from archivebox.core.takeover_util import enter_single_runner_gate
    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
    from archivebox.machine.models import Process
    from archivebox.core.models import Snapshot
@ -446,16 +446,15 @@ def run_snapshot_worker(snapshot_id: str) -> int:

    snapshot = None
    try:
-        with maintain_runner_lease(current):
-            with foreground_shutdown_signals(), foreground_parent_watchdog():
-                for _ in range(10):
-                    snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-                    if snapshot.retry_at is None:
-                        snapshot.update_and_requeue(retry_at=timezone.now())
-                    elif snapshot.retry_at > timezone.now():
-                        break
-                    if not run_due_snapshot(snapshot, lock_seconds=60):
-                        break
+        with foreground_shutdown_signals(), foreground_parent_watchdog():
+            for _ in range(10):
+                snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+                if snapshot.retry_at is None:
+                    snapshot.update_and_requeue(retry_at=timezone.now())
+                elif snapshot.retry_at > timezone.now():
+                    break
+                if not run_due_snapshot(snapshot, lock_seconds=60):
+                    break
        return 0
    except KeyboardInterrupt:
        try:
--- a/archivebox/core/takeover_util.py
+++ b/archivebox/core/takeover_util.py
@ -1,14 +1,25 @@
+"""Coordinate local foreground processes without pretending to provide a distributed lock.
+
+ArchiveBox wants one orchestrator per collection, but Process rows can only prove
+liveness for PIDs visible on the current machine/PID namespace. We therefore
+enforce one active runner per ``(Machine, DATA_DIR)`` locally, retire stale rows
+from sequential containers on that machine, and only warn about rows owned by a
+different machine. Foreign-machine rows must never block progress or be killed:
+multi-machine coordination belongs in the Crawl/Snapshot CAS claim layer, not in
+process takeover.
+
+These helpers only hand local supervisord/runner ownership between CLI parents.
+They must not hold database transactions or filesystem locks while work runs.
+"""
+
 from __future__ import annotations

 import time
 import sys
-import threading
 from collections.abc import Callable
-from contextlib import contextmanager
-from datetime import timedelta
 from pathlib import Path

-from django.db import DatabaseError, IntegrityError, close_old_connections
+from django.db import IntegrityError
 from django.utils import timezone
 from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
@ -16,27 +27,6 @@ from archivebox.config.common import rprint
 RUNNER_ACTIVE_WORKER_TYPE = "worker_runner"
 RUNNER_WAITING_WORKER_TYPE = "runner_waiting"
 RUNNER_GATE_WORKER_TYPES = (RUNNER_ACTIVE_WORKER_TYPE, RUNNER_WAITING_WORKER_TYPE, "")
-RUNNER_LEASE_HEARTBEAT_SECONDS = 5.0
-RUNNER_LEASE_TIMEOUT = timedelta(seconds=30)
-
-
-def runtime_stack_owner_types():
-    from archivebox.machine.models import Process
-
-    return (
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ORCHESTRATOR,
-    )
-
-
-def foreground_runner_owner_types():
-    from archivebox.machine.models import Process
-
-    return (
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ADD,
-        Process.TypeChoices.UPDATE,
-    )


 def current_command(process_type: str, *, data_dir: str | Path, url: str | None = None):
@ -47,31 +37,8 @@ def current_command(process_type: str, *, data_dir: str | Path, url: str | None
    return proc


-def live_processes(*, process_type: str, data_dir: str | Path, url: str | None = None):
-    from archivebox.machine.models import Machine, Process
-
-    qs = Process.objects.filter(
-        machine=Machine.current(),
-        process_type=process_type,
-        status=Process.StatusChoices.RUNNING,
-        pwd=str(data_dir),
-    )
-    if url is not None:
-        qs = qs.filter(url=url)
-    return [proc for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50) if proc.is_running]
-
-
-def newest_live_process(*, process_type: str, data_dir: str | Path, url: str | None = None):
-    processes = live_processes(process_type=process_type, data_dir=data_dir, url=url)
-    return processes[0] if processes else None
-
-
-def command_is_newest(command, *, process_type: str, data_dir: str | Path, url: str | None = None) -> bool:
-    leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
-    return bool(leader and leader.id == command.id)
-
-
 def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
+    """Return the live local parent allowed to own the server runtime stack."""
    from archivebox.machine.models import Machine, Process

    machine = Machine.current()
@ -79,7 +46,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
        machine=machine,
        status=Process.StatusChoices.RUNNING,
        pwd=str(data_dir),
-        process_type__in=runtime_stack_owner_types(),
+        process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ORCHESTRATOR),
    )
    if exclude_id is not None:
        base_qs = base_qs.exclude(id=exclude_id)
@ -108,6 +75,7 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:


 def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
+    """Return the newest live local parent allowed to borrow runner/sonic."""
    from archivebox.machine.models import Machine, Process

    machine = Machine.current()
@ -115,7 +83,7 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
        machine=machine,
        status=Process.StatusChoices.RUNNING,
        pwd=str(data_dir),
-        process_type__in=foreground_runner_owner_types(),
+        process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ADD, Process.TypeChoices.UPDATE),
    )
    if exclude_id is not None:
        qs = qs.exclude(id=exclude_id)
@ -131,26 +99,6 @@ def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
    return bool(owner and owner.id == command.id)


-def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
-    try:
-        from archivebox.workers.supervisord_util import active_supervisord_runtime_components
-
-        components = active_supervisord_runtime_components()
-    except Exception:
-        components = []
-
-    names = list(components)
-    if not names and owner is not None:
-        from archivebox.machine.models import Process
-
-        if owner.process_type == Process.TypeChoices.SERVER:
-            names = ["orchestrator", "server"]
-        elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
-            names = ["orchestrator"]
-
-    return ", ".join(dict.fromkeys(names)) or "runtime stack"
-
-
 def ensure_daemon_stack(*, reason: str = ""):
    from archivebox.config.common import get_config
    from archivebox.workers.supervisord_util import (
@ -186,115 +134,65 @@ def ensure_daemon_stack(*, reason: str = ""):
    return start_worker(supervisor, sonic_worker)


-def healthy_orchestrator(*, data_dir: str | Path):
-    from archivebox.machine.models import Machine, Process
-    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+def live_runner_processes(*, data_dir: str | Path):
+    """Return locally verifiable runners and warn about unsupported overlap.

-    supervisor = get_existing_supervisord_process()
-    worker = get_worker(supervisor, "worker_runner") if supervisor else None
-    if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
-        return worker
-
-    for proc in Process.objects.filter(
-        machine=Machine.current(),
-        process_type=Process.TypeChoices.ORCHESTRATOR,
-        status=Process.StatusChoices.RUNNING,
-        pwd=str(data_dir),
-    ).order_by("-created_at"):
-        if proc.is_running:
-            return proc
-    return None
-
-
-def _runner_sort_key(process):
-    return (process.started_at or process.created_at, process.created_at, str(process.id))
-
-
-def live_runner_processes(*, data_dir: str | Path, exclude_id=None):
+    A Process row from another machine is observability only: its PID cannot be
+    checked or signalled here, so it neither joins the local election nor gets
+    mutated. A row for this same Machine from another PID namespace represents
+    a previous sequential container under the supported model; warn, retire the
+    unreachable row, and let the new container continue.
+    """
    from archivebox.machine.models import Machine, Process

    machine = Machine.current()
    Process.cleanup_stale_running(machine=machine)
    qs = Process.objects.filter(
-        machine=machine,
        status=Process.StatusChoices.RUNNING,
        process_type=Process.TypeChoices.ORCHESTRATOR,
        worker_type__in=RUNNER_GATE_WORKER_TYPES,
        pwd=str(data_dir),
    )
-    if exclude_id is not None:
-        qs = qs.exclude(id=exclude_id)
-    now = timezone.now()
+    foreign_machine_exists = qs.exclude(machine=machine).exists()
+    qs = qs.filter(machine=machine)
    live = []
+    foreign_namespace_ids = []
    for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
-        if not process.shares_pid_namespace and process.modified_at < now - RUNNER_LEASE_TIMEOUT:
-            # A PID from another container cannot be inspected or signaled
-            # safely. Its DB heartbeat is the only cross-namespace liveness
-            # proof, so release the runner gate only after that lease expires.
-            exited = Process.objects.filter(
-                pk=process.pk,
-                status=Process.StatusChoices.RUNNING,
-                modified_at=process.modified_at,
-            ).update(
-                status=Process.StatusChoices.EXITED,
-                exit_code=process.exit_code if process.exit_code is not None else 0,
-                ended_at=now,
-                retry_at=None,
-                modified_at=now,
-            )
-            if exited:
-                continue
-            process.refresh_from_db()
+        if not process.shares_pid_namespace:
+            foreign_namespace_ids.append(process.id)
+            continue
        if process.is_running:
            live.append(process)
+    if foreign_machine_exists or foreign_namespace_ids:
+        rprint(
+            "[bold yellow]WARNING: Multiple orchestrators sharing a single collection is not officially supported! "
+            "Corruption may occur if you run two ArchiveBox workers on the same collection at once.[/bold yellow]",
+            file=sys.stderr,
+            soft_wrap=True,
+        )
+    if foreign_namespace_ids:
+        now = timezone.now()
+        Process.objects.filter(id__in=foreign_namespace_ids, status=Process.StatusChoices.RUNNING).update(
+            status=Process.StatusChoices.EXITED,
+            exit_code=0,
+            ended_at=now,
+            retry_at=None,
+            modified_at=now,
+        )
    return live


-@contextmanager
-def maintain_runner_lease(command, *, interval: float = RUNNER_LEASE_HEARTBEAT_SECONDS):
-    """Keep the active runner row fresh for takeover across PID namespaces."""
-    from archivebox.machine.models import Process
-
-    stopped = threading.Event()
-
-    def heartbeat() -> None:
-        close_old_connections()
-        try:
-            while not stopped.wait(interval):
-                try:
-                    updated = Process.objects.filter(
-                        pk=command.pk,
-                        status=Process.StatusChoices.RUNNING,
-                        worker_type=RUNNER_ACTIVE_WORKER_TYPE,
-                    ).update(modified_at=timezone.now())
-                    if not updated:
-                        return
-                except DatabaseError:
-                    # A transient SQLite lock must not terminate the runner.
-                    # The timeout allows several missed heartbeats before a
-                    # cross-namespace contender may reclaim the gate.
-                    continue
-        finally:
-            close_old_connections()
-
-    heartbeat_thread = threading.Thread(target=heartbeat, name="archivebox-runner-lease", daemon=True)
-    heartbeat_thread.start()
-    try:
-        yield
-    finally:
-        stopped.set()
-        heartbeat_thread.join(timeout=interval + 1.0)
-
-
 def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout: float = 5.0) -> bool:
    """
-    Admit exactly one active runner for this DATA_DIR using Process rows.
+    Admit one active runner for this Machine and DATA_DIR using Process rows.

    The current process is a real OS process while it waits, so we keep its
    Process row RUNNING but mark worker_type=runner_waiting. Only the process
    that wins takeover is promoted to worker_type=worker_runner, which is
-    protected by a partial unique DB constraint. Older runners are terminated
-    and fully waited out before promotion, so the runner work loop never overlaps.
+    protected by a partial unique DB constraint scoped to (Machine, DATA_DIR).
+    Older locally verifiable runners are terminated and fully waited out before
+    promotion, so runner work never overlaps on one machine. Foreign machines
+    are intentionally outside this gate and only produce a warning above.
    """
    from archivebox.machine.models import Process

@ -304,12 +202,7 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
        pwd=str(data_dir),
        timeout=CONSTANTS.MAX_HOOK_RUNTIME_SECONDS,
    )
-    last_waiting_heartbeat = time.monotonic()
-    announced_cross_namespace_runners = set()
    while True:
-        if time.monotonic() - last_waiting_heartbeat >= RUNNER_LEASE_HEARTBEAT_SECONDS:
-            command.heartbeat()
-            last_waiting_heartbeat = time.monotonic()
        runners = live_runner_processes(data_dir=data_dir)
        if all(process.id != command.id for process in runners):
            command.refresh_from_db()
@ -321,7 +214,7 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
            )
            runners = live_runner_processes(data_dir=data_dir)

-        newest = max(runners, key=_runner_sort_key)
+        newest = max(runners, key=lambda process: (process.started_at or process.created_at, process.created_at, str(process.id)))
        if newest.id != command.id:
            rprint(
                f"[yellow][*] Newer ArchiveBox runner pid={newest.pid} is taking over; exiting this runner.[/yellow]",
@ -332,16 +225,8 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
        older_runners = [process for process in runners if process.id != command.id]
        if older_runners:
            for process in older_runners:
-                if process.shares_pid_namespace:
-                    rprint(f"[yellow][*] Stopping older ArchiveBox runner process (pid={process.pid})...[/yellow]", file=sys.stderr)
-                    process.kill_tree(graceful_timeout=graceful_timeout)
-                else:
-                    if process.id not in announced_cross_namespace_runners:
-                        rprint(
-                            "[yellow][*] Waiting for older ArchiveBox runner in another PID namespace to stop...[/yellow]",
-                            file=sys.stderr,
-                        )
-                        announced_cross_namespace_runners.add(process.id)
+                rprint(f"[yellow][*] Stopping older ArchiveBox runner process (pid={process.pid})...[/yellow]", file=sys.stderr)
+                process.kill_tree(graceful_timeout=graceful_timeout)
            time.sleep(0.1)
            continue

@ -361,24 +246,9 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
            time.sleep(0.1)


-def standby_until_leader_needed(command, *, process_type: str, data_dir: str | Path, url: str | None = None, interval: float = 2.0) -> None:
-    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
-
-    announced = False
-    while not command_is_newest(command, process_type=process_type, data_dir=data_dir, url=url):
-        reap_foreground_supervisord_process()
-        if not announced:
-            leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
-            leader_pid = leader.pid if leader else "unknown"
-            rprint(f"[yellow][*] Standing by; newer ArchiveBox process pid={leader_pid} is running the orchestrator and server.[/yellow]")
-            announced = True
-        time.sleep(interval)
-    command.modified_at = timezone.now()
-    command.save(update_fields=["modified_at"])
-
-
 def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
-    from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
+    from archivebox.machine.models import Process
+    from archivebox.workers.supervisord_util import active_supervisord_runtime_components, reap_foreground_supervisord_process

    announced = False
    previous_owner_pid = None
@ -387,7 +257,16 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
        if not announced:
            owner = runtime_stack_owner(data_dir=data_dir)
            owner_pid = owner.pid if owner else "unknown"
-            components = runtime_stack_component_label(owner=owner, data_dir=data_dir)
+            try:
+                component_names = list(active_supervisord_runtime_components())
+            except Exception:
+                component_names = []
+            if not component_names and owner is not None:
+                if owner.process_type == Process.TypeChoices.SERVER:
+                    component_names = ["orchestrator", "server"]
+                elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
+                    component_names = ["orchestrator"]
+            components = ", ".join(dict.fromkeys(component_names)) or "runtime stack"
            previous_owner_pid = owner_pid
            rprint(
                f"[yellow][*] A newer archivebox process took over the {components} "
--- a/archivebox/machine/models.py
+++ b/archivebox/machine/models.py
@ -1187,6 +1187,11 @@ class Process(ModelWithDeleteAfter, models.Model):
            models.Index(fields=["machine", "status", "process_type"], name="mach_proc_running_idx"),
        ]
        constraints = [
+            # This is deliberately machine-scoped. It prevents two locally
+            # verifiable runners from working the same collection, while not
+            # claiming to coordinate independent hosts that share a database.
+            # Cross-machine work ownership belongs to short Crawl/Snapshot CAS
+            # claims so PostgreSQL deployments can support that model later.
            models.UniqueConstraint(
                fields=["machine", "pwd"],
                condition=Q(status="running", process_type="orchestrator", worker_type="worker_runner"),
@ -1393,7 +1398,7 @@ class Process(ModelWithDeleteAfter, models.Model):
        self.save(update_fields=updates)

    def heartbeat(self) -> None:
-        """Touch modified_at so standby/leader selection can see this parent is alive."""
+        """Keep a long-lived watcher visible in recent-process monitoring."""
        self.save(update_fields=["modified_at"])

    def mark_exited(self, *, exit_code: int = 0) -> None:
--- a/archivebox/services/runner.py
+++ b/archivebox/services/runner.py
@ -484,6 +484,11 @@ class CrawlRunner:
                await self.enqueue_pending_snapshots_from_projection()

    async def heartbeat_active_leases(self) -> None:
+        # These are resumable work-item leases, not orchestrator-election
+        # heartbeats. Each update is a short autocommit statement; network and
+        # filesystem work continues outside a database transaction. A future
+        # PostgreSQL multi-machine runner uses these Crawl/Snapshot claims as
+        # its coordination boundary while SQLite keeps one local orchestrator.
        if self._run_task is None:
            return
        now_monotonic = time.monotonic()
--- a/archivebox/tests/test_cli_mcp.py
+++ b/archivebox/tests/test_cli_mcp.py
@ -1,11 +1,18 @@
 #!/usr/bin/env python3
-"""
-Tests for archivebox mcp command.
-"""
+"""Tests for the ArchiveBox MCP server and CLI entry point."""

+import json
+import os
+
+import pytest
+
+from archivebox.mcp.server import MCPServer
 from archivebox.tests.conftest import run_archivebox_cmd


+pytestmark = pytest.mark.django_db(transaction=True)
+
+
 def test_mcp_help_runs_successfully(tmp_path):
    """The mcp command should be registered and expose help."""

@ -13,3 +20,150 @@ def test_mcp_help_runs_successfully(tmp_path):

    assert result.returncode == 0
    assert "mcp" in result.stdout.lower()
+
+
+def test_mcp_stdio_handles_handshake_notification_and_ping(initialized_archive):
+    requests = [
+        {
+            "jsonrpc": "2.0",
+            "id": 1,
+            "method": "initialize",
+            "params": {
+                "protocolVersion": "2025-11-25",
+                "capabilities": {},
+                "clientInfo": {"name": "archivebox-test", "version": "1"},
+            },
+        },
+        {"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}},
+        {"jsonrpc": "2.0", "id": 2, "method": "ping", "params": {}},
+    ]
+    result = run_archivebox_cmd(
+        ["mcp"],
+        cwd=initialized_archive,
+        input="".join(json.dumps(request) + "\n" for request in requests),
+        default_cli_env=True,
+    )
+    responses = [json.loads(line) for line in result.stdout.splitlines()]
+
+    assert result.returncode == 0
+    assert [response["id"] for response in responses] == [1, 2]
+    assert responses[0]["result"]["protocolVersion"] == "2025-11-25"
+    assert responses[1]["result"] == {}
+
+
+def test_mcp_exposes_six_focused_tools():
+    tools = MCPServer().handle_tools_list({})["tools"]
+    tools_by_name = {tool["name"]: tool for tool in tools}
+
+    assert set(tools_by_name) == {"add", "search", "crawl", "snapshot", "archiveresult", "shell"}
+    assert len(tools) == 6
+    assert all("outputSchema" in tool for tool in tools)
+
+    crawl_schema = tools_by_name["crawl"]["inputSchema"]
+    assert crawl_schema["properties"]["action"]["enum"] == ["create", "delete", "list", "update"]
+    assert crawl_schema["properties"]["urls"]["type"] == "array"
+    assert crawl_schema["properties"]["records"]["type"] == "array"
+    assert tools_by_name["search"]["annotations"]["readOnlyHint"] is True
+    assert tools_by_name["shell"]["inputSchema"]["required"] == ["code"]
+    assert tools_by_name["shell"]["annotations"]["destructiveHint"] is True
+
+
+def test_mcp_crawl_create_returns_structured_json(initialized_archive):
+    os.chdir(initialized_archive)
+    result = MCPServer().handle_tools_call(
+        {
+            "name": "crawl",
+            "arguments": {
+                "action": "create",
+                "urls": ["https://mcp-test.example.com/"],
+                "depth": 1,
+                "tag": "mcp-test",
+            },
+        },
+    )
+
+    assert result["isError"] is False, result
+    assert result["structuredContent"]["success"] is True
+    assert result["structuredContent"]["error"] is None
+    assert result["structuredContent"]["command"] == "archivebox crawl create"
+    assert result["structuredContent"]["exitCode"] == 0
+    assert result["structuredContent"]["records"][0]["urls"] == "https://mcp-test.example.com/"
+    assert result["structuredContent"]["records"][0]["max_depth"] == 1
+    assert json.loads(result["content"][0]["text"]) == result["structuredContent"]
+
+
+def test_mcp_snapshot_update_accepts_records_without_a_jsonl_pipeline(initialized_archive):
+    os.chdir(initialized_archive)
+    server = MCPServer()
+    created = server.handle_tools_call(
+        {
+            "name": "snapshot",
+            "arguments": {
+                "action": "create",
+                "urls": ["https://mcp-update.example.com/"],
+            },
+        },
+    )
+    assert created["isError"] is False, created
+    snapshot = created["structuredContent"]["records"][0]
+
+    updated = server.handle_tools_call(
+        {
+            "name": "snapshot",
+            "arguments": {
+                "action": "update",
+                "records": [{"id": snapshot["id"]}],
+                "tag": "updated-through-mcp",
+            },
+        },
+    )
+
+    assert updated["isError"] is False
+    assert updated["structuredContent"]["records"][0]["id"] == snapshot["id"]
+    assert "updated-through-mcp" in updated["structuredContent"]["records"][0]["tags"]
+
+
+def test_mcp_cli_errors_are_structured_for_agents(initialized_archive):
+    os.chdir(initialized_archive)
+    result = MCPServer().handle_tools_call(
+        {
+            "name": "crawl",
+            "arguments": {"action": "create"},
+        },
+    )
+
+    assert result["isError"] is True
+    assert result["structuredContent"]["success"] is False
+    assert result["structuredContent"]["exitCode"] == 1
+    assert "No URLs provided" in result["structuredContent"]["error"]
+    assert json.loads(result["content"][0]["text"]) == result["structuredContent"]
+
+
+def test_mcp_invalid_action_is_a_protocol_error():
+    response = MCPServer().handle_request(
+        {
+            "jsonrpc": "2.0",
+            "id": 1,
+            "method": "tools/call",
+            "params": {"name": "crawl", "arguments": {"action": "bogus"}},
+        },
+    )
+
+    assert response["error"]["code"] == -32602
+    assert "Choose one of: create, delete, list, update" in response["error"]["message"]
+
+
+def test_mcp_shell_runs_python_through_archivebox_shell(initialized_archive):
+    os.chdir(initialized_archive)
+    result = MCPServer().handle_tools_call(
+        {
+            "name": "shell",
+            "arguments": {
+                "code": "from archivebox.core.models import Snapshot; print(f'shell_ok={Snapshot.objects.count() >= 0}')",
+            },
+        },
+    )
+
+    assert result["isError"] is False, result
+    assert result["structuredContent"]["command"] == "archivebox shell"
+    assert result["structuredContent"]["stdout"] == "shell_ok=True\n"
--- a/archivebox/tests/test_cli_run.py
+++ b/archivebox/tests/test_cli_run.py
@ -10,7 +10,6 @@ Tests cover:
 import os
 import signal
 import subprocess
-from datetime import timedelta

 import psutil
 import pytest
@ -734,7 +733,7 @@ class TestRunDaemonMode:
                cleanup_process_group(proc.pid)
                proc.wait(timeout=15)

-    def test_run_daemon_recovers_stopped_runner_from_previous_pid_namespace(self, initialized_archive, db):
+    def test_run_daemon_retires_runner_from_previous_pid_namespace(self, initialized_archive, db):
        from django.utils import timezone

        from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE
@ -753,8 +752,6 @@ class TestRunDaemonMode:
                started_at=timezone.now(),
                env={PROCESS_PID_NAMESPACE_KEY: f"{get_current_pid_namespace()}-stopped-container"},
            )
-            Process.objects.filter(pk=stopped_runner.pk).update(modified_at=timezone.now() - timedelta(minutes=5))
-
        queued = run_archivebox_cmd(["crawl", "create", create_test_url()], cwd=initialized_archive, env=env, timeout=60)
        assert queued.returncode == 0, queued.stderr or queued.stdout

@ -774,6 +771,7 @@ class TestRunDaemonMode:

        try:
            wait_for_log(daemon_log, "[Crawl#", timeout=10)
+            assert "Multiple orchestrators sharing a single collection is not officially supported" in daemon_log.read_text()
            with use_archivebox_db(initialized_archive):
                stopped_runner.refresh_from_db()
                assert stopped_runner.status == Process.StatusChoices.EXITED
--- a/archivebox/tests/test_takeover_util.py
+++ b/archivebox/tests/test_takeover_util.py
@ -7,7 +7,6 @@ import re
 import signal
 import subprocess
 import sys
-import time
 from pathlib import Path

 import pytest
@ -963,27 +962,32 @@ def test_runtime_stack_owner_allows_top_level_runner_when_no_parent_command_exis
        _stop_archivebox_shells([proc])


-def test_active_runner_lease_heartbeat_updates_persisted_process(tmp_path, initialized_archive):
-    from django.utils import timezone
-
-    from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, maintain_runner_lease
+def test_foreign_machine_runner_only_warns(tmp_path, initialized_archive, capsys):
+    from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, live_runner_processes
    from archivebox.machine.models import Machine

-    runner = Process.objects.create(
-        machine=Machine.current(),
+    foreign_machine = Machine.objects.create(
+        guid="foreign-machine",
+        hostname="foreign-host",
+        hw_manufacturer="Test",
+        hw_product="Test",
+        hw_uuid="foreign-hardware",
+        os_arch="x86_64",
+        os_family="linux",
+        os_platform="linux",
+        os_release="test",
+        os_kernel="test",
+    )
+    foreign_runner = Process.objects.create(
+        machine=foreign_machine,
        process_type=Process.TypeChoices.ORCHESTRATOR,
        worker_type=RUNNER_ACTIVE_WORKER_TYPE,
        pwd=str(tmp_path),
-        pid=os.getpid(),
-        started_at=timezone.now(),
+        pid=1,
        status=Process.StatusChoices.RUNNING,
    )
-    initial_heartbeat = runner.modified_at

-    with maintain_runner_lease(runner, interval=0.01):
-        deadline = time.monotonic() + 2.0
-        while runner.modified_at <= initial_heartbeat and time.monotonic() < deadline:
-            time.sleep(0.01)
-            runner.refresh_from_db()
-
-    assert runner.modified_at > initial_heartbeat
+    assert live_runner_processes(data_dir=tmp_path) == []
+    foreign_runner.refresh_from_db()
+    assert foreign_runner.status == Process.StatusChoices.RUNNING
+    assert "Multiple orchestrators sharing a single collection is not officially supported" in capsys.readouterr().err
--- a/bin/collect_ui_screenshots.sh
+++ b/bin/collect_ui_screenshots.sh
@ -18,6 +18,8 @@ ARCHIVE_PID=""
 CREATED_TEMP_USER=0
 CREATE_API_TOKEN=0
 CREATE_WEBHOOK=0
+ABXPKG_LIB_DIR=""
+SCREENSHOT_CHROME_BINARY=""
 CAPTURE_ROOT="$(mktemp -d)"
 MANIFEST_FILE="$CAPTURE_ROOT/manifest.jsonl"
 PERSONAS_DIR="$CAPTURE_ROOT/personas"
@ -203,6 +205,13 @@ fi
 # the Sweeting.me example below runs through its own real foreground runner.
 stop_background_runner

+ABXPKG_LIB_DIR="$(uv run --no-cache --project "$REPO_DIR" abx-dl config --get ABXPKG_LIB_DIR | sed 's/^[^=]*=//; s/^"//; s/"$//')"
+SCREENSHOT_CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
+if [[ ! -x "$SCREENSHOT_CHROME_BINARY" ]]; then
+    echo "[!] abx-dl projected Chromium was not found at $SCREENSHOT_CHROME_BINARY" >&2
+    exit 1
+fi
+
 VIEWS=(
    "Login|$ADMIN_BASE_URL/admin/login/|/admin/login/|archivebox/templates/admin/login.html"
    "Public snapshot list|$PUBLIC_BASE_URL/public/|/public/|archivebox/core/views.py"
@ -225,9 +234,11 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
            SCREENSHOT_HEIGHT=1000 \
            node "$REPO_DIR/bin/take_screenshot.js" "$url" "$CAPTURE_ROOT/snapshot-header-state.png" >/dev/null
    fi
+    view_timing_report=""
    while IFS='|' read -r profile viewport_width viewport_height; do
        filename="$(printf '%02d' "$capture_index")-$slug-$profile.png"
        capture_dir="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/$profile"
+        timing_report_path="$view_timing_report"
        capture_env=(
            "RESOLUTION=$viewport_width,$viewport_height"
            "CHROME_RESOLUTION=$viewport_width,$viewport_height"
@ -272,6 +283,8 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
                    SCREENSHOT_SNAPSHOT_HEADER=expanded \
                    SCREENSHOT_EXPECT_LIVE_PROGRESS=1 \
                    node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
+                view_timing_report="$capture_dir/report.json"
+                timing_report_path="$view_timing_report"
            fi
        elif [[ "$profile" == "desktop" || "$capture_mode" == wait-replay:* || -z "${ABXPKG_LIB_DIR:-}" ]]; then
            capture_log="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")-$profile-abx-dl.log"
@ -311,6 +324,8 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
                    SCREENSHOT_VARIANTS_JSON="$responsive_variants" \
                    SCREENSHOT_COLLAPSE_FILTERS=1 \
                    node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
+                view_timing_report="$capture_dir/report.json"
+                timing_report_path="$view_timing_report"
                uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" validate \
                    "$capture_dir/report.json" "$expected_path"
            fi
@ -324,6 +339,7 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
        cp "$screenshot_path" "$PUBLIC_OUTPUT_DIR/$filename"
        UI_SCREENSHOT_NAME="$name" UI_SCREENSHOT_URL="$url" UI_SCREENSHOT_SOURCE="$source" \
            UI_SCREENSHOT_FILENAME="$filename" UI_SCREENSHOT_PROFILE="$profile" \
+            UI_SCREENSHOT_TIMING_REPORT="$timing_report_path" \
            uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" append \
                "$MANIFEST_FILE" "$OUTPUT_DIR/$filename"
    done <<<"$CAPTURE_PROFILES"
@ -331,12 +347,6 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
    # Capture the public views before login because the real admin-login hint
    # intentionally redirects authenticated personas away from /public/.
    if [[ "$capture_index" == "2" ]]; then
-        ABXPKG_LIB_DIR="$(uv run --no-cache --project "$REPO_DIR" abx-dl config --get ABXPKG_LIB_DIR | sed 's/^[^=]*=//; s/^"//; s/"$//')"
-        SCREENSHOT_CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
-        if [[ ! -x "$SCREENSHOT_CHROME_BINARY" ]]; then
-            echo "[!] abx-dl projected Chromium was not found at $SCREENSHOT_CHROME_BINARY" >&2
-            exit 1
-        fi
        echo "[*] Logging in through the real $ACTIVE_PERSONA browser persona"
        NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
            CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
--- a/bin/generate_ui_screenshot_gallery.py
+++ b/bin/generate_ui_screenshot_gallery.py
@ -112,6 +112,12 @@ def append_manifest(manifest_path: Path, screenshot_path: Path) -> None:
        "width": dimensions[0],
        "height": dimensions[1],
    }
+    timing_report_path = os.environ.get("UI_SCREENSHOT_TIMING_REPORT", "").strip()
+    if timing_report_path:
+        timing_report = json.loads(Path(timing_report_path).read_text(encoding="utf-8"))
+        ttfb_ms = timing_report.get("checks", {}).get("ttfbMs")
+        if isinstance(ttfb_ms, int | float):
+            item["ttfb_ms"] = round(ttfb_ms)
    with manifest_path.open("a", encoding="utf-8") as manifest:
        manifest.write(json.dumps(item) + "\n")

@ -179,6 +185,8 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
        if parsed_url.fragment:
            route = f"{route}#{parsed_url.fragment}"
        source_url = f"{source_base_url}{capture['source']}"
+        ttfb_values = [variant["ttfb_ms"] for variant in variants.values() if isinstance(variant.get("ttfb_ms"), int | float)]
+        ttfb_text = f" · ~{round(sum(ttfb_values) / len(ttfb_values))}ms TTFB" if ttfb_values else ""
        markdown_cells = []
        html_figures = []
        for profile, (width, height) in CAPTURE_PROFILES.items():
@ -206,7 +214,7 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
                (
                    f"## {capture['name']}",
                    "",
-                    f"View: [`{route}`]({capture['url']}) · [View code]({source_url})",
+                    f"View: [`{route}`]({capture['url']}) · [View code]({source_url}){ttfb_text}",
                    "",
                    "<table><thead><tr>",
                    "".join(f"<th>{profile.title()}</th>" for profile in CAPTURE_PROFILES),
@ -219,7 +227,7 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
        html_sections.append(
            f"<article><h2>{html.escape(capture['name'])}</h2>"
            f'<p><a href="{html.escape(capture["url"])}"><code>{html.escape(route)}</code></a> · '
-            f'<a href="{html.escape(source_url)}">View code</a></p>'
+            f'<a href="{html.escape(source_url)}">View code</a>{html.escape(ttfb_text)}</p>'
            f'<div class="shots">{"".join(html_figures)}</div></article>',
        )

--- a/bin/take_screenshot.js
+++ b/bin/take_screenshot.js
@ -116,6 +116,28 @@ async function main() {
    await Promise.all(restoredPages.map((restoredPage) => restoredPage.close()));
    page.setDefaultTimeout(45000);

+    const client = await page.createCDPSession();
+    const documentRequests = new Map();
+    await client.send('Network.enable');
+    client.on('Network.requestWillBeSent', (event) => {
+      if (event.type !== 'Document') return;
+      documentRequests.set(event.requestId, {
+        url: event.request.url,
+        requestTimestamp: event.timestamp,
+      });
+    });
+    client.on('Network.responseReceived', (event) => {
+      if (event.type !== 'Document') return;
+      const record = documentRequests.get(event.requestId);
+      if (!record) return;
+      record.responseTimestamp = event.timestamp;
+      record.status = event.response.status;
+      record.responseUrl = event.response.url;
+      if (Number.isFinite(record.requestTimestamp) && Number.isFinite(record.responseTimestamp)) {
+        record.ttfbMs = Math.round((record.responseTimestamp - record.requestTimestamp) * 1000);
+      }
+    });
+
    if (process.env.SCREENSHOT_SNAPSHOT_VIEW || process.env.SCREENSHOT_SNAPSHOT_HEADER || process.env.SCREENSHOT_COLLAPSE_FILTERS === '1' || process.env.SCREENSHOT_RESET_FILTERS === '1') {
      await page.evaluateOnNewDocument((snapshotView, snapshotHeader, collapseFilters, resetFilters) => {
        if (snapshotView) localStorage.setItem('preferred_snapshot_view_mode', snapshotView);
@ -281,6 +303,18 @@ async function main() {
    }));
    checks.status = navigationResponse ? navigationResponse.status() : null;
    checks.finalUrl = page.url();
+    const navigationUrl = navigationResponse?.url() || page.url();
+    const matchingDocumentRequests = [...documentRequests.values()]
+      .filter((record) => record.status === checks.status && Number.isFinite(record.ttfbMs))
+      .filter((record) => {
+        const responseUrl = record.responseUrl || record.url || '';
+        return responseUrl === navigationUrl || responseUrl === checks.finalUrl || responseUrl.split('#')[0] === checks.finalUrl.split('#')[0];
+      });
+    const timingRecord = matchingDocumentRequests.at(-1)
+      || [...documentRequests.values()].filter((record) => Number.isFinite(record.ttfbMs)).at(-1);
+    if (timingRecord) {
+      checks.ttfbMs = timingRecord.ttfbMs;
+    }

    let frameChecks = null;
    const embeddedFrameHandle = await page.$('.crawl-snapshots-embed iframe');
--- a/docs/Docker.md
+++ b/docs/Docker.md
@ -192,13 +192,12 @@ Never enable on-demand TLS or request individual certificates for `snap-*` hostn

 ### Setup

-Fetch and run the ArchiveBox Docker image to create your initial archive.
+Fetch and run the ArchiveBox Docker image. Starting the server creates the initial archive automatically.

 ```bash
 docker pull archivebox/archivebox:dev

 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev init
 docker run -d --name archivebox -v "$PWD:/data" -p 8000:8000 archivebox/archivebox:dev
 ```

--- a/docs/Roadmap.md
+++ b/docs/Roadmap.md
@ -17,7 +17,7 @@
 - right now, the paths of the extractor output are scattered all over the codebase, e.g. `output.pdf` (should be moved to constants at the top of the plugin config file)
 - make out_dir, link_dir, extractor_dir, naming consistent across codebase
 - remove `timestamps` as primary keys in favor of hashes, UUIDs, or some other slug https://github.com/ArchiveBox/ArchiveBox/issues/74
- - create a migration system for folder layout independent of the index (`mv` is atomic at the FS level, so we just need a `transaction.atomic(): move(oldpath, newpath); snap.data_dir = newpath; snap.save()`)
+ - create a migration system for folder layout independent of the index
 - make `Tag` a real model `ManyToMany` with Snapshots
 - allow multiple Snapshots of the same site over time + CLI / UI to manage those, + migration from old style `#2020-01-01` hack to proper versioned snapshots
 - upgrade from Django 3 to Django 5 https://github.com/ArchiveBox/ArchiveBox/issues/988
--- a/docs/Upgrading.md
+++ b/docs/Upgrading.md
@ -18,12 +18,11 @@ archivebox install
 archivebox update --migrate-only
 archivebox status

-# Docker Compose install
+# Docker Compose upgrade
 cd ~/archivebox
 docker compose down
 docker compose pull
 docker compose run --rm archivebox init
-docker compose run --rm archivebox install
 docker compose run --rm archivebox update --migrate-only
 docker compose up -d
 ```
@ -35,7 +34,7 @@ docker compose up -d
 2. **Read the release notes carefully** for any instructions or extra steps around upgrading for each release you're skipping or installing
 3. **Stop any running ArchiveBox server, scheduler, and worker processes**, then back up the entire collection data directory before upgrading. `archivebox config --get ...` and a database-only backup do not include archived outputs.
   `cd ~/archivebox && tar -czf "archivebox-data-$(date +%s).tar.gz" data/`
-4. Follow the steps below for your installation method, then run `archivebox init`, `archivebox install`, and `archivebox update --migrate-only` inside the collection
+4. Follow the steps below for your installation method. Bare-metal installs run `archivebox init`, `archivebox install`, and `archivebox update --migrate-only` inside the collection; Docker images already include runtime dependencies.
 5. Confirm the upgrade succeeded and check for any orphan/corrupted snapshots with `archivebox status`

 💬 [Open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new/choose) in our bug tracker if you experience any problems with upgrading/merging/modifying collections.
@ -49,7 +48,7 @@ docker compose up -d

 **ℹ️ How it works internally:**

-The same command is used for initializing a new archive and upgrading an existing database. `archivebox init` is idempotent and can safely be run multiple times; it applies database migrations and prepares collection-level state. `archivebox install` resolves runtime dependencies for the new version. `archivebox update --migrate-only` performs filesystem migrations and reconciles Snapshot metadata with the current layout without scheduling normal archive maintenance jobs. `archivebox status` checks collection health afterward.
+The same command is used for initializing a new archive and upgrading an existing database. `archivebox init` is idempotent and can safely be run multiple times; it applies database migrations and prepares collection-level state. For bare-metal installs, `archivebox install` resolves runtime dependencies for the new version; Docker images include those dependencies at build time. `archivebox update --migrate-only` performs filesystem migrations and reconciles Snapshot metadata with the current layout without scheduling normal archive maintenance jobs. `archivebox status` checks collection health afterward.

 There are three main areas on disk that ArchiveBox modifies during upgrades:
 - `index.sqlite3` contains the SQLite3 DB index that gets upgraded automatically by Django based on the changes in [`archivebox/core/models.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/models.py).
@ -76,7 +75,6 @@ cd ~/archivebox        # or wherever your folder containing docker-compose.yml i
 docker compose down    # stop the currently running ArchiveBox containers
 docker compose pull    # pull the latest image version from Docker Hub
 docker compose run --rm archivebox init
-docker compose run --rm archivebox install
 docker compose run --rm archivebox update --migrate-only
 docker compose up -d
 ```
@ -97,7 +95,6 @@ docker stop CONTAINER_ID
 cd ~/archivebox/data  # or wherever your existing collection is stored
 docker pull archivebox/archivebox:dev
 docker run --rm -v $PWD:/data -it archivebox/archivebox:dev init
-docker run --rm -v $PWD:/data -it archivebox/archivebox:dev install
 docker run --rm -v $PWD:/data -it archivebox/archivebox:dev update --migrate-only

 # restart the archivebox server container if needed
--- a/docs/apidocs/archivebox/archivebox.core.takeover_util.md
+++ b/docs/apidocs/archivebox/archivebox.core.takeover_util.md
@ -15,30 +15,10 @@
 :class: autosummary longtable
 :align: left

-* - {py:obj}`runtime_stack_owner_types <archivebox.core.takeover_util.runtime_stack_owner_types>`
-  - ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner_types
-    :summary:
-    ```
-* - {py:obj}`foreground_runner_owner_types <archivebox.core.takeover_util.foreground_runner_owner_types>`
-  - ```{autodoc2-docstring} archivebox.core.takeover_util.foreground_runner_owner_types
-    :summary:
-    ```
 * - {py:obj}`current_command <archivebox.core.takeover_util.current_command>`
  - ```{autodoc2-docstring} archivebox.core.takeover_util.current_command
    :summary:
    ```
-* - {py:obj}`live_processes <archivebox.core.takeover_util.live_processes>`
-  - ```{autodoc2-docstring} archivebox.core.takeover_util.live_processes
-    :summary:
-    ```
-* - {py:obj}`newest_live_process <archivebox.core.takeover_util.newest_live_process>`
-  - ```{autodoc2-docstring} archivebox.core.takeover_util.newest_live_process
-    :summary:
-    ```
-* - {py:obj}`command_is_newest <archivebox.core.takeover_util.command_is_newest>`
-  - ```{autodoc2-docstring} archivebox.core.takeover_util.command_is_newest
-    :summary:
-    ```
 * - {py:obj}`runtime_stack_owner <archivebox.core.takeover_util.runtime_stack_owner>`
  - ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner
    :summary:
@ -55,22 +35,10 @@
  - ```{autodoc2-docstring} archivebox.core.takeover_util.command_owns_foreground_runner
    :summary:
    ```
-* - {py:obj}`runtime_stack_component_label <archivebox.core.takeover_util.runtime_stack_component_label>`
-  - ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_component_label
-    :summary:
-    ```
 * - {py:obj}`ensure_daemon_stack <archivebox.core.takeover_util.ensure_daemon_stack>`
  - ```{autodoc2-docstring} archivebox.core.takeover_util.ensure_daemon_stack
    :summary:
    ```
-* - {py:obj}`healthy_orchestrator <archivebox.core.takeover_util.healthy_orchestrator>`
-  - ```{autodoc2-docstring} archivebox.core.takeover_util.healthy_orchestrator
-    :summary:
-    ```
-* - {py:obj}`_runner_sort_key <archivebox.core.takeover_util._runner_sort_key>`
-  - ```{autodoc2-docstring} archivebox.core.takeover_util._runner_sort_key
-    :summary:
-    ```
 * - {py:obj}`live_runner_processes <archivebox.core.takeover_util.live_runner_processes>`
  - ```{autodoc2-docstring} archivebox.core.takeover_util.live_runner_processes
    :summary:
@ -79,10 +47,6 @@
  - ```{autodoc2-docstring} archivebox.core.takeover_util.enter_single_runner_gate
    :summary:
    ```
-* - {py:obj}`standby_until_leader_needed <archivebox.core.takeover_util.standby_until_leader_needed>`
-  - ```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_leader_needed
-    :summary:
-    ```
 * - {py:obj}`standby_until_runtime_stack_needed <archivebox.core.takeover_util.standby_until_runtime_stack_needed>`
  - ```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_runtime_stack_needed
    :summary:
@ -145,20 +109,6 @@

 ````

-````{py:function} runtime_stack_owner_types()
-:canonical: archivebox.core.takeover_util.runtime_stack_owner_types
-
-```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner_types
-```
-````
-
-````{py:function} foreground_runner_owner_types()
-:canonical: archivebox.core.takeover_util.foreground_runner_owner_types
-
-```{autodoc2-docstring} archivebox.core.takeover_util.foreground_runner_owner_types
-```
-````
-
 ````{py:function} current_command(process_type: str, *, data_dir: str | pathlib.Path, url: str | None = None)
 :canonical: archivebox.core.takeover_util.current_command

@ -166,27 +116,6 @@
 ```
 ````

-````{py:function} live_processes(*, process_type: str, data_dir: str | pathlib.Path, url: str | None = None)
-:canonical: archivebox.core.takeover_util.live_processes
-
-```{autodoc2-docstring} archivebox.core.takeover_util.live_processes
-```
-````
-
-````{py:function} newest_live_process(*, process_type: str, data_dir: str | pathlib.Path, url: str | None = None)
-:canonical: archivebox.core.takeover_util.newest_live_process
-
-```{autodoc2-docstring} archivebox.core.takeover_util.newest_live_process
-```
-````
-
-````{py:function} command_is_newest(command, *, process_type: str, data_dir: str | pathlib.Path, url: str | None = None) -> bool
-:canonical: archivebox.core.takeover_util.command_is_newest
-
-```{autodoc2-docstring} archivebox.core.takeover_util.command_is_newest
-```
-````
-
 ````{py:function} runtime_stack_owner(*, data_dir: str | pathlib.Path, exclude_id=None)
 :canonical: archivebox.core.takeover_util.runtime_stack_owner

@ -215,13 +144,6 @@
 ```
 ````

-````{py:function} runtime_stack_component_label(*, owner=None, data_dir: str | pathlib.Path) -> str
-:canonical: archivebox.core.takeover_util.runtime_stack_component_label
-
-```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_component_label
-```
-````
-
 ````{py:function} ensure_daemon_stack(*, reason: str = '')
 :canonical: archivebox.core.takeover_util.ensure_daemon_stack

@ -229,21 +151,7 @@
 ```
 ````

-````{py:function} healthy_orchestrator(*, data_dir: str | pathlib.Path)
-:canonical: archivebox.core.takeover_util.healthy_orchestrator
-
-```{autodoc2-docstring} archivebox.core.takeover_util.healthy_orchestrator
-```
-````
-
-````{py:function} _runner_sort_key(process)
-:canonical: archivebox.core.takeover_util._runner_sort_key
-
-```{autodoc2-docstring} archivebox.core.takeover_util._runner_sort_key
-```
-````
-
-````{py:function} live_runner_processes(*, data_dir: str | pathlib.Path, exclude_id=None)
+````{py:function} live_runner_processes(*, data_dir: str | pathlib.Path)
 :canonical: archivebox.core.takeover_util.live_runner_processes

 ```{autodoc2-docstring} archivebox.core.takeover_util.live_runner_processes
@ -257,13 +165,6 @@
 ```
 ````

-````{py:function} standby_until_leader_needed(command, *, process_type: str, data_dir: str | pathlib.Path, url: str | None = None, interval: float = 2.0) -> None
-:canonical: archivebox.core.takeover_util.standby_until_leader_needed
-
-```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_leader_needed
-```
-````
-
 ````{py:function} standby_until_runtime_stack_needed(command, *, data_dir: str | pathlib.Path, interval: float = 2.0) -> dict[str, object]
 :canonical: archivebox.core.takeover_util.standby_until_runtime_stack_needed

--- a/etc/package.json
+++ b/etc/package.json
@ -1,6 +1,6 @@
 {
  "name": "archivebox",
-  "version": "0.9.35rc362",
+  "version": "0.9.35rc365",
  "repository": "github:ArchiveBox/ArchiveBox",
  "license": "MIT",
  "dependencies": {
--- a/publicsite/index.html
+++ b/publicsite/index.html
@ -99,8 +99,9 @@
 <pre><span class="ab-dim"># Docker Compose is the recommended setup</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">mkdir -p ~/archivebox/data &amp;&amp; cd ~/archivebox</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">curl -fsSL 'https://docker-compose.archivebox.io' &gt; docker-compose.yml</span>
-<span class="ab-prompt">$</span> <span class="ab-cmd">docker compose run archivebox init</span>
-<span class="ab-out">-> created ./data/index.sqlite3</span>
+<span class="ab-prompt">$</span> <span class="ab-cmd">docker compose up -d --wait</span>
+<span class="ab-out">-> initialized ./data/index.sqlite3</span>
+<span class="ab-ok">ok listening on http://127.0.0.1:8000</span>
 <span class="ab-prompt">$</span> <span class="ab-cursor" aria-hidden="true">&nbsp;</span></pre>
          </div>
        </div>
@ -207,16 +208,15 @@
        <div class="step">
          <span>3</span>
          <div>
-            <h3>Initialize and start</h3>
-            <div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">docker compose run archivebox init</span>
-<span class="ab-prompt">$</span> <span class="ab-cmd">docker compose up</span></pre></div></div>
+            <h3>Start ArchiveBox (initializes automatically)</h3>
+            <div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">docker compose up -d --wait</span></pre></div></div>
          </div>
        </div>
        <div class="step">
          <span>4</span>
          <div>
            <h3>Add your first URL</h3>
-            <div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">docker compose run archivebox add 'https://example.com'</span></pre></div></div>
+            <div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">docker compose exec archivebox archivebox add 'https://example.com'</span></pre></div></div>
          </div>
        </div>
      </div>
--- a/pyproject.toml
+++ b/pyproject.toml
@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.35rc362"
+version = "0.9.35rc365"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -342,7 +342,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"


 [tool.bumpver]
-current_version = "v0.9.35rc362"
+current_version = "v0.9.35rc365"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"
--- a/uv.lock
+++ b/uv.lock
@ -120,7 +120,7 @@ wheels = [

 [[package]]
 name = "archivebox"
-version = "0.9.35rc362"
+version = "0.9.35rc365"
 source = { editable = "." }
 dependencies = [
    { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },