Merge remote-tracking branch 'origin/dev' into claude/archivebox-postgresql-support-xrlgim

# Conflicts:
#	.github/workflows/test-parallel.yml
#	.github/workflows/test.yml
#	uv.lock

.github/configs/ci-linux-env.json

@@ -1,28 +0,0 @@
-{
-  "properties": {
-    "CC_BINARY": {
-      "default": "cc"
-    },
-    "LDAPSEARCH_BINARY": {
-      "default": "ldapsearch"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{CC_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{LDAPSEARCH_BINARY}",
-      "binproviders": "env",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapsearch",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ]
-}

.github/configs/ci-tooling.json

@@ -86,6 +86,10 @@
     }
   },
   "ci_binaries": [
+    {
+      "name": "{BASH_BINARY}",
+      "binproviders": "env,apt,brew"
+    },
     {
       "name": "{PYTHON_BINARY}",
       "binproviders": "env"
@@ -99,6 +103,18 @@
       "binproviders": "env"
     }
   ],
+  "bsd_binaries": [
+    {
+      "name": "{BASH_BINARY}",
+      "binproviders": "env,pyinfra"
+    }
+  ],
+  "freebsd_binaries": [
+    {
+      "name": "{SUDO_BINARY}",
+      "binproviders": "env,pyinfra"
+    }
+  ],
   "docker_binaries": [
     {
       "name": "{DOCKER_BINARY}",

.github/scripts/clone_abx_repo.sh

@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-set -Eeuo pipefail
-
-repo_name="$1"
-target_dir="${2:-$repo_name}"
-repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-lock_file="$repo_root/uv.lock"
-tooling_config="$repo_root/.github/configs/ci-tooling.json"
-
-locked_version() {
-    local wanted="$1" line package=""
-    while IFS= read -r line; do
-        case "$line" in
-            '[[package]]') package="" ;;
-            "name = \"${wanted}\"") package="$wanted" ;;
-            'version = "'*'"')
-                if [[ "$package" == "$wanted" ]]; then
-                    line="${line#version = \"}"
-                    printf '%s\n' "${line%\"}"
-                    return 0
-                fi
-                ;;
-        esac
-    done < "$lock_file"
-    return 1
-}
-
-version="$(locked_version "$repo_name")"
-[[ -n "$version" ]] || { echo "Could not find ${repo_name} in uv.lock" >&2; exit 1; }
-
-abxpkg_version="$(locked_version abxpkg)"
-[[ -n "$abxpkg_version" ]] || { echo "Could not find abxpkg in uv.lock" >&2; exit 1; }
-
-ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-${RUNNER_TEMP:-/tmp}/archivebox-clone-abxpkg}"
-mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-uv run --no-project --with "abxpkg==$abxpkg_version" abxpkg env \
-    --install \
-    --lib="$ABXPKG_LIB_DIR" \
-    --deps-from="$tooling_config:git_binaries" \
-    >/dev/null
-git_binary="$ABXPKG_LIB_DIR/env/bin/git"
-[[ -L "$git_binary" ]]
-[[ -x "$git_binary" ]]
-
-echo "Cloning ArchiveBox/${repo_name}@v${version} into ${target_dir}"
-"$git_binary" clone --depth=1 --branch "v${version}" "https://github.com/ArchiveBox/${repo_name}.git" "$target_dir"

.github/scripts/discover_test_matrix.py

@@ -1,107 +1,32 @@
 #!/usr/bin/env python3
-"""Build deterministic GitHub Actions matrices from every discovered test file."""
+"""Discover every ArchiveBox test file for one CI matrix."""
 
-import argparse
 import json
-import re
 from pathlib import Path
 
 
-CHROMIUM_PATTERN = re.compile(
-    rb"chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue",
-    re.IGNORECASE,
-)
-SONIC_PATTERN = re.compile(
-    rb"""shutil\.which\(["']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic""",
-    re.IGNORECASE,
-)
+def main() -> None:
+    root = Path.cwd().resolve()
+    archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
 
-
-def contains(pattern: re.Pattern[bytes], paths: list[Path]) -> bool:
-    return any(pattern.search(path.read_bytes()) for path in paths)
-
-
-def archivebox_matrix(root: Path) -> list[dict[str, object]]:
-    tests = sorted((root / "archivebox/tests").glob("test_*.py"))
-    if not tests:
+    if not archivebox_tests:
         raise SystemExit("No ArchiveBox tests discovered")
 
-    shard_count = min(16, len(tests))
-    matrix = []
-    assigned: list[Path] = []
-    for shard in range(shard_count):
-        shard_tests = tests[shard::shard_count]
-        assigned.extend(shard_tests)
-        matrix.append(
-            {
-                "name": f"main/shard-{shard + 1}",
-                "paths": [path.relative_to(root).as_posix() for path in shard_tests],
-                "needs_chromium": contains(CHROMIUM_PATTERN, shard_tests),
-                "needs_sonic": contains(SONIC_PATTERN, shard_tests),
-            },
-        )
-
-    if sorted(assigned) != tests or len(assigned) != len(set(assigned)):
-        raise SystemExit("ArchiveBox tests were not assigned exactly once")
-    print(f"Assigned {len(tests)} test files exactly once across {shard_count} shards")
-    return matrix
-
-
-def plugin_matrix(root: Path) -> list[dict[str, object]]:
-    plugins_root = root / "abx-plugins/abx_plugins/plugins"
-    suite_dirs = sorted(path for path in plugins_root.glob("*/tests") if path.is_dir())
-    root_tests = sorted((root / "abx-plugins/tests").glob("test_*.py"))
-    if not suite_dirs or not root_tests:
-        raise SystemExit("Plugin suites or root tests were not discovered")
-
     matrix: list[dict[str, object]] = []
-    expected = list(root_tests)
-    for suite_dir in suite_dirs:
-        suite_tests = sorted(suite_dir.rglob("test_*.py"))
-        if not suite_tests:
-            raise SystemExit(f"No tests found in {suite_dir}")
-        expected.extend(suite_tests)
-        plugin = suite_dir.parent.name
+    for path in archivebox_tests:
         matrix.append(
             {
-                "plugin": plugin,
-                "name": f"plugin/{plugin}",
-                "test_path": suite_dir.relative_to(root).as_posix(),
-                "config_path": (suite_dir.parent / "config.json").relative_to(root).as_posix(),
-                "needs_chromium": contains(re.compile(rb"chrom", re.IGNORECASE), suite_tests),
-                "needs_sonic": plugin == "search_backend_sonic",
+                "name": f"main/{path.stem.removeprefix('test_')}",
+                "path": path.relative_to(root).as_posix(),
+                "extra": "ldap" if path.name == "test_auth_ldap.py" else "",
             },
         )
 
-    matrix.append(
-        {
-            "plugin": "root",
-            "name": "plugin/root",
-            "test_path": "abx-plugins/tests",
-            "config_path": "abx-plugins/abx_plugins/plugins/base/config.json",
-            "needs_chromium": contains(re.compile(rb"chrom|archivewebpage", re.IGNORECASE), root_tests),
-            "needs_sonic": False,
-        },
-    )
+    discovered_paths = [str(entry["path"]) for entry in matrix]
+    if len(discovered_paths) != len(set(discovered_paths)):
+        raise SystemExit("Tests were not discovered exactly once")
 
-    assigned = []
-    for entry in matrix:
-        assigned.extend(sorted((root / str(entry["test_path"])).rglob("test_*.py")))
-    if sorted(assigned) != sorted(expected) or len(assigned) != len(set(assigned)):
-        raise SystemExit("Plugin tests were not assigned exactly once")
-    print(
-        f"Assigned {len(suite_dirs)} plugin suites and {len(root_tests)} root test files exactly once",
-    )
-    return matrix
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("scope", choices=("archivebox", "plugins"))
-    parser.add_argument("--workspace", type=Path, default=Path.cwd())
-    args = parser.parse_args()
-    root = args.workspace.resolve()
-    matrix = archivebox_matrix(root) if args.scope == "archivebox" else plugin_matrix(root)
+    print(f"Discovered {len(matrix)} test files exactly once")
     print(json.dumps(matrix, separators=(",", ":")))
 
 

.github/scripts/docs_http_server.py

@@ -1,34 +0,0 @@
-from __future__ import annotations
-
-import argparse
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from pathlib import Path
-
-
-class DocsRequestHandler(BaseHTTPRequestHandler):
-    def do_GET(self) -> None:
-        body = f"<!doctype html><title>ArchiveBox docs fixture</title><p>{self.path}</p>\n".encode()
-        self.send_response(200)
-        self.send_header("Content-Type", "text/html; charset=utf-8")
-        self.send_header("Content-Length", str(len(body)))
-        self.end_headers()
-        self.wfile.write(body)
-
-    def log_message(self, format: str, *args: object) -> None:
-        return
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--ready-fifo", type=Path, required=True)
-    args = parser.parse_args()
-
-    with ThreadingHTTPServer(("127.0.0.1", 0), DocsRequestHandler) as server:
-        host, port = server.server_address
-        with args.ready_fifo.open("w") as ready_fifo:
-            ready_fifo.write(f"http://{host}:{port}\n")
-        server.serve_forever()
-
-
-if __name__ == "__main__":
-    main()

.github/workflows/ci.yml

@@ -23,13 +23,13 @@ jobs:
     uses: ./.github/workflows/lint.yml
     secrets: inherit
 
-  complete-tests:
-    name: Complete test suite
+  install-cli-platform:
+    name: Install and CLI platform compatibility
     uses: ./.github/workflows/test.yml
     secrets: inherit
 
-  sharded-plugin-tests:
-    name: Sharded and plugin tests
+  tests:
+    name: Discovered test matrix
     uses: ./.github/workflows/test-parallel.yml
     secrets: inherit
 
@@ -60,8 +60,8 @@ jobs:
     if: ${{ always() }}
     needs:
       - lint
-      - complete-tests
-      - sharded-plugin-tests
+      - install-cli-platform
+      - tests
       - documentation
       - codeql
       - python-artifacts
@@ -71,8 +71,8 @@ jobs:
       - name: Verify every required lane succeeded
         env:
           LINT_RESULT: ${{ needs.lint.result }}
-          COMPLETE_TESTS_RESULT: ${{ needs.complete-tests.result }}
-          SHARDED_PLUGIN_TESTS_RESULT: ${{ needs.sharded-plugin-tests.result }}
+          INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
+          TESTS_RESULT: ${{ needs.tests.result }}
           DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
           CODEQL_RESULT: ${{ needs.codeql.result }}
           PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
@@ -80,8 +80,8 @@ jobs:
         run: |
           for result in \
             "$LINT_RESULT" \
-            "$COMPLETE_TESTS_RESULT" \
-            "$SHARDED_PLUGIN_TESTS_RESULT" \
+            "$INSTALL_CLI_PLATFORM_RESULT" \
+            "$TESTS_RESULT" \
             "$DOCUMENTATION_RESULT" \
             "$CODEQL_RESULT" \
             "$PYTHON_ARTIFACTS_RESULT" \

.github/workflows/docker.yml

@@ -193,9 +193,6 @@ jobs:
           context: ./
           file: ./Dockerfile
           builder: ${{ steps.buildx.outputs.name }}
-          tags: |
-            ${{ env.DOCKERHUB_IMAGE }}
-            ${{ env.GHCR_IMAGE }}
           labels: ${{ steps.docker_meta.outputs.labels }}
           build-args: |
             ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
@@ -207,7 +204,7 @@ jobs:
           outputs: |
             type=docker,name=${{ matrix.local_tag }}
             type=oci,dest=${{ runner.temp }}/archivebox-image.tar
-            type=image,push-by-digest=true,name-canonical=true,push=true
+            type=image,"name=${{ env.DOCKERHUB_IMAGE }},${{ env.GHCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true
 
       - name: Build pull request image
         if: ${{ !inputs.push_digests }}

.github/workflows/docs.yml

@@ -7,65 +7,14 @@ env:
   PYTHONIOENCODING: utf-8
   USE_COLOR: "False"
   SHOW_PROGRESS: "False"
-  ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD: "archivebox-docs-ci-only"
 
 jobs:
-  docs-matrix:
+  discover:
+    name: Discover documentation examples
     runs-on: ubuntu-24.04
     timeout-minutes: 5
     outputs:
-      matrix: ${{ steps.matrix.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-      - id: matrix
-        run: |
-          uv run --no-project python - <<'PY' >> "$GITHUB_OUTPUT"
-          import json
-          import tomllib
-
-          with open("docs/codeblocks.toml", "rb") as manifest_file:
-              ci = tomllib.load(manifest_file)["ci"]
-
-          include = []
-          for environment, runner in ci["standard"].items():
-              if environment == "core":
-                  include.extend(
-                      {
-                          "environment": environment,
-                          "runner": runner,
-                          "core_shard": shard,
-                          "job_name": f"core/{shard}",
-                          "validate_manifest": shard == "metadata",
-                      }
-                      for shard in ci["core_shards"]
-                  )
-              else:
-                  include.append(
-                      {
-                          "environment": environment,
-                          "runner": runner,
-                          "core_shard": "",
-                          "job_name": environment,
-                          "validate_manifest": False,
-                      },
-                  )
-
-          print("matrix=" + json.dumps({"include": include}, separators=(",", ":")))
-          PY
-
-  docs-standard:
-    name: docs/${{ matrix.job_name }}
-    needs: docs-matrix
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 5
-    env:
-      DOCS_CORE_SHARD: ${{ matrix.core_shard }}
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJSON(needs.docs-matrix.outputs.matrix) }}
+      matrix: ${{ steps.inventory.outputs.matrix }}
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
@@ -77,213 +26,150 @@ jobs:
       - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
           version: "0.11.3"
-      - name: Install ArchiveBox
-        run: uv sync --locked --dev --all-extras
-      - name: Prepare abxpkg environment
-        shell: bash
+      - name: Install ArchiveBox development dependencies
+        run: uv sync --locked --dev
+      - name: Resolve documentation tools through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
         run: |
-          {
-            echo "ABXPKG_LIB_DIR=${{ runner.temp }}/archivebox-docs-lib"
-            echo "ARCHIVEBOX_PROJECT_DIR=$GITHUB_WORKSPACE"
-          } >> "$GITHUB_ENV"
-          echo "${{ runner.temp }}/archivebox-docs-lib/env/bin" >> "$GITHUB_PATH"
-      - name: Resolve Node.js through abxpkg
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
           export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          tooling_env="$(
-            uv run --no-sync abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries"
-          )"
-          JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$tooling_env" >> "$GITHUB_ENV"
-
-          node_binary="$ABXPKG_LIB_DIR/env/bin/node"
+          uv run --no-sync abxpkg env \
+            --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
+            --deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
+            --deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
+          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
+          mmdc_binary="$ABXPKG_LIB_DIR/env/bin/mmdc"
+          nginx_binary="$ABXPKG_LIB_DIR/env/bin/nginx"
+          test -L "$bash_binary"
+          test -L "$mmdc_binary"
+          test -L "$nginx_binary"
+          test -x "$bash_binary"
+          test -x "$mmdc_binary"
+          test -x "$nginx_binary"
           {
-            echo "JQ_BINARY=$JQ_BINARY"
-            echo "NODE_BINARY=$node_binary"
+            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
+            echo "BASH_BINARY=$bash_binary"
           } >> "$GITHUB_ENV"
-          test -L "$node_binary"
-          test -x "$node_binary"
-          "$node_binary" --version
-      - name: Resolve documentation shell tools through abxpkg
-        shell: bash
+      - id: inventory
+        name: Validate the complete authored documentation inventory
         run: |
-          set -Eeuo pipefail
-          shell_env="$(
-            uv run --no-sync abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries"
-          )"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$shell_env" >> "$GITHUB_ENV"
+          matrix="$(uv run --no-sync python docs/test_codeblocks_manifest.py matrix)"
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
 
-          curl_binary="$ABXPKG_LIB_DIR/env/bin/curl"
-          echo "CURL_BINARY=$curl_binary" >> "$GITHUB_ENV"
-          test -L "$curl_binary"
-          test -x "$curl_binary"
-      - name: Resolve Docker through abxpkg
-        if: matrix.environment == 'docker'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          docker_env="$(
-            uv run --no-sync abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
-          )"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$docker_env" >> "$GITHUB_ENV"
-
-          docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
-          echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
-          test -L "$docker_binary"
-          test -x "$docker_binary"
-          "$docker_binary" --version
-      - name: Start bound local documentation site
-        shell: bash
-        run: |
-          ready_fifo="${{ runner.temp }}/archivebox-docs-http-ready"
-          mkfifo "$ready_fifo"
-          uv run --no-sync python .github/scripts/docs_http_server.py --ready-fifo "$ready_fifo" &
-          server_pid=$!
-          IFS= read -r docs_url < "$ready_fifo"
-          {
-            echo "ARCHIVEBOX_DOCS_SERVER_PID=$server_pid"
-            echo "ARCHIVEBOX_DOCS_URL_ONE=$docs_url/collection-one"
-            echo "ARCHIVEBOX_DOCS_URL_TWO=$docs_url/collection-two"
-            echo "ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT=18000"
-            echo "ARCHIVEBOX_DOCS_STATIC_PORT=18001"
-          } >> "$GITHUB_ENV"
-      - name: Resolve documentation validators through abxpkg
-        if: matrix.validate_manifest
-        run: |
-          validators_env="$(
-            uv run abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
-          )"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$validators_env" >> "$GITHUB_ENV"
-      - name: Resolve merge tools through abxpkg
-        if: matrix.environment == 'merge'
-        run: |
-          uv run abxpkg install rsync --lib "$ABXPKG_LIB_DIR" --binproviders env,apt,brew
-          uv run abxpkg install sqlite3 --lib "$ABXPKG_LIB_DIR" --binproviders env,apt,brew
-      - name: Validate documentation inventory and structured fences
-        if: matrix.validate_manifest
-        run: uv run --no-sync pytest -q docs/test_codeblocks_manifest.py
-      - name: Initialize documentation collection
-        if: contains(fromJSON('["core", "macos", "root"]'), matrix.environment)
-        shell: bash
-        run: |
-          docs_data_dir="${{ runner.temp }}/archivebox-docs-data"
-          mkdir -p "$docs_data_dir"
-          (cd "$docs_data_dir" && uv run --project "$GITHUB_WORKSPACE" --no-sync archivebox init)
-          echo "ARCHIVEBOX_DOCS_DATA_DIR=$docs_data_dir" >> "$GITHUB_ENV"
-      - name: Build local documentation image
-        if: matrix.environment == 'docker'
-        run: '"$DOCKER_BINARY" build --tag archivebox-docs-ci .'
-      - name: Run documentation code blocks
-        if: matrix.environment != 'root' && !contains(fromJSON('["core", "macos"]'), matrix.environment)
-        run: |
-          mapfile -t docs_paths < <(
-            uv run --no-sync python - <<'PY'
-          from pathlib import Path
-
-          print(*(str(path) for path in sorted(Path("docs").rglob("*.md")) if not path.is_symlink()), sep="\n")
-          PY
-          )
-          uv run --no-sync pytest -vv --tb=long README.md AGENTS.md skills "${docs_paths[@]}" --docs-environment=${{ matrix.environment }}
-      - name: Run collection documentation code blocks
-        if: contains(fromJSON('["core", "macos"]'), matrix.environment)
-        working-directory: ${{ env.ARCHIVEBOX_DOCS_DATA_DIR }}
-        run: |
-          docs_paths=()
-          if [[ -n "$DOCS_CORE_SHARD" ]]; then
-            mapfile -t docs_paths < <(
-              uv run --project "$GITHUB_WORKSPACE" --no-sync python - "$DOCS_CORE_SHARD" <<'PY'
-          import os
-          from pathlib import Path
-          import sys
-          import tomllib
-
-          workspace = Path(os.environ["GITHUB_WORKSPACE"])
-          with (workspace / "docs" / "codeblocks.toml").open("rb") as manifest_file:
-              shard_paths = tomllib.load(manifest_file)["ci"]["core_shards"][sys.argv[1]]
-          print(*(workspace / path for path in shard_paths), sep="\n")
-          PY
-            )
-          else
-            while IFS= read -r docs_path; do
-              docs_paths+=("$docs_path")
-            done < <(
-              uv run --project "$GITHUB_WORKSPACE" --no-sync python - <<'PY'
-          import os
-          from pathlib import Path
-
-          docs_dir = Path(os.environ["GITHUB_WORKSPACE"]) / "docs"
-          print(*(str(path) for path in sorted(docs_dir.rglob("*.md")) if not path.is_symlink()), sep="\n")
-          PY
-            )
-            docs_paths=("$GITHUB_WORKSPACE/README.md" "$GITHUB_WORKSPACE/AGENTS.md" "$GITHUB_WORKSPACE/skills" "${docs_paths[@]}")
-          fi
-          uv run --project "$GITHUB_WORKSPACE" --no-sync pytest -vv --tb=long "${docs_paths[@]}" --docs-environment=${{ matrix.environment }}
-      - name: Run root documentation code blocks
-        if: matrix.environment == 'root'
-        working-directory: ${{ env.ARCHIVEBOX_DOCS_DATA_DIR }}
-        run: |
-          uv_bin="$ABXPKG_LIB_DIR/env/bin/uv"
-          test -L "$uv_bin"
-          test -x "$uv_bin"
-          mapfile -t docs_paths < <(
-            uv run --project "$GITHUB_WORKSPACE" --no-sync python - <<'PY'
-          import os
-          from pathlib import Path
-
-          docs_dir = Path(os.environ["GITHUB_WORKSPACE"]) / "docs"
-          print(*(str(path) for path in sorted(docs_dir.rglob("*.md")) if not path.is_symlink()), sep="\n")
-          PY
-          )
-          "$SUDO_BINARY" --preserve-env=PATH,ABXPKG_LIB_DIR,ARCHIVEBOX_PROJECT_DIR,ARCHIVEBOX_DOCS_URL_ONE,ARCHIVEBOX_DOCS_URL_TWO,ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT,ARCHIVEBOX_DOCS_STATIC_PORT,ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD "$uv_bin" run --project "$GITHUB_WORKSPACE" --no-sync pytest -vv --tb=long "$GITHUB_WORKSPACE/README.md" "$GITHUB_WORKSPACE/AGENTS.md" "$GITHUB_WORKSPACE/skills" "${docs_paths[@]}" --docs-environment=${{ matrix.environment }}
-      - name: Stop local documentation site
-        if: always()
-        run: kill "$ARCHIVEBOX_DOCS_SERVER_PID"
-
-  docs-freebsd:
-    name: docs/freebsd
-    runs-on: ubuntu-24.04
-    timeout-minutes: 5
+  examples:
+    name: docs/${{ matrix.name }}
+    needs: discover
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-      - uses: vmactions/freebsd-vm@77ed28d336d03fe19a3f4f7266c1d2c4714dd79d # v1.5.2
+        with:
+          submodules: true
+          fetch-depth: 1
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        with:
+          python-version: "3.13"
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        with:
+          version: "0.11.3"
+      - name: Install ArchiveBox
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        run: uv sync --locked --dev
+      - name: Resolve documentation runtime through abxpkg
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
+        run: |
+          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
+          dependency_sources=(
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries"
+          )
+          if [[ '${{ matrix.environment }}' == ubuntu ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
+          elif [[ '${{ matrix.environment }}' == docker ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
+          elif [[ '${{ matrix.environment }}' == macos ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")
+          fi
+          uv run --no-sync abxpkg env \
+            --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            "${dependency_sources[@]}"
+          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
+          python_binary="$ABXPKG_LIB_DIR/env/bin/python"
+          sudo_binary="$ABXPKG_LIB_DIR/env/bin/sudo"
+          uv_binary="$ABXPKG_LIB_DIR/env/bin/uv"
+          test -L "$bash_binary"
+          test -x "$bash_binary"
+          test -L "$python_binary"
+          test -x "$python_binary"
+          test -L "$sudo_binary"
+          test -x "$sudo_binary"
+          test -L "$uv_binary"
+          test -x "$uv_binary"
+          {
+            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
+            echo "BASH_BINARY=$bash_binary"
+            echo "PYTHON_BINARY=$python_binary"
+            echo "SUDO_BINARY=$sudo_binary"
+            echo "UV_BINARY=$uv_binary"
+          } >> "$GITHUB_ENV"
+          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
+          if [[ '${{ matrix.environment }}' == docker ]]; then
+            docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
+            test -L "$docker_binary"
+            test -x "$docker_binary"
+            echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
+          elif [[ '${{ matrix.environment }}' == macos ]]; then
+            test -L "$ABXPKG_LIB_DIR/env/bin/brew"
+            test -x "$ABXPKG_LIB_DIR/env/bin/brew"
+          fi
+      - name: Build the exact checkout's documentation image
+        if: ${{ matrix.environment == 'docker' }}
+        run: '"$DOCKER_BINARY" build --tag archivebox/archivebox:dev .'
+      - name: Run the Ubuntu, Docker, or macOS documentation lane
+        if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
+        run: uv run --no-sync python docs/test_codeblocks_manifest.py run-environment '${{ matrix.environment }}'
+      - name: Run the root documentation lane
+        if: ${{ matrix.environment == 'root' }}
+        run: |
+          "$SUDO_BINARY" \
+            --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
+            "$UV_BINARY" run --no-sync python docs/test_codeblocks_manifest.py run-environment root
+      - name: Run the FreeBSD documentation lane
+        if: ${{ matrix.environment == 'freebsd' }}
+        uses: vmactions/freebsd-vm@77ed28d336d03fe19a3f4f7266c1d2c4714dd79d # v1.5.2
         with:
           usesh: true
           prepare: pkg install -y py313-uv
-          run: uv run --no-project --with pytest --with pytest-codeblocks pytest -o addopts=--codeblocks -vv --tb=long README.md AGENTS.md skills $(uv run --no-project python -c 'from pathlib import Path; print(*(str(path) for path in sorted(Path("docs").rglob("*.md")) if not path.is_symlink()))') --docs-environment=freebsd
-
-  docs-openbsd:
-    name: docs/openbsd
-    runs-on: ubuntu-24.04
-    timeout-minutes: 5
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-      - uses: vmactions/openbsd-vm@c941015845c0f0c429676840963dc63b226d4f69 # v1.4.5
+          run: |
+            abxpkg_version="$(uv run --no-project python -c 'import tomllib; dependencies = tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"]; print(next(dependency.split("==", 1)[1] for dependency in dependencies if dependency.startswith("abxpkg==")))')"
+            export ABXPKG_LIB_DIR="/tmp/archivebox-docs-lib"
+            uvx --from "abxpkg[pyinfra]==$abxpkg_version" abxpkg env \
+              --install \
+              --lib="$ABXPKG_LIB_DIR" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:bsd_binaries" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:freebsd_binaries"
+            BASH_BINARY="$ABXPKG_LIB_DIR/env/bin/bash" uv run --no-project python docs/test_codeblocks_manifest.py run-environment freebsd
+      - name: Run the OpenBSD documentation lane
+        if: ${{ matrix.environment == 'openbsd' }}
+        uses: vmactions/openbsd-vm@c941015845c0f0c429676840963dc63b226d4f69 # v1.4.5
         with:
           usesh: true
           prepare: pkg_add uv
-          run: uv run --no-project --with pytest --with pytest-codeblocks pytest -o addopts=--codeblocks -vv --tb=long README.md AGENTS.md skills $(uv run --no-project python -c 'from pathlib import Path; print(*(str(path) for path in sorted(Path("docs").rglob("*.md")) if not path.is_symlink()))') --docs-environment=openbsd
+          run: |
+            abxpkg_version="$(uv run --no-project python -c 'import tomllib; dependencies = tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"]; print(next(dependency.split("==", 1)[1] for dependency in dependencies if dependency.startswith("abxpkg==")))')"
+            export ABXPKG_LIB_DIR="/tmp/archivebox-docs-lib"
+            uvx --from "abxpkg[pyinfra]==$abxpkg_version" abxpkg env \
+              --install \
+              --lib="$ABXPKG_LIB_DIR" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:bsd_binaries"
+            BASH_BINARY="$ABXPKG_LIB_DIR/env/bin/bash" uv run --no-project python docs/test_codeblocks_manifest.py run-environment openbsd

.github/workflows/lint.yml

@@ -5,7 +5,6 @@ on:
 
 env:
   UV_NO_SOURCES: "1"
-  PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
 
 jobs:
   lint:
@@ -31,19 +30,8 @@ jobs:
       - name: Verify checked dependency lock
         run: uv lock --check
 
-      - name: Clone abxpkg
-        run: bash .github/scripts/clone_abx_repo.sh abxpkg
-
-      - name: Clone abx-plugins
-        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
-
-      - name: Clone abx-dl
-        run: bash .github/scripts/clone_abx_repo.sh abx-dl
-
       - name: Install dependencies with uv
-        run: |
-          uv venv
-          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[sonic,debug]"
+        run: uv sync --locked --dev --extra sonic --extra debug
 
       - name: Run prek
         run: uv run --no-sync --no-sources prek run --all-files

.github/workflows/test-parallel.yml

@@ -9,15 +9,13 @@ env:
   USE_COLOR: False
   UV_NO_SOURCES: "1"
   CI_PYTHON_VERSION: "3.13.14"
-  UV_CACHE_DIR: ${{ github.workspace }}/.uv-cache
 
 jobs:
   discover-tests:
-    name: Discover test files
+    name: Discover every test
     runs-on: ubuntu-24.04
     outputs:
-      test-files: ${{ steps.set-matrix.outputs.test-files }}
-      plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
+      tests: ${{ steps.discover.outputs.tests }}
 
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -25,78 +23,63 @@ jobs:
           submodules: true
           fetch-depth: 1
 
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: ${{ env.CI_PYTHON_VERSION }}
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
           version: "0.11.3"
           enable-cache: false
 
-      - name: Resolve matrix tools through abxpkg
+      - name: Resolve discovery tools through abxpkg
         env:
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
         run: |
           set -Eeuo pipefail
           ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
           test -n "$ABXPKG_VERSION"
           mkdir -p "$ABXPKG_LIB_DIR/env/bin"
+          export PYTHON_BINARY="$pythonLocation/bin/python"
+          test -x "$PYTHON_BINARY"
           uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
             --install \
             --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-            >/dev/null
-          JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
-          PYTHON_BINARY="$ABXPKG_LIB_DIR/env/bin/python"
-          test -L "$JQ_BINARY"
-          test -x "$JQ_BINARY"
-          test -L "$PYTHON_BINARY"
-          test -x "$PYTHON_BINARY"
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
           {
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-            echo "JQ_BINARY=$JQ_BINARY"
-            echo "PYTHON_BINARY=$PYTHON_BINARY"
+            echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
+            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
           } >> "$GITHUB_ENV"
-          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
+          test -L "$ABXPKG_LIB_DIR/env/bin/python"
+          test -x "$ABXPKG_LIB_DIR/env/bin/python"
+          test -L "$ABXPKG_LIB_DIR/env/bin/jq"
+          test -x "$ABXPKG_LIB_DIR/env/bin/jq"
 
-      - name: Discover test files
-        id: set-matrix
-        shell: bash
+      - name: Discover every test exactly once
+        id: discover
         run: |
           set -Eeuo pipefail
-          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py archivebox)
+          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
           test "${#output[@]}" -eq 2
-          json_array="${output[1]}"
-          echo "test-files=$json_array" >> "$GITHUB_OUTPUT"
+          echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
           echo "${output[0]}"
-          echo "$json_array" | "$JQ_BINARY" '.'
+          "$JQ_BINARY" -e 'length > 0 and length <= 256' <<< "${output[1]}"
 
-      - name: Clone abx-plugins
-        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
-
-      - name: Discover plugin tests
-        id: set-plugin-matrix
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py plugins)
-          test "${#output[@]}" -eq 2
-          json_array="${output[1]}"
-          echo "plugin-tests=$json_array" >> "$GITHUB_OUTPUT"
-          echo "${output[0]}"
-          echo "$json_array" | "$JQ_BINARY" '.'
-
-  prepare-python-dependencies:
-    name: Prepare Python 3.13.14 dependency cache
+  tests:
+    name: ${{ matrix.test.name }}
+    needs: discover-tests
     runs-on: ubuntu-24.04
     env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
-      LIB_DIR: /tmp/abx-lib
-      ABXPKG_LIB_DIR: /tmp/abx-lib
+      ABXPKG_LIB_DIR: ${{ github.workspace }}/.abx-lib
+      CHROME_HEADLESS: "true"
+      DATA_DIR: /tmp/archivebox-test-data
+      PERSONAS_DIR: /tmp/abx-personas
+      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
+
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
 
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -104,406 +87,45 @@ jobs:
           submodules: true
           fetch-depth: 1
 
-      - name: Set up Python ${{ env.CI_PYTHON_VERSION }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: ${{ env.CI_PYTHON_VERSION }}
           architecture: x64
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
           version: "0.11.3"
-          enable-cache: false
 
-      - name: Clone abxpkg
-        run: bash .github/scripts/clone_abx_repo.sh abxpkg
-
-      - name: Clone abx-plugins
-        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
-
-      - name: Clone abx-dl
-        run: bash .github/scripts/clone_abx_repo.sh abx-dl
-
-      - name: Restore or create the SHA-specific uv cache
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
-        with:
-          path: ${{ env.UV_CACHE_DIR }}
-          key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }}
-
-      - name: Bootstrap local abxpkg
+      - name: Resolve optional build dependencies through abxpkg
+        if: matrix.test.extra != ''
         run: |
           set -Eeuo pipefail
-          uv venv
-          uv pip install -e ./abxpkg
-
-      - name: Resolve Linux build dependencies through abxpkg
-        run: |
-          set -Eeuo pipefail
-          mkdir -p "$LIB_DIR/env/bin"
-          echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
-          uv run --no-sync --no-sources abxpkg env \
+          ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          test -n "$ABXPKG_VERSION"
+          uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
             --install \
             --no-cache \
             --json \
             --lib="$ABXPKG_LIB_DIR" \
             --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries"
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --json \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-env.json:required_binaries"
 
-      - name: Populate the full editable dependency cache
+      - name: Install test dependencies
+        env:
+          TEST_EXTRA: ${{ matrix.test.extra }}
         run: |
           set -Eeuo pipefail
-          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
-          uv run --no-sync --no-sources python -c 'import ldap; print(ldap.__version__)'
-
-  run-tests:
-    name: ${{ matrix.test.name }}
-    runs-on: ubuntu-24.04
-    needs: [discover-tests, prepare-python-dependencies]
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
-      CHROME_HEADLESS: "true"
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
-
-    strategy:
-      fail-fast: false
-      matrix:
-        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
-        python: ["3.13"]
-
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ matrix.python }}
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      - name: Clone abxpkg
-        run: bash .github/scripts/clone_abx_repo.sh abxpkg
-
-      - name: Clone abx-plugins
-        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
-
-      - name: Clone abx-dl
-        run: bash .github/scripts/clone_abx_repo.sh abx-dl
-
-      - name: Cache uv
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
-        with:
-          path: ${{ env.UV_CACHE_DIR }}
-          key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }}
-          fail-on-cache-miss: true
-
-      - name: Install dependencies with uv
-        run: |
-          set -Eeuo pipefail
-          uv venv
-          uv pip install --offline --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
-
-      - name: Resolve Node.js through abxpkg
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -Eeuo pipefail
-          mkdir -p "$LIB_DIR/env/bin"
-          export PATH="$LIB_DIR/env/bin:$PATH"
-          {
-            echo "LIB_DIR=$LIB_DIR"
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-          } >> "$GITHUB_ENV"
-          echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
-
-          tooling_env="$(
-            uv run --no-sync --no-sources abxpkg env \
-              --install \
-              --json \
-              --lib="$LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries"
-          )"
-          JQ_BINARY="$LIB_DIR/env/bin/jq"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$tooling_env" >> "$GITHUB_ENV"
-
-          node_binary="$LIB_DIR/env/bin/node"
-          {
-            echo "JQ_BINARY=$JQ_BINARY"
-            echo "NODE_BINARY=$node_binary"
-          } >> "$GITHUB_ENV"
-          test -L "$JQ_BINARY"
-          test -x "$JQ_BINARY"
-          test -L "$node_binary"
-          test -x "$node_binary"
-          "$node_binary" --version
-
-      - name: Resolve core test binaries through abxpkg
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR/env/bin"
-          {
-            echo "LIB_DIR=$LIB_DIR"
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-          } >> "$GITHUB_ENV"
-          echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --json \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/wget/config.json:required_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/git/config.json:required_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_ripgrep/config.json:required_binaries" \
-            | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            >> "$GITHUB_ENV"
-
-      - name: Resolve Chrome through abxpkg
-        if: ${{ matrix.test.needs_chromium }}
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          {
-            echo "LIB_DIR=$LIB_DIR"
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-          } >> "$GITHUB_ENV"
-
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --json \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \
-            | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            >> "$GITHUB_ENV"
-
-      - name: Resolve Sonic through abxpkg
-        if: ${{ matrix.test.needs_sonic }}
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          {
-            echo "LIB_DIR=$LIB_DIR"
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-          } >> "$GITHUB_ENV"
-
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --json \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \
-            | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            >> "$GITHUB_ENV"
-
-      - name: Install PostgreSQL server binaries
-        if: contains(toJson(matrix.test.paths), 'test_postgres_backend')
-        run: |
-          set -Eeuo pipefail
-          if ! ls /usr/lib/postgresql/*/bin/initdb >/dev/null 2>&1 && ! command -v initdb >/dev/null 2>&1; then
-            sudo apt-get update && sudo apt-get install -y postgresql
+          extra_args=()
+          if [[ -n "$TEST_EXTRA" ]]; then
+            extra_args=(--extra "$TEST_EXTRA")
           fi
+          uv sync --locked --dev "${extra_args[@]}"
 
-      - name: Run test - ${{ matrix.test.name }}
+      - name: Run ${{ matrix.test.name }}
         env:
-          TEST_PATHS_JSON: ${{ toJson(matrix.test.paths) }}
+          TEST_PATH: ${{ matrix.test.path }}
         run: |
           set -Eeuo pipefail
-          mapfile -t test_paths < <("$JQ_BINARY" -r '.[]' <<<"$TEST_PATHS_JSON")
-          [[ "${#test_paths[@]}" -gt 0 ]]
-          mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -vs "${test_paths[@]}" --basetemp="$GITHUB_WORKSPACE/tests/out"
-
-  plugin-tests:
-    name: ${{ matrix.plugin.name }}
-    runs-on: ubuntu-24.04
-    needs: [discover-tests, prepare-python-dependencies]
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
-      CHROME_HEADLESS: "true"
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
-
-    strategy:
-      fail-fast: false
-      matrix:
-        plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }}
-
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Set up Python ${{ env.CI_PYTHON_VERSION }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ env.CI_PYTHON_VERSION }}
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      - name: Clone abxpkg
-        run: bash .github/scripts/clone_abx_repo.sh abxpkg
-
-      - name: Clone abx-plugins
-        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
-
-      - name: Clone abx-dl
-        run: bash .github/scripts/clone_abx_repo.sh abx-dl
-
-      - name: Cache uv
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
-        with:
-          path: ${{ env.UV_CACHE_DIR }}
-          key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }}
-          fail-on-cache-miss: true
-
-      - name: Install dependencies with uv
-        run: |
-          set -Eeuo pipefail
-          uv venv
-          uv pip install --offline --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
-
-      - name: Resolve Node.js through abxpkg
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -Eeuo pipefail
-          mkdir -p "$LIB_DIR/env/bin"
-          export PATH="$LIB_DIR/env/bin:$PATH"
-          {
-            echo "LIB_DIR=$LIB_DIR"
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-          } >> "$GITHUB_ENV"
-          echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
-
-          tooling_env="$(
-            uv run --no-sync --no-sources abxpkg env \
-              --install \
-              --json \
-              --lib="$LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries"
-          )"
-          JQ_BINARY="$LIB_DIR/env/bin/jq"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$tooling_env" >> "$GITHUB_ENV"
-
-          node_binary="$LIB_DIR/env/bin/node"
-          {
-            echo "JQ_BINARY=$JQ_BINARY"
-            echo "NODE_BINARY=$node_binary"
-          } >> "$GITHUB_ENV"
-          test -L "$JQ_BINARY"
-          test -x "$JQ_BINARY"
-          test -L "$node_binary"
-          test -x "$node_binary"
-          "$node_binary" --version
-
-      - name: Resolve plugin dependencies through abxpkg
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR/env/bin"
-          {
-            echo "LIB_DIR=$LIB_DIR"
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-          } >> "$GITHUB_ENV"
-          echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --json \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/${{ matrix.plugin.config_path }}:required_binaries" \
-            | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            >> "$GITHUB_ENV"
-
-      - name: Resolve Chrome through abxpkg
-        if: ${{ matrix.plugin.needs_chromium }}
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          {
-            echo "LIB_DIR=$LIB_DIR"
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-          } >> "$GITHUB_ENV"
-
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --json \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \
-            | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            >> "$GITHUB_ENV"
-
-      - name: Resolve Sonic through abxpkg
-        if: ${{ matrix.plugin.needs_sonic }}
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          {
-            echo "LIB_DIR=$LIB_DIR"
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-          } >> "$GITHUB_ENV"
-
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --json \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \
-            | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            >> "$GITHUB_ENV"
-
-      - name: Run plugin tests - ${{ matrix.plugin.name }}
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
-        run: |
-          set -Eeuo pipefail
-          test_path="$GITHUB_WORKSPACE/${{ matrix.plugin.test_path }}"
-          test -d "$test_path"
-          test_count="$("$LIB_DIR/env/bin/python" -c 'import pathlib, sys; print(sum(1 for _ in pathlib.Path(sys.argv[1]).rglob("test_*.py")))' "$test_path")"
-          test "$test_count" -gt 0
-          DATA_DIR="$(mktemp -d -t archivebox_plugin_tests.XXXXXX)"
-          export DATA_DIR
-          plugin_tmpdir="$(mktemp -d -t archivebox_plugin_run.XXXXXX)"
-          cd "$plugin_tmpdir"
-          uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources python -m pytest \
-            "$test_path" -p no:django -v --tb=short
+          test -f "$TEST_PATH"
+          mkdir -p "$DATA_DIR"
+          uv run --no-sync --no-sources pytest -vs "$TEST_PATH" \
+            --basetemp="$RUNNER_TEMP/pytest"

.github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: Integration Tests
+name: Install and CLI Compatibility
 
 on:
   workflow_call:
@@ -12,10 +12,9 @@ env:
   UV_NO_SOURCES: "1"
 
 jobs:
-  python_tests:
+  install_cli_platform:
+    name: CLI install / ${{ matrix.os_name }} / Python ${{ matrix.python }}
     runs-on: ${{ matrix.os }}
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
 
     strategy:
       fail-fast: false
@@ -48,44 +47,23 @@ jobs:
         with:
           version: "0.11.3"
 
-      - name: Clone abxpkg
-        run: bash .github/scripts/clone_abx_repo.sh abxpkg
-
-      - name: Clone abx-plugins
-        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
-
-      - name: Clone abx-dl
-        run: bash .github/scripts/clone_abx_repo.sh abx-dl
-
-      ### Install dependencies
-      - name: Cache uv
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-uv-
-
-      - name: Bootstrap local abxpkg
-        run: |
-          set -Eeuo pipefail
-          uv venv
-          uv pip install -e ./abxpkg
-
       - name: Prepare abxpkg environment
         env:
           LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
         run: |
           set -Eeuo pipefail
+          ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          test -n "$ABXPKG_VERSION"
           mkdir -p "$LIB_DIR/env/bin"
           {
+            echo "ABXPKG_VERSION=$ABXPKG_VERSION"
             echo "LIB_DIR=$LIB_DIR"
             echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
           } >> "$GITHUB_ENV"
           echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
 
-      - name: Resolve Node.js through abxpkg
+      - name: Resolve CI tools through abxpkg
         env:
           LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
@@ -93,25 +71,18 @@ jobs:
           set -Eeuo pipefail
           export PATH="$LIB_DIR/env/bin:$PATH"
           tooling_env="$(
-            uv run --no-sync --no-sources abxpkg env \
+            uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
               --install \
               --json \
               --lib="$LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries"
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
           )"
           JQ_BINARY="$LIB_DIR/env/bin/jq"
           "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
             <<< "$tooling_env" >> "$GITHUB_ENV"
-
-          node_binary="$LIB_DIR/env/bin/node"
-          {
-            echo "JQ_BINARY=$JQ_BINARY"
-            echo "NODE_BINARY=$node_binary"
-          } >> "$GITHUB_ENV"
-          test -L "$node_binary"
-          test -x "$node_binary"
-          "$node_binary" --version
+          echo "JQ_BINARY=$JQ_BINARY" >> "$GITHUB_ENV"
+          test -L "$JQ_BINARY"
+          test -x "$JQ_BINARY"
 
       - name: Resolve Linux build dependencies through abxpkg
         if: runner.os == 'Linux'
@@ -120,19 +91,12 @@ jobs:
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
         run: |
           set -Eeuo pipefail
-          uv run --no-sync --no-sources abxpkg env \
+          uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
             --install \
             --no-cache \
             --json \
             --lib="$LIB_DIR" \
             --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries"
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --json \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-env.json:required_binaries" \
-            | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            >> "$GITHUB_ENV"
 
       - name: Resolve host Homebrew through abxpkg
         if: runner.os == 'macOS'
@@ -142,7 +106,7 @@ jobs:
         run: |
           set -Eeuo pipefail
           brew_env="$(
-            uv run --no-sync --no-sources abxpkg env \
+            uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
               --install \
               --json \
               --lib="$LIB_DIR" \
@@ -164,14 +128,14 @@ jobs:
           brew_root="$(dirname "$(dirname "$brew_target")")"
           export ABXPKG_BREW_ROOT="$brew_root"
 
-          uv run --no-sync --no-sources abxpkg env \
+          uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
             --install \
             --json \
             --lib="$LIB_DIR" \
             --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-build.json:required_binaries"
 
           PATH="$brew_root/opt/openldap/bin:$PATH" \
-            uv run --no-sync --no-sources abxpkg env \
+            uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
               --install \
               --json \
               --lib="$LIB_DIR" \
@@ -180,6 +144,14 @@ jobs:
               >> "$GITHUB_ENV"
 
           ldapvc_target="$(readlink "$LIB_DIR/env/bin/ldapvc")"
+          while [[ -L "$ldapvc_target" ]]; do
+            next_ldapvc_target="$(readlink "$ldapvc_target")"
+            if [[ "$next_ldapvc_target" == /* ]]; then
+              ldapvc_target="$next_ldapvc_target"
+            else
+              ldapvc_target="$(dirname "$ldapvc_target")/$next_ldapvc_target"
+            fi
+          done
           test -x "$ldapvc_target"
           openldap_prefix="$(dirname "$(dirname "$ldapvc_target")")"
           test -f "$openldap_prefix/include/ldap.h"
@@ -192,8 +164,7 @@ jobs:
           } >> "$GITHUB_ENV"
 
       - name: Install dependencies with uv
-        run: |
-          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+        run: uv sync --locked --dev --all-extras
 
       - name: ArchiveBox full install check
         run: |
@@ -205,27 +176,8 @@ jobs:
           uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox version
           uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status
 
-      - name: Install PostgreSQL server binaries
-        if: matrix.os_name == 'macOS' || matrix.python == '3.14'
-        run: |
-          set -Eeuo pipefail
-          if [ "$RUNNER_OS" = "Linux" ]; then
-            if ! ls /usr/lib/postgresql/*/bin/initdb >/dev/null 2>&1 && ! command -v initdb >/dev/null 2>&1; then
-              sudo apt-get update && sudo apt-get install -y postgresql
-            fi
-          else
-            if ! command -v initdb >/dev/null 2>&1 && ! ls /opt/homebrew/opt/postgresql*/bin/initdb >/dev/null 2>&1; then
-              brew install postgresql@17
-            fi
-          fi
-
-      - name: Run consolidated core suite
-        if: matrix.os_name == 'macOS' || matrix.python == '3.14'
-        run: |
-          mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -q archivebox/tests --basetemp="$GITHUB_WORKSPACE/tests/out/${{ matrix.os_name }}-python-${{ matrix.python }}"
-
   docker_tests:
+    name: Docker CLI and mount compatibility
     runs-on: ubuntu-24.04
 
     steps:

AGENTS.md

@@ -18,19 +18,19 @@ ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev
 ## Development Setup
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"
-uv sync --project "$project_dir" --dev --all-extras
-cd "$archivebox_data" && uv run --project "$project_dir" --no-sync archivebox init --install
+uv sync --dev --all-extras
+mkdir -p data
+cd data
+uv run --project .. archivebox init --install
 ```
 
 Run collection commands from inside an initialized data directory:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"
-cd "$archivebox_data"
-uv run --project "$project_dir" --no-sync archivebox init --install && uv run --project "$project_dir" --no-sync archivebox status && uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}" && uv run --project "$project_dir" --no-sync archivebox run
+cd data
+uv run --project .. archivebox status
+uv run --project .. archivebox add --plugins=parse_txt_urls 'https://example.com/'
+uv run --project .. archivebox run
 ```
 
 ## User-Facing Setup
@@ -38,12 +38,11 @@ uv run --project "$project_dir" --no-sync archivebox init --install && uv run --
 Recommended CLI install:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-tool_root="$(mktemp -d)"; export UV_TOOL_DIR="$tool_root/tools" UV_TOOL_BIN_DIR="$tool_root/bin"
-uv tool install --force "$project_dir"
-export PLUGINS=parse_txt_urls
-archivebox_data="$(mktemp -d)"
-cd "$archivebox_data" && "$UV_TOOL_BIN_DIR/archivebox" init --install && "$UV_TOOL_BIN_DIR/archivebox" add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
+uv tool install --python 3.13 --prerelease allow archivebox
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init --install
+archivebox add --plugins=parse_txt_urls 'https://example.com/'
 ```
 
 Alternative install methods:
@@ -55,18 +54,18 @@ Alternative install methods:
 
 ## Basic Usage
 
-<!--pytest-codeblocks:cont-->
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"
-cd "$archivebox_data" && uv run --project "$project_dir" --no-sync archivebox init --install
-uv run --project "$project_dir" --no-sync archivebox version && uv run --project "$project_dir" --no-sync archivebox help && uv run --project "$project_dir" --no-sync archivebox status
-uv run --project "$project_dir" --no-sync archivebox install
-uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/docs-basic-usage}"
-uv run --project "$project_dir" --no-sync archivebox list --json --with-headers
-uv run --project "$project_dir" --no-sync archivebox search 'example' && uv run --project "$project_dir" --no-sync archivebox update --filter-type=domain example.com
-uv run --project "$project_dir" --no-sync archivebox remove --yes --delete --filter-type=exact "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/docs-basic-usage}"
-uv run --project "$project_dir" --no-sync archivebox run
+cd ~/archivebox/data
+archivebox version
+archivebox help
+archivebox status
+archivebox install
+archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
+archivebox list --json --with-headers
+archivebox search 'example'
+archivebox update --filter-type=domain example.com
+archivebox remove --yes --delete --filter-type=exact 'https://example.com/docs-basic-usage'
+archivebox run
 ```
 
 ## Verification
@@ -74,9 +73,8 @@ uv run --project "$project_dir" --no-sync archivebox run
 Use targeted tests for focused work:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-.}"
-uv run --project "$project_dir" --no-sync pytest "$project_dir/archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options" -q
-(cd "$project_dir" && uv run --no-sync prek run --all-files)
+uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
+uv run prek run --all-files
 ```
 
 Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

README.md

@@ -100,7 +100,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
 <sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
-<sub>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
+<sub>Set <code>BASE_URL</code> to change the public base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically. <code>BIND_ADDR</code> only controls the local listen address.</sub>
 </details>
 <br/>
 
@@ -165,7 +165,7 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 
 # Quickstart
 
-**🖥&nbsp; [Supported OSs](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#supported-systems):** Linux/BSD, macOS, Windows (Docker) &nbsp; **👾&nbsp; CPUs:** `amd64` (`x86_64`), `arm64`, `arm7`<br/>
+**🖥&nbsp; [Supported OSs](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#supported-systems):** Linux/BSD, macOS, Windows (Docker) &nbsp; **👾&nbsp; CPUs:** `amd64` (`x86_64`), `arm64`, `arm7` <sup>(raspi>=3)</sup><br/>
 
 <br/>
 
@@ -280,7 +280,7 @@ archivebox help
 
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
 <br/>
-<sub>See the <a href="https://github.com/ArchiveBox/pip-archivebox"><code>pip-archivebox</code></a> repo for more details about this distribution.</sub>
+<sub>See the <a href="https://docs.astral.sh/uv/guides/tools/"><code>uv tool</code> documentation</a> for more details about this installation method.</sub>
 <br/><br/>
 </details>
 
@@ -353,11 +353,11 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <summary><img src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" height="28px" align="top"/> <code>pacman</code> / <img src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" height="28px" align="top"/> <code>pkg</code> / <img src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" height="28px" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
 <br/>
 
-> *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*
+> *Warning: These are contributed by external volunteers and may lag behind the official `uv` and Docker channels.*
 
 <ul>
 <li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>, maintained by <a href="https://github.com/jasongodev"><code>@jasongodev</code></a>)</li>
-<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
+<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>uv</code> under-the-hood)</li>
 <li>Nix: <a href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix"><code>nix-env --install archivebox</code></a> (contributed by <a href="https://github.com/siraben"><code>@siraben</code></a>)</li>
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
@@ -470,23 +470,7 @@ For more discussion on managed and paid hosting options see here: <a href="https
 ArchiveBox commands can be run in a terminal [directly on your host](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), or via [Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1)/[Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage).  
 <sup>(depending on how you chose to install it above)</sup>
 
-<!--
 ```bash
-set -euo pipefail
-__archivebox_docs_home="$(mktemp -d)"
-export HOME="$__archivebox_docs_home"
-export PLUGINS=parse_txt_urls
-mkdir -p ~/archivebox/data
-cd ~/archivebox/data
-archivebox init
-archivebox version
-archivebox help
-test -f index.sqlite3
-test -d archive
-rm -rf "$__archivebox_docs_home"
-```
--->
-```console
 mkdir -p ~/archivebox/data   # create a new data dir anywhere
 cd ~/archivebox/data         # IMPORTANT: cd into the directory
 
@@ -577,7 +561,7 @@ docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
 <pre lang="bash"><code style="white-space: pre-line">
 archivebox shell           # explore the Python library API in a REPL
 sqlite3 ./index.sqlite3    # run SQL queries directly on your index
-ls ./archive/*/index.html  # or inspect snapshot data directly on the filesystem
+find ./archive/users -path '*/snapshots/*/*/*/index.html'  # inspect snapshot data directly
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage">Python Shell</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage">SQL API</a>, and <a href="https://github.com/ArchiveBox/ArchiveBox#archive-layout">Disk Layout</a> wikis. ➡️</i>
 </details>
@@ -602,7 +586,7 @@ docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 
 <sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
-<sup>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
+<sup>Set <code>BASE_URL</code> to change the public base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically. <code>BIND_ADDR</code> only controls the local listen address.</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
 <br/><br/>
@@ -610,10 +594,10 @@ docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 
 <pre lang="bash"><code style="white-space: pre-line">
 archivebox config --set PUBLIC_ADD_VIEW=True   # allow guests to submit URLs 
-archivebox config --set PUBLIC_SNAPSHOTS=True  # allow guests to see snapshot content
+archivebox config --set PERMISSIONS=public     # make newly added snapshots public
 archivebox config --set PUBLIC_INDEX=True      # allow guests to see list of all snapshots
 # or
-docker compose run archivebox config --set ...
+docker compose run archivebox config --set PERMISSIONS=public
 
 # restart the server to apply any config changes
 </code></pre>
@@ -690,58 +674,19 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e1e5bd78-b0b6-45dc-914c-e1046fee4bc4" width="330px" align="right" style="float: right"/>
 
 
-<!--
-```bash
-set -euo pipefail
-__archivebox_docs_home="$(mktemp -d)"
-export HOME="$__archivebox_docs_home"
-mkdir -p ~/archivebox/data ~/Downloads
-cd ~/archivebox/data
-archivebox init
-cat > ~/Downloads/some_feed.xml <<'EOF'
-<?xml version="1.0" encoding="UTF-8"?>
-<rss version="2.0">
-  <channel>
-    <title>ArchiveBox docs test feed</title>
-    <link>https://example.com/</link>
-    <description>ArchiveBox docs test feed</description>
-    <item>
-      <title>Feed item</title>
-      <link>https://example.com/from-feed</link>
-      <guid>https://example.com/from-feed</guid>
-    </item>
-  </channel>
-</rss>
-EOF
-```
--->
-<!--pytest-codeblocks:cont-->
 ```bash
 # archivebox add --help
-archivebox add --plugins=parse_txt_urls 'https://example.com/some/page'
-archivebox add --depth=1 --plugins=parse_rss_urls < "$HOME/Downloads/some_feed.xml"
-archivebox add --plugins=parse_txt_urls 'https://example.com/docs-example'
-echo 'http://example.com' | archivebox add --plugins=parse_txt_urls
-echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add --plugins=parse_txt_urls
+archivebox add 'https://example.com/some/page'
+archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
+archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
+echo 'http://example.com' | archivebox add
+echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
 
 # if using Docker, add -i when piping stdin:
 # echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox:dev add
 # if using Docker Compose, add -T when piping stdin / stdout:
 # echo 'https://example.com' | docker compose run -T archivebox add
 ```
-<!--pytest-codeblocks:cont-->
-<!--
-```bash
-archivebox list --json > snapshots.json
-grep -q 'https://example.com/some/page' snapshots.json
-grep -q 'https://example.com/from-feed' snapshots.json
-grep -q 'https://example.com/docs-example' snapshots.json
-grep -q 'http://example.com' snapshots.json
-test -d archive/users/system/crawls
-test "$(grep -c '\"url\"' snapshots.json)" -ge 4
-rm -rf "$__archivebox_docs_home"
-```
--->
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
 
@@ -798,10 +743,10 @@ ArchiveBox can be configured via environment variables, by using the `archivebox
 archivebox config --get CHROME_BINARY           # view a specific value
 <br/>
 archivebox config --set CHROME_BINARY=chromium  # persist a config using CLI
+# OR edit ArchiveBox.conf and add this under its existing [ARCHIVING_CONFIG] section:
+CHROME_BINARY=chromium
 # OR
-echo CHROME_BINARY=chromium >> ArchiveBox.conf  # persist a config using file
-# OR
-env CHROME_BINARY=chromium archivebox ...       # run with a one-off config
+env CHROME_BINARY=chromium archivebox version   # run with a one-off config
 </code></pre>
 <sub>These methods also work the same way when run inside Docker, see the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker Configuration</a> wiki page for details.</sub>
 </details><br/>
@@ -819,7 +764,7 @@ TIMEOUT=240                # default: 60    add more seconds on slower networks
 CHECK_SSL_VALIDITY=False   # default: True  False = allow saving URLs w/ bad SSL
 <br/>
 PUBLIC_INDEX=True          # default: True  whether anon users can view index
-PUBLIC_SNAPSHOTS=True      # default: True  whether anon users can view pages
+PERMISSIONS=public         # default: public visibility for newly added snapshots
 PUBLIC_ADD_VIEW=False      # default: False whether anon users can add new URLs
 <br/>
 USER_AGENT="Mozilla/5.0 ..."  # change this to get around bot blocking
@@ -872,9 +817,7 @@ These optional subdependencies used for archiving sites include:
 <li>and more as we grow...</li>
 </ul>
 
-You don't need to install every dependency to use ArchiveBox. ArchiveBox will automatically disable extractors that rely on dependencies that aren't installed, based on what is configured and available in your <code>$PATH</code>.
-  
-If not using Docker, make sure to keep the dependencies up-to-date yourself and check that ArchiveBox isn't reporting any incompatibility with the versions you install.
+You don't need to install every dependency by hand. ArchiveBox resolves every extractor dependency through <code>abxpkg</code>: it uses a compatible host installation when one is already available, and otherwise installs and manages the dependency for you.
 
 <pre lang="bash"><code style="white-space: pre-line"># install uv + archivebox first (see Quickstart instructions above)
 <br/>
@@ -910,7 +853,7 @@ All <code>archivebox</code> CLI commands are designed to be run from inside an A
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data   # just an example, can be anywhere
 archivebox init</code></pre>
 
-The on-disk layout is optimized to be easy to browse by hand and durable long-term. The main index is a standard <code>index.sqlite3</code> database in the root of the data folder (it can also be <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#2-export-and-host-it-as-static-html">exported as static JSON/HTML</a>), and the archive snapshots are organized by date-added timestamp in the <code>data/archive/</code> subfolder.
+The on-disk layout is optimized to be easy to browse by hand and durable long-term. The main index is a standard <code>index.sqlite3</code> database in the root of the data folder (it can also be <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#2-export-and-host-it-as-static-html">exported as static JSON/HTML</a>). Snapshot data is organized by user, date, domain, and UUID under <code>data/archive/users/</code>.
 
 <img src="https://user-images.githubusercontent.com/511499/117453293-c7b91600-af12-11eb-8a3f-aa48b0f9da3c.png" width="400px" align="right" style="float: right"/>
 
@@ -919,18 +862,23 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
     index.sqlite3
     ArchiveBox.conf
     archive/
-        ...
-        1617687755/
-            index.html
-            index.json
-            screenshot.png
-            media/some_video.mp4
-            warc/1617687755.warc.gz
-            git/somerepo.git
-            ...
+        1617687755 -> users/admin/snapshots/20210406/example.com/SNAPSHOT_UUID/
+        users/
+            admin/
+                snapshots/
+                    20210406/
+                        example.com/
+                            SNAPSHOT_UUID/
+                                index.html
+                                index.jsonl
+                                screenshot/screenshot.png
+                                ytdlp/media/some_video.mp4
+                                wget/warc/example.com.warc.gz
+                                git/somerepo.git
+                                ...
 </code></pre>
 
-Each snapshot subfolder <code>data/archive/TIMESTAMP/</code> includes a static <code>index.json</code> and <code>index.html</code> describing its contents, and the snapshot extractor outputs are plain files within the folder.
+Each snapshot subfolder includes static metadata and plain extractor output files. ArchiveBox also maintains a backwards-compatible <code>data/archive/TIMESTAMP</code> symlink for each snapshot.
 
 <h4>Learn More</h4>
 <ul>
@@ -964,7 +912,7 @@ archivebox list --json --with-headers > index.json     # export to json blob
 archivebox list --csv=timestamp,url,title > index.csv  # export to csv spreadsheet
 
 # (if using Docker Compose, add the -T flag when piping)
-# docker compose run -T archivebox list --html 'https://example.com' > index.json
+# docker compose run -T archivebox list --html 'https://example.com' > index.html
 </code></pre>
 
 The paths in the static exports are relative, make sure to keep them next to your `./archive` folder when backing them up or viewing them.
@@ -974,7 +922,7 @@ The paths in the static exports are relative, make sure to keep them next to you
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#2-export-and-host-it-as-static-html">Wiki: Publishing Your Archive (Exporting as Static HTML)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#publishing">Wiki: Security Overview (Publishing)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#public_index--public_snapshots--public_add_view">Wiki: Configuration (<code>PUBLIC_INDEX</code>, <code>PUBLIC_SNAPSHOTS</code>, <code>PUBLIC_ADD_VIEW</code>)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#permissions">Wiki: Configuration (<code>PERMISSIONS</code>, <code>PUBLIC_INDEX</code>, <code>PUBLIC_ADD_VIEW</code>)</a></li>
 </ul>
 
 </details>
@@ -1005,7 +953,7 @@ archivebox add 'https://vimeo.com/somePrivateVideo'
 
 # restrict the main index, Snapshot content, and Add Page to authenticated users as-needed:
 archivebox config --set PUBLIC_INDEX=False
-archivebox config --set PUBLIC_SNAPSHOTS=False
+archivebox config --set PERMISSIONS=private
 archivebox config --set PUBLIC_ADD_VIEW=False
 archivebox manage createsuperuser
 </code></pre>
@@ -1021,8 +969,8 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir">Wiki: Configuration (<code>CHROME_USER_DATA_DIR</code>)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#cookies_file">Wiki: Configuration (<code>COOKIES_FILE</code>)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Personas">Wiki: Personas (browser profiles and cookies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>
 
 </details>
@@ -1031,7 +979,7 @@ archivebox manage createsuperuser
 
 ### Security Risks of Viewing Archived JS
 
-Be aware that malicious archived JS can access the contents of other pages in your archive when viewed. Because the Web UI serves all viewed snapshots from a single domain, they share a request context and **typical CSRF/CORS/XSS/CSP protections do not work to prevent cross-site request attacks**. See the [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) page and [Issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239) for more details.
+Archived JavaScript is untrusted content. The default <code>SERVER_SECURITY_MODE=safe-subdomains-fullreplay</code> serves replay content on isolated snapshot subdomains so it cannot share the admin UI's cookies or origin. If your deployment cannot use wildcard <code>*.archivebox.localhost</code> subdomains, use <code>safe-onedomain-nojsreplay</code>, which keeps one origin but disables JavaScript replay. See the [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview) and [Issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239) for details.
 
 
 <br/>
@@ -1039,18 +987,16 @@ Be aware that malicious archived JS can access the contents of other pages in yo
 <summary><i>Expand to see risks and mitigations...</i></summary>
 
 
-<pre lang="bash"><code style="white-space: pre-line"># visiting an archived page with malicious JS:
-https://127.0.0.1:8000/archive/1602401954/example.com/index.html
+<pre lang="bash"><code style="white-space: pre-line"># Default: full replay on isolated snapshot subdomains
+archivebox config --set SERVER_SECURITY_MODE=safe-subdomains-fullreplay
 
-# example.com/index.js can now make a request to read everything from:
-https://127.0.0.1:8000/index.html
-https://127.0.0.1:8000/archive/*
-# then example.com/index.js can send it off to some evil server
+# Alternative for deployments without wildcard subdomains: disable JS replay
+archivebox config --set SERVER_SECURITY_MODE=safe-onedomain-nojsreplay
 </code></pre>
 
 <blockquote>
 <p><em>NOTE: Only the <code>wget</code> &amp; <code>dom</code> extractor methods execute archived JS when viewing snapshots, all other archive methods produce static output that does not execute JS on viewing.</em><br/>
-<em>If you are worried about these issues ^ you should disable these extractors using:<br/> <code>archivebox config --set SAVE_WGET=False SAVE_DOM=False</code>.</em></p>
+<em>If you do not need JavaScript-capable replay at all, you can also disable those extractors with:<br/> <code>archivebox config --set WGET_ENABLED=False DOM_ENABLED=False</code>.</em></p>
 </blockquote>
 
 <h4>Learn More</h4>
@@ -1077,7 +1023,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 <ul>
 <li>Set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#curl_user_agent"><code>CHROME_USER_AGENT</code>, <code>WGET_USER_AGENT</code>, <code>CURL_USER_AGENT</code></a> to impersonate a real browser (by default, ArchiveBox reveals that it's a bot when using the default user agent settings)</li>
-<li>Set up a logged-in browser session for archiving using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile"><code>CHROME_USER_DATA_DIR</code> &amp; <code>COOKIES_FILE</code></a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Personas">importing a dedicated browser profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1090,7 +1036,7 @@ In the future we plan on adding support for running JS scripts during archiving
 
 ### Saving Multiple Snapshots of a Single URL
 
-ArchiveBox appends a hash with the current date `https://example.com#2020-10-24` to differentiate when a single URL is archived multiple times.
+ArchiveBox can preserve multiple snapshots of the same URL. The default <code>ONLY_NEW=True</code> skips URLs already in the collection; use <code>--no-only-new</code> when you intentionally want another snapshot.
 
 
 <br/>
@@ -1099,16 +1045,13 @@ ArchiveBox appends a hash with the current date `https://example.com#2020-10-24`
 <br/>
 
 
-Because ArchiveBox uniquely identifies snapshots by URL, it must use a workaround to take multiple snapshots of the same URL (otherwise they would show up as a single Snapshot entry). It makes the URLs of repeated snapshots unique by adding a hash with the archive date at the end:
+Each re-archive creates a distinct Snapshot row for the same URL:
 
-<pre lang="bash"><code style="white-space: pre-line">archivebox add 'https://example.com#2020-10-24'
-...
-archivebox add 'https://example.com#2020-10-25'
+<pre lang="bash"><code style="white-space: pre-line">archivebox add 'https://example.com'
+archivebox add --no-only-new 'https://example.com'
 </code></pre>
 
-The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI is a shortcut for this hash-date multi-snapshotting workaround.
-
-Improved support for saving multiple snapshots of a single URL without this hash-date workaround will be <a href="https://github.com/ArchiveBox/ArchiveBox/issues/179">added eventually</a> (along with the ability to view diffs of the changes between runs).
+The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI performs the same explicit re-archive.
 
 <h4>Learn More</h4>
 
@@ -1235,7 +1178,7 @@ ArchiveBox's stance is that duplication of other people's content is only ethica
 
 In the U.S., <a href="https://guides.library.oregonstate.edu/copyright/libraries">libraries, researchers, and archivists</a> are allowed to duplicate copyrighted materials under <a href="https://libguides.ala.org/copyright/fairuse">"fair use"</a> for <a href="https://guides.cuny.edu/cunyfairuse/librarians#:~:text=One%20of%20these%20specified%20conditions,may%20be%20liable%20for%20copyright">private study, scholarship, or research</a>. Archive.org's non-profit preservation work is <a href="https://blog.archive.org/2024/03/01/fair-use-in-action-at-the-internet-archive/">covered under fair use</a> in the US, and they properly handle <a href="https://cardozoaelj.com/2015/03/20/use-of-copyright-law-to-take-down-revenge-porn/">unethical content</a>/<a href="https://help.archive.org/help/rights/">DMCA</a>/<a href="https://gdpr.eu/right-to-be-forgotten/#:~:text=An%20individual%20has%20the%20right,that%20individual%20withdraws%20their%20consent.">GDPR</a> removal requests to maintain good standing in the eyes of the law.
 
-As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use software like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).
+As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use software like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal counsel if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and placing branding overrides in your collection's fixed <code>custom_templates/</code> directory).
 
 </details>
 <br/>
@@ -1285,7 +1228,6 @@ ArchiveBox is neither the highest fidelity nor the simplest tool available for s
 
 <br/>
 
-
 ## Internet Archiving Ecosystem
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/78d8a725-97f4-47f5-b983-1f62843ddc51" width="14%" align="right" style="float: right"/>
@@ -1384,7 +1326,7 @@ All contributions to ArchiveBox are welcomed! Check our [issues](https://github.
 
 For low hanging fruit / easy first tickets, see: <a href="https://github.com/ArchiveBox/ArchiveBox/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22help+wanted%22">ArchiveBox/Issues `#good first ticket` `#help wanted`</a>.
 
-**Python API Documentation:** https://docs.archivebox.io/en/dev/archivebox.html#module-archivebox.main
+**Python API Documentation:** https://docs.archivebox.io/dev/apidocs/
 
 **Internal Architecture Diagrams:** https://github.com/ArchiveBox/ArchiveBox/wiki/ArchiveBox-Architecture-Diagrams
 
@@ -1397,7 +1339,7 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc
 
 First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/
 
-```console
+```bash
 git clone https://github.com/ArchiveBox/monorepo
 cd monorepo
 ./bin/setup.sh
@@ -1418,7 +1360,7 @@ Repos included in monorepo setup:
 
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine
 
-```console
+```bash
 # Install ArchiveBox runtime dependencies
 mkdir -p data && cd data
 archivebox init
@@ -1433,7 +1375,7 @@ archivebox server 0.0.0.0:8000
 
 #### 2. Option B: Build the docker container and use that for development instead
 
-```console
+```bash
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
 # inside the container will reload and pick up your changes
@@ -1463,7 +1405,7 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 
 <details><summary><i>Click to expand...</i></summary>
 
-```console
+```bash
 # set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
 
@@ -1487,7 +1429,7 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj
 
 If you're looking for the latest `dev` Docker image, it's often available pre-built on Docker Hub, simply pull and use `archivebox/archivebox:dev`.
 
-```console
+```bash
 docker pull archivebox/archivebox:dev
 docker run archivebox/archivebox:dev version
 # verify the BUILD_TIME and COMMIT_HASH in the output are recent
@@ -1497,7 +1439,7 @@ docker run archivebox/archivebox:dev version
 
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:
 
-```console
+```bash
 # docker-compose.yml:
 services:
     archivebox:
@@ -1509,9 +1451,8 @@ services:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
 docker run -it -v $PWD:/data archivebox:dev init
 
-# or with pip:
-pip install 'git+https://github.com/pirate/ArchiveBox@dev'
-npm install 'git+https://github.com/ArchiveBox/ArchiveBox.git#dev'
+# or with uv:
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 archivebox install
 ```
 
@@ -1521,7 +1462,7 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
-```console
+```bash
 ./bin/lint.sh
 ./bin/test.sh
 ```
@@ -1534,10 +1475,9 @@ archivebox install
 
 <details><summary><i>Click to expand...</i></summary>
 
-```console
+```bash
 # generate the database migrations after changes to models.py
-cd archivebox/
-./manage.py makemigrations
+archivebox manage makemigrations
 
 # enter a python shell or a SQL shell
 cd path/to/test/data/
@@ -1546,8 +1486,7 @@ archivebox manage dbshell
 
 # generate a graph of the ORM models
 brew install graphviz
-pip install pydot graphviz
-archivebox manage graph_models -a -o orm.png
+uv run --with pydot --with graphviz archivebox manage graph_models -a -o orm.png
 open orm.png
 
 # list all models with field db info and methods
@@ -1555,7 +1494,7 @@ archivebox manage list_model_info --all --signature --db-type --field-class
 
 # print all django settings
 archivebox manage print_settings
-archivebox manage print_settings --format=yaml    # pip install pyyaml
+uv run --with pyyaml archivebox manage print_settings --format=yaml
 
 # autogenerate an admin.py from given app models
 archivebox manage admin_generator core > core/admin.py
@@ -1569,9 +1508,8 @@ archivebox manage runscript testdata
 archivebox manage reset_db
 
 # use django-tui to interactively explore commands
-uv pip install django-tui
 # ensure django-tui is in INSTALLED_APPS: core/settings.py
-archivebox manage tui
+uv run --with django-tui archivebox manage tui
 ```
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/dc3e9f8c-9544-46e0-a7f0-30f571b72022" width="600px" alt="ArchiveBox ORM models relatinoship graph"/>
@@ -1598,7 +1536,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>
 
 (Normally CI takes care of this, but these scripts can be run to do it manually)
-```console
+```bash
 ./bin/build.sh
 
 # or individually:

archivebox/cli/archivebox_add.py

@@ -235,6 +235,7 @@ def add(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
                             interactive_interrupts=True,
+                            config=get_config(crawl=crawl),
                         )
                         crawl.refresh_from_db(fields=["status", "retry_at"])
                         if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:

archivebox/cli/archivebox_update.py

@@ -232,7 +232,6 @@ def update(
     from archivebox.core.takeover_util import (
         command_owns_foreground_runner,
         current_command,
-        ensure_daemon_stack,
         foreground_runner_owner,
         standby_until_foreground_runner_needed,
     )
@@ -253,11 +252,9 @@ def update(
         standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
         raise_if_shutdown_requested()
 
-    def run_scoped_runner(*args: str, ensure_daemon_reason: str | None = None) -> None:
+    def run_scoped_runner(*args: str) -> None:
         while True:
             wait_for_turn()
-            if ensure_daemon_reason:
-                ensure_daemon_stack(reason=ensure_daemon_reason)
             exit_code = run_runner_worker(
                 list(args),
                 name=f"worker_runner_update_{os.getpid()}",
@@ -398,7 +395,6 @@ def update(
                     if full_update_empty:
                         print("[*] No snapshots found; skipping search indexing backfill.")
                     else:
-                        ensure_daemon_stack(reason="search indexing")
                         search_plugins = _get_search_indexing_plugins()
                         if not search_plugins:
                             print("[*] No search indexing plugins are available, nothing to backfill.")
@@ -502,7 +498,6 @@ def update(
                     else:
                         run_scoped_runner(
                             *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
-                            ensure_daemon_reason="search indexing" if do_index else None,
                         )
 
                 if not continuous:

archivebox/core/settings.py

@@ -76,7 +76,7 @@ INSTALLED_APPS = [
     "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
     "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
     "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
-    "abx_plugins.plugins.opencode",
+    "archivebox.opencode",
     # 3rd-party apps from PyPI that need to be loaded last
     "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
     "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)

archivebox/core/urls.py

@@ -28,7 +28,7 @@ from archivebox.core.views import (
 )
 from archivebox.progressmonitor.views import live_progress_view
 from archivebox.search.views import public_snapshot_search_stream_view
-from abx_plugins.plugins.opencode.views import opencode_proxy_view
+from archivebox.opencode.views import opencode_proxy_view
 
 CONFIG = get_config()
 DEBUG = CONFIG.DEBUG or ("--debug" in sys.argv)
@@ -38,7 +38,7 @@ urlpatterns = [
     path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
     path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
     path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
-    re_path(r"^admin/agent/?(?=$|opencode)", include("abx_plugins.plugins.opencode.urls")),
+    re_path(r"^admin/agent/?(?=$|opencode)", include("archivebox.opencode.urls")),
     re_path(r"^(?P<path>assets/.*)$", opencode_proxy_view, name="opencode-assets"),
     path("public/search-stream/", public_snapshot_search_stream_view, name="public-search-stream"),
     path("public/", PublicIndexView.as_view(), name="public-index"),

archivebox/mcp/README.md

@@ -19,9 +19,7 @@ This is a lightweight, stateless MCP server that dynamically introspects Archive
 ### Start the MCP Server
 
 ```bash
-request='{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}'
-response="$(printf '%s\n' "$request" | "$UV_BINARY" run --project "$ARCHIVEBOX_PROJECT_DIR" --no-sync archivebox mcp)"
-"$JQ_BINARY" -e '.id == 1 and .result.serverInfo.name == "archivebox-mcp"' <<< "$response"
+printf '%s\n' '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' | archivebox mcp
 ```
 
 The server runs in stdio mode, reading JSON-RPC 2.0 requests from stdin and writing responses to stdout.
@@ -30,20 +28,11 @@ The server runs in stdio mode, reading JSON-RPC 2.0 requests from stdin and writ
 
 ```python
 import json
-import os
 import subprocess
 
 request = {"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}
 completed = subprocess.run(
-    [
-        os.environ["UV_BINARY"],
-        "run",
-        "--project",
-        os.environ["ARCHIVEBOX_PROJECT_DIR"],
-        "--no-sync",
-        "archivebox",
-        "mcp",
-    ],
+    ["archivebox", "mcp"],
     input=json.dumps(request) + "\n",
     capture_output=True,
     text=True,

archivebox/opencode/__init__.py

@@ -0,0 +1 @@
+"""ArchiveBox OpenCode admin integration."""

archivebox/opencode/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class OpencodeConfig(AppConfig):
+    name = "archivebox.opencode"
+    label = "opencode_plugin"

archivebox/opencode/templates/opencode/agent.html

@@ -0,0 +1,211 @@
+{% extends "admin/base.html" %}
+{% load i18n %}
+
+{% block title %}Agent{% endblock %}
+
+{% block breadcrumbs %}
+    <div class="breadcrumbs">
+        <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+        &rsaquo; Agent
+    </div>
+{% endblock %}
+
+{% block extrastyle %}
+    {{ block.super }}
+    <style>
+        body.opencode-agent #content {
+            padding: 0;
+        }
+        body.opencode-agent #content-main {
+            height: calc(100vh - 150px);
+            min-height: 560px;
+        }
+        .opencode-agent-error {
+            margin: 24px;
+            color: #0f172a;
+            font: 14px -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+        }
+        .opencode-agent-error code {
+            color: #0369a1;
+        }
+        .opencode-agent-frame {
+            width: 100%;
+            height: 100%;
+            border: 0;
+            display: block;
+        }
+        .opencode-agent-shell {
+            position: relative;
+            height: 100%;
+        }
+        .opencode-agent-welcome {
+            position: absolute;
+            inset: 0;
+            z-index: 20;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            padding: 24px;
+            background: rgba(0, 0, 0, 0.48);
+            color: #111827;
+        }
+        .opencode-agent-welcome[hidden] {
+            display: none;
+        }
+        .opencode-agent-welcome-panel {
+            width: min(720px, 100%);
+            max-height: min(720px, calc(100vh - 190px));
+            overflow: auto;
+            border-radius: 8px;
+            border: 1px solid #d1d5db;
+            background: Canvas;
+            color: CanvasText;
+            box-shadow: 0 20px 60px rgba(0, 0, 0, 0.32);
+            padding: 24px;
+            font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+        }
+        .opencode-agent-welcome-panel h1 {
+            margin: 0 0 10px;
+            font-size: 22px;
+            line-height: 1.25;
+        }
+        .opencode-agent-welcome-panel p {
+            margin: 0 0 14px;
+        }
+        .opencode-agent-welcome-panel ul {
+            margin: 0 0 16px 20px;
+            padding: 0;
+        }
+        .opencode-agent-welcome-panel li {
+            margin: 0 0 8px;
+        }
+        .opencode-agent-welcome-warning {
+            margin: 16px 0;
+            padding: 12px 14px;
+            border: 1px solid #f59e0b;
+            border-radius: 6px;
+            background: color-mix(in srgb, #f59e0b 12%, Canvas);
+        }
+        .opencode-agent-welcome-actions {
+            display: flex;
+            justify-content: flex-end;
+            gap: 8px;
+        }
+        .opencode-agent-welcome button {
+            cursor: pointer;
+            border: 1px solid #2563eb;
+            border-radius: 6px;
+            background: #2563eb;
+            color: #fff;
+            padding: 8px 14px;
+            font-weight: 600;
+        }
+        @media (max-width: 700px) {
+            .opencode-agent-welcome {
+                padding: 12px;
+            }
+            .opencode-agent-welcome-panel {
+                max-height: calc(100vh - 170px);
+                padding: 18px;
+            }
+        }
+    </style>
+{% endblock %}
+
+{% block bodyclass %}{{ block.super }} opencode-agent{% endblock %}
+
+{% block content %}
+    <div id="content-main">
+        {% if error %}
+            <div class="opencode-agent-error">
+                {{ error }}
+                {% if command %}<br><br><code>{{ command }}</code>{% endif %}
+            </div>
+        {% else %}
+            <script>
+                (() => {
+                    const workdir = "{{ workdir|escapejs }}";
+                    const recentSessionId = "{{ recent_session_id|escapejs }}";
+                    const merge = (left, right) => {
+                        if (!left || typeof left !== "object" || Array.isArray(left)) return right;
+                        const out = {...left};
+                        for (const [key, value] of Object.entries(right)) {
+                            out[key] = value && typeof value === "object" && !Array.isArray(value) ? merge(out[key], value) : value;
+                        }
+                        return out;
+                    };
+                    const seed = (key, value) => {
+                        try {
+                            localStorage.setItem(key, JSON.stringify(merge(JSON.parse(localStorage.getItem(key) || "{}"), value)));
+                        } catch {
+                            localStorage.setItem(key, JSON.stringify(value));
+                        }
+                    };
+                    const setJSON = (key, value) => localStorage.setItem(key, JSON.stringify(value));
+                    const page = {
+                        activeProject: workdir,
+                        activeWorkspace: workdir,
+                        gettingStartedDismissed: true,
+                        workspaceExpanded: {[workdir]: true},
+                    };
+                    if (recentSessionId) {
+                        page.lastProjectSession = {[workdir]: {directory: workdir, id: recentSessionId, at: Date.now()}};
+                    }
+                    seed("opencode.global.dat:layout.page", page);
+                    seed("opencode.global.dat:layout", {
+                        sidebar: {opened: true, width: 280},
+                        fileTree: {opened: true, width: 280, tab: "all"},
+                    });
+                    const serverKey = `${location.origin}/admin/agent/opencode`;
+                    setJSON("opencode.global.dat:server", {
+                        list: [],
+                        projects: {
+                            [serverKey]: [{worktree: workdir, expanded: true}],
+                        },
+                        lastProject: {
+                            [serverKey]: workdir,
+                        },
+                    });
+                    localStorage.setItem("opencode.settings.dat:defaultServerUrl", serverKey);
+                })();
+            </script>
+            <div class="opencode-agent-shell">
+                <iframe src="{{ proxy_url }}" title="OpenCode Agent" class="opencode-agent-frame"></iframe>
+                <div class="opencode-agent-welcome" id="opencode-agent-welcome" hidden>
+                    <div class="opencode-agent-welcome-panel" role="dialog" aria-modal="true" aria-labelledby="opencode-agent-welcome-title">
+                        <h1 id="opencode-agent-welcome-title">ArchiveBox AI Agent</h1>
+                        <p>This agent can work directly with your ArchiveBox collection. It can inspect the crawl database, create and monitor crawls, run maintenance operations, answer research questions, query archived data, and help with complex collection workflows.</p>
+                        <p>Example prompts:</p>
+                        <ul>
+                            <li>Archive this list of URLs with depth 0, then report which ones failed and why.</li>
+                            <li>Find snapshots from the last month that failed PDF or screenshot extraction and retry them.</li>
+                            <li>Create a constrained crawl for this site, avoid login/privacy/sitemap URLs, and watch the logs as it runs.</li>
+                            <li>Summarize what my collection contains about this topic and link to the best saved pages.</li>
+                        </ul>
+                        <div class="opencode-agent-welcome-warning">
+                            The agent has unrestricted access to this collection and can make destructive edits. Review requests carefully. Permission settings can be changed from the OpenCode gear icon in the lower left.
+                        </div>
+                        <div class="opencode-agent-welcome-actions">
+                            <button type="button" id="opencode-agent-welcome-dismiss">Start using Agent</button>
+                        </div>
+                    </div>
+                </div>
+            </div>
+            <script>
+                (() => {
+                    const key = "archivebox.opencode.agentWelcomeDismissed.v1";
+                    const welcome = document.getElementById("opencode-agent-welcome");
+                    const dismiss = document.getElementById("opencode-agent-welcome-dismiss");
+                    if (!welcome || !dismiss) return;
+                    if (localStorage.getItem(key) !== "1") {
+                        welcome.hidden = false;
+                    }
+                    dismiss.addEventListener("click", () => {
+                        localStorage.setItem(key, "1");
+                        welcome.hidden = true;
+                    });
+                })();
+            </script>
+        {% endif %}
+    </div>
+{% endblock %}

archivebox/opencode/urls.py

@@ -0,0 +1,13 @@
+from django.urls import path, re_path
+
+from archivebox.opencode.views import agent_view, opencode_proxy_view
+
+
+urlpatterns = [
+    path("", agent_view, name="opencode-agent"),
+    re_path(
+        r"^opencode(?:/(?P<path>.*))?$",
+        opencode_proxy_view,
+        name="opencode-proxy",
+    ),
+]

archivebox/opencode/views.py

@@ -0,0 +1,716 @@
+from __future__ import annotations
+
+import atexit
+import base64
+import os
+import re
+import shutil
+import subprocess
+import threading
+import time
+from pathlib import Path
+from typing import Any
+from urllib.parse import urljoin, urlsplit
+
+import httpx
+import requests
+from abx_plugins.plugins import opencode as opencode_plugin
+from archivebox.config import CONSTANTS
+from archivebox.config.common import get_config
+from archivebox.core.routes_util import build_admin_url, get_api_base_url, get_base_url
+from django.http import (
+    Http404,
+    HttpRequest,
+    HttpResponse,
+    HttpResponseForbidden,
+    StreamingHttpResponse,
+)
+from django.shortcuts import redirect, render
+from django.views.decorators.csrf import csrf_exempt
+
+
+_PROCESS: subprocess.Popen | None = None
+_PROCESS_LOCK = threading.Lock()
+_PROXY_PREFIX = "/admin/agent/opencode"
+_PROXY_PREFIX_REGEX = _PROXY_PREFIX.replace("/", r"\/")
+_PROXY_PREFIX_NO_SLASH_REGEX = _PROXY_PREFIX.lstrip("/").replace("/", r"\/")
+_CONFIG_PATH = Path(opencode_plugin.__file__).with_name("config.json")
+
+_TEXT_CONTENT_TYPES = (
+    "text/",
+    "application/javascript",
+    "application/json",
+    "application/x-javascript",
+)
+_HOP_BY_HOP_HEADERS = {
+    "connection",
+    "keep-alive",
+    "proxy-authenticate",
+    "proxy-authorization",
+    "te",
+    "trailers",
+    "transfer-encoding",
+    "upgrade",
+}
+_ARCHIVEBOX_SKILL = """---
+name: archivebox
+description: Use ArchiveBox's CLI and local REST API from an ArchiveBox collection.
+---
+
+You are running inside an ArchiveBox collection directory.
+
+- ArchiveBox collection directory: {archivebox_data_dir}
+- ArchiveBox BASE_URL: {archivebox_base_url}
+- ArchiveBox Admin URL: {archivebox_admin_url}
+- ArchiveBox REST API URL: {archivebox_api_url}
+- Prefer the `archivebox` CLI for authenticated changes, e.g. `archivebox add`, `archivebox schedule`, `archivebox update`, and `archivebox shell`.
+- Run ArchiveBox CLI commands from the ArchiveBox collection directory above.
+- Get command help with `archivebox list --help`, `archivebox add --help`, `archivebox schedule --help`, etc. Do not use `archivebox help <command>`.
+- Use `--depth=0` by default. Only use recursive crawling when the user explicitly asks for it; use `--depth=1` when you need pages one hop out.
+- Before any recursive crawl, constrain scope with ArchiveBox config such as `CRAWL_MAX_URLS`, `CRAWL_MAX_SIZE`, `SNAPSHOT_MAX_*`, `URL_ALLOWLIST`, `URL_DENYLIST`, and related limits.
+- Respect the configured `archivebox config --get ONLY_NEW` behavior unless the user explicitly says otherwise. Remind users that expected crawl URLs can be skipped when the collection already contains snapshots with the same URL.
+- Always audit newly discovered crawl URLs before letting a crawl run broadly. Treat junk URLs such as privacy policies, legal pages, tag archives, sitemap files, feeds, login/logout URLs, and other low-value boilerplate as unwanted unless the user explicitly asked to archive them.
+- Always watch crawl output and logs as the crawl progresses, and correct errors early instead of waiting until the crawl finishes.
+- If a crawl contains bad URLs, pause it, edit the crawl's `urls` field to remove them, delete any unneeded snapshots already created under that crawl, then resume the crawl.
+- Use `archivebox shell -c '...'` or `archivebox shell <<'PY' ... PY` for Django ORM work. Shell Plus prints an import banner first; keep stderr visible while debugging.
+- Use full ArchiveBox module paths in shell code: `from archivebox.crawls.models import Crawl, CrawlSchedule` and `from archivebox.core.models import Snapshot, ArchiveResult`.
+- If a model/field/relation is unclear, inspect `_meta.fields` before guessing, e.g. `archivebox shell -c "from archivebox.crawls.models import Crawl; print([f.name for f in Crawl._meta.fields])"`.
+- Use `archivebox config --get BASE_URL` only to verify the configured base URL; prefer the seeded URLs above for API/admin requests.
+- Use `$ARCHIVEBOX_API_URL` for REST API inspection when helpful. Do not assume admin session cookies authenticate API subdomain requests; prefer CLI/shell for authenticated mutations unless the admin provides or asks you to create an API token.
+- Discover REST endpoints from `${{ARCHIVEBOX_API_URL}}v1/openapi.json`; crawl endpoints live under `/api/v1/crawls/`, snapshots under `/api/v1/core/`.
+- Do not bypass ArchiveBox auth, expose API keys, or modify config unless the admin explicitly asks.
+- After creating crawls or snapshots, report the crawl/snapshot IDs and the exact command or API request used.
+"""
+
+
+def _stop_owned_process(process: subprocess.Popen | None = None) -> None:
+    global _PROCESS
+    owned_process = process or _PROCESS
+    if owned_process is None:
+        return
+    if owned_process.poll() is None:
+        owned_process.terminate()
+        try:
+            owned_process.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            owned_process.kill()
+            owned_process.wait()
+    if _PROCESS is owned_process:
+        _PROCESS = None
+
+
+atexit.register(_stop_owned_process)
+
+
+def _machine_config() -> dict[str, Any]:
+    resolved = get_config()
+    return dict(resolved.model_dump(mode="json"))
+
+
+def _archivebox_data_dir_default() -> Path:
+    return Path(CONSTANTS.DATA_DIR)
+
+
+def _archivebox_route_urls(request: HttpRequest, route_config) -> tuple[str, str, str]:
+    base_url = get_base_url(
+        request=request,
+        config=route_config,
+    ).rstrip("/")
+    admin_url = build_admin_url(
+        "/admin/",
+        request=request,
+        config=route_config,
+    ).rstrip("/")
+    api_url = f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/"
+    return base_url, admin_url, api_url
+
+
+def _config_value(config: dict, key: str, default):
+    value = config.get(key, default)
+    if value in (None, ""):
+        return default
+    return value
+
+
+def _opencode_enabled(config: dict) -> bool:
+    value = config.get("OPENCODE_ENABLED", False)
+    if isinstance(value, bool):
+        return value
+    return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _require_enabled(config: dict) -> None:
+    if not _opencode_enabled(config):
+        raise Http404
+
+
+def _require_superuser(request: HttpRequest):
+    user = getattr(request, "user", None)
+    if user is None:
+        return redirect(f"/admin/login/?next={request.get_full_path()}")
+    if (
+        bool(getattr(user, "is_authenticated", False))
+        and bool(getattr(user, "is_active", False))
+        and bool(getattr(user, "is_superuser", False))
+    ):
+        return None
+    if bool(getattr(user, "is_authenticated", False)):
+        return HttpResponseForbidden(
+            b"ArchiveBox agent access requires a superuser account.",
+        )
+    return redirect(f"/admin/login/?next={request.get_full_path()}")
+
+
+def _origin_allowed(request: HttpRequest, path: str | None = None) -> bool:
+    if request.method in {"GET", "HEAD", "OPTIONS", "TRACE"}:
+        return True
+
+    expected_host = request.get_host()
+    pty_connect = bool(
+        path and path.startswith("pty/") and path.endswith("/connect-token"),
+    )
+    if pty_connect:
+        return True
+
+    origin = _request_header(request, "Origin")
+    if origin:
+        return _same_host(origin, expected_host)
+
+    referer = _request_header(request, "Referer")
+    if referer:
+        return _same_host(referer, expected_host)
+
+    fetch_site = _request_header(request, "Sec-Fetch-Site")
+    if fetch_site:
+        return fetch_site in {"same-origin", "same-site", "none"}
+
+    return False
+
+
+def _same_host(value: str, expected_host: str) -> bool:
+    parsed = urlsplit(value)
+    return parsed.scheme in {"http", "https"} and parsed.netloc == expected_host
+
+
+def _settings(config: dict) -> dict:
+    host = str(_config_value(config, "OPENCODE_HOST", "127.0.0.1"))
+    port = int(_config_value(config, "OPENCODE_PORT", 4096))
+    default_data_dir = _archivebox_data_dir_default()
+    workdir = Path(
+        str(_config_value(config, "OPENCODE_WORKDIR", default_data_dir)),
+    ).expanduser()
+    opencode_dir = Path(
+        str(_config_value(config, "OPENCODE_STATE_DIR", workdir / "opencode")),
+    ).expanduser()
+    binary = str(_config_value(config, "OPENCODE_BINARY", "opencode"))
+    timeout = int(_config_value(config, "OPENCODE_TIMEOUT", 30))
+    return {
+        "host": host,
+        "port": port,
+        "origin": f"http://{host}:{port}",
+        "workdir": workdir,
+        "opencode_dir": opencode_dir,
+        "config_home": opencode_dir / "config",
+        "data_home": opencode_dir / "data",
+        "state_home": opencode_dir / "state",
+        "cache_home": opencode_dir / "cache",
+        "home": opencode_dir / "home",
+        "binary": binary,
+        "config": config,
+        "timeout": timeout,
+    }
+
+
+def _resolve_binary(binary: str, config: dict) -> tuple[Any, Any, dict[str, str]]:
+    try:
+        from abxpkg import BinProvider
+        from abx_plugins.plugins.base.utils import load_required_binary_from_config
+
+        binary_environ = os.environ.copy()
+        lib_dir = config.get("ABXPKG_LIB_DIR")
+        if lib_dir:
+            binary_environ["ABXPKG_LIB_DIR"] = str(lib_dir)
+        loaded_dependencies = [
+            load_required_binary_from_config(
+                required_binary,
+                _CONFIG_PATH,
+                global_config=config,
+                environ=binary_environ,
+                install=False,
+            )
+            for required_binary in (
+                str(config.get("NODE_BINARY") or "node"),
+                str(config.get("GIT_BINARY") or "git"),
+                binary,
+            )
+        ]
+    except Exception as err:
+        raise RuntimeError(
+            f"OpenCode dependency is not installed from required_binaries: {err}",
+        ) from err
+
+    if any(not loaded.loaded_abspath for loaded in loaded_dependencies):
+        raise RuntimeError(
+            "OpenCode dependency is not installed from required_binaries.",
+        )
+
+    providers = [loaded.loaded_binprovider for loaded in loaded_dependencies if loaded.loaded_binprovider is not None]
+    binary_env = BinProvider.build_exec_env(
+        providers=providers,
+        base_env=binary_environ,
+    )
+    return (
+        loaded_dependencies[-1],
+        loaded_dependencies[1],
+        binary_env,
+    )
+
+
+def _project_route(workdir: Path, session_id: str = "") -> str:
+    encoded = base64.b64encode(str(workdir.resolve()).encode()).decode()
+    encoded = encoded.replace("+", "-").replace("/", "_").rstrip("=")
+    route = f"{_PROXY_PREFIX}/{encoded}/session"
+    return f"{route}/{session_id}" if session_id else route
+
+
+def _ensure_project_files(settings: dict) -> None:
+    workdir = settings["workdir"].resolve()
+    workdir.mkdir(parents=True, exist_ok=True)
+    git_marker = workdir / ".git" / "not-a-git"
+    if git_marker.exists():
+        # Current OpenCode hangs on the legacy fake marker, so remove only
+        # that invalid shape before initializing the real worktree.
+        shutil.rmtree(git_marker.parent)
+
+    editable_skill_path = settings["opencode_dir"] / "SKILL.md"
+    editable_skill_path.parent.mkdir(parents=True, exist_ok=True)
+    if not editable_skill_path.exists():
+        editable_skill_path.write_text(
+            _ARCHIVEBOX_SKILL.format(
+                archivebox_data_dir=workdir,
+                archivebox_base_url=settings.get("archivebox_base_url", ""),
+                archivebox_admin_url=settings.get("archivebox_admin_url", ""),
+                archivebox_api_url=settings.get("archivebox_api_url", ""),
+            ),
+        )
+
+    opencode_skill_path = settings["config_home"] / "opencode" / "skills" / "archivebox" / "SKILL.md"
+    opencode_skill_path.parent.mkdir(parents=True, exist_ok=True)
+    if opencode_skill_path.resolve() != editable_skill_path.resolve():
+        if opencode_skill_path.exists() or opencode_skill_path.is_symlink():
+            opencode_skill_path.unlink()
+        opencode_skill_path.symlink_to(editable_skill_path)
+
+
+def _ensure_default_session(settings: dict) -> str:
+    workdir = settings["workdir"].resolve()
+    params = {"directory": str(workdir)}
+    timeout = settings["timeout"]
+    project = requests.post(
+        f"{settings['origin']}/project/git/init",
+        params=params,
+        timeout=timeout,
+    )
+    project.raise_for_status()
+    project_data = project.json()
+    if Path(str(project_data.get("worktree") or "/")).resolve() != workdir:
+        raise RuntimeError(
+            f"OpenCode initialized the wrong project worktree: {project_data.get('worktree')!r}",
+        )
+
+    sessions = requests.get(
+        f"{settings['origin']}/session",
+        params={**params, "roots": "true", "limit": 55},
+        timeout=timeout,
+    )
+    sessions.raise_for_status()
+    session_data = sessions.json()
+    if not isinstance(session_data, list):
+        raise RuntimeError("OpenCode returned an invalid project session list.")
+    for session_data_item in session_data:
+        if not isinstance(session_data_item, dict):
+            continue
+        session_id = str(session_data_item.get("id") or "")
+        session_directory = session_data_item.get("directory")
+        if session_id and session_directory and Path(str(session_directory)).resolve() == workdir:
+            return session_id
+
+    session = requests.post(
+        f"{settings['origin']}/session",
+        params=params,
+        json={},
+        timeout=timeout,
+    )
+    session.raise_for_status()
+    session_data = session.json()
+    session_id = str(session_data.get("id") or "")
+    session_directory = session_data.get("directory")
+    if not session_id or not session_directory or Path(str(session_directory)).resolve() != workdir:
+        raise RuntimeError(
+            "OpenCode did not create a session for the requested worktree.",
+        )
+    return session_id
+
+
+def _recent_session_id(settings: dict) -> str:
+    workdir = str(settings["workdir"].resolve())
+    try:
+        response = requests.get(
+            f"{settings['origin']}/session",
+            params={"directory": workdir, "roots": "true", "limit": 1},
+            timeout=settings["timeout"],
+        )
+        response.raise_for_status()
+        sessions = response.json()
+        if sessions:
+            return str(sessions[0].get("id") or "")
+    except requests.RequestException:
+        pass
+    return ""
+
+
+def _health(settings: dict) -> bool:
+    try:
+        response = requests.get(
+            f"{settings['origin']}/global/health",
+            timeout=2,
+        )
+        return response.status_code == 200
+    except requests.RequestException:
+        return False
+
+
+def _ensure_opencode(settings: dict) -> tuple[bool, str]:
+    global _PROCESS
+    started_process: subprocess.Popen | None = None
+    workdir = settings["workdir"].resolve()
+    try:
+        binary, git_binary, binary_env = _resolve_binary(
+            settings["binary"],
+            settings["config"],
+        )
+    except RuntimeError as err:
+        return False, str(err)
+
+    env = {
+        **os.environ,
+        **binary_env,
+        "ARCHIVEBOX_BASE_URL": str(settings.get("archivebox_base_url", "")),
+        "ARCHIVEBOX_ADMIN_URL": str(settings.get("archivebox_admin_url", "")),
+        "ARCHIVEBOX_API_URL": str(settings.get("archivebox_api_url", "")),
+        "BROWSER": "false",
+        "GIT_CEILING_DIRECTORIES": str(workdir),
+        "HOME": str(settings["home"]),
+        "OPENCODE_DISABLE_PROJECT_CONFIG": "true",
+        "XDG_CONFIG_HOME": str(settings["config_home"]),
+        "XDG_DATA_HOME": str(settings["data_home"]),
+        "XDG_STATE_HOME": str(settings["state_home"]),
+        "XDG_CACHE_HOME": str(settings["cache_home"]),
+    }
+
+    with _PROCESS_LOCK:
+        settings["workdir"].mkdir(parents=True, exist_ok=True)
+        settings["config_home"].mkdir(parents=True, exist_ok=True)
+        settings["data_home"].mkdir(parents=True, exist_ok=True)
+        settings["state_home"].mkdir(parents=True, exist_ok=True)
+        settings["cache_home"].mkdir(parents=True, exist_ok=True)
+        settings["home"].mkdir(parents=True, exist_ok=True)
+        _ensure_project_files(settings)
+
+        if not (workdir / ".git").exists():
+            try:
+                git_init = git_binary.exec(
+                    cmd=("init", "--quiet"),
+                    cwd=workdir,
+                    env=env,
+                    timeout=settings["timeout"],
+                )
+            except (AssertionError, OSError, subprocess.SubprocessError) as err:
+                return False, f"OpenCode project initialization failed: {err}"
+            if git_init.returncode != 0:
+                output = (git_init.stderr or git_init.stdout or "").strip()
+                return (
+                    False,
+                    f"OpenCode project initialization failed: {output or f'git exited with {git_init.returncode}'}",
+                )
+
+        if _health(settings):
+            try:
+                _ensure_default_session(settings)
+            except (requests.RequestException, RuntimeError, ValueError) as err:
+                return False, f"OpenCode project initialization failed: {err}"
+            return True, ""
+
+        binary_abspath = binary.loaded_abspath
+        if binary.loaded_binprovider is not None:
+            binary_abspath = binary.loaded_binprovider._exec_bin_abspath(
+                Path(binary.loaded_abspath),
+            )
+        cmd = [
+            str(binary_abspath),
+            "serve",
+            "--hostname",
+            settings["host"],
+            "--port",
+            str(settings["port"]),
+        ]
+        try:
+            _PROCESS = subprocess.Popen(
+                cmd,
+                cwd=workdir,
+                env=env,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                start_new_session=True,
+            )
+            started_process = _PROCESS
+        except FileNotFoundError:
+            return False, f"OpenCode binary not found: {settings['binary']}"
+
+        deadline = time.monotonic() + settings["timeout"]
+        while time.monotonic() < deadline:
+            if _health(settings):
+                try:
+                    _ensure_default_session(settings)
+                except (requests.RequestException, RuntimeError, ValueError) as err:
+                    _stop_owned_process(started_process)
+                    return False, f"OpenCode project initialization failed: {err}"
+                return True, ""
+            if started_process and started_process.poll() is not None:
+                if _PROCESS is started_process:
+                    _PROCESS = None
+                return False, "OpenCode exited before the web server became ready."
+            time.sleep(0.25)
+
+        _stop_owned_process(started_process)
+        return False, "Timed out waiting for OpenCode to start."
+
+
+def agent_view(request: HttpRequest):
+    config = _machine_config()
+    _require_enabled(config)
+    auth_response = _require_superuser(request)
+    if auth_response:
+        return auth_response
+
+    settings = _settings(config)
+    route_config = request.__dict__.get("archivebox_config")
+    base_url, admin_url, api_url = _archivebox_route_urls(request, route_config)
+    settings["archivebox_base_url"] = base_url
+    settings["archivebox_admin_url"] = admin_url
+    settings["archivebox_api_url"] = api_url
+    ok, error = _ensure_opencode(settings)
+    from archivebox.core.admin_site import archivebox_admin
+
+    recent_session_id = _recent_session_id(settings) if ok else ""
+    context = {
+        **archivebox_admin.each_context(request),
+        "title": "Agent",
+        "error": "" if ok else error,
+        "command": f"{settings['binary']} serve --hostname {settings['host']} --port {settings['port']}" if error else "",
+        # OpenCode 1.17+ keeps durable sessions on the explicit
+        # /<dirBase64>/session/<sessionId> route. We still seed localStorage
+        # below because the sidebar state uses it, but the iframe itself must
+        # open the durable session URL so a fresh browser does not land on the
+        # transient new-session route and appear to have lost prior sessions.
+        "proxy_url": _project_route(settings["workdir"], recent_session_id),
+        "workdir": str(settings["workdir"].resolve()),
+        "recent_session_id": recent_session_id,
+    }
+    return render(
+        request,
+        "opencode/agent.html",
+        context,
+        status=200 if ok else 502,
+    )
+
+
+def _proxy_url(settings: dict, path: str | None) -> str:
+    rel = "/" if not path else f"/{path}"
+    return urljoin(settings["origin"], rel)
+
+
+def _request_header(request: HttpRequest, name: str) -> str | None:
+    meta = getattr(request, "META", {})
+    if name == "Content-Type":
+        value = meta.get("CONTENT_TYPE")
+    elif name == "Content-Length":
+        value = meta.get("CONTENT_LENGTH")
+    else:
+        value = meta.get(f"HTTP_{name.upper().replace('-', '_')}")
+    return str(value) if value else None
+
+
+def _request_headers(request: HttpRequest, settings: dict) -> dict[str, str]:
+    forwarded = {}
+    for key in ("Accept", "Accept-Language", "Content-Type", "Range", "User-Agent"):
+        value = _request_header(request, key)
+        if value:
+            forwarded[key] = value
+    return forwarded
+
+
+def _request_params(request: HttpRequest) -> tuple[tuple[str, str], ...]:
+    if hasattr(request.GET, "lists"):
+        return tuple((key, str(value)) for key, values in request.GET.lists() for value in values)
+    return tuple((key, str(value)) for key, value in dict(request.GET).items())
+
+
+async def _event_chunks(request: HttpRequest, settings: dict, path: str | None):
+    timeout = httpx.Timeout(settings["timeout"], read=None)
+    url = _proxy_url(settings, path)
+    method = request.method or "GET"
+    async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
+        async with client.stream(
+            method,
+            url,
+            params=_request_params(request),
+            headers=_request_headers(request, settings),
+        ) as upstream:
+            async for chunk in upstream.aiter_raw(chunk_size=512):
+                yield chunk
+
+
+def _rewrite_text(body: bytes, settings: dict) -> bytes:
+    text = body.decode("utf-8", errors="replace")
+    text = text.replace(settings["origin"], _PROXY_PREFIX)
+    text = text.replace("location.origin", f'location.origin+"{_PROXY_PREFIX}"')
+    text = text.replace(
+        "k(k5,{get component(){return t.router??Az},",
+        f'k(k5,{{base:"{_PROXY_PREFIX}",get component(){{return t.router??Az}},',
+    )
+    text = text.replace('"/assets/', f'"{_PROXY_PREFIX}/assets/')
+    text = text.replace("'/assets/", f"'{_PROXY_PREFIX}/assets/")
+    proxy_path = rf'(\1.replace(/^{_PROXY_PREFIX_REGEX}(?=\/|$)/,"")||"/")'
+    text = re.sub(r"\b(window\.location\.pathname)\b", proxy_path, text)
+    text = re.sub(r"(?<![.\w])(location\.pathname)\b", proxy_path, text)
+    text = text.replace(
+        'window.history.replaceState(nz(o),"",r):window.history.pushState(o,"",r)',
+        (
+            f'window.history.replaceState(nz(o),"",r.startsWith("{_PROXY_PREFIX}")?r:r.startsWith("/")?"{_PROXY_PREFIX}"+r:r):'
+            f'window.history.pushState(o,"",r.startsWith("{_PROXY_PREFIX}")?r:r.startsWith("/")?"{_PROXY_PREFIX}"+r:r)'
+        ),
+    )
+    text = text.replace(
+        'const BL="modulepreload",UL=function(t){return"/"+t}',
+        f'const BL="modulepreload",UL=function(t){{return"{_PROXY_PREFIX}/"+t}}',
+    )
+    text = re.sub(
+        rf"""(?P<prefix>\b(?:href|src|action)=["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
+        rf"\g<prefix>{_PROXY_PREFIX}/",
+        text,
+    )
+    text = re.sub(
+        rf"""(?P<prefix>\b(?:fetch|EventSource)\(["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
+        rf"\g<prefix>{_PROXY_PREFIX}/",
+        text,
+    )
+    text = re.sub(
+        rf"""(?P<prefix>\burl\(["']?)/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
+        rf"\g<prefix>{_PROXY_PREFIX}/",
+        text,
+    )
+    return text.encode("utf-8")
+
+
+def _response_headers(upstream: requests.Response, settings: dict) -> dict[str, str]:
+    headers = {}
+    for key, value in upstream.headers.items():
+        lower = key.lower()
+        if lower in _HOP_BY_HOP_HEADERS or lower in {
+            "content-length",
+            "content-encoding",
+            "x-frame-options",
+        }:
+            continue
+        if lower == "location":
+            if value.startswith(settings["origin"]):
+                value = value.replace(settings["origin"], _PROXY_PREFIX, 1)
+            elif value.startswith("/"):
+                value = f"{_PROXY_PREFIX}{value}"
+        headers[key] = value
+    return headers
+
+
+@csrf_exempt
+def opencode_proxy_view(request: HttpRequest, path: str | None = None):
+    config = _machine_config()
+    _require_enabled(config)
+    auth_response = _require_superuser(request)
+    if auth_response:
+        return auth_response
+    if not _origin_allowed(request, path):
+        return HttpResponseForbidden(
+            b"Cross-origin OpenCode agent requests are blocked.",
+        )
+
+    settings = _settings(config)
+    route_config = request.__dict__.get("archivebox_config")
+    base_url, admin_url, api_url = _archivebox_route_urls(request, route_config)
+    settings["archivebox_base_url"] = base_url
+    settings["archivebox_admin_url"] = admin_url
+    settings["archivebox_api_url"] = api_url
+    ok, error = _ensure_opencode(settings)
+    if not ok:
+        return HttpResponse(
+            error.encode(),
+            status=502,
+            content_type="text/plain; charset=utf-8",
+        )
+
+    if request.method == "GET" and (path or "").endswith("/event"):
+        response = StreamingHttpResponse(
+            _event_chunks(request, settings, path),
+            content_type="text/event-stream",
+        )
+        response.headers["Cache-Control"] = "no-store"
+        response.headers["X-Accel-Buffering"] = "no"
+        return response
+
+    try:
+        method = request.method or "GET"
+        upstream = requests.request(
+            method,
+            _proxy_url(settings, path),
+            params=_request_params(request),
+            data=request.body if method not in {"GET", "HEAD"} else None,
+            headers=_request_headers(request, settings),
+            stream=True,
+            timeout=(settings["timeout"], None),
+            allow_redirects=False,
+        )
+    except requests.RequestException as err:
+        return HttpResponse(
+            str(err).encode(),
+            status=502,
+            content_type="text/plain; charset=utf-8",
+        )
+
+    content_type = upstream.headers.get("Content-Type", "")
+    is_event_stream = content_type.startswith("text/event-stream")
+    is_text = not is_event_stream and any(content_type.startswith(prefix) for prefix in _TEXT_CONTENT_TYPES)
+    headers = _response_headers(upstream, settings)
+    if is_text:
+        body = _rewrite_text(upstream.content, settings)
+        response = HttpResponse(
+            body,
+            status=upstream.status_code,
+            content_type=content_type or "text/plain; charset=utf-8",
+        )
+    elif is_event_stream:
+        response = StreamingHttpResponse(
+            upstream.iter_lines(chunk_size=1),
+            status=upstream.status_code,
+            content_type=content_type or "text/event-stream",
+        )
+    else:
+        response = StreamingHttpResponse(
+            upstream.iter_content(chunk_size=64 * 1024),
+            status=upstream.status_code,
+            content_type=content_type or "application/octet-stream",
+        )
+    for key, value in headers.items():
+        response.headers[key] = value
+    response.headers["Cache-Control"] = "no-store"
+    return response

archivebox/tests/conftest.py

@@ -1590,9 +1590,8 @@ def resolve_abxpkg_binary_env(
     *binary_names: str,
     env: dict[str, str] | None = None,
     deps_from: Path | list[Path] | tuple[Path, ...] | None = None,
-    install: bool = True,
 ) -> dict[str, str]:
-    """Resolve real test dependencies through abxpkg and return its exported env."""
+    """Resolve already-available test dependencies through abxpkg."""
     command_env = dict(env) if env is not None else os.environ.copy()
     command_env["ABXPKG_LIB_DIR"] = str(lib_dir)
     command = [
@@ -1601,8 +1600,6 @@ def resolve_abxpkg_binary_env(
         "--json",
         f"--lib={lib_dir}",
     ]
-    if install:
-        command.append("--install")
     deps_configs = [deps_from] if isinstance(deps_from, Path) else list(deps_from or ())
     command.extend(f"--deps-from={config}:required_binaries" for config in deps_configs)
     command.extend(binary_names)
@@ -1625,7 +1622,6 @@ def resolve_abxpkg_chrome_env(lib_dir: Path, env: dict[str, str] | None = None)
         lib_dir,
         env=env,
         deps_from=chrome_config,
-        install=False,
     )
     chrome_binary = Path(payload["CHROME_BINARY"])
     node_binary = Path(payload["NODE_BINARY"])

archivebox/tests/test_archive_result_service.py

@@ -37,6 +37,7 @@ def _run_shipped_snapshot_hook(
     import asyncio
 
     from abx_dl.services.process_service import ProcessService as HookProcessService
+    from abx_plugins.plugins.base.utils import get_hydrated_required_binaries
     from archivebox.core.models import ArchiveResult
     from archivebox.machine.models import Process
     from archivebox.services.archive_result_service import ArchiveResultService
@@ -45,6 +46,15 @@ def _run_shipped_snapshot_hook(
     hook_path = Path(str(files(f"abx_plugins.plugins.{plugin}").joinpath(hook_name)))
     projected_hook_name = event_hook_name or hook_name
     hook_config = hook_path.parent / "config.json"
+    for required_binary in get_hydrated_required_binaries(
+        hook_config,
+        environ={**os.environ, "ABXPKG_LIB_DIR": str(lib_dir)},
+    ):
+        install_real_binary(
+            required_binary["name"],
+            binproviders=required_binary["binproviders"],
+            overrides=required_binary.get("overrides"),
+        )
     binary_env = resolve_abxpkg_binary_env(lib_dir, deps_from=hook_config)
     output_dir = Path(snapshot.output_dir) / plugin
     output_dir.mkdir(parents=True, exist_ok=True)
@@ -509,7 +519,6 @@ def test_process_started_hydrates_binary_and_iface_from_existing_binary_records(
     mercury_env = resolve_abxpkg_binary_env(
         lib_dir,
         deps_from=mercury_config,
-        install=False,
     )
     mercury_path = Path(mercury_env["MERCURY_BINARY"])
     provider_path = Path(binary.abspath)
@@ -582,20 +591,21 @@ def test_process_started_hydrates_binary_and_iface_from_existing_binary_records(
 
 @pytest.mark.django_db(transaction=True)
 def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp_path, hermetic_lib_dir):
-    from archivebox.machine.models import NetworkInterface
+    from archivebox.machine.models import Binary, NetworkInterface
     from archivebox.machine.models import Process as MachineProcess
     from archivebox.services.process_service import ProcessService as ArchiveBoxProcessService
+    from archivebox.services.runner import run_install
     from abx_dl.services.process_service import ProcessService as DlProcessService
 
-    iface = NetworkInterface.current()
-    machine = iface.machine
-
     lib_dir = hermetic_lib_dir
-    chrome_config = Path(str(files("abx_plugins.plugins.chrome").joinpath("config.json")))
-    node_env = resolve_abxpkg_binary_env(lib_dir, deps_from=chrome_config)
-    node_path = Path(node_env["NODE_BINARY"])
-    node = install_real_binary("node", machine=machine)
-    assert Path(node.abspath).resolve() == node_path.resolve()
+    run_install(plugin_names=["chrome"])
+    installed_node_ids = set(
+        Binary.objects.filter(name="node", status=Binary.StatusChoices.INSTALLED).values_list("id", flat=True),
+    )
+    assert installed_node_ids
+    iface = NetworkInterface.current()
+    node_env = resolve_abxpkg_binary_env(lib_dir, "node")
+    node_path = lib_dir / "env" / "bin" / "node"
 
     hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js")))
     crawl_dir = tmp_path / "crawl"
@@ -618,7 +628,7 @@ def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp
                 env={
                     **node_env,
                     "ABXPKG_LIB_DIR": str(lib_dir),
-                    "NODE_BINARY": node.abspath,
+                    "NODE_BINARY": str(node_path),
                     "CRAWL_DIR": str(crawl_dir),
                     "SNAP_DIR": str(crawl_dir / "snapshot"),
                     "CHROME_USER_DATA_DIR": str(output_dir / "profile"),
@@ -646,7 +656,11 @@ def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp
         pwd=str(output_dir),
         cmd=[str(hook_path)],
     )
-    assert process.binary_id == node.id
+    assert process.binary_id is not None
+    assert process.binary_id in installed_node_ids
+    assert process.binary.name == "node"
+    assert process.binary.status == process.binary.StatusChoices.INSTALLED
+    assert Path(process.binary.abspath).resolve() == node_path.resolve()
     assert process.iface_id == iface.id
     assert process.exit_code == 0, process.stderr
     assert "chrome zombies. cpu usage:" in process.stdout

archivebox/tests/test_cli_server.py

@@ -39,8 +39,16 @@ from archivebox.tests.conftest import (
 def _resolve_sonic_env(data_dir: Path) -> dict[str, str]:
     from abx_plugins import get_plugins_dir
 
+    lib_dir = data_dir / "lib"
+    install_result = run_archivebox_cmd(
+        ["install", "search_backend_sonic"],
+        cwd=data_dir,
+        env={"ABXPKG_LIB_DIR": str(lib_dir)},
+        default_cli_env=True,
+    )
+    assert install_result.returncode == 0, install_result.stderr or install_result.stdout
     config = Path(get_plugins_dir()) / "search_backend_sonic" / "config.json"
-    resolved = resolve_abxpkg_binary_env(data_dir / "lib", deps_from=config)
+    resolved = resolve_abxpkg_binary_env(lib_dir, deps_from=config)
     assert Path(resolved["SONIC_BINARY"]).is_file()
     return resolved
 
@@ -244,6 +252,34 @@ def test_server_daemon_starts_real_plugin_owned_sonic_worker(initialized_archive
     assert "sonic" in state["worker_sonic"]["name"]
 
 
+@pytest.mark.timeout(300)
+@pytest.mark.django_db(transaction=True)
+def test_foreground_runner_starts_enabled_plugin_daemon_before_snapshot_hooks(initialized_archive, recursive_test_site):
+    from archivebox.core.models import ArchiveResult
+    from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+    env = cli_env(
+        PLUGINS="wget",
+        SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1",
+        SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
+        ABXPKG_LIB_DIR=str(initialized_archive / "lib"),
+    )
+    result = run_archivebox_cmd(
+        ["add", "--depth=0", "--plugins=wget,search_backend_sonic", recursive_test_site["root_url"]],
+        cwd=initialized_archive,
+        env=env,
+        timeout=300,
+    )
+
+    assert result.returncode == 0, result.stderr or result.stdout
+    with use_archivebox_db(initialized_archive):
+        sonic_result = ArchiveResult.objects.get(plugin="search_backend_sonic")
+    assert sonic_result.status == ArchiveResult.StatusChoices.SUCCEEDED
+    assert sonic_result.output_str.endswith("kb text indexed")
+    supervisord_log = (initialized_archive / "logs" / "supervisord.log").read_text(encoding="utf-8", errors="replace")
+    assert "spawned: 'worker_sonic' with pid" in supervisord_log
+
+
 def test_server_daemon_restarts_runner_killed_by_signal(archivebox_daemon_server):
     server = archivebox_daemon_server(
         SEARCH_BACKEND_ENGINE="sqlite",

archivebox/tests/test_crawl_runner.py

@@ -6,7 +6,7 @@ import sys
 import pytest
 from asgiref.sync import sync_to_async
 
-from archivebox.tests.conftest import resolve_abxpkg_binary_env
+from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary_env
 
 pytestmark = pytest.mark.django_db
 
@@ -412,6 +412,7 @@ def test_machine_service_persists_only_derived_config_events(tmp_path, hermetic_
     machine = Machine.current()
     machine.config = {}
     machine.save(update_fields=["config"])
+    install_real_binary("wget", machine=machine, binproviders="env,apt,brew")
     resolve_abxpkg_binary_env(hermetic_lib_dir, "wget")
     wget_binary = hermetic_lib_dir / "env" / "bin" / "wget"
     assert wget_binary.is_symlink()
@@ -487,6 +488,7 @@ def test_load_run_state_uses_real_lib_dir_for_machine_binary_config(tmp_path, he
     resolved_lib_dir = get_config(include_machine=False).ABXPKG_LIB_DIR
     assert resolved_lib_dir == hermetic_lib_dir, f"ABXPKG_LIB_DIR override not applied: {resolved_lib_dir!r} != {hermetic_lib_dir!r}"
 
+    install_real_binary("wget", binproviders="env,apt,brew")
     resolve_abxpkg_binary_env(resolved_lib_dir, "wget")
     wget_binary = resolved_lib_dir / "env" / "bin" / "wget"
     assert wget_binary.is_symlink()
@@ -788,6 +790,7 @@ def test_wait_for_snapshot_tasks_returns_after_completed_tasks_are_pruned():
     asyncio.run(run_test())
 
 
+@pytest.mark.django_db(transaction=True)
 def test_abx_process_service_background_process_finishes_after_process_exit(tmp_path, recursive_test_site, hermetic_lib_dir):
     from abx_dl.events import ProcessCompletedEvent, ProcessEvent
     from abx_dl.orchestrator import create_bus
@@ -808,6 +811,7 @@ def test_abx_process_service_background_process_finishes_after_process_exit(tmp_
     plugin_output_dir.mkdir(parents=True)
     hook_path = Path(str(files("abx_plugins.plugins.wget").joinpath("on_Snapshot__06_wget.finite.bg.py")))
     wget_config = Path(str(files("abx_plugins.plugins.wget").joinpath("config.json")))
+    install_real_binary("wget", binproviders="env,apt,brew")
     hook_env = resolve_abxpkg_binary_env(hermetic_lib_dir, deps_from=wget_config)
 
     async def run_test():

archivebox/tests/test_hooks.py

@@ -18,7 +18,7 @@ from pathlib import Path
 
 import pytest
 
-from archivebox.tests.conftest import resolve_abxpkg_binary_env
+from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary_env
 
 # Set up Django before importing any Django-dependent modules
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.settings")
@@ -183,16 +183,20 @@ class TestJSONLParsing:
 class TestRequiredBinaryConfigHandling:
     """Test that required_binaries keep configured XYZ_BINARY values intact."""
 
-    def test_binary_env_var_absolute_path_handling(self, tmp_path):
+    @pytest.mark.django_db(transaction=True)
+    def test_binary_env_var_absolute_path_handling(self, hermetic_lib_dir):
         """abxpkg should expose the resolved binary as an absolute path."""
-        resolved = resolve_abxpkg_binary_env(tmp_path / "lib", deps_from=WGET_CONFIG)
+        install_real_binary("wget", binproviders="env,apt,brew")
+        resolved = resolve_abxpkg_binary_env(hermetic_lib_dir, deps_from=WGET_CONFIG)
 
         assert Path(resolved["WGET_BINARY"]).is_absolute()
         assert Path(resolved["WGET_BINARY"]).is_file()
 
-    def test_binary_env_var_name_only_handling(self, tmp_path):
+    @pytest.mark.django_db(transaction=True)
+    def test_binary_env_var_name_only_handling(self, hermetic_lib_dir):
         """The projected command name should execute the resolved host binary."""
-        lib_dir = tmp_path / "lib"
+        lib_dir = hermetic_lib_dir
+        install_real_binary("wget", binproviders="env,apt,brew")
         resolve_abxpkg_binary_env(lib_dir, deps_from=WGET_CONFIG)
         projection = lib_dir / "env" / "bin" / "wget"
         result = subprocess.run([projection, "--version"], capture_output=True, text=True)
@@ -362,9 +366,13 @@ class TestHookExecution:
         assert records[0]["type"] == "ArchiveResult"
         assert records[0]["status"] == "succeeded"
 
-    def test_js_hook_execution(self, tmp_path):
+    @pytest.mark.django_db(transaction=True)
+    def test_js_hook_execution(self, tmp_path, hermetic_lib_dir):
         """A shipped JavaScript hook should execute through projected Node."""
-        lib_dir = tmp_path / "lib"
+        from archivebox.services.runner import run_install
+
+        lib_dir = hermetic_lib_dir
+        run_install(plugin_names=["chrome"])
         chrome_config = Path(
             str(files("abx_plugins.plugins.chrome").joinpath("config.json")),
         )
@@ -405,10 +413,12 @@ class TestHookExecution:
         assert "chrome zombies" in result.stdout
 
     @pytest.mark.django_db(transaction=True)
-    def test_real_js_hook_runs_through_abxpkg_node_projection(self, tmp_path):
+    def test_real_js_hook_runs_through_abxpkg_node_projection(self, tmp_path, hermetic_lib_dir):
         from archivebox.plugins.hooks import run_hook
+        from archivebox.services.runner import run_install
 
-        lib_dir = tmp_path / "lib"
+        lib_dir = hermetic_lib_dir
+        run_install(plugin_names=["chrome"])
         node_env = resolve_abxpkg_binary_env(lib_dir, deps_from=CHROME_CONFIG)
         node_projection = lib_dir / "env" / "bin" / "node"
         crawl_dir = tmp_path / "crawl"
@@ -470,8 +480,9 @@ class TestDependencyRecordOutput:
     """Test Binary JSONL emitted by the real CLI and persisted model."""
 
     @pytest.mark.django_db(transaction=True)
-    def test_binary_cli_emits_resolved_dependency_record(self, initialized_archive, tmp_path):
-        wget_path = resolve_abxpkg_binary_env(tmp_path / "lib", deps_from=WGET_CONFIG)["WGET_BINARY"]
+    def test_binary_cli_emits_resolved_dependency_record(self, initialized_archive, hermetic_lib_dir):
+        install_real_binary("wget", binproviders="env,apt,brew")
+        wget_path = resolve_abxpkg_binary_env(hermetic_lib_dir, deps_from=WGET_CONFIG)["WGET_BINARY"]
         version = subprocess.run([wget_path, "--version"], capture_output=True, text=True, check=True).stdout.split()[2]
         from archivebox.tests.conftest import parse_jsonl_output, run_archivebox_cmd
 
@@ -573,11 +584,13 @@ class TestPluginMetadata:
 
 
 @pytest.mark.django_db(transaction=True)
-def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_path):
+def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_path, hermetic_lib_dir):
     """Hook subprocesses must get a real NODE_MODULES_DIR even when NODE_PATH has multiple entries."""
     from archivebox.plugins.hooks import run_hook
+    from archivebox.services.runner import run_install
 
-    lib_dir = tmp_path / "lib"
+    lib_dir = hermetic_lib_dir
+    run_install(plugin_names=["chrome"])
     chrome_config = Path(str(files("abx_plugins.plugins.chrome").joinpath("config.json")))
     node_env = resolve_abxpkg_binary_env(
         lib_dir,

archivebox/tests/test_machine_models.py

@@ -36,7 +36,7 @@ from archivebox.machine.models import (
     PROCESS_TIMEOUT_GRACE,
 )
 from archivebox.machine.detect import unknown_if_blank
-from archivebox.tests.conftest import resolve_abxpkg_binary_env
+from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary_env
 
 pytestmark = pytest.mark.django_db(transaction=True)
 
@@ -209,6 +209,7 @@ class TestMachineModel:
     def test_machine_from_jsonl_update(self, hermetic_lib_dir):
         """Machine.from_json() should update machine config."""
         Machine.current()  # Ensure machine exists
+        install_real_binary("wget", binproviders="env,apt,brew")
         resolve_abxpkg_binary_env(hermetic_lib_dir, "wget")
         wget_path = hermetic_lib_dir / "env" / "bin" / "wget"
         assert wget_path.is_symlink()
@@ -231,6 +232,7 @@ class TestMachineModel:
         import survive. Only ``_BINARY`` paths get validated/dropped on import.
         """
         Machine.current()  # Ensure machine exists
+        install_real_binary("wget", binproviders="env,apt,brew")
         resolve_abxpkg_binary_env(hermetic_lib_dir, "wget")
         wget_path = hermetic_lib_dir / "env" / "bin" / "wget"
         assert wget_path.is_symlink()
@@ -264,6 +266,8 @@ class TestMachineModel:
         """
         import archivebox.machine.models as models
 
+        install_real_binary("node", binproviders="env,apt,brew")
+        install_real_binary("wget", binproviders="env,apt,brew")
         resolve_abxpkg_binary_env(hermetic_lib_dir, "node", "wget")
         chrome_path = hermetic_lib_dir / "env" / "bin" / "node"
         node_path = hermetic_lib_dir / "env" / "bin" / "wget"
@@ -309,6 +313,7 @@ class TestMachineModel:
 
         lib_dir = get_config(include_machine=False).ABXPKG_LIB_DIR
         assert lib_dir == hermetic_lib_dir
+        install_real_binary("node", binproviders="env,apt,brew")
         resolve_abxpkg_binary_env(lib_dir, "node")
         chrome_path = lib_dir / "env" / "bin" / "node"
         machine = Machine.current()

archivebox/tests/test_opencode_agent.py

@@ -75,7 +75,7 @@ def opencode_archive_config(initialized_archive):
 
 @pytest.fixture
 def live_opencode(opencode_archive_config):
-    from abx_plugins.plugins.opencode import views
+    from archivebox.opencode import views
 
     install = run_archivebox_cmd(
         ["install", "opencode", "--binproviders=env,pnpm"],
@@ -114,7 +114,7 @@ def live_opencode(opencode_archive_config):
 
 def test_opencode_disabled_route_does_not_start_server(client, initialized_archive):
     from archivebox.machine.models import Machine
-    from abx_plugins.plugins.opencode import views
+    from archivebox.opencode import views
 
     os.chdir(initialized_archive)
     Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
@@ -163,7 +163,7 @@ def test_opencode_proxy_blocks_cross_site_fetch_metadata(admin_client, db, live_
 
 
 def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode):
-    from abx_plugins.plugins.opencode import views
+    from archivebox.opencode import views
 
     response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
     recent_session_id = views._recent_session_id(live_opencode.settings)
@@ -238,7 +238,7 @@ def test_opencode_starts_with_isolated_state(live_opencode):
 
 
 def test_opencode_state_dir_is_separate_from_workdir(tmp_path):
-    from abx_plugins.plugins.opencode import views
+    from archivebox.opencode import views
 
     workdir = tmp_path / "data"
     settings = views._settings({"OPENCODE_WORKDIR": str(workdir)})
@@ -258,7 +258,7 @@ def test_opencode_state_dir_is_separate_from_workdir(tmp_path):
 
 
 def test_opencode_rewrites_vite_preload_assets():
-    from abx_plugins.plugins.opencode import views
+    from archivebox.opencode import views
 
     body = b'const BL="modulepreload",UL=function(t){return"/"+t};const icon="/assets/sprite.svg#anthropic"'
     rewritten = views._rewrite_text(body, {"origin": "http://127.0.0.1:4096"}).decode()

archivebox/tests/test_search.py

@@ -688,8 +688,16 @@ class TestSearchBackendsE2E:
         from abx_plugins import get_plugins_dir
 
         plugins_dir = Path(get_plugins_dir())
+        lib_dir = initialized_archive / "lib"
+        install_result = run_archivebox_cmd(
+            ["install", "search_backend_ripgrep", "search_backend_sonic"],
+            cwd=initialized_archive,
+            env={"ABXPKG_LIB_DIR": str(lib_dir)},
+            default_cli_env=True,
+        )
+        assert install_result.returncode == 0, install_result.stderr or install_result.stdout
         binary_env = resolve_abxpkg_binary_env(
-            initialized_archive / "lib",
+            lib_dir,
             deps_from=[
                 plugins_dir / "search_backend_ripgrep" / "config.json",
                 plugins_dir / "search_backend_sonic" / "config.json",

archivebox/tests/test_ui_live_progress.py

@@ -77,19 +77,21 @@ def real_second_snapshot_hook_process(snapshot, tmp_path):
 
 
 @pytest.fixture
-def real_crawl_setup_process(snapshot, tmp_path):
+def real_crawl_setup_process(snapshot, hermetic_lib_dir):
     from archivebox.plugins.hooks import run_hook
+    from archivebox.services.runner import run_install
 
     hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js")))
     config_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("config.json")))
-    binary_env = resolve_abxpkg_binary_env(tmp_path / "lib", deps_from=config_path)
+    run_install(plugin_names=["chrome"])
+    binary_env = resolve_abxpkg_binary_env(hermetic_lib_dir, deps_from=config_path)
     output_dir = Path(snapshot.crawl.output_dir) / "chrome"
     process = run_hook(
         hook_path,
         output_dir,
         config={
             **binary_env,
-            "ABXPKG_LIB_DIR": str(tmp_path / "lib"),
+            "ABXPKG_LIB_DIR": str(hermetic_lib_dir),
             "CRAWL_DIR": str(snapshot.crawl.output_dir),
             "SNAP_DIR": str(snapshot.output_dir),
             "CHROME_USER_DATA_DIR": str(output_dir / "profile"),

archivebox/workers/supervisord_util.py

@@ -918,9 +918,17 @@ def run_runner_worker(
     name: str = "worker_runner_once",
     interactive_interrupts: bool = False,
     keep_running=None,
+    config=None,
 ) -> int:
+    from archivebox.config.common import get_config
+
     supervisor = get_or_create_supervisord_process(daemonize=False)
     worker = RUNNER_ONCE_WORKER(args, name=name)
+    workers = [(worker, False)]
+
+    sonic_worker = get_sonic_supervisord_worker_from_plugin(config if config is not None else get_config())
+    if sonic_worker is not None:
+        workers.insert(0, (sonic_worker, False))
     log_path = Path(worker["stdout_logfile"])
     if not log_path.is_absolute():
         log_path = CONSTANTS.DATA_DIR / log_path
@@ -928,7 +936,7 @@ def run_runner_worker(
     log_path.touch()
     log_handle = log_path.open()
     log_handle.seek(0, 2)
-    sync_supervisord_workers(supervisor, [(worker, False)], prune=False)
+    sync_supervisord_workers(supervisor, workers, prune=False)
     final_states = {"STOPPED", "EXITED", "FATAL", "UNKNOWN"}
     forwarded_interrupt = False
     try:

bin/setup.sh

@@ -25,7 +25,7 @@ ARCHIVEBOX_PYTHON="${ARCHIVEBOX_PYTHON:-3.13}"
 ARCHIVEBOX_PACKAGE="${ARCHIVEBOX_PACKAGE:-git+https://github.com/ArchiveBox/ArchiveBox.git@${ARCHIVEBOX_BRANCH}}"
 ARCHIVEBOX_PLATFORM="${ARCHIVEBOX_PLATFORM:-}"
 ARCHIVEBOX_COMPOSE_URL="${ARCHIVEBOX_COMPOSE_URL:-https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/${ARCHIVEBOX_BRANCH}/docker-compose.yml}"
-ABXPKG_PACKAGE="${ABXPKG_PACKAGE:-abxpkg==1.11.288}"
+ABXPKG_PACKAGE="${ABXPKG_PACKAGE:-abxpkg==1.11.293}"
 ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-$HOME/.cache/archivebox/setup-abxpkg}"
 BOOTSTRAP_UV_BINARY=""
 UV_BINARY=""

bin/test.sh

@@ -12,8 +12,8 @@ IFS=$'\n'
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
 
-source "$DIR/.venv/bin/activate"
-
 mkdir -p "$DIR/tests/out"
-pytest -s --basetemp="$DIR/tests/out" "$@"
-exec ./bin/test_plugins.sh
+if [ "$#" -eq 0 ]; then
+    set -- archivebox/tests
+fi
+exec uv run --project "$DIR" --no-sync --no-sources pytest -s --basetemp="$DIR/tests/out" "$@"

bin/test_plugins.sh

@@ -1,362 +0,0 @@
-#!/bin/bash
-# Run ArchiveBox plugin tests with coverage
-#
-# All plugin tests use pytest and are located in pluginname/tests/test_*.py
-#
-# Usage: ./bin/test_plugins.sh [plugin_name] [--no-coverage] [--coverage-report]
-#
-# Examples:
-#   ./bin/test_plugins.sh                     # Run all plugin tests with coverage
-#   ./bin/test_plugins.sh chrome              # Run chrome plugin tests with coverage
-#   ./bin/test_plugins.sh parse_*             # Run all parse_* plugin tests with coverage
-#   ./bin/test_plugins.sh --no-coverage       # Run all tests without coverage
-#   ./bin/test_plugins.sh --coverage-report   # Just show coverage report without running tests
-#
-# For running individual hooks with coverage:
-#   NODE_V8_COVERAGE=./coverage/js "$ABXPKG_LIB_DIR/env/bin/node" <hook>.js [args]  # JS hooks
-#   coverage run --parallel-mode <hook>.py [args]         # Python hooks
-#
-# Coverage results are saved to .coverage (Python) and coverage/js (JavaScript):
-#   coverage combine && coverage report
-#   coverage json
-#   ./bin/test_plugins.sh --coverage-report
-
-set -euo pipefail
-
-# Color codes
-GREEN='\033[0;32m'
-RED='\033[0;31m'
-YELLOW='\033[1;33m'
-NC='\033[0m' # No Color
-
-# Save root directory first
-ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
-PLUGINS_DIR="${ABX_PLUGINS_DIR:-$(uv run --project "$ROOT_DIR" --no-sync --no-sources python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir())')}"
-
-resolve_node_binary() {
-    export ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-$ROOT_DIR/.venv/abxpkg}"
-    mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-    uv run --no-sync --no-sources abxpkg env \
-        --install \
-        --lib="$ABXPKG_LIB_DIR" \
-        --deps-from="$ROOT_DIR/.github/configs/ci-tooling.json:node_binaries" \
-        >/dev/null
-    NODE_BINARY="$ABXPKG_LIB_DIR/env/bin/node"
-    test -L "$NODE_BINARY"
-    test -x "$NODE_BINARY"
-}
-
-# Parse arguments
-PLUGIN_FILTER=""
-ENABLE_COVERAGE=true
-COVERAGE_REPORT_ONLY=false
-
-for arg in "$@"; do
-    if [ "$arg" = "--no-coverage" ]; then
-        ENABLE_COVERAGE=false
-    elif [ "$arg" = "--coverage-report" ]; then
-        COVERAGE_REPORT_ONLY=true
-    else
-        PLUGIN_FILTER="$arg"
-    fi
-done
-
-# Function to show JS coverage report (inlined from convert_v8_coverage.js)
-show_js_coverage() {
-    local plugin_root="$1"
-    local coverage_dir="$2"
-
-    if [ ! -d "$coverage_dir" ] || ! uv run --no-sync --no-sources python - "$coverage_dir" <<'PY'
-import os
-import sys
-
-raise SystemExit(0 if any(os.scandir(sys.argv[1])) else 1)
-PY
-    then
-        echo "No JavaScript coverage data collected"
-        echo "(JS hooks may not have been executed during tests)"
-        return
-    fi
-
-    resolve_node_binary
-    "$NODE_BINARY" - "$plugin_root" "$coverage_dir" << 'ENDJS'
-const fs = require('fs');
-const path = require('path');
-const pluginRoot = path.resolve(process.argv[2]).replace(/\\/g, '/');
-const coverageDir = process.argv[3];
-
-const files = fs.readdirSync(coverageDir).filter(f => f.startsWith('coverage-') && f.endsWith('.json'));
-if (files.length === 0) {
-    console.log('No coverage files found');
-    process.exit(0);
-}
-
-const coverageByFile = {};
-
-files.forEach(file => {
-    const data = JSON.parse(fs.readFileSync(path.join(coverageDir, file), 'utf8'));
-    data.result.forEach(script => {
-        const url = script.url;
-        if (url.startsWith('node:') || url.includes('node_modules')) return;
-
-        if (!coverageByFile[url]) {
-            coverageByFile[url] = { totalRanges: 0, executedRanges: 0 };
-        }
-
-        script.functions.forEach(func => {
-            func.ranges.forEach(range => {
-                coverageByFile[url].totalRanges++;
-                if (range.count > 0) coverageByFile[url].executedRanges++;
-            });
-        });
-    });
-});
-
-const allFiles = Object.keys(coverageByFile).sort();
-const pluginFiles = allFiles.filter(url => url.replace(/\\/g, '/').includes(pluginRoot));
-const otherFiles = allFiles.filter(url => !url.startsWith('node:') && !url.replace(/\\/g, '/').includes(pluginRoot));
-
-console.log('Total files with coverage: ' + allFiles.length + '\n');
-console.log('Plugin files: ' + pluginFiles.length);
-console.log('Node internal: ' + allFiles.filter(u => u.startsWith('node:')).length);
-console.log('Other: ' + otherFiles.length + '\n');
-
-console.log('JavaScript Coverage Report');
-console.log('='.repeat(80));
-console.log('');
-
-if (otherFiles.length > 0) {
-    console.log('Non-plugin files with coverage:');
-    otherFiles.forEach(url => console.log('  ' + url));
-    console.log('');
-}
-
-if (pluginFiles.length === 0) {
-    console.log('No plugin files covered');
-    process.exit(0);
-}
-
-let totalRanges = 0, totalExecuted = 0;
-
-pluginFiles.forEach(url => {
-    const cov = coverageByFile[url];
-    const pct = cov.totalRanges > 0 ? (cov.executedRanges / cov.totalRanges * 100).toFixed(1) : '0.0';
-    const normalizedUrl = url.replace(/\\/g, '/');
-    const displayPath = normalizedUrl.includes(pluginRoot) ? normalizedUrl.slice(normalizedUrl.indexOf(pluginRoot)) : url;
-    console.log(displayPath + ': ' + pct + '% (' + cov.executedRanges + '/' + cov.totalRanges + ' ranges)');
-    totalRanges += cov.totalRanges;
-    totalExecuted += cov.executedRanges;
-});
-
-console.log('');
-console.log('-'.repeat(80));
-const overallPct = totalRanges > 0 ? (totalExecuted / totalRanges * 100).toFixed(1) : '0.0';
-console.log('Total: ' + overallPct + '% (' + totalExecuted + '/' + totalRanges + ' ranges)');
-ENDJS
-}
-
-show_pytest_log() {
-    uv run --no-sync --no-sources python - "$1" <<'PY'
-from collections import deque
-from pathlib import Path
-import sys
-
-ignored_prefixes = ("platform", "cachedir", "rootdir", "configfile", "plugins:")
-lines = (
-    line
-    for line in Path(sys.argv[1]).read_text(errors="replace").splitlines()
-    if not line.startswith(ignored_prefixes)
-)
-print(*deque(lines, maxlen=100), sep="\n")
-PY
-}
-
-combine_parallel_coverage() {
-    if compgen -G "$ROOT_DIR/.coverage.*" >/dev/null; then
-        uv run --no-sync --no-sources coverage combine
-    fi
-}
-
-# If --coverage-report only, just show the report and exit
-if [ "$COVERAGE_REPORT_ONLY" = true ]; then
-    cd "$ROOT_DIR" || exit 1
-    echo "=========================================="
-    echo "Python Coverage Summary"
-    echo "=========================================="
-    combine_parallel_coverage
-    uv run --no-sync --no-sources coverage report --include="*/abx_plugins/plugins/*" --omit="*/tests/*"
-    echo ""
-
-    echo "=========================================="
-    echo "JavaScript Coverage Summary"
-    echo "=========================================="
-    show_js_coverage "$PLUGINS_DIR" "$ROOT_DIR/coverage/js"
-    echo ""
-
-    echo "For detailed coverage reports:"
-    echo "  Python:     coverage report --show-missing --include='*/abx_plugins/plugins/*' --omit='*/tests/*'"
-    echo "  Python:     coverage json  # LLM-friendly format"
-    echo "  Python:     coverage html  # Interactive HTML report"
-    exit 0
-fi
-
-# Set DATA_DIR for tests (required by abxpkg and plugins)
-# Use temp dir to isolate tests from project files
-if [ -z "${DATA_DIR:-}" ]; then
-    DATA_DIR="$(mktemp -d -t archivebox_plugin_tests.XXXXXX)"
-    export DATA_DIR
-    # Clean up on exit
-    trap 'rm -rf "$DATA_DIR"' EXIT
-fi
-
-# Reset coverage data if collecting coverage
-if [ "$ENABLE_COVERAGE" = true ]; then
-    echo "Resetting coverage data..."
-    cd "$ROOT_DIR" || exit 1
-    uv run --no-sync --no-sources coverage erase
-    rm -rf "$ROOT_DIR/coverage/js" 2>/dev/null
-    mkdir -p "$ROOT_DIR/coverage/js"
-
-    # Enable Python subprocess coverage
-    export COVERAGE_PROCESS_START="$ROOT_DIR/pyproject.toml"
-    export PYTHONPATH="$ROOT_DIR${PYTHONPATH:+:$PYTHONPATH}"  # For sitecustomize.py
-
-    # Enable Node.js V8 coverage (built-in, no packages needed)
-    export NODE_V8_COVERAGE="$ROOT_DIR/coverage/js"
-
-    echo "Python coverage: enabled (subprocess support)"
-    echo "JavaScript coverage: enabled (NODE_V8_COVERAGE=$NODE_V8_COVERAGE)"
-    echo ""
-fi
-
-cd "$ROOT_DIR" || exit 1
-
-echo "=========================================="
-echo "ArchiveBox Plugin Tests"
-echo "=========================================="
-echo ""
-
-if [ -n "$PLUGIN_FILTER" ]; then
-    echo "Filter: $PLUGIN_FILTER"
-else
-    echo "Running all plugin tests"
-fi
-
-if [ "$ENABLE_COVERAGE" = true ]; then
-    echo "Coverage: enabled"
-else
-    echo "Coverage: disabled"
-fi
-echo ""
-
-# Track results
-TOTAL_PLUGINS=0
-PASSED_PLUGINS=0
-FAILED_PLUGINS=0
-
-# Find and run plugin tests
-mapfile -t TEST_DIRS < <(
-    uv run --no-sync --no-sources python - "$PLUGINS_DIR" "$PLUGIN_FILTER" <<'PY'
-from pathlib import Path
-import sys
-
-plugins_dir = Path(sys.argv[1])
-plugin_filter = sys.argv[2] or "*"
-test_dirs = [path for path in sorted(plugins_dir.glob(f"{plugin_filter}*/tests")) if path.is_dir()]
-if test_dirs:
-    print(*(str(path) for path in test_dirs), sep="\n")
-PY
-)
-
-if [ "${#TEST_DIRS[@]}" -eq 0 ]; then
-    echo -e "${RED}No plugin tests found${NC}" >&2
-    [ -n "$PLUGIN_FILTER" ] && echo "Pattern: $PLUGIN_FILTER"
-    exit 1
-fi
-
-for test_dir in "${TEST_DIRS[@]}"; do
-    # Check if there are any Python test files
-    if ! compgen -G "${test_dir}/test_*.py" > /dev/null 2>&1; then
-        echo -e "${RED}No test_*.py files found in ${test_dir}${NC}" >&2
-        exit 1
-    fi
-
-    plugin_dir="${test_dir%/tests}"
-    plugin_name="${plugin_dir##*/}"
-    TOTAL_PLUGINS=$((TOTAL_PLUGINS + 1))
-
-    echo -e "${YELLOW}[RUNNING]${NC} $plugin_name"
-
-    # Build pytest command with optional coverage
-    PYTEST_CMD=(uv run --project "$ROOT_DIR" --no-sync --no-sources python -m pytest "$test_dir" -p no:django -v --tb=short)
-    if [ "$ENABLE_COVERAGE" = true ]; then
-        PYTEST_CMD+=(--cov="$plugin_dir" --cov-append --cov-branch)
-        echo "[DEBUG] NODE_V8_COVERAGE before pytest: $NODE_V8_COVERAGE"
-        uv run --no-sync --no-sources python -c "import os; print('[DEBUG BASH->PYTHON] NODE_V8_COVERAGE:', os.environ.get('NODE_V8_COVERAGE', 'NOT_SET'))"
-    fi
-
-    LOG_FILE=$(mktemp -t "archivebox_plugin_${plugin_name}.XXXXXX.log")
-    PLUGIN_TMPDIR=$(mktemp -d -t "archivebox_plugin_${plugin_name}.XXXXXX")
-    if (
-        cd "$PLUGIN_TMPDIR"
-        TMPDIR="$PLUGIN_TMPDIR" "${PYTEST_CMD[@]}"
-    ) >"$LOG_FILE" 2>&1; then
-        show_pytest_log "$LOG_FILE"
-        echo -e "${GREEN}[PASSED]${NC} $plugin_name"
-        PASSED_PLUGINS=$((PASSED_PLUGINS + 1))
-    else
-        show_pytest_log "$LOG_FILE"
-        echo -e "${RED}[FAILED]${NC} $plugin_name"
-        FAILED_PLUGINS=$((FAILED_PLUGINS + 1))
-    fi
-    rm -f "$LOG_FILE"
-    rm -rf "$PLUGIN_TMPDIR"
-    echo ""
-done
-
-# Print summary
-echo "=========================================="
-echo "Test Summary"
-echo "=========================================="
-echo -e "Total plugins tested: $TOTAL_PLUGINS"
-echo -e "${GREEN}Passed:${NC}              $PASSED_PLUGINS"
-echo -e "${RED}Failed:${NC}              $FAILED_PLUGINS"
-echo ""
-
-if [ $TOTAL_PLUGINS -eq 0 ]; then
-    echo -e "${RED}No tests ran${NC}" >&2
-    exit 1
-elif [ $FAILED_PLUGINS -eq 0 ]; then
-    echo -e "${GREEN}✓ All plugin tests passed!${NC}"
-
-    # Show coverage summary if enabled
-    if [ "$ENABLE_COVERAGE" = true ]; then
-        echo ""
-        echo "=========================================="
-        echo "Python Coverage Summary"
-        echo "=========================================="
-        # Coverage data is in ROOT_DIR, combine and report from there
-        cd "$ROOT_DIR" || exit 1
-        # Copy coverage data from plugins dir if it exists
-        combine_parallel_coverage
-        uv run --no-sync --no-sources coverage report --include="*/abx_plugins/plugins/*" --omit="*/tests/*"
-        echo ""
-
-        echo "=========================================="
-        echo "JavaScript Coverage Summary"
-        echo "=========================================="
-        show_js_coverage "$PLUGINS_DIR" "$ROOT_DIR/coverage/js"
-        echo ""
-
-        echo "For detailed coverage reports (from project root):"
-        echo "  Python:     coverage report --show-missing --include='*/abx_plugins/plugins/*' --omit='*/tests/*'"
-        echo "  Python:     coverage json  # LLM-friendly format"
-        echo "  Python:     coverage html  # Interactive HTML report"
-        echo "  JavaScript: ./bin/test_plugins.sh --coverage-report"
-    fi
-
-    exit 0
-else
-    echo -e "${RED}✗ Some plugin tests failed${NC}"
-    exit 1
-fi

conftest.py

@@ -1,80 +0,0 @@
-from __future__ import annotations
-
-import tomllib
-from pathlib import Path
-from typing import Any
-
-import pytest
-
-
-DOCS_MANIFEST = Path(__file__).parent / "docs" / "codeblocks.toml"
-
-
-def _load_docs_manifest() -> dict[str, Any]:
-    with DOCS_MANIFEST.open("rb") as manifest_file:
-        return tomllib.load(manifest_file)
-
-
-def pytest_addoption(parser: pytest.Parser) -> None:
-    parser.addoption(
-        "--docs-environment",
-        action="store",
-        default=None,
-        help="Run Markdown code blocks assigned to one docs CI environment.",
-    )
-
-
-def pytest_configure(config: pytest.Config) -> None:
-    manifest = _load_docs_manifest()
-    for environment in manifest["environments"]:
-        config.addinivalue_line(
-            "markers",
-            f"docs_environment_{environment}: Markdown code block assigned to the {environment} CI environment",
-        )
-
-
-def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
-    manifest = _load_docs_manifest()
-    environments = set(manifest["environments"])
-    selected_environment = config.getoption("--docs-environment")
-    if selected_environment is not None and selected_environment not in environments:
-        raise pytest.UsageError(
-            f"Unknown docs environment {selected_environment!r}; expected one of {sorted(environments)}",
-        )
-
-    file_environments = manifest["files"]
-    block_environments = manifest["blocks"]
-    collected_nodeids: set[str] = set()
-    collected_paths: set[str] = set()
-    deselected: list[pytest.Item] = []
-    selected: list[pytest.Item] = []
-
-    for item in items:
-        if item.path.suffix != ".md":
-            selected.append(item)
-            continue
-
-        nodeid = item.nodeid
-        collected_nodeids.add(nodeid)
-        relative_path = item.path.resolve().relative_to(config.rootpath.resolve()).as_posix()
-        collected_paths.add(relative_path)
-        canonical_nodeid = f"{relative_path}::{nodeid.partition('::')[2]}"
-        environment = block_environments.get(canonical_nodeid, file_environments.get(relative_path))
-        if environment is None:
-            raise pytest.UsageError(f"Markdown code block has no docs environment: {nodeid}")
-        if environment not in environments:
-            raise pytest.UsageError(f"Markdown code block has unknown docs environment {environment!r}: {nodeid}")
-
-        item.add_marker(f"docs_environment_{environment}")
-        if selected_environment is not None and environment != selected_environment:
-            deselected.append(item)
-        else:
-            selected.append(item)
-
-    stale_blocks = {nodeid for nodeid in block_environments if nodeid.partition("::")[0] in collected_paths} - collected_nodeids
-    if stale_blocks and collected_nodeids:
-        raise pytest.UsageError(f"Docs manifest contains stale code block ids: {sorted(stale_blocks)}")
-
-    if deselected:
-        config.hook.pytest_deselected(items=deselected)
-        items[:] = selected

docs/ArchiveBox-Architecture-Diagrams.md

@@ -1,200 +1,121 @@
 # ArchiveBox Architecture Diagrams
 
-## High-Level System Execution Flow
+This page is a map of the current execution and persistence paths. The implementation lives primarily in:
+
+- `archivebox/cli/` for CLI entry points
+- `archivebox/services/runner.py` for crawl and snapshot execution
+- `archivebox/crawls/models.py` for the `Crawl` model and state machine
+- `archivebox/core/models.py` for `Snapshot`, `ArchiveResult`, and the `Snapshot` state machine
+- `archivebox/services/` for bus event projectors
+- `abxpkg` and `abx-plugins` for binary resolution and plugin hooks
+
+## High-Level Execution Flow
+
+```mermaid
+flowchart TD
+    ENTRY["CLI, Web UI, REST API, or scheduler"] --> CRAWL["Create or resume a Crawl row"]
+    CRAWL --> RUNNER["run_crawl() / CrawlRunner"]
+    RUNNER --> DISCOVER["Create or select Snapshot rows"]
+    DISCOVER --> EVENTS["Emit crawl and snapshot lifecycle events"]
+    EVENTS --> PLUGINS["Run selected abx-plugin hooks"]
+    PLUGINS --> PROCESSES["Persist Process rows and hook output"]
+    PROCESSES --> RESULTS["Project ArchiveResult rows"]
+    RESULTS --> FILES["Write snapshot output files"]
+    RESULTS --> SNAPSTATE["Seal or requeue Snapshot"]
+    SNAPSTATE --> CRAWLSTATE["Seal, pause, or continue Crawl"]
+
+    EVENTS --> BINREQ["BinaryRequestEvent"]
+    BINREQ --> ABXPKG["abxpkg resolution"]
+    ABXPKG --> HOST["Compatible host binary"]
+    ABXPKG --> MANAGED["Managed install fallback"]
+    HOST --> ENV["Project resolved binary into LIB_DIR/env/bin"]
+    MANAGED --> ENV
+
+    CRAWL -.-> DB["SQLite database"]
+    PROCESSES -.-> DB
+    RESULTS -.-> DB
+    FILES -.-> STORAGE["archive/users/... snapshot storage"]
+```
+
+ArchiveBox has one normal crawl execution path. CLI commands and web/API actions create or select database rows, then call the same runner. The runner emits lifecycle events, abx-plugin hooks do the extraction work, and service projectors persist processes and results.
+
+Binary discovery and installation always goes through abxpkg. Compatible host binaries are preferred; managed providers are the fallback. Resolved binaries are projected into `LIB_DIR/env/bin` before programmatic use. `LIB_DIR/bin` is only a convenience directory for humans.
+
+## Persistent Data
+
+```mermaid
+flowchart LR
+    DATA["ArchiveBox data directory"] --> DB["index.sqlite3"]
+    DATA --> ARCHIVE["archive/users/<user>/snapshots/<date>/<domain>/<uuid>/"]
+    DATA --> SOURCES["sources/"]
+    DATA --> LOGS["logs/"]
+    DATA --> LIB["lib/env/bin/ resolved binaries"]
+
+    ARCHIVE --> PLUGINOUT["Plugin-namespaced outputs"]
+    ARCHIVE --> META["Snapshot metadata and indexes"]
+```
+
+The database is the source of truth for model state. Snapshot directories contain captured artifacts and rendered metadata. Older collections may also contain legacy timestamp-named snapshot directories.
+
+## `Crawl` State Machine
+
+Implemented by `Crawl` and `CrawlMachine` in `archivebox/crawls/models.py`.
 
 ```mermaid
 stateDiagram-v2
-    archivebox.cli.main(sys.argv)
-    state Supervisord {
-        Scheduler
-        state Orchestrator {
-            [*] --> TICK
-            TICK --> SPAWN_ACTORS: queued > 0
-            SPAWN_ACTORS --> TICK
-            TICK --> IDLE: queued == 0
-            IDLE --> TICK: 1s
-        }
-    }
-
-    note left of archivebox.cli.main(sys.argv)
-        archivebox entrypoint
-    end note
-
-    state "archivebox.cli.SUBCOMMAND" as MAIN_THREAD
-
-    archivebox.cli.main(sys.argv) --> run_subcommand(sys.argv)
-    run_subcommand(sys.argv) --> setup_django()
-    setup_django() --> Supervisord: spawns in background
-    setup_django() --> MAIN_THREAD: runs in foreground   
-
-    MAIN_THREAD --> archivebox.main.SUBCOMMAND
-    archivebox.main.SUBCOMMAND --> Storage: add_to_queue()
-
-    state Actors {
-        CrawlActor --> Crawl: tick()
-        SnapshotActor --> Snapshot: tick()
-        ArchiveResultActors --> ArchiveResult: tick()
-    }
-
-    state "State Machines" as JOBS {
-
-        state Crawl {
-            state "QUEUED" as CRAWL_QUEUED
-            state "STARTED" as CRAWL_STARTED
-            state "SEALED" as CRAWL_SEALED
-            CRAWL_QUEUED --> CRAWL_STARTED: create_root_snapshot()
-            CRAWL_STARTED --> CRAWL_SEALED: is_finished
-        }
-
-        state Snapshot {
-            state "QUEUED" as SNAP_QUEUED
-            state "STARTED" as SNAP_STARTED
-            state "SEALED" as SNAP_SEALED
-            SNAP_QUEUED --> SNAP_STARTED: create_pending_archiveresults()
-            SNAP_STARTED --> SNAP_SEALED: is_finished
-        }
-
-        state ArchiveResult {
-            QUEUED --> STARTED: run_extractor()
-            STARTED --> BACKOFF: is_temp_error
-            BACKOFF --> STARTED: is_retry_past
-            STARTED --> FAILED: is_fatal_error
-            STARTED --> SUCCEEDED: is_succeded
-        }
-        
-        
-        note right of ArchiveResult
-            exec_crome()
-        end note
-        
-        note right of ArchiveResult
-            exec_wget()
-        end note
-        
-        note right of ArchiveResult
-            exec_curl()
-        end note
-        
-        note right of ArchiveResult
-            ... other extractor subprocesses ...
-        end note
-    }
-    
-    state Storage {
-        state "DB" as SQLITE_DB
-        sources/
-        archive/
-        state "index.json" as INDEX_JSONS
-    }
-
-    Storage: Storage
-
-    Orchestrator --> Actors: spawns subprocesses
-    
-    Crawl --> Snapshot: create_root_snapshot()
-    Snapshot --> ArchiveResult: create_pending_archiveresults()
-    
-    Crawl --> Storage: .save()
-    Snapshot --> Storage: .save()
-    ArchiveResult --> Storage: .save()
-    
-    Storage --> Actors: get_queue()
-
-
+    [*] --> QUEUED
+    QUEUED --> STARTED: tick and valid URLs
+    QUEUED --> QUEUED: tick and not ready
+    QUEUED --> SEALED: all existing snapshots finished
+    STARTED --> SEALED: all snapshots finished
+    QUEUED --> PAUSED: pause requested
+    STARTED --> PAUSED: pause requested
+    PAUSED --> QUEUED: resume requested
+    PAUSED --> PAUSED: tick
+    QUEUED --> SEALED: explicit seal
+    STARTED --> SEALED: explicit seal
+    PAUSED --> SEALED: explicit seal
+    SEALED --> [*]
 ```
 
----
+A crawl owns a set of snapshots. Entering `STARTED` creates or discovers those snapshots; sealing waits for their normal lifecycle to finish. Pausing also schedules child snapshots to pause, and resuming returns the crawl to the runnable queue.
 
-## State Diagrams for Main Models
+## `Snapshot` State Machine
 
-
-### `Crawl`
-
-- `crawls/models.py`: `Crawl`
-- `crawls/statemachines.py`: `CrawlMachine`
+Implemented by `Snapshot` and `SnapshotMachine` in `archivebox/core/models.py`.
 
 ```mermaid
 stateDiagram-v2
-    STARTED --> SEALED: tick [is_finished]
-    STARTED --> STARTED: tick [!is_finished]    
-    QUEUED --> STARTED: tick [can_start]
-    QUEUED --> QUEUED: tick [!can_start]
-
-    
-    note left of QUEUED
-        Crawl created
-    end note
-    
-    note right of STARTED
-        create_root_snapshot()
-        crawl.retry_at = now + 5s
-    end note
+    [*] --> QUEUED
+    QUEUED --> STARTED: tick and URL is ready
+    QUEUED --> QUEUED: tick and not ready
+    QUEUED --> SEALED: all existing results finished
+    STARTED --> SEALED: all hook results finished
+    QUEUED --> PAUSED: pause requested
+    STARTED --> PAUSED: pause requested
+    PAUSED --> QUEUED: resume requested
+    PAUSED --> PAUSED: tick
+    QUEUED --> SEALED: explicit seal
+    STARTED --> SEALED: explicit seal
+    PAUSED --> SEALED: explicit seal
+    SEALED --> [*]
 ```
 
+The runner creates one queued `ArchiveResult` per selected hook, executes those hooks through the shared event bus, and seals the snapshot after every result reaches a final status. The narrow search-index maintenance operation on an already sealed snapshot is the intentional exception; it does not reopen or invent a second general lifecycle path.
 
-## `Snapshot`
+## `ArchiveResult` Projection
 
-- `core/models.py`: `Snapshot`
-- `core/statemachines.py`: `SnapshotMachine`
+`ArchiveResult` is not driven by a separate Python state machine. The runner creates queued rows, and `ArchiveResultService` projects `ArchiveResultEvent` and `ProcessCompletedEvent` data into them.
 
 ```mermaid
-stateDiagram-v2
-    STARTED --> SEALED: tick [is_finished]
-    STARTED --> STARTED: tick [!is_finished]
-    QUEUED --> STARTED: tick [can_start]
-    QUEUED --> QUEUED: tick [!can_start]
-    
-    note left of QUEUED
-        Snapshot created
-    end note
-    
-    note right of STARTED
-        create_pending_archiveresults(extractors)
-        snapshot.retry_at = now + 60s
-    end note
+flowchart LR
+    QUEUED["queued"] --> STARTED["started"]
+    STARTED --> SUCCEEDED["succeeded"]
+    STARTED --> FAILED["failed"]
+    STARTED --> SKIPPED["skipped"]
+    STARTED --> NORESULTS["noresults"]
+    STARTED -. recoverable wait .-> BACKOFF["backoff"]
+    BACKOFF -. resumed work .-> STARTED
 ```
 
-
-### `ArchiveResult`
-
-- `core/models.py`: `ArchiveResult`
-- `core/statemachines.py`: `ArchiveResultMachine`
-
-<img width="1740" alt="image" src="https://github.com/user-attachments/assets/23d596ab-6c8a-440a-b49b-a2432f37abb3">
-
-```mermaid
-stateDiagram-v2
-    QUEUED --> QUEUED: tick [!can_start]
-    QUEUED --> STARTED: tick [can_start]
-    STARTED --> STARTED: tick [!is_finished]
-    STARTED --> BACKOFF: tick [is_backoff]
-    STARTED --> FAILED: tick [is_failed]
-    STARTED --> SUCCEEDED: tick [is_succeeded]
-    BACKOFF --> BACKOFF: tick [!can_start]
-    BACKOFF --> STARTED: tick [can_start]
-    
-    note left of QUEUED
-        ArchiveResult created
-    end note
-    
-    note left of STARTED
-        start_ts = now
-        retry_at = now + 60s
-        create_output_dir()
-        run_extractor()
-    end note
-    
-    note right of BACKOFF
-        retry_at = now + 60s
-    end note
-    
-    note right of SUCCEEDED
-        end_ts = now
-        retry_at = None
-    end note
-    
-    note right of FAILED
-        end_ts = now
-        retry_at = None
-    end note
-```
+`succeeded`, `failed`, `skipped`, and `noresults` are final result statuses. Each row identifies the plugin and hook that produced it and stores structured output, file metadata, timing, and error details.

docs/Changelog.md

@@ -27,11 +27,10 @@
    - https://github.com/ArchiveBox/ArchiveBox/releases
  - easy migration from previous versions
    ```bash
-   export PLUGINS=parse_txt_urls
-   archive_dir="$(mktemp -d)"
-   cd "$archive_dir"
+   cd path/to/your/archive/folder
    archivebox init
-   archivebox add --plugins=parse_txt_urls 'https://example.com'
+   archivebox add 'https://example.com'
+   archivebox add 'https://getpocket.com/users/USERNAME/feed/all' --depth=1
    ```
  - full transition to Django Sqlite DB with migrations (making upgrades between versions much safer now)
  - maintains an intuitive and helpful CLI that's backwards-compatible with all previous archivebox data versions

docs/Chromium-Install.md

@@ -1,50 +1,21 @@
 # Chrome / Chromium Setup
 
-By default, ArchiveBox looks for any existing installed version of Chrome/Chromium and uses it if found.  You can optionally install a specific version and set the environment variable `CHROME_BINARY` to force ArchiveBox to use that one, e.g.:  
-
- - `CHROME_BINARY=google-chrome-beta`
- - `CHROME_BINARY=/usr/bin/chromium-browser`
- - `CHROME_BINARY='/Applications/Chromium.app/Contents/MacOS/Chromium'`
- - `CHROME_BINARY='~/Library/Caches/ms-playwright/chromium-857950/chrome-mac/Chromium.app/Contents/MacOS/Chromium'`
-
-If you don't already have Chrome installed, I recommend installing Chromium instead of Google Chrome, as it's the open-source fork of Chrome that doesn't send as much tracking data to Google.
-
-**Detect or install a compatible Chrome/Chromium:**
-
-<img src="https://imgur.zervice.io/FxFoIMH.jpg" width="25%" align="right"/> 
+ArchiveBox resolves Chrome through `abxpkg`, just like every other runtime binary. It checks compatible browsers already installed on the host first. When it finds one, it projects that exact browser into the managed runtime environment; otherwise it installs a compatible managed Chromium build.
 
 ```bash
-export PLUGINS=chrome
-test_root="$(mktemp -d)"
-export HOME="$test_root/home"
-mkdir -p "$HOME"
-archivebox_data="$test_root/data"
-mkdir -p "$archivebox_data"
-cd "$archivebox_data"
-archivebox init
 archivebox install chrome
 archivebox version
 ```
 
-## Installing Chromium
+The resolved browser is always available through `./lib/env/bin/chromium` inside the collection. `archivebox version` shows whether it came from the host or a managed provider, along with the exact version and path.
 
-### ⭐️ Any OS (recommended)
+If you need to select a specific compatible browser already installed on the host, set `CHROME_BINARY` and let the same installer validate and project it:
 
-ArchiveBox uses `abxpkg` to prefer a compatible browser already installed on the host. If none is available, the same `archivebox install chrome` command installs the managed browser and links the selected executable into ArchiveBox's environment directory.
-
-### macOS
-
-If a compatible Chrome app is already installed, `archivebox install chrome` detects and uses it without installing another copy.
-
-### Ubuntu/Debian
-If a compatible `chromium` or `chromium-browser` is already installed, `archivebox install chrome` detects and uses it. Otherwise it installs a compatible managed build.
-
-## Installing Google Chrome
-
-### macOS
-If `/Applications/Google Chrome.app` is compatible, ArchiveBox detects it automatically.
-### Ubuntu/Debian
-If a compatible `google-chrome` is already installed, ArchiveBox detects it automatically.
+```bash
+archivebox config --set CHROME_BINARY=google-chrome
+archivebox install chrome
+archivebox version
+```
 
 ## Troubleshooting Chromium Install
 
@@ -56,7 +27,7 @@ If you encounter problems setting up Google Chrome or Chromium, see the [Trouble
 
 You may choose to set up a Chrome/Chromium user profile in order to use your cookies/sessions to log into sites behind authentication/paywall during archiving.
 
-*Note: not all extractors use Chrome (e.g. `wget`, `mercury`, `media`), so [`COOKIES_FILE`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration/#cookies_file) should be set up as well after this.*
+*Note: not all extractors use Chrome (e.g. `wget`, `mercury`, `media`). Importing a dedicated host browser profile into a persona also exports its cookies for those extractors; directly logging in through a new ArchiveBox Chrome profile does not.*
 
 > [!WARNING]
 > **We strongly recommend you use [separate burner credentials dedicated to archiving](https://docs.sweeting.me/s/cookie-dilemma),** e.g. don't provide cookies for your normal daily Facebook/Instagram/Google/etc. accounts as server responses and page content will often contain your name/email/PII, session cookies, private tokens, etc. which then get preserved in your snapshots for eternity.
@@ -78,12 +49,10 @@ If using ArchiveBox in Docker, the easiest way to set up session credentials is
 ```yaml
 services:
     archivebox:
-        # ...
+        ...
         volumes:
-            # ...
-            - ./data/personas/Default:/data/personas/Default
+            ...
         environment:
-            - CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile
             - DISPLAY=novnc:0.0
             
     novnc:
@@ -98,23 +67,31 @@ services:
 
 2. Start the `novnc` window server container
 ```bash
-docker compose config --quiet
+docker compose up -d novnc
+# wait a few seconds for novnc to start...
 ```
 
 3. Start ArchiveBox's Chrome inside Docker
 ```bash
-docker compose run --rm archivebox archivebox version
+docker compose run archivebox persona create personal
+docker compose run archivebox /data/lib/env/bin/chromium --user-data-dir=/data/personas/personal/chrome_profile --profile-directory=Default --disable-gpu --disable-features=dbus --disable-dev-shm-usage --start-maximized --no-sandbox --disable-setuid-sandbox --no-zygote --disable-sync --no-first-run
 ```
-After confirming the image sees Chromium, launch the reported browser path with `--user-data-dir=/data/personas/Default/chrome_profile` and the display/security flags appropriate for your container. Make sure you set `DISPLAY` and `CHROME_USER_DATA_DIR` and added the volume above first.
+<small>(make sure you set `DISPLAY` and keep the normal persistent `/data` volume from the Compose setup!)</small>
 
 4. Open [`http://localhost:8080/vnc.html`](http://localhost:8080/vnc.html) in your browser. You should see a remote linux desktop shown with Chrome open, allowing you to remote-control ArchiveBox's browser. Use it to log into any sites where you want to save credentials.
 
-5. ✅ Close the browser, stop & remove novnc, and then run archivebox normally. It will use the profile stored in `CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile` going forward, you should now be able to archive sites as if you were logged in!
+5. ✅ Close the browser, stop & remove novnc, and then select the `personal` persona when archiving. Chrome-based extractors will use the saved profile and should see the sites as logged in.
 
 ```bash
 # stop the archivebox and novnc containers
+docker compose down
 docker compose down --remove-orphans
-docker compose run --rm archivebox add --index-only 'https://example.com/profile-check'
+# edit docker-compose.yml to remove/comment out the novnc: section
+
+# test it all out by archiving something hosted on one of the domains you logged in to
+docker compose run archivebox add --persona=personal 'https://private.example.com/some/site/requiring/login.html'
+# check the SingleFile, Screenshot, DOM, or PDF snapshot output (only these use the Chrome profile)
+# make sure the content appears as your logged-in user would see it
 ```
 
 Under the hood this uses [Xvfb](https://www.x.org/releases/X11R7.6/doc/man/man1/Xvfb.1.xhtml) + [Fluxbox](http://www.fluxbox.org/) + [`novnc`](https://github.com/theasp/docker-novnc) to provide a virtual display, window manager, and VNC server + novnc websocket viewer.
@@ -124,56 +101,43 @@ Under the hood this uses [Xvfb](https://www.x.org/releases/X11R7.6/doc/man/man1/
 
 If running ArchiveBox on your local machine without Docker, this process is fairly easy.
 
-First, tell archivebox where you want to store your Chrome profile.
+First, create a persona to hold the dedicated Chrome profile.
 
 ```bash
-test_root="$(mktemp -d)"
-export HOME="$test_root/home"
-mkdir -p "$HOME"
-archivebox_data="$test_root/data"
-mkdir -p "$archivebox_data"
-cd "$archivebox_data"
-archivebox init
-profile_dir="$archivebox_data/personas/Default/chrome_profile"
-archivebox config --set "CHROME_USER_DATA_DIR=$profile_dir"
+archivebox persona create personal
 ```
 
-Then run Chrome (with that profile dir) to open a visible browser window where you can log into things, e.g.:
+Then install/resolve Chrome and launch the projected browser with that profile dir:
 
-<!--pytest-codeblocks:cont-->
 ```bash
 archivebox install chrome
-chrome_binary="$(archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
-test -x "$chrome_binary"
-archivebox config --get CHROME_USER_DATA_DIR | grep -Fq "$profile_dir"
-"$chrome_binary" --version | grep -Eiq 'chrome|chromium'
+./lib/env/bin/chromium --user-data-dir="$PWD/personas/personal/chrome_profile"
 ```
 
 Once it's open, log in to all the sites you want to be logged in to for archiving, then close/quit Chrome.
 
-✅ All ArchiveBox extractors that use Chrome (e.g. Screenshot, PDF, DOM, Singlefile) should now use that profile.  
-*Don't forget to set up [`COOKIES_FILE`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration/#cookies_file) for the rest!*
+✅ Chrome-based extractors (e.g. Screenshot, PDF, DOM, Singlefile) use that profile whenever you archive with `--persona=personal`.
+
+Directly logging in through this profile does not generate a `cookies.txt` for non-Chrome extractors. If those extractors need the same login state, use the recommended [`archivebox persona create --import=chrome personal`](https://github.com/ArchiveBox/ArchiveBox/wiki/Personas) workflow with a dedicated host browser profile instead; the import copies the Chrome profile and exports its cookies together.
 
 <br/>
 
 ### Non-Docker Setup (Remote Host)
 
-You must set up the profile using the exact same version of chrome that ArchiveBox is running (which can be found with `archivebox version`).
-You can download the latest chromium with `pip install playwright && playwright install --with-deps chromium`, or get older versions of Chrome from https://chromium.cypress.io.
+You must set up the profile using the exact same version of Chrome that ArchiveBox is running. Run `archivebox install chrome` and `archivebox version` on each machine so `abxpkg` selects and validates the browser.
 
 **General steps:**
 
 1. Make sure you are running the same OS and have the same version of Chrome installed as the host running ArchiveBox
-2. Follow the `Non-Docker Setup (Local Host)` setups above to create a Chrome profile locally
-3. Rsync your chrome profile from your local machine to the remote archivebox host  
-   `rsync --archive /path/to/profile remotehost:/path/to/profile/on/remote/host`
-4. Configure ArchiveBox on the remote host to use the `rsync`'ed Chrome profile  
-   `archivebox config --set CHROME_USER_DATA_DIR=/path/to/profile/on/remote/host`
+2. Follow the `Non-Docker Setup (Local Host)` steps above to create the `personal` persona and Chrome profile locally
+3. Create the same persona from the ArchiveBox data directory on the remote host: `archivebox persona create personal`
+4. Rsync the persona's Chrome profile from your local collection into the matching remote persona: `rsync --archive ~/archivebox/data/personas/personal/chrome_profile/ remotehost:~/archivebox/data/personas/personal/chrome_profile/`
 
-You may need to run `chown -R archivebox /path/to/profile/on/remote/host` on the remote host to make the profile editable by the `archivebox` user on that machine.
+You may need to run `chown -R archivebox ~/archivebox/data/personas/personal/chrome_profile` on the remote host to make the profile editable by the `archivebox` user on that machine.
 
-✅ All ArchiveBox extractors that use Chrome (e.g. Screenshot, PDF, DOM, Singlefile) should now use that profile.  
-*Don't forget to set up [`COOKIES_FILE`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration/#cookies_file) for the rest!*
+✅ Chrome-based extractors (e.g. Screenshot, PDF, DOM, Singlefile) use that profile whenever you archive with `--persona=personal`.
+
+If non-Chrome extractors need the same login state, prefer importing a dedicated host browser profile with `archivebox persona create --import=chrome personal` so the persona receives both the Chrome profile and an exported `cookies.txt`.
 
 ---
 

docs/Configuration.md

@@ -4,23 +4,11 @@ Configuration of ArchiveBox is done by using the `archivebox config` command, mo
 
 *Some equivalent examples of setting some configuration options:*
 ```bash
-set -euo pipefail
-examples_root="$(mktemp -d)"
-trap 'rm -rf "$examples_root"' EXIT
-
-# Persist a value through the CLI.
-mkdir -p "$examples_root/cli" && cd "$examples_root/cli"
-archivebox init
 archivebox config --set TIMEOUT=120
-
-# Or write the same value in a different collection's config file.
-mkdir -p "$examples_root/file" && cd "$examples_root/file"
-archivebox init
-printf '\n[ARCHIVING_CONFIG]\nTIMEOUT=120\n' >> ArchiveBox.conf
-archivebox config --get TIMEOUT
-
-# Or override the value for one command without persisting it.
-env TIMEOUT=120 archivebox add --index-only "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
+# OR edit ArchiveBox.conf and add this under its existing [ARCHIVING_CONFIG] section:
+TIMEOUT=120
+# OR
+env TIMEOUT=120 archivebox add ~/Downloads/bookmarks_export.html
 ```
 
 Environment variables seed process-level defaults. Persisted Machine, Persona, Crawl, and Snapshot settings can override them depending on scope, and existing Crawl config is not silently overwritten by later environment changes. Runtime-derived values like crawl/snapshot output dirs are resolved fresh for each run instead of being stored in frozen crawl config. For more examples see [Usage: Configuration](Usage#run-archivebox-with-configuration-options)...
@@ -62,9 +50,8 @@ Controls what happens when you `add` a URL that **already has a Snapshot** in yo
 Equivalent to the `--only-new` / `--no-only-new` flag on `archivebox add`:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-example_url="${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"; uv run --project "$project_dir" --no-sync archivebox add --plugins=wget "$example_url"
-uv run --project "$project_dir" --no-sync archivebox add --plugins=wget --no-only-new "$example_url"
+archivebox add https://example.com                    # honors ONLY_NEW (default True)
+archivebox add --no-only-new https://example.com      # force a re-archive even if already in the index
 ```
 
 > [!NOTE]
@@ -135,8 +122,8 @@ You can generate a `cookies.txt` using a [browser extension](https://chromewebst
 The recommended path is to create a persona and let it manage cookies + Chrome profile state for you:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync archivebox persona create personal
-uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls --persona=personal "${ARCHIVEBOX_DOCS_URL_ONE:-https://members.example.com/feed}"
+archivebox persona create --import=chrome personal
+archivebox add --persona=personal https://members.example.com/feed
 ```
 
 > [!WARNING]
@@ -262,9 +249,9 @@ Retention policy: automatically delete Crawls, Snapshots, ArchiveResults, and Pr
 Accepted units: `h`/`hr`/`hour`, `d`/`day`, `w`/`week`, `mo`/`month`, `y`/`yr`/`year`. The minimum non-zero duration is `1h`. Examples:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync archivebox config --set DELETE_AFTER=24h
-uv run --project "$project_dir" --no-sync archivebox config --set DELETE_AFTER=30d
-uv run --project "$project_dir" --no-sync archivebox config --set DELETE_AFTER=6mo
+archivebox config --set DELETE_AFTER=24h     # daily rolling buffer
+archivebox config --set DELETE_AFTER=30d     # 30-day retention
+archivebox config --set DELETE_AFTER=6mo     # 6 months
 ```
 
 `DELETE_AFTER` can be set globally, per-persona, per-crawl, or per-snapshot — the most-specific value wins. When a Snapshot is created, its `delete_at` timestamp is computed from the effective `DELETE_AFTER` and persisted; the retention sweeper then deletes rows whose `delete_at` is in the past.
@@ -303,7 +290,7 @@ Comma-separated **whitelist** of plugins to load and run for this archiving run.
 When set, only the listed plugins (plus any plugins they declare as `required_plugins` in their `config.json` — e.g. picking `singlefile` automatically pulls in `chrome`) participate in the run. Equivalent to the CLI flag:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync archivebox add --plugins=wget,favicon,screenshot "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
+archivebox add --plugins=wget,favicon,screenshot https://example.com
 ```
 
 The admin "Add" form and REST API both write to `PLUGINS` when you select extractors — there is no separate "enabled set" config knob; `PLUGINS` is the single source of truth for which plugins run on any given Crawl, Snapshot, or Persona scope.
@@ -322,7 +309,7 @@ Useful for one-off runs ("just grab a screenshot and skip everything else") or f
 #### `ADMIN_USERNAME` / `ADMIN_PASSWORD`
 **Possible Values:** [`None`]/`"admin"`/...
 
-Only used on first run / initial setup in Docker. ArchiveBox will create an admin superuser with the specified username and password when both options are present in the environment at startup. After the user exists, changing these values has no effect — use `archivebox manage changepassword <username>` or the Django admin UI instead.
+Used on first run / initial setup in any installation method. ArchiveBox will create an admin superuser with the specified username and password when both options are present during `archivebox init`. After the user exists, changing these values has no effect — use `archivebox manage changepassword <username>` or the Django admin UI instead.
 
 > [!WARNING]
 > Setting `ADMIN_PASSWORD` via environment variable bakes the secret into your shell history, Docker inspect output, and process listings. For long-lived deployments, set it once during provisioning, create the user, then unset the variable.
@@ -342,8 +329,8 @@ More info:
 Server-wide toggles for whether login is required to use each public area of ArchiveBox.
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync archivebox config --set PUBLIC_INDEX=True
-uv run --project "$project_dir" --no-sync archivebox config --set PUBLIC_ADD_VIEW=False
+archivebox config --set PUBLIC_INDEX=True        # allow viewing the snapshot index without login
+archivebox config --set PUBLIC_ADD_VIEW=False    # require login to submit new URLs via the web UI
 ```
 
 - `PUBLIC_INDEX` (default `True`) — when on, anonymous visitors can browse the snapshot list page. Individual snapshot visibility is still gated by each Snapshot's own [`PERMISSIONS`](#permissions) field.
@@ -373,8 +360,6 @@ The `host:port` socket the ArchiveBox web server actually listens on. **This is
 - `127.0.0.1:8000` (default) — listen only on the loopback interface. Safest when you're running a reverse proxy on the same host and don't want the server reachable directly from the network.
 - `0.0.0.0:8000` — listen on **all** IPv4 interfaces. Required when running in Docker without `--network=host`, or when you want the server reachable from other machines on your LAN without a reverse proxy.
 - `[::]:8000` — listen on all IPv6 interfaces (most modern OSes will accept v4-mapped connections too).
-- `unix:/path/to/archivebox.sock` — bind to a Unix socket instead of a TCP port (useful for nginx/Caddy on the same host).
-
 IPv6 literal addresses must be bracketed: `[::1]:8000`, not `::1:8000`.
 
 > [!NOTE]
@@ -447,12 +432,6 @@ Number of rows to render per page on the Snapshot and ArchiveResult list views (
 
 Free-form text rendered in the footer of every archive page. Useful for adding a takedown contact, an org disclaimer, or attribution. Plain text — no HTML.
 
----
-#### `CUSTOM_TEMPLATES_DIR`
-**Possible Values:** [`data/custom_templates`]/`/path/to/custom_templates`/...
-
-Path to a directory containing custom HTML / CSS / image overrides for the default ArchiveBox templates. Files placed here shadow the built-in templates of the same path, letting you rebrand the UI without forking. See the Django template loader docs for the resolution order.
-
 ---
 #### `REVERSE_PROXY_USER_HEADER`
 **Possible Values:** [`Remote-User`]/`X-Remote-User`/`X-Forwarded-User`/...
@@ -500,7 +479,7 @@ URL users are redirected to after logging out. The default `/` keeps users on Ar
 Master switch for LDAP authentication. When `True`, ArchiveBox loads the `django-auth-ldap` backend and validates that `LDAP_SERVER_URI`, `LDAP_BIND_DN`, `LDAP_BIND_PASSWORD`, and `LDAP_USER_BASE` are all set — startup fails fast otherwise.
 
 ```bash
-archivebox_spec="${ARCHIVEBOX_PROJECT_DIR:+$ARCHIVEBOX_PROJECT_DIR[ldap]}"; archivebox_spec="${archivebox_spec:-archivebox[ldap] @ git+https://github.com/ArchiveBox/ArchiveBox.git@dev}"; tool_root="$(mktemp -d)"; UV_TOOL_DIR="$tool_root/tools" UV_TOOL_BIN_DIR="$tool_root/bin" uv tool install --python 3.13 --upgrade "$archivebox_spec"
+uv tool install --python 3.13 --upgrade 'archivebox[ldap] @ git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 ```
 
 Then set these configuration values:
@@ -798,7 +777,7 @@ Which search backend engine to use when running `archivebox search` and renderin
 
 - **`ripgrep`** *(default)* — Pure filesystem grep across each Snapshot's archived output (HTML, text, metadata) via the [`search_backend_ripgrep`](https://archivebox.github.io/abx-plugins/#search_backend_ripgrep) plugin. No extra daemon, no extra database to maintain — just install `rg` and it works. Slow on very large collections (each query re-scans the disk) but always 100% correct: results reflect what's actually on disk *right now*, no stale index. Best choice for small-to-medium collections (≲50k snapshots) and for users who don't want to run extra services.
 
-- **`sonic`** — Fast, suggest-style fuzzy search via a running [Sonic](https://github.com/valeriansaliou/sonic) daemon (configured via the [`search_backend_sonic`](https://archivebox.github.io/abx-plugins/#search_backend_sonic) plugin). ArchiveBox pushes text into Sonic at index time and queries it at search time. Sub-millisecond queries even at very large scale, but you have to run and maintain the Sonic process (Docker compose has it built in). Best choice for large collections (≳100k snapshots) when query latency matters.
+- **`sonic`** — Fast, suggest-style fuzzy search via a running [Sonic](https://github.com/valeriansaliou/sonic) daemon (configured via the [`search_backend_sonic`](https://archivebox.github.io/abx-plugins/#search_backend_sonic) plugin). ArchiveBox pushes text into Sonic at index time and queries it at search time. Sub-millisecond queries even at very large scale; ArchiveBox starts the managed service automatically when this backend is selected. Best choice for large collections (≳100k snapshots) when query latency matters.
 
 - **`sqlite`** — FTS5 full-text index stored alongside ArchiveBox's main `index.sqlite3`, configured via the [`search_backend_sqlite`](https://archivebox.github.io/abx-plugins/#search_backend_sqlite) plugin. No extra processes, no extra binary — uses the SQLite already shipped with Python. Faster than `ripgrep` on large collections, slightly slower than `sonic`, but no daemon to babysit. Good middle ground for users who want a real index without operational overhead.
 
@@ -839,7 +818,7 @@ Whether to colorize console output with ANSI escape codes. Defaults to `True` wh
 Override to **force-off** when piping `archivebox` output into a log file or cron-mail wrapper that doesn't strip ANSI codes (otherwise you'll see `^[[31m...^[[0m` litter throughout your logs). Override to **force-on** for tools like `script(1)` or some CI runners that don't report as a TTY but *do* render ANSI correctly.
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; USE_COLOR=False uv run --project "$project_dir" --no-sync archivebox add --plugins=wget "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}" >> archive.log; test -s archive.log
+USE_COLOR=False archivebox add https://example.com >> archive.log
 ```
 
 *Related options:* [`SHOW_PROGRESS`](#show_progress), [`DEBUG`](#debug)
@@ -853,12 +832,7 @@ Whether to render live progress bars during long-running operations (archiving,
 Override to **force-off** in environments where the auto-detection is fooled into thinking it has a TTY (some Docker setups, Kubernetes log collectors, `tmux`/`screen` pipes) but the redrawing carriage-return output ends up as garbage in your logs.
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-printf '%s\n' \
-    "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/one}" \
-    "${ARCHIVEBOX_DOCS_URL_TWO:-https://example.com/two}" > urls.txt
-SHOW_PROGRESS=False uv run --project "$project_dir" --no-sync archivebox add --plugins=wget < urls.txt
+SHOW_PROGRESS=False archivebox add < urls.txt
 ```
 
 *Related options:* [`USE_COLOR`](#use_color)
@@ -1131,10 +1105,10 @@ A handful of *core* options (documented above on this page) act as the **fallbac
 All plugin options can be set via the same three mechanisms as core options — env var, `ArchiveBox.conf`, or `archivebox config --set` — and inspected with `archivebox config`:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync archivebox config
-uv run --project "$project_dir" --no-sync archivebox config --get SCREENSHOT_RESOLUTION
-uv run --project "$project_dir" --no-sync archivebox config --set SCREENSHOT_RESOLUTION=1920,1080
-uv run --project "$project_dir" --no-sync archivebox config --search wget
+archivebox config                                  # show every option (core + every installed plugin)
+archivebox config --get SCREENSHOT_RESOLUTION      # read one value
+archivebox config --set SCREENSHOT_RESOLUTION=1920,1080
+archivebox config --search wget                    # search options by name/description
 ```
 
 ### Why is plugin config documented separately?

docs/Docker.md

@@ -2,7 +2,7 @@
 
 ## Overview
 
-Running ArchiveBox with Docker allows you to manage it in a container without exposing it to the rest of your system. ArchiveBox generally works the same in Docker as it does outside Docker. You can even use `pip`-installed ArchiveBox and Docker ArchiveBox in tandem, as they both share the same data directory format.
+Running ArchiveBox with Docker allows you to manage it in a container without exposing it to the rest of your system. ArchiveBox generally works the same in Docker as it does outside Docker. You can even use `uv`-installed ArchiveBox and Docker ArchiveBox in tandem, as they both share the same data directory format.
 
 <img src="https://imgur.zervice.io/qFAPRwC.png" width="20%" align="right"/>
 
@@ -68,15 +68,9 @@ docker compose run archivebox init
 docker compose run archivebox manage createsuperuser
 ```
 
-To use [Sonic](https://github.com/valeriansaliou/sonic) for improved full-text search, download this config & uncomment the sonic service in `docker-compose.yml`:
+To use [Sonic](https://github.com/valeriansaliou/sonic) for improved full-text search, select it as the search backend. ArchiveBox installs and starts the managed service automatically:
 ```bash
-# download the sonic config file into your data folder (e.g. ~/archivebox)
-curl -fsSL 'https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/etc/sonic.cfg' > sonic.cfg
-
-# then uncomment the sonic-related sections in docker-compose.yml
-nano docker-compose.yml
-
-# to backfill any existing archive data into the search index, run:
+docker compose run archivebox config --set SEARCH_BACKEND_ENGINE=sonic
 docker compose run archivebox update --index-only
 ```
 
@@ -113,7 +107,7 @@ docker compose run -T archivebox add < ~/Downloads/example_urls.txt
 docker compose run archivebox add --depth=1 /data/sources/example_urls.txt
 
 # OR pipe in URLs from a remote source
-curl 'https://example.com/some/rss/feed.xml' | docker compose run archivebox add
+curl 'https://example.com/some/rss/feed.xml' | docker compose run -T archivebox add
 docker compose run archivebox add --depth=1 'https://example.com/some/rss/feed.xml'
 ```
 
@@ -130,16 +124,16 @@ docker compose run archivebox add --depth=1 'https://example.com/some/feed.rss'
 
 ### Accessing the data
 
-The outputted archive data is stored in `data/` (relative to the project root), or whatever folder path you specified in the `docker-compose.yml` `volumes:` section. Make sure the `data/` folder on the host has permissions initially set to `777` so that the ArchiveBox command is able to set it to the specified `OUTPUT_PERMISSIONS` config setting on the first run.
+The outputted archive data is stored in `data/` (relative to the project root), or whatever folder path you specified in the `docker-compose.yml` `volumes:` section. The mounted directory must be writable by its current owner; the entrypoint detects that non-root owner and runs ArchiveBox with matching permissions.
 
-To access the results directly via the filesystem, open `./data/archive/<timestamp>/index.html` (timestamp is shown in output of previous command).
+To access a result directly via the filesystem, follow its backwards-compatible `./data/archive/<timestamp>` symlink, or browse the canonical `./data/archive/users/<user>/snapshots/<date>/<domain>/<uuid>/` tree.
 
 Alternatively, to use the web UI, start the server with:
 ```bash
 docker compose up         # add -d to run in the background
 ```
 
-Then open [`http://127.0.0.1:8000`](http://127.0.0.1:8000).
+Then open [`http://web.archivebox.localhost:8000`](http://web.archivebox.localhost:8000) for the public UI or [`http://admin.archivebox.localhost:8000`](http://admin.archivebox.localhost:8000) for the admin UI.
 
 <br/>
 
@@ -150,29 +144,29 @@ ArchiveBox running with `docker compose` accepts all the same config options as
 The recommended way configure ArchiveBox in Docker Compose is using `archivebox config --set ...` or by editing `ArchiveBox.conf`.
 ```bash
 docker compose run archivebox config --set TIMEOUT=120
-# OR
-echo 'TIMEOUT=120' >> ./data/ArchiveBox.conf
+# OR edit ./data/ArchiveBox.conf and add this under its existing [ARCHIVING_CONFIG] section:
+TIMEOUT=120
 
 # plugin-specific options work the same way (see https://archivebox.github.io/abx-plugins/)
-docker compose run archivebox config --set MEDIA_MAX_SIZE=750mb
+docker compose run archivebox config --set YTDLP_MAX_SIZE=750m
 ```
 This will apply the config to all containers or archivebox instances that access the collection.
 
 If you're only running one container, or if you want to scope config options to only apply to a particular container, you can set them in that container's `environment:` section:
 
 ```yaml
-# ...
+...
 
 services:
     archivebox:
-        # ...
+        ...
         environment:
             - USE_COLOR=False
             - SHOW_PROGRESS=False
             - CHECK_SSL_VALIDITY=False
             - RESOLUTION=1900,1820
             - MEDIA_TIMEOUT=512000
-        # ...
+        ...
 ```
 
 You can also specify an env file via CLI when running compose using `docker compose --env-file=/path/to/config.env ...` although you must specify the variables in the `environment:` section that you want to have passed down to the ArchiveBox container from the passed env file.
@@ -261,7 +255,7 @@ docker run -it -v /media/USB-DRIVE/archivebox/data:/data archivebox/archivebox:d
 Then to view your data, you can look in the folder on the host `/media/USB-DRIVE/archivebox/data`, or use the Web UI:
 ```bash
 docker run -it -v /media/USB_DRIVE/archivebox/data:/data -p 8000:8000 archivebox/archivebox:dev
-# then open https://127.0.0.1:8000
+# then open http://web.archivebox.localhost:8000
 ```
 
 <br/>
@@ -273,8 +267,8 @@ The easiest way is to use `archivebox config --set KEY=value` or edit `./Archive
 For example, this sets `TIMEOUT=120` as a persistent setting for the collection:
 ```bash
 docker run -it -v $PWD:/data archivebox/archivebox:dev config --set TIMEOUT=120
-# OR
-echo 'TIMEOUT=120' >> ./ArchiveBox.conf
+# OR edit ./ArchiveBox.conf and add this under its existing [ARCHIVING_CONFIG] section:
+TIMEOUT=120
 ```
 
 ArchiveBox in Docker also accepts config as environment variables, see more on the [[Configuration]] page (and the [abx-plugins config reference](https://archivebox.github.io/abx-plugins/) for per-plugin options).

docs/Install.md

@@ -24,7 +24,7 @@ ArchiveBox is primarily distributed as a Python package installed with `uv`, but
 **CPU Architectures:** `amd64` (`x86_64`), `arm64` (`aarch64`), `arm7`  
 *(Including 64-bit Intel/AMD, M1/M2/etc. Macs, Raspberry Pi >= 3)*
 
-* [**macOS:**](#macos) >=10.12 (with `pip`)
+* [**macOS:**](#macos) >=10.12 (with `uv` or Homebrew)
 * [**Linux:**](#ubuntudebian) Ubuntu (>= 18.04), Debian (>= 10), etc. (with `apt`)
 * [**BSD:**](#bsd) FreeBSD, OpenBSD, NetBSD etc (with `pkg`)
 
@@ -36,7 +36,7 @@ Other systems are not officially supported but may work with degraded functional
  * **Windows:** Via [[Docker]], Docker in WSL2, or WSL2 without Docker (not recommended)
  * [Other UNIX systems:](https://github.com/ArchiveBox/ArchiveBox#-package-manager-setup) Arch, Nix, Guix, Fedora, SUSE, Arch, CentOS, etc.
 
-<sub>Note: On `arm7` the `playwright` package is not available, so `chromium` must be installed manually if needed.</sub>
+<sub>Note: Some managed binary providers do not publish `arm7` builds. Run `archivebox install` to see which compatible host or managed providers are available for your platform.</sub>
 
 <br/>
 
@@ -80,8 +80,8 @@ If you're on Linux with `apt` or FreeBSD with `pkg` there is an optional auto-se
 *(or scroll further down for manual install instructions)*
 
 ```bash
-set -euo pipefail; setup_script="$(mktemp)"; curl -fsSL "file://${ARCHIVEBOX_PROJECT_DIR:-$PWD}/bin/setup.sh" > "$setup_script"
-bash -n "$setup_script"; cmp "$setup_script" "${ARCHIVEBOX_PROJECT_DIR:-$PWD}/bin/setup.sh"
+curl -fsSL 'https://get.archivebox.io' | bash
+# shortcut to run https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/bin/setup.sh
 ``` 
 The script explains what it installs beforehand, and will prompt for user confirmation before making any changes to your system. The script uses Docker if already installed, but you can decline and it will install ArchiveBox using `uv` instead.
 
@@ -113,9 +113,9 @@ See our [Dependencies](https://github.com/ArchiveBox/ArchiveBox#dependencies) do
 
 <br/>
 
-### 1. Install base system dependencies needed for your OS
+### 1. Install `uv` or the ArchiveBox OS package
 
-*Be aware, you'll need to keep all these packages up-to-date yourself over time!*
+ArchiveBox itself is the only tool you need to bootstrap manually. After that, `archivebox install` resolves every runtime dependency through `abxpkg`, preferring compatible host binaries and installing managed ones only when needed.
 
 <img src="https://imgur.zervice.io/Ue9BI7n.png" width="30px" align="right"/>
 
@@ -124,17 +124,9 @@ See our [Dependencies](https://github.com/ArchiveBox/ArchiveBox#dependencies) do
 Make sure you have [Homebrew](https://brew.sh/) installed first.
 
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-brew install uv node git wget curl ffmpeg yt-dlp ripgrep sonic
-tool_root="$(mktemp -d)"; export UV_TOOL_DIR="$tool_root/tools" UV_TOOL_BIN_DIR="$tool_root/bin"
-uv tool install --python 3.13 --upgrade "$project_dir"
-archivebox_data="$(mktemp -d)"; cd "$archivebox_data"
-"$UV_TOOL_BIN_DIR/archivebox" init
-"$UV_TOOL_BIN_DIR/archivebox" install
-"$UV_TOOL_BIN_DIR/archivebox" version
-brew list --versions uv node git wget curl ffmpeg yt-dlp ripgrep sonic
-brew info ffmpeg >/dev/null
-brew info --cask chromium >/dev/null
+brew install uv
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
+archivebox install
 ```
 
 <img src="https://assets.ubuntu.com/v1/c5cb0f8e-picto-ubuntu.svg" width="30px" align="right"/>
@@ -144,14 +136,14 @@ brew info --cask chromium >/dev/null
 Use the third-party ArchiveBox apt repo for the simplest bare-metal install:
 
 ```bash
-set -euo pipefail; echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' > /etc/apt/sources.list.d/archivebox.list
-apt-get update
-apt-get install -y archivebox
-archivebox_data="$(mktemp -d)"; cd "$archivebox_data"
+echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' | sudo tee /etc/apt/sources.list.d/archivebox.list
+sudo apt update
+sudo apt install archivebox
+
+mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
 archivebox install
-archivebox add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
-archivebox status
+archivebox add 'https://example.com'
 ```
 
 The apt package is a thin dev-channel wrapper around the normal Python install
@@ -166,17 +158,19 @@ if you want it to install missing system packages via apt.
 #### FreeBSD
 
 ```bash
-set -euo pipefail; pkg install -y python313 git wget curl yt-dlp ripgrep py313-sqlite3 npm-node22 ffmpeg
-pkg install -y chromium
-python3.13 --version; node --version; git --version
-wget --version; curl --version; yt-dlp --version; rg --version
-ffmpeg -version; chromium --version
+sudo pkg install curl
+curl -LsSf https://astral.sh/uv/install.sh | sh
+export PATH="$HOME/.local/bin:$PATH"
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
+archivebox install
 ```
 
 #### OpenBSD
 
 ```bash
-set -euo pipefail; pkg_add python313 node wget git curl yt-dlp ffmpeg ripgrep chromium; python3.13 --version; node --version; chromium --version
+doas pkg_add uv
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
+archivebox install
 ```
 
 #### Arch Linux / Nix / Guix / etc. Other OSs
@@ -193,10 +187,11 @@ See the [Quickstart](https://github.com/ArchiveBox/ArchiveBox#-package-manager-s
 If you are not using the apt package above, install ArchiveBox with `uv`.
 
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-tool_root="$(mktemp -d)"; export UV_TOOL_DIR="$tool_root/tools" UV_TOOL_BIN_DIR="$tool_root/bin"
-uv tool install --python 3.13 --upgrade "$project_dir"
-"$UV_TOOL_BIN_DIR/archivebox" --help
+# get the dev version of ArchiveBox
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
+
+# if the optional ldap extra must compile locally on Debian/Ubuntu, install its headers and retry
+# sudo apt install build-essential libldap2-dev libsasl2-dev
 ```
 
 <br/>
@@ -205,14 +200,21 @@ uv tool install --python 3.13 --upgrade "$project_dir"
 
 Finish installing runtime dependencies for the enabled ArchiveBox plugins.
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"
-cd "$archivebox_data"
-uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox install
-uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
-uv run --project "$project_dir" --no-sync archivebox version
-uv run --project "$project_dir" --no-sync archivebox help
+# create a new empty folder anywhere to hold your collection, and cd into it
+mkdir -p ~/archivebox/data && cd ~/archivebox/data
+
+# instantiate the directory as an archivebox collection dir
+archivebox init
+
+# auto-install runtime dependencies such as Chromium, yt-dlp, SingleFile, etc.
+archivebox install
+
+# archive a first URL
+archivebox add 'https://example.com'
+
+# ✅ see a final detailed breakdown of all the installed dependencies and commands available
+archivebox version
+archivebox help
 ```
 
 <br/>
@@ -221,13 +223,15 @@ uv run --project "$project_dir" --no-sync archivebox help
 
 Make sure the `uv`-installed version of `archivebox` is available in your `$PATH`.
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"; cd "$archivebox_data"
-uv run --project "$project_dir" --no-sync archivebox init
-uv tool list
-uv run --project "$project_dir" --no-sync archivebox version
-uv run --project "$project_dir" --no-sync archivebox status
-uv run --project "$project_dir" --no-sync archivebox help
+uv tool list             # show info about uv-installed tools
+
+echo $PATH               # show the directories your system is searching for binaries
+type -a archivebox       # show all installed archivebox binaries available
+
+cd ~/archivebox/data
+archivebox version       # ⭐️ show lots of useful info about installed dependencies and more
+archivebox status
+archivebox help
 ```
 (ensure the version shown is the most recent available from [Releases](https://github.com/ArchiveBox/ArchiveBox/releases))  
   
@@ -244,22 +248,22 @@ If you have issues getting Chromium / Google Chrome or other dependencies workin
 For guides on how to import URLs from different sources into ArchiveBox, check out [Input Formats](https://github.com/ArchiveBox/ArchiveBox#input-formats) and [Preparing URLs](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive). ➡️
 
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
+cd ~/archivebox/data
 ```
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-printf '%s\n' "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}" > bookmarks_export.html
-uv run --project "$project_dir" --no-sync archivebox add --help; uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls < bookmarks_export.html
+# feed in your URLs to start archiving!
+archivebox add --help
+archivebox add < ~/Downloads/bookmarks_export.html
 ```
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox list
-uv run --project "$project_dir" --no-sync archivebox status
+# inspect the newly added Snapshots via the CLI
+archivebox list
+archivebox status
 ```
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox server --help
-printf 'Open http://localhost:%s\n' "${ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT:-8000}"
+# OR start the webserver and view them in the Web UI
+archivebox server 0.0.0.0:8000
+open http://web.archivebox.localhost:8000
 ```
 See our [[Usage]] Wiki documentation page for more examples.
 
@@ -267,15 +271,16 @@ See our [[Usage]] Wiki documentation page for more examples.
 
 ### Next Steps: *Upgrading Archivebox to a new version*
 
-Make sure all apt/brew/pkg/etc. dependencies from above are installed & up-to-date first.
+Upgrade ArchiveBox itself first; `archivebox install` will then re-resolve compatible host binaries and update any managed runtime dependencies.
 
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-tool_root="$(mktemp -d)"; export UV_TOOL_DIR="$tool_root/tools" UV_TOOL_BIN_DIR="$tool_root/bin"
-uv tool install --python 3.13 --upgrade "$project_dir"
-archivebox_data="$(mktemp -d)"; cd "$archivebox_data"
-"$UV_TOOL_BIN_DIR/archivebox" init
-"$UV_TOOL_BIN_DIR/archivebox" install
+# get the dev version of ArchiveBox
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
+
+# run init inside any data directories to migrate the index to the latest version
+cd ~/archivebox/data
+archivebox init          # update collection index & apply any migrations
+archivebox install       # update runtime dependencies to latest versions
 ```
 
 Check our more detailed [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives) documentation and [Release Notes](https://github.com/ArchiveBox/ArchiveBox/releases) if you run into any problems. ➡️

docs/Merging-Collections.md

@@ -1,60 +1,52 @@
 # Merging Collections
 
-Two or more existing ArchiveBox collection dirs can be merged together by simply combining the contents of `archive/*` and re-running `archivebox init` to pull the new Snapshots into the index.
+Current ArchiveBox collections cannot be merged safely by copying their `archive/users/...` trees: the database owns the Crawl, Snapshot, user, permission, and state-machine records, and `archivebox init` intentionally does not import orphaned current-layout directories. For current collections, use a database-aware migration or export the source URLs and re-archive them into the destination collection. Copying current Snapshot directories alone is a backup operation, not a merge.
+
+The workflow below is retained for **legacy collections whose real Snapshot directories are `archive/<timestamp>/`**. `archivebox update` can import those legacy directories into a fresh index.
 
 > [!WARNING]
-> Snapshot folders are identified by their timestamp (in milliseconds), this is normally not a problem for archives collected on one machine, but when merging archives from two different instances that ran at the same time it means there is a small chance of conflicts. Check the contents of `archive/` before merging, and backup any directories that may conflict before proceeding.
+> Back up every collection before merging. Confirm that the source entries are real legacy timestamp directories, not compatibility symlinks into `archive/users/...`, and inspect path conflicts instead of allowing one collection to overwrite another.
 
-1. Run `archivebox init` and `archivebox status` in each existing collection to apply migrations and confirm that both collections use the current ArchiveBox version. The complete example below creates two temporary collections so the merge can be reproduced safely; replace those paths with your existing collection paths.
+1. Upgrade both old collections to the most recent ArchiveBox version (following instructions above)
   ```bash
-  set -euo pipefail
-  merge_root="$(mktemp -d)"
-  trap 'rm -rf "$merge_root"' EXIT
-  collection_one="$merge_root/archivebox1"
-  collection_two="$merge_root/archivebox2"
-  merged_collection="$merge_root/archivebox_new"
-
-  mkdir -p "$collection_one" "$collection_two"
-  cd "$collection_one"
+  cd /path/to/archivebox1/data
   archivebox init
-  archivebox add --plugins=wget "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/?collection=one}"
   archivebox status
 
-  cd "$collection_two"
+  cd /path/to/archivebox2/data
   archivebox init
-  archivebox add --plugins=wget "${ARCHIVEBOX_DOCS_URL_TWO:-https://example.com/?collection=two}"
   archivebox status
+
+  # ... repeat the same for each collection if merging more than two
   ```
 
 2. Create a new empty archivebox collection in a new folder somewhere, this will hold the new merged collection
-  <!--pytest-codeblocks:cont-->
   ```bash
-  mkdir -p "$merged_collection"
-  cd "$merged_collection"
+  mkdir -p /path/to/archivebox_new/data
+  cd /path/to/archivebox_new/data
   archivebox init
   ```
 
-3. Copy everything under `./archive/*` in each old collection into the new collection's `./archive/` folder
-  <!--pytest-codeblocks:cont-->
+3. Copy the real legacy `archive/<timestamp>/` directories from each old collection into the new collection's `archive/` folder.
   ```bash
-  rsync --archive "$collection_one/archive/" "$merged_collection/archive/"
-  rsync --archive "$collection_two/archive/" "$merged_collection/archive/"
+  rsync --archive --info=progress2 /path/to/archivebox1/data/archive/ /path/to/archivebox_new/data/archive/
+  rsync --archive --info=progress2 /path/to/archivebox2/data/archive/ /path/to/archivebox_new/data/archive/
+  # ...repeat the same for each collection if merging more than two
   ```
 
-4. Run `archivebox update` in the new merged collection to import the copied Snapshot directories and regenerate the index
-  <!--pytest-codeblocks:cont-->
+4. Run `archivebox update` in the new collection to import the legacy directories
   ```bash
-  cd "$merged_collection"
-  archivebox update --index-only
+  cd /path/to/archivebox_new/data
+  archivebox update
   ```
 
 5. The new collection should now contain all the entries from the old collections combined
-  <!--pytest-codeblocks:cont-->
   ```bash
-  cd "$merged_collection"
+  cd /path/to/archivebox_new/data
   archivebox status
 
-  test "$(find archive/users/system/snapshots -name index.jsonl | wc -l | tr -d ' ')" -eq 2
+  # optionally force an update of the snapshot index files (normally done lazily)
+  archivebox update --index-only
   ```
   For more information about why Snapshot index files are usually updated lazily, see: https://github.com/ArchiveBox/ArchiveBox/issues/962
 
@@ -69,12 +61,8 @@ If you need to automate changes to the ArchiveBox DB (for example adding a User
 Note, this is often unnecessary for modifying ArchiveBox on a host that doesn't have the CLI installed, as you can also copy the `index.sqlite3` to a local machine that has it, do the modifications locally, then copy the modified db back into place on the host. (Docker/CLI/GUI/Web ArchiveBox all share the same DB schema/format)
 
 ```bash
-set -euo pipefail
-collection="$(mktemp -d)"
-trap 'rm -rf "$collection"' EXIT
-cd "$collection"
-archivebox init
-sqlite3 index.sqlite3 'SELECT COUNT(*) FROM core_snapshot;'
+cd ~/archivebox/data    # cd into your archivebox collection dir
+sqlite3 index.sqlite3   # open the db with sqlite3 shell
 ```
 
 #### Example: Modifying an existing user's email
@@ -91,31 +79,23 @@ WHERE username = 'someUsernameHere';
 
 1. First, generate the hashed password in a Python shell using Django's `make_password` function.
 
-This can be done on any machine with Python 3+, it doesn't have to have ArchiveBox installed.
+Use the Django version bundled with the ArchiveBox installation that owns the collection:
   ```bash
-uv run python -c "from django.contrib.auth.hashers import PBKDF2PasswordHasher; print(PBKDF2PasswordHasher().encode('somePasswordHere', 'someSaltHere'))"
-```
-```python
-from django.contrib.auth.hashers import PBKDF2PasswordHasher
-
-hasher = PBKDF2PasswordHasher()
-password_hash = hasher.encode("somePasswordHere", "someSaltHere")
-assert hasher.verify("somePasswordHere", password_hash)
+  archivebox shell -c "from django.contrib.auth.hashers import make_password; print(make_password('somePasswordHere', 'someSaltHere', 'pbkdf2_sha256'))"
+  ```
+```python3
+>>> from django.contrib.auth.hashers import make_password
+>>> make_password('somePasswordHere', 'someSaltHere', 'pbkdf2_sha256')         # choose a password and a salt (can be anything 12 chars long)
+'pbkdf2_sha256$...$someSaltHere$...'
 ```
 2. Use the generated hashed password to insert a new User row in the SQLite3 database directly:
   ```bash
-set -euo pipefail
-collection="$(mktemp -d)"
-trap 'rm -rf "$collection"' EXIT
-cd "$collection"
-archivebox init
-password_hash="$(uv run python -c "from django.contrib.auth.hashers import PBKDF2PasswordHasher; print(PBKDF2PasswordHasher().encode('somePasswordHere', 'someSaltHere'))")"
-sqlite3 index.sqlite3 "INSERT INTO auth_user (password, last_login, is_superuser, username, first_name, last_name, email, is_staff, is_active, date_joined) VALUES ('$password_hash', NULL, 0, 'someUsername', '', '', 'someEmail@example.com', 0, 1, CURRENT_TIMESTAMP);"
-test "$(sqlite3 index.sqlite3 "SELECT COUNT(*) FROM auth_user WHERE username='someUsername';")" -eq 1
+cd ~/archivebox/data    # cd into your archivebox collection dir
+sqlite3 index.sqlite3   # open the db with sqlite3 shell
 ```
 ```sql
 INSERT INTO "auth_user" ("password", "last_login", "is_superuser", "username", "first_name", "last_name", "email", "is_staff", "is_active", "date_joined")
-VALUES ('pbkdf2_sha256$216000$someSaltHere$+2beZufc3JUXnmn0tG+2peJEBh7MjxPYmT3YfIFzEl0=', NULL, 0, 'someUsername', '', '', 'someEmail@example.com', 0, 1, '2022-03-22 23:34:02.333042')
+VALUES ('GENERATED_PASSWORD_HASH', NULL, 0, 'someUsername', '', '', 'someEmail@example.com', 0, 1, '2022-03-22 23:34:02.333042')
 ```
   Replace the values above with the desired username, email, and password hash from python output^.
 

docs/Publishing-Your-Archive.md

@@ -7,87 +7,32 @@ There are two ways to publish your archive: using the `archivebox server` or by
 ## 1. Use the built-in web server
 
 ```bash
-set -euo pipefail
-publish_root="$(mktemp -d)"
-server_pid=""
-log_pid=""
-cleanup() {
-    if [ -n "$server_pid" ]; then
-        kill "$server_pid"
-        wait "$server_pid" || true
-    fi
-    if [ -n "$log_pid" ]; then
-        kill "$log_pid" 2>/dev/null || true
-    fi
-    rm -rf "$publish_root"
-}
-trap cleanup EXIT
-mkdir -p "$publish_root/data"
-cd "$publish_root/data"
-archivebox init
-
 # set the permissions depending on how public/locked down you want it to be
 archivebox config --set PUBLIC_INDEX=True
 archivebox config --set PUBLIC_ADD_VIEW=True
 archivebox config --set PERMISSIONS=public        # default visibility of newly created snapshots (was: PUBLIC_SNAPSHOTS=True)
+archivebox config --set BASE_URL=https://archive.example.com
+archivebox config --set SERVER_SECURITY_MODE=safe-subdomains-fullreplay
 
-# create an admin username and password for yourself (set your own value first)
-: "${ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD:?Set ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD to a unique password}"
-DJANGO_SUPERUSER_USERNAME="${ADMIN_USERNAME:-archivebox-docs-admin}" \
-DJANGO_SUPERUSER_EMAIL="${ADMIN_EMAIL:-archivebox-docs@example.com}" \
-DJANGO_SUPERUSER_PASSWORD="$ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD" \
-archivebox manage createsuperuser --noinput
+# create an admin username and password for yourself
+archivebox manage createsuperuser
 
 # then start the webserver and open the web UI in your browser
-server_port="${ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT:-8000}"
-server_log="$publish_root/archivebox-server.log"
-server_output_fifo="$publish_root/archivebox-server-output"
-server_ready_fifo="$publish_root/archivebox-server-ready"
-mkfifo "$server_output_fifo" "$server_ready_fifo"
-awk -v log="$server_log" -v ready_fifo="$server_ready_fifo" -v pattern="Listening on TCP" '
-    { print >> log; fflush(log) }
-    !ready && $0 ~ pattern { print "ready" > ready_fifo; close(ready_fifo); ready=1 }
-' <"$server_output_fifo" &
-log_pid=$!
-PYTHONUNBUFFERED=1 archivebox server "0.0.0.0:$server_port" >"$server_output_fifo" 2>&1 &
-server_pid=$!
-IFS= read -r readiness < "$server_ready_fifo"
-test "$readiness" = "ready"
-curl --fail --silent --show-error "http://127.0.0.1:$server_port/" >/dev/null
+archivebox server 0.0.0.0:8000
+open https://web.archive.example.com
 ```
 
 This server is enabled out-of-the-box if you're using `docker-compose` to run ArchiveBox.
 If hosting publicly, it's essential to place an SSL termination server in front of ArchiveBox. The bundled compose file includes opt-in `https` (Traefik) and `tunnel` (Cloudflare Tunnel) profiles, or you can bring your own reverse proxy such as [`traefik`](https://github.com/traefik/traefik), [`caddy`](https://caddyserver.com/docs/automatic-https#activation), or [`cloudflared`](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/).
 
 > [!TIP]
-> Advanced: You can use nginx to serve the static `/archive/` dir directly from the filesystem to increase performance.  
-> To protect the `/admin/` dashboard, it should ideally be served from a [different domain](#security-concerns) using redirects.
+> Advanced: You can use nginx to serve a static export directly from the filesystem. Do not proxy live replay paths back onto the admin origin; use ArchiveBox's security-mode routing.
 
 <br/>
 
 ## 2. Export and host it as static HTML
 
 ```bash
-set -euo pipefail
-publish_root="$(mktemp -d)"
-server_pid=""
-log_pid=""
-cleanup() {
-    if [ -n "$server_pid" ]; then
-        kill "$server_pid"
-        wait "$server_pid" || true
-    fi
-    if [ -n "$log_pid" ]; then
-        kill "$log_pid" 2>/dev/null || true
-    fi
-    rm -rf "$publish_root"
-}
-trap cleanup EXIT
-mkdir -p "$publish_root/data"
-cd "$publish_root/data"
-archivebox init
-archivebox add --index-only "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
-
 archivebox list --html --with-headers > index.html
 archivebox list --json --with-headers > index.json
 
@@ -95,22 +40,8 @@ archivebox list --json --with-headers > index.json
 # e.g. github pages or another static hosting provider
 
 # you can also serve it with the simple python HTTP server
-server_port="${ARCHIVEBOX_DOCS_STATIC_PORT:-8001}"
-server_log="$publish_root/static-server.log"
-server_output_fifo="$publish_root/static-server-output"
-server_ready_fifo="$publish_root/static-server-ready"
-mkfifo "$server_output_fifo" "$server_ready_fifo"
-awk -v log="$server_log" -v ready_fifo="$server_ready_fifo" -v pattern="Serving HTTP on" '
-    { print >> log; fflush(log) }
-    !ready && $0 ~ pattern { print "ready" > ready_fifo; close(ready_fifo); ready=1 }
-' <"$server_output_fifo" &
-log_pid=$!
-uv run --no-project python -u -m http.server --bind 0.0.0.0 --directory . "$server_port" >"$server_output_fifo" 2>&1 &
-server_pid=$!
-IFS= read -r readiness < "$server_ready_fifo"
-test "$readiness" = "ready"
-curl --fail --silent --show-error "http://127.0.0.1:$server_port/index.html" >/dev/null
-curl --fail --silent --show-error "http://127.0.0.1:$server_port/index.json" >/dev/null
+python3 -m http.server --bind 0.0.0.0 --directory . 8000
+open http://127.0.0.1:8000
 ```
 
 Here's a sample nginx configuration that works to serve your static archive folder:
@@ -126,7 +57,7 @@ location / {
 
 Make sure you're not running any content as CGI or PHP, you only want to serve static files!
 
-Urls look like: `https://demo.archivebox.io/archive/1493350273/en.wikipedia.org/wiki/Dining_philosophers_problem.html`
+Legacy timestamp URLs remain available through compatibility symlinks, for example: `https://demo.archivebox.io/archive/1493350273/wget/en.wikipedia.org/wiki/Dining_philosophers_problem.html`
 
 <br/>
 
@@ -137,25 +68,13 @@ Urls look like: `https://demo.archivebox.io/archive/1493350273/en.wikipedia.org/
 ## Security Concerns
 
 > [!CAUTION]
-> Re-hosting untrusted archived content on a domain can potentially compromise *all apps on that domain*!  
-> (including other subdomains)
+> Re-hosting untrusted archived content on the same origin as an authenticated application can compromise that application.
 
-Make sure you thoroughly understand the dangers of [hosting untrusted HTML/JS/CSS that may be captured during archiving](https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy), and how viewing it can enable [CSRF attacks](https://en.wikipedia.org/wiki/Cross-site_request_forgery) across all apps on the same domain. If a logged-in user happens to visit an archived page with malicious Javascript embedded, it would allow the JS to hijack any cookies on the domain and pretend to be them, potentially exfiltrating or modifying other Snapshots/data on your server.
+Make sure you understand the dangers of [hosting untrusted HTML/JS/CSS](https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy). The default `SERVER_SECURITY_MODE=safe-subdomains-fullreplay` separates the admin, web, and API control planes from replay content, and gives each Snapshot its own replay subdomain. Admin cookies are scoped away from those replay origins.
 
-(This is why we don't support serving ArchiveBox from a subdirectory like `myapps.example.com/archivebox/`, it's too dangerous to share domains)
+This mode requires wildcard DNS and TLS for `*.archive.example.com`. If your deployment cannot provide wildcard subdomains, use `SERVER_SECURITY_MODE=safe-onedomain-nojsreplay`, which keeps one origin but disables JavaScript replay.
 
-The industry standard approach is to use a separate domain for untrusted content, for example Github uses `githubusercontent.com` and Google uses `googleusercontent.com` for all user-uploaded files. If hosting ArchiveBox publicly, do the same and keep it on an isolated domain in order to mitigate potential damage of leaked cookies, CORS, and CSRF attack.  
-
-### Protecting the Admin Dashboard
-
-To protect the Admin dashboard, it's also recommended to serve all content under `/archive/` on a separate domain from `/admin/`. We do this on our servers using a simple redirect rule in nginx/cloudflare like so:
-
-- https://demo.archivebox.io: only serves `/`, redirects `/archive/*` to `demo-static.`
-- https://demo-static.archivebox.io: only serves `/archive/`, redirects everything else to `demo.`
-
-<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/8d855976-3b4a-4fa8-ad52-999b3c3deba4" width="800px" alt="Cloudflare redirect rule for /archive/ to another domain"/>
-
-> Note: This is still recommended, but less critical if your `/archive/` folder does not contain any archived JS (e.g. if you set [`WGET_ENABLED=False`](https://archivebox.github.io/abx-plugins/#wget) and [`DOM_ENABLED=False`](https://archivebox.github.io/abx-plugins/#dom)).
+Do not serve ArchiveBox from a shared subdirectory such as `myapps.example.com/archivebox/`; it cannot provide the required origin isolation. If you do not need JavaScript-capable replay, you can also disable the relevant extractors with `WGET_ENABLED=False` and `DOM_ENABLED=False`.
 
 More info:
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
@@ -183,7 +102,7 @@ Archiving for personal backups, research, and some other use-cases are covered b
 
 Please modify the [`FOOTER_INFO`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info) config variable to add your contact info to the footer of your index.
 
-Note: ArchiveBox prevents search engines from indexing your archives using [`/robots.txt`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/static/robots.txt#L2) by default. It's not recommended to [disable](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir) this as it often leads to a flood of automated takedown requests and abuse reports to your hosting provider (from anti-piracy bots that scan for cloned copyrighted content via search engines).
+Note: ArchiveBox prevents search engines from indexing your archives using [`/robots.txt`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/static/robots.txt#L2) by default. It is not recommended to override this file in the collection's fixed `custom_templates/` directory, as public indexing often leads to automated takedown requests and abuse reports.
 
 *Keep in mind individuals, companies, schools, and libraries all have different copyright exemptions in different countries. Double check the specific laws for your situation in your own jurisdiction!*
 

docs/Quickstart.md

@@ -32,9 +32,9 @@ Follow the links here to find instructions for exporting a list of URLs from eac
  - [Safari Bookmarks](http://imgur.zervice.io/AtcvUZA.png)
  - [Opera Bookmarks](http://help.opera.com/Windows/12.10/en/importexport.html)
  - [Internet Explorer Bookmarks](https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows)
- - Chrome History: `./bin/export_browser_history.sh --chrome`
- - Firefox History: `./bin/export_browser_history.sh --firefox`
- - Safari History: `./bin/export_browser_history.sh --safari`
+ - Chrome History: `bash ./bin/export_browser_history.sh --chrome`
+ - Firefox History: `bash ./bin/export_browser_history.sh --firefox`
+ - Safari History: `bash ./bin/export_browser_history.sh --safari`
  - Other File or URL: (e.g. RSS feed url, text file path) pass as second argument in the next step
 
  (If any of these links are broken, please submit an issue and I'll fix it)
@@ -43,17 +43,17 @@ Follow the links here to find instructions for exporting a list of URLs from eac
 
 Pass in URLs directly, import a list of links from a file, or import from a feed URL. All via stdin:
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"
-cd "$archivebox_data"
-uv run --project "$project_dir" --no-sync archivebox init
-printf '%s\n' "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/one}" > your_urls.txt
-uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls < your_urls.txt
-curl -fsSL "${ARCHIVEBOX_DOCS_URL_TWO:-https://example.com/two}" | uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls
-uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
-uv run --project "$project_dir" --no-sync archivebox list --json
-uv run --project "$project_dir" --no-sync archivebox status
-uv run --project "$project_dir" --no-sync archivebox search example
+archivebox add < your_urls.txt
+
+# or if using plain Docker
+docker run -v $PWD:/data -i archivebox/archivebox:dev add < your_urls.txt
+
+# or if using Docker Compose
+docker compose run -T archivebox add < your_urls.txt
+
+# any text containing URLs can ingested via stdin or as args
+curl -fsSL 'https://getpocket.com/users/YOURUSERNAME/feed/all' | archivebox add
+archivebox add 'https://example.com'
 ```
 
 ## ✅ Done!
@@ -62,16 +62,16 @@ Open `./archive` to view your archive data in the filesystem.
 
 You can also use the interactive Web UI to view/manage/add links to your archive:
 ```bash
-docker_data="$(mktemp -d)"
-docker run --rm -v "$docker_data:/data" archivebox-docs-ci init
-docker run --rm -v "$docker_data:/data" archivebox-docs-ci add --plugins=parse_txt_urls 'https://example.com/'
-docker run --rm -v "$docker_data:/data" archivebox-docs-ci list --json
-compose_file="$(mktemp)"
-printf 'services:\n  archivebox:\n    image: archivebox-docs-ci\n    volumes:\n      - %s:/data\n' "$docker_data" > "$compose_file"
-docker compose -f "$compose_file" run --rm archivebox status
-docker compose -f "$compose_file" run --rm archivebox server --help
-docker run --rm -v "$docker_data:/data" archivebox-docs-ci server --help
-docker version
+# with plain Docker:
+docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
+
+# with Docker Compose:
+docker compose up -d
+
+# or without Docker:
+archivebox server
+
+open http://web.archivebox.localhost:8000
 ```
 
 ---
@@ -79,7 +79,7 @@ docker version
 **Next Steps:**
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync archivebox help
+archivebox help   # see info about all the available commands
 ```
 
  - Read [[Usage]] to learn about the various CLI and web UI functions

docs/Scheduled-Archiving.md

@@ -14,14 +14,14 @@ One-shot foreground flows such as `archivebox add ...` continue to process only
 ## CLI Usage
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"
-cd "$archivebox_data" && uv run --project "$project_dir" --no-sync archivebox init
-PLUGINS=parse_txt_urls uv run --project "$project_dir" --no-sync archivebox schedule --every=daily --depth=1 "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/feed.xml}"
-PLUGINS=parse_txt_urls uv run --project "$project_dir" --no-sync archivebox schedule --every='0 */6 * * *' "${ARCHIVEBOX_DOCS_URL_TWO:-https://example.com/feed.xml}"
-uv run --project "$project_dir" --no-sync archivebox schedule --show
-uv run --project "$project_dir" --no-sync archivebox schedule --run-all && uv run --project "$project_dir" --no-sync archivebox schedule --clear
-uv run --project "$project_dir" --no-sync archivebox schedule --foreground --help
+cd ~/archivebox/data
+
+archivebox schedule --every=daily --depth=1 https://example.com/feed.xml
+archivebox schedule --every='0 */6 * * *' https://example.com/feed.xml
+archivebox schedule --show
+archivebox schedule --clear
+archivebox schedule --run-all
+archivebox schedule --foreground
 ```
 
 Accepted schedule formats:
@@ -43,7 +43,7 @@ With the new orchestrator flow, you only need the main `archivebox` service:
 services:
   archivebox:
     image: archivebox/archivebox:dev
-    command: server --quick-init 0.0.0.0:8000
+    command: server --init 0.0.0.0:8000
     volumes:
       - ./data:/data
 ```
@@ -51,8 +51,8 @@ services:
 Create schedules with:
 
 ```bash
-compose_file="$(mktemp)"; docker_data="$(mktemp -d)"; printf 'services:\n  archivebox:\n    image: archivebox-docs-ci\n    volumes:\n      - %s:/data\n' "$docker_data" > "$compose_file"; docker compose -f "$compose_file" run --rm archivebox init
-docker compose -f "$compose_file" run --rm archivebox schedule --every=weekly --depth=1 https://example.com/feed.xml && docker compose -f "$compose_file" run --rm archivebox schedule --show
+docker compose run --rm archivebox schedule --every=weekly --depth=1 https://example.com/feed.xml
+docker compose run --rm archivebox schedule --show
 ```
 
 If the main `archivebox server` container is already running, its orchestrator will pick up future scheduled runs automatically. There is no scheduler sidecar to restart.
@@ -62,25 +62,25 @@ If the main `archivebox server` container is already running, its orchestrator w
 Archive a Twitter mirror once a week:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync archivebox schedule --every=weekly --depth=1 'https://nitter.net/ArchiveBoxApp'
+archivebox schedule --every=weekly --depth=1 'https://nitter.net/ArchiveBoxApp'
 ```
 
 Archive a subreddit and linked discussions once a week:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync archivebox config --set URL_ALLOWLIST='^http(s)?:\/\/(.+)?teddit\.net\/?.*$'
-uv run --project "$project_dir" --no-sync archivebox schedule --every=weekly --depth=1 'https://teddit.net/r/DataHoarder/'
+archivebox config --set URL_ALLOWLIST='^http(s)?:\/\/(.+)?teddit\.net\/?.*$'
+archivebox schedule --every=weekly --depth=1 'https://teddit.net/r/DataHoarder/'
 ```
 
 Archive Hacker News every day:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync archivebox config --set URL_DENYLIST='^http(s)?:\/\/(.+\.)?(youtube\.com)|(amazon\.com)\/.*$'
-uv run --project "$project_dir" --no-sync archivebox schedule --every=daily --depth=1 'https://news.ycombinator.com'
+archivebox config --set URL_DENYLIST='^http(s)?:\/\/(.+\.)?(youtube\.com)|(amazon\.com)\/.*$'
+archivebox schedule --every=daily --depth=1 'https://news.ycombinator.com'
 ```
 
 Queue a daily maintenance update:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync archivebox schedule --every=day
+archivebox schedule --every=day
 ```

docs/Security-Overview.md

@@ -6,11 +6,11 @@
 ## Web UI Permissions
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox config --set PUBLIC_INDEX=False && uv run --project "$project_dir" --no-sync archivebox config --set PUBLIC_ADD_VIEW=False
-uv run --project "$project_dir" --no-sync archivebox config --set PERMISSIONS=private
-uv run --project "$project_dir" --no-sync archivebox manage createsuperuser --help && uv run --project "$project_dir" --no-sync archivebox manage changepassword --help
+archivebox config --set PUBLIC_INDEX=False      # require login to access the list of Snapshots
+archivebox config --set PUBLIC_ADD_VIEW=False   # require log-in to submit new URLs for archiving
+archivebox config --set PERMISSIONS=private     # default new snapshots to login-required (was: PUBLIC_SNAPSHOTS=False)
+
+archivebox manage [createsuperuser|changepassword] # create/modify admin UI users
 ```
 
 See [[Setting Up Authentication]] for more...
@@ -30,10 +30,10 @@ This is the default (lax) mode, intended for archiving public (non-secret) URLs
 The default mode should not be used for archiving entire browser history or authenticated private content like Google Docs, paywalled content, invite-only subreddits, private photo share urls, etc.
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox config --set ARCHIVEDOTORG_ENABLED=True
-uv run --project "$project_dir" --no-sync archivebox config --set CHROME_ISOLATION=snapshot
-uv run --project "$project_dir" --no-sync archivebox config --set COOKIES_FILE=None
+# (these are the defaults)
+archivebox config --set ARCHIVEDOTORG_ENABLED=True   # see https://archivebox.github.io/abx-plugins/#archivedotorg
+archivebox persona create public
+archivebox add --persona=public 'https://example.com'
 ```
 
 
@@ -44,12 +44,12 @@ uv run --project "$project_dir" --no-sync archivebox config --set COOKIES_FILE=N
 ArchiveBox is able to archive content that requires authentication or cookies, but it comes with some caveats. Create dedicated logins for archiving to access paywalled content, private forums, LAN-only content, etc. then share them with ArchiveBox via Chrome profile + cookies.txt file.
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cookies_file="$(mktemp)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync archivebox persona create personal
-uv run --project "$project_dir" --no-sync archivebox config --set ARCHIVEDOTORG_ENABLED=False && uv run --project "$project_dir" --no-sync archivebox config --set COOKIES_FILE="$cookies_file"
-uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls --persona=personal "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
+archivebox config --set ARCHIVEDOTORG_ENABLED=False
+archivebox persona create --import=chrome personal
+archivebox add --persona=personal 'https://members.example.com/'
 ```
 
-To get started, set [`CHROME_USER_DATA_DIR`](https://archivebox.github.io/abx-plugins/#chrome) and [`COOKIES_FILE`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#cookies_file) to point to a Chrome user folder that has your sessions and a wget `cookies.txt` file respectively.
+To get started, import a dedicated browser profile into a [persona](https://github.com/ArchiveBox/ArchiveBox/wiki/Personas). A persona keeps its Chrome profile and `cookies.txt` together and applies the same identity consistently across extractors.
 
 ➡️ For full instructions on setting up a Chromium user profile see here: https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile
 
@@ -77,21 +77,13 @@ If you're importing private links or authenticated content, you probably don't w
 ### Publishing
 
 > [!CAUTION]
-> Re-hosting untrusted archived content on a domain can potentially compromise *all apps on that domain*!  
-> (including other subdomains)
+> Re-hosting untrusted archived content on the same origin as an authenticated application can compromise that application.
 
-Make sure you thoroughly understand the dangers of [hosting untrusted HTML/JS/CSS that may be captured during archiving](https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy), and how viewing it can enable [CSRF attacks](https://en.wikipedia.org/wiki/Cross-site_request_forgery) across all apps on the same domain. If a logged-in user happens to visit an archived page with malicious Javascript embedded, it would allow the JS to hijack any cookies on the domain and pretend to be them, potentially exfiltrating or modifying other Snapshots/data on your server.
+Make sure you understand the dangers of [hosting untrusted HTML/JS/CSS](https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy). The default `SERVER_SECURITY_MODE=safe-subdomains-fullreplay` separates admin, web, and API control-plane origins from replay content, and gives each Snapshot its own replay subdomain so archived JavaScript cannot share admin cookies.
 
-(This is why we don't support serving ArchiveBox from a subdirectory like `myapps.example.com/archivebox/`, it's too dangerous to share domains)
+This mode requires wildcard DNS and TLS for your configured `BASE_URL`. If your deployment cannot provide wildcard subdomains, use `SERVER_SECURITY_MODE=safe-onedomain-nojsreplay`, which keeps one origin but disables JavaScript replay.
 
-The industry standard approach is to use a separate domain for untrusted content, for example Github uses `githubusercontent.com` and Google uses `googleusercontent.com` for all user-uploaded files. If hosting ArchiveBox publicly, do the same and keep it on an isolated domain in order to mitigate potential damage of leaked cookies, CORS, and CSRF attacks.  
-
-To protect the Admin dashboard, it's also recommended to serve all content under `/archive/` on a separate domain from `/admin/`. We do this on our servers using a simple redirect rule in nginx/cloudflare like so:
-
-- https://demo.archivebox.io: only serves `/`, redirects `/archive/*` to `demo-static.`
-- https://demo-static.archivebox.io: only serves `/archive/`, redirects everything else to `demo.`
-
-<img width="400" alt="Cloudflare redirect rule for /archive/ to be served by a separate domain" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/9c77f503-0d97-4a8d-810f-1f4400c7aa3e">
+Do not serve ArchiveBox from a shared subdirectory such as `myapps.example.com/archivebox/`; it cannot provide the required origin isolation.
 
 Published archives automatically include a `robots.txt` `Disallow: /` to block search engines from indexing them. You may still wish to publish your contact info in the index footer though using [`FOOTER_INFO`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info) so that you can respond to any DMCA and copyright takedown notices if you accidentally rehost copyrighted content.
 
@@ -110,7 +102,7 @@ More info:
 
 <br/>
 
-## Do not run as root
+## Run ArchiveBox as an unprivileged user
 
 <img src="https://imgur.zervice.io/yDqJc4I.jpg" width="150px" align="right"/>
 
@@ -130,23 +122,21 @@ More info:
 >   
 > If you must use `exec` for some reason (e.g. if you only have access to a live container shell), you can run `su archivebox` within the shell, or add the arg `--user=archivebox` after `exec`.
 
-Do not run ArchiveBox as root for a number of reasons:
- - Chrome will execute as root and fail immediately because Chrome sandboxing is pointless when the data directory is opened as root (do not set [`CHROME_SANDBOX=False`](https://archivebox.github.io/abx-plugins/#chrome) just to bypass that error!)
+ArchiveBox drops privileges to the collection owner when it starts as root and can do so safely, including in the official Docker image. Do not bypass that boundary or force runtime dependencies to stay privileged:
+ - Browser sandboxing cannot provide its normal protection when the browser itself runs as root
  - All dependencies will be run as root, if any of them have a vulnerability that's exploited by sites you're archiving you're opening yourself up to full system compromise
  - ArchiveBox does lots of HTML parsing, filesystem access, and shell command execution.  A bug in any one of those subsystems could potentially lead to deleted/damaged data on your hard drive, or full system compromise unless restricted to a user that only has permissions to access the directories needed
  - Do you really trust a project created by a Github user called `@pirate` 😉? Why give a random program off the internet root access to your entire system? (I don't have malicious intent, I'm just saying in principle you should not be running random Github projects as root)
 
 **Instead, you should run ArchiveBox under a separate user account with less privileged access:**
 ```bash
-getent group archivebox >/dev/null || groupadd --system archivebox
-created_archivebox_user=false; if ! id archivebox >/dev/null 2>&1; then useradd --system --gid archivebox --create-home archivebox; created_archivebox_user=true; fi; trap 'if [ "$created_archivebox_user" = true ]; then userdel --remove archivebox >/dev/null 2>&1 || true; fi' EXIT
-archivebox_home="$(getent passwd archivebox | cut -d: -f6)"; mkdir -p "$archivebox_home/data"; chown -R archivebox:archivebox "$archivebox_home"
-uv_binary="$(command -v uv)"; sudo -u archivebox env HOME="$archivebox_home" DATA_DIR="$archivebox_home/data" "$uv_binary" run --project "$ARCHIVEBOX_PROJECT_DIR" --no-sync archivebox init
-sudo -u archivebox env HOME="$archivebox_home" DATA_DIR="$archivebox_home/data" "$uv_binary" run --project "$ARCHIVEBOX_PROJECT_DIR" --no-sync archivebox add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
+useradd -r -g archivebox -G audio,video archivebox  # the audio & video groups are used by chrome
+mkdir -p /home/archivebox/data
+chown -R archivebox:archivebox /home/archivebox
+...
+sudo -u archivebox archivebox add ...
 ```
 
-~~If you absolutely must run it as root for some reason, a footgun is provided: you can set `ALLOW_ROOT=True` via environment variable or in your ArchiveBox.conf file.~~ This footgun option was removed (I'm sorry, the support burden of helping people who messed up their systems by running everything as root was too high).
-
 <img src="https://imgur.zervice.io/ca1he6I.png" width="40px" align="right"/>
 
 <br/>
@@ -169,11 +159,11 @@ More info:
 
 ### Filesystem
 
-How much are you planning to archive?  Only a few bookmarked articles, or thousands of pages of browsing history a day?  If it's only 1-50 pages a day, you can probably just stick it in a normal folder on your hard drive, but if you want to go over 100 pages a day, you will likely want to put your archive on a compressed/deduplicated/encrypted disk image or filesystem like ZFS. Other distributed/networked/checksummed filesystems that have also been reported to work (but are not technically officially supported) include SMB, NFS, Ceph, Unraid, and BTRFS. Make sure the filesystem you're using supports FSYNC. Some filesystems are unable to store more than a certain number of directory entries, and your total number of snapshots in `./archive` may be capped as a result. Some other filesystems begin to have performance degradations but continue to function when the directory entry count gets too high. Generally this isn't an issue unless you have more than ~20,000 Snapshot folders in `./archive`.
+How much are you planning to archive? Only a few bookmarked articles, or thousands of pages of browsing history a day? If it's only 1-50 pages a day, you can probably use a normal folder on your hard drive, but at higher volume you may want a compressed/deduplicated/encrypted filesystem like ZFS. Other distributed/networked/checksummed filesystems reported to work include SMB, NFS, Ceph, Unraid, and BTRFS. The database and config must remain on a local filesystem with reliable FSYNC. Current Snapshot directories are sharded under `archive/users/<user>/snapshots/<date>/<domain>/<uuid>/`, avoiding the old single-directory scaling limit.
 
 #### Purging entries
 
-Unless `--yes --delete` is passed to `archivebox remove`, Snapshots removed from the index remain in the filesystem and their `./archive/<timestamp>` folders need to be deleted manually to be fully removed. Imported URLs are also logged separately in `./sources`, `./logs`, and the Sonic full-text index `./sonic` and should be removed manually as well to clear all traces of a URL added by accident. You can search for a URL on the filesystem you're trying to remove using `grep -a -r "https://example.com/url/to/search/for"`.
+`archivebox remove --yes URL` deletes matching Snapshot rows and schedules their Snapshot directories for cleanup through the normal state-machine path. The legacy `--delete` flag is accepted only for CLI compatibility and does not change that behavior. Original imports and operational history may still appear in `sources/`, `logs/`, or an external search backend; remove those separately if your goal is to erase every trace of a URL.
 
 #### Permissions
 

docs/Setting-Up-Storage.md

@@ -5,19 +5,19 @@
 
 <br/>
 
-ArchiveBox supports a wide range of local and remote filesystems using `rclone` and/or Docker storage plugins. The examples below use [Docker Compose bind mounts](https://docs.docker.com/storage/bind-mounts/) to demonstrate the concepts, you can adapt them to your OS and environment needs.
+ArchiveBox supports a wide range of local and remote filesystems using `rclone` and/or Docker storage plugins. The examples below use [Docker Compose bind mounts](https://docs.docker.com/storage/bind-mounts/) to demonstrate the concepts; adapt the host paths, ownership, and provider settings to your environment.
 
 Example [`docker-compose.yml`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docker-compose.yml) storage setup:
 ```yaml
 services:
     archivebox:
-        # ...
+        # ...other service settings...
         volumes:
             # your index db, config, logs, etc. should be stored on a local SSD (usually <10Gb)
             - ./data:/data
 
             # but bulk archive/ content can be located on an HDD or remote filesystem
-            - /mnt/archivebox-s3/data/archive:/data/archive
+            - /mnt/archivebox-archive:/data/archive
 ```
 
 <h4>Related Docs</h4>
@@ -46,12 +46,11 @@ services:
 
 <a name="zfs"></a>
 
-### `ZFS` (recommended for best experience on Linux/BSD) ⭐️
+### `ZFS` (recommended for experienced Linux/BSD operators) ⭐️
 
 > [!TIP]
-> *This is the recommended filesystem for ArchiveBox on Linux, macOS, and BSD (w/wo Docker).*  
-> [`apt install zfsutils-linux`](https://openzfs.github.io/openzfs-docs/Getting%20Started/Ubuntu/index.html)  
-> <sub>Provides RAID, compression, encryption, deduping, 0-cost point-in-time backups, remote sync, integrity verification, and more...</sub>
+> *ZFS is a good choice when you already operate OpenZFS and want checksumming, compression, snapshots, replication, and optional encryption or disk redundancy.*
+> On Ubuntu, follow the official [OpenZFS installation guide](https://openzfs.github.io/openzfs-docs/Getting%20Started/Ubuntu/index.html). macOS and BSD installation and property support differ, so use the guide for your OS.
 
 - https://openzfs.github.io/openzfs-docs/
 - https://openzfs.github.io/openzfs-docs/man/v2.2/8/zpool-create.8.html
@@ -59,31 +58,38 @@ services:
 - https://docs.docker.com/storage/storagedriver/zfs-driver/
 - https://www.ixsystems.com/blog/fast-dedup-is-a-valentines-gift-to-the-openzfs-and-truenas-communities/
 
+> [!CAUTION]
+> Creating a pool erases the selected disks. The two-disk example below creates a mirror; replace both `/dev/disk/by-id/...` placeholders with the persistent IDs of empty disks you intend to erase.
+
 ```bash
-set -euo pipefail; apt-get update -qq
-apt-get install -y zfsutils-linux
-command -v zpool
-command -v zfs
-zpool --version
-zfs --version
-zpool create --help >/dev/null
-zfs create --help >/dev/null
-work_dir="$(mktemp -d)"
-disk_one="$work_dir/disk1.img"
-disk_two="$work_dir/disk2.img"
-truncate -s 128M "$disk_one"
-truncate -s 128M "$disk_two"
-test "$(stat -c %s "$disk_one")" -eq 134217728
-test "$(stat -c %s "$disk_two")" -eq 134217728
-printf '%s\n' 'zpool create -f -O mountpoint=/mnt/archivebox archivebox /dev/disk/by-uuid/disk1 /dev/disk/by-uuid/disk2'
-printf '%s\n' 'zfs create -o mountpoint=/mnt/archivebox/data archivebox/data'
-printf '%s\n' 'zfs create -o encryption=on -o keysource=passphrase,prompt archivebox/encrypted'
-zpool status >/dev/null 2>&1 || test ! -e /dev/zfs
-test -d "$work_dir"
-rm -f "$disk_one"
-rm -f "$disk_two"
-rmdir "$work_dir"
-test ! -e "$work_dir"
+# Create a mirrored pool without forcing ZFS's safety checks.
+sudo zpool create \
+    -O mountpoint=none \
+    -O compression=lz4 \
+    -O dnodesize=auto \
+    -O atime=off \
+    -O xattr=sa \
+    -O acltype=posixacl \
+    -O aclinherit=passthrough \
+    archivebox mirror \
+    /dev/disk/by-id/DISK_ONE \
+    /dev/disk/by-id/DISK_TWO
+
+# Create the unencrypted ArchiveBox data dataset.
+sudo zfs create \
+    -o mountpoint=/mnt/archivebox/data \
+    archivebox/data
+```
+
+To encrypt a new dataset, use this command **instead of** the unencrypted `zfs create` command above. ZFS encryption must be selected when the dataset is created.
+
+```bash
+sudo zfs create \
+    -o mountpoint=/mnt/archivebox/data \
+    -o encryption=on \
+    -o keyformat=passphrase \
+    -o keylocation=prompt \
+    archivebox/data
 ```
 
 <a name="ntfs"></a><a name="hfs"></a><a name="btrfs"></a>
@@ -111,7 +117,7 @@ test ! -e "$work_dir"
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/6124b92a-df5a-47c4-b3c2-006ebd28785b" alt="local filesystem icon" width="80px" align="right"/>
 
-ArchiveBox supports many common types of remote filesystems using RClone, FUSE, Docker Storage providers, and Docker Volume Plugins.  
+ArchiveBox supports many common types of remote filesystems using Rclone, FUSE, Docker storage providers, and Docker volume plugins.
 
 The `data/archive/` subfolder contains the bulk archived content, and it supports being stored on a slower remote server (SMB/NFS/SFTP/etc.) or object store (S3/B2/R2/etc.). For data integrity and performance reasons, the rest of the `data/` directory (`data/ArchiveBox.conf`, `data/logs`, etc.) must be stored locally while ArchiveBox is running.
 
@@ -136,9 +142,10 @@ services:
 
 volumes:
     archivebox-archive:
+        driver: local
         driver_opts:
             type: "nfs"
-            o: "addr=some-remote-server.example.com,nolock,soft,rw,nfsvers=4"
+            o: "addr=some-remote-server.example.com,rw,nfsvers=4"
             device: ":/archivebox-archive"
 ```
 
@@ -171,18 +178,28 @@ volumes:
 
 ### Amazon S3 / Backblaze B2 / Google Drive / etc. (RClone)
 
+ArchiveBox stores snapshot content under `data/archive/users/<user>/snapshots/<date>/<domain>/<uuid>/` and keeps backwards-compatible `data/archive/<timestamp>` symlinks. Object-storage mounts must enable Rclone's VFS symlink translation so both parts of this layout work.
+
+Install the `rclone` binary through `abxpkg`:
+
 ```bash
-set -euo pipefail; apt-get update -qq; apt-get install -y rclone fuse3
-fuse_conf_backup="$(mktemp)"; test ! -e /etc/fuse.conf || cp /etc/fuse.conf "$fuse_conf_backup"
-trap 'if test -s "$fuse_conf_backup"; then cp "$fuse_conf_backup" /etc/fuse.conf; else rm -f /etc/fuse.conf; fi' EXIT
-grep -qxF user_allow_other /etc/fuse.conf 2>/dev/null || printf '%s\n' user_allow_other >> /etc/fuse.conf
-rclone version; fusermount3 --version
+uv tool install abxpkg
+abxpkg install rclone
+abxpkg run rclone version
+```
+
+Then install the FUSE 3 system integration supplied by your OS. For example, on Ubuntu:
+
+```bash
+sudo apt-get install fuse3
+grep -qxF user_allow_other /etc/fuse.conf ||
+    printf '%s\n' user_allow_other | sudo tee -a /etc/fuse.conf
 ```
 
 Then define your remote storage config `~/.config/rclone/rclone.conf`:
 
 > [!TIP]
-> You can also create `rclone.conf` using the RClone Web GUI: `rclone rcd --rc-web-gui`
+> You can also create `rclone.conf` using the Rclone Web GUI: `abxpkg run rclone rcd --rc-web-gui`
 
 ```ini
 # Example rclone.conf using Amazon S3 for storage:
@@ -194,7 +211,7 @@ secret_access_key = YYY
 region = us-east-1
 ```
 
-#### RClone Config Examples
+#### Rclone Config Examples
 
 - [SMB](https://rclone.org/smb/) / [Ceph](https://rclone.org/s3/#ceph) / [SFTP](https://rclone.org/sftp/) / [FTP](https://rclone.org/ftp/) / [WebDAV (e.g. Nextcloud)](https://rclone.org/webdav/)
 - [Google Drive](https://rclone.org/drive/) / [Dropbox](https://rclone.org/dropbox/) / [OneDrive](https://rclone.org/onedrive/)
@@ -212,35 +229,68 @@ region = us-east-1
 
 <br/>
 
-#### Option A: Running RClone on Bare Metal host
+#### Option A: Running Rclone on a bare-metal host
 
 1. *If Needed:* Transfer any existing local archive data to the remote volume first
+
+> [!CAUTION]
+> Stop ArchiveBox before migrating its archive directory. `rclone sync` makes the remote destination match the local source and can delete files already present at the destination. Run it with `--dry-run` first, make a separate backup, and do not move the local copy until `rclone check` succeeds.
+
 ```bash
-set -euo pipefail; source_archive="$(mktemp -d)"; remote_archive="$(mktemp -d)"; printf 'ArchiveBox storage test\n' > "$source_archive/snapshot.txt"; rclone sync --fast-list --transfers 20 "$source_archive/" "$remote_archive/"
-cmp "$source_archive/snapshot.txt" "$remote_archive/snapshot.txt"; mv "$source_archive" "$source_archive.localbackup"; test -f "$source_archive.localbackup/snapshot.txt"
+abxpkg run rclone sync \
+    --dry-run \
+    --links \
+    --fast-list \
+    --transfers 20 \
+    --progress \
+    /opt/archivebox/data/archive/ \
+    archivebox-s3:data/archive/
+
+# Remove --dry-run only after reviewing the proposed changes, then verify them.
+abxpkg run rclone sync \
+    --links \
+    --fast-list \
+    --transfers 20 \
+    --progress \
+    /opt/archivebox/data/archive/ \
+    archivebox-s3:data/archive/
+abxpkg run rclone check --links /opt/archivebox/data/archive/ archivebox-s3:data/archive/
+
+mv /opt/archivebox/data/archive /opt/archivebox/data/archive.localbackup
+mkdir -p /opt/archivebox/data/archive
 ```
 2. **Mount the remote storage volume as FUSE filesystem**
-```text
-rclone mount
-    --allow-other \                # essential, allows Docker to access FUSE mounts
-    --uid 911 --gid 911 \          # 911 is the default used by ArchiveBox
-    --vfs-cache-mode=full \        # cache both file metadata and contents
-    --transfers=16 --checkers=4 \  # use 16 threads for transfers & 4 for checking
-    archivebox-s3/data/archive:/opt/archivebox/data/archive         # remote:local
+
+Run the mount as the numeric user that owns the local ArchiveBox collection. The command stays in the foreground so a service manager can supervise it.
+
+```bash
+abxpkg run rclone mount \
+    archivebox-s3:data/archive/ \
+    /opt/archivebox/data/archive/ \
+    --allow-other \
+    --vfs-cache-mode=full \
+    --vfs-links \
+    --transfers=16 \
+    --checkers=4
 ```
 
-See here for full more detailed instructions here: [RClone Documentation: The `rclone mount` command](https://rclone.org/commands/rclone_mount/)
+See [Rclone's `rclone mount` documentation](https://rclone.org/commands/rclone_mount/) for service-manager and cache-size configuration.
 
 > [!TIP]
-> You can use any RClone FUSE mounts as a normal volumes (bind mount) for Docker ArchiveBox, typically no storage plugin is needed as long as `allow-other` is setup properly.
+> You can use an existing Rclone FUSE mount as a normal Docker bind mount. A separate storage plugin is usually unnecessary when `user_allow_other` and `--allow-other` are configured correctly.
 
-`docker run -v $PWD:/data -v /opt/archivebox/data/archive:/data/archive`
+```bash
+docker run --rm \
+    -v "$PWD:/data" \
+    -v /opt/archivebox/data/archive:/data/archive \
+    archivebox/archivebox:dev status
+```
 
 `docker-compose.yml`:
 ```yaml
 services:
     archivebox:
-        # ...
+        # ...other service settings...
         volumes:
             - ./data:/data
             - /opt/archivebox/data/archive:/data/archive
@@ -248,16 +298,24 @@ services:
 
 <br/>
 
-#### Option B: Running RClone with Docker Storage Plugin
+#### Option B: Running Rclone with the Docker storage plugin
 
-*This is only needed if you are unable to `Option A` for compatibility or performance reasons, or if you prefer defining your remote storage config in `docker-compose.yml` instead of `rclone.conf`.*
+*This Linux Docker Engine option is only needed if you cannot use Option A for compatibility or performance reasons, or if you prefer defining your remote storage in `docker-compose.yml`.*
 
-See here for full instructions: [RClone Documentation: Docker Plugin](https://rclone.org/docker/)
+See here for full instructions: [Rclone Documentation: Docker Plugin](https://rclone.org/docker/)
 
 1. First, install the [Rclone Docker Volume Plugin](https://rclone.org/docker/#installing-as-managed-plugin) for your CPU architecture (e.g. `amd64` or `arm64`):
+
 ```bash
-set -euo pipefail; installed_rclone_plugin=false; docker plugin inspect rclone >/dev/null 2>&1 || { docker plugin install rclone/docker-volume-rclone:amd64 --grant-all-permissions --alias rclone; installed_rclone_plugin=true; }; trap 'if [ "$installed_rclone_plugin" = true ]; then docker plugin disable --force rclone; docker plugin rm rclone; fi' EXIT
-docker plugin inspect rclone --format '{{.Name}} {{.Enabled}}' | grep -q '^rclone true$'
+sudo mkdir -p \
+    /var/lib/docker-plugins/rclone/config \
+    /var/lib/docker-plugins/rclone/cache
+sudo install -m 600 \
+    ~/.config/rclone/rclone.conf \
+    /var/lib/docker-plugins/rclone/config/rclone.conf
+
+# Replace amd64 with arm64 on ARM hosts.
+docker plugin install rclone/docker-volume-rclone:amd64 --grant-all-permissions --alias rclone
 ```
 
 2. Then, [create a volume using the Docker CLI](https://rclone.org/docker/#creating-volumes-via-cli) or [define one using Docker Compose / Swarm](https://rclone.org/docker/#using-with-swarm-or-compose):
@@ -274,10 +332,11 @@ volumes:
     archivebox-s3:
         driver: rclone
         driver_opts:
-            remote: 'archivebox-s3/data/archive'
+            remote: 'archivebox-s3:data/archive'
             allow_other: 'true'
             vfs_cache_mode: full
-            poll_interval: 0
+            vfs_links: 'true'
+            # Match these to the numeric owner of ./data; 911:911 is the image default.
             uid: 911
             gid: 911
             transfers: 16
@@ -287,7 +346,8 @@ volumes:
 
 To start the container and verify the filesystem is accessible within it:
 ```bash
-set -euo pipefail; docker_data="$(mktemp -d)"; docker run --rm -v "$docker_data:/data" archivebox-docs-ci init; docker run --rm -v "$docker_data:/data" archivebox-docs-ci /bin/bash -c 'ls -lah /data/archive/ | tee /data/archive/.write_test.txt'; test -s "$docker_data/archive/.write_test.txt"
+docker compose run --rm archivebox \
+    /bin/bash -c 'touch /data/archive/.write_test && rm /data/archive/.write_test'
 ```
 
 <br/>

docs/Setting-up-Authentication.md

@@ -36,18 +36,18 @@ Use these options to set up your desired permissions for non-admin guest users:
 You need a user account to access the Admin UI, you can run the commands below to create/edit a user from the CLI:
 
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"
-uv run --project "$project_dir" --no-sync archivebox init
-DJANGO_SUPERUSER_PASSWORD=archivebox-docs-password uv run --project "$project_dir" --no-sync archivebox manage createsuperuser --noinput --username archivebox-docs --email docs@example.com
-uv run --project "$project_dir" --no-sync archivebox manage shell -c "from django.contrib.auth import get_user_model; user=get_user_model().objects.get(username='archivebox-docs'); user.set_password('archivebox-docs-new-password'); user.save()"
-uv run --project "$project_dir" --no-sync archivebox manage shell -c "from django.contrib.auth import authenticate; assert authenticate(username='archivebox-docs', password='archivebox-docs-new-password') is not None"
+archivebox manage createsuperuser
+archivebox manage changepassword <username>
+
+# equivalent: docker compose run archivebox manage [...]
+# equivalent: docker run -v $PWD:/data archivebox/archivebox manage [...]
 ```
 
 > [!TIP]
 > If using Docker, you can set [`ADMIN_USERNAME` & `ADMIN_PASSWORD`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#admin_username--admin_password) to auto-create an admin account on first run.
 
-Existing users can be managed from the Admin UI here: [`/admin/auth/user/`](http://127.0.0.1:8000/admin/auth/user/),  
-and you can change your password in the UI here: [`/admin/password_change/`](http://127.0.0.1:8000/admin/password_change/).
+Existing users can be managed from the Admin UI here: [`/admin/auth/user/`](http://admin.archivebox.localhost:8000/admin/auth/user/),
+and you can change your password here: [`/admin/password_change/`](http://admin.archivebox.localhost:8000/admin/password_change/).
 
 <br/>
 <br/>
@@ -60,18 +60,15 @@ Set these ArchiveBox configuration values based on your reverse proxy setup and
 ```bash
 # REQUIRED: the header where your upstream reverse proxy will place the authenticated user's username/email
 # EXAMPLE: Cf-Access-Authenticated-User-Email (if using Cloudflare Access / Zero Trust)
-set -euo pipefail; export REVERSE_PROXY_USER_HEADER=X-Remote-User
+REVERSE_PROXY_USER_HEADER=X-Remote-User
 
 # REQUIRED: the IP/CIDR of your upstream reverse proxy server
 # WARNING: make sure this range contains ONLY your reverse proxy server!
 # ArchiveBox will completely trust any IP in this range for authentication
-export REVERSE_PROXY_WHITELIST=192.0.2.3/32
+REVERSE_PROXY_WHITELIST=192.0.2.3/32
 
 # OPTIONAL: redirect users to an external URL after they log out
-export LOGOUT_REDIRECT_URL=https://auth.yourcompany.example.com/after/logout
-test "$REVERSE_PROXY_USER_HEADER" = X-Remote-User
-test "$REVERSE_PROXY_WHITELIST" = 192.0.2.3/32
-test "$LOGOUT_REDIRECT_URL" = https://auth.yourcompany.example.com/after/logout
+LOGOUT_REDIRECT_URL=https://auth.yourcompany.example.com/after/logout
 ```
 
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#reverse_proxy_user_header
@@ -87,22 +84,22 @@ test "$LOGOUT_REDIRECT_URL" = https://auth.yourcompany.example.com/after/logout
 
 First, install the `ldap` add-on to use this feature (not needed for Docker Archivebox).
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; tool_root="$(mktemp -d)"; UV_TOOL_DIR="$tool_root/tools" UV_TOOL_BIN_DIR="$tool_root/bin" uv tool install --python 3.13 --upgrade "$project_dir[ldap]"; "$tool_root/bin/archivebox" --help
+uv tool install --python 3.13 --upgrade 'archivebox[ldap] @ git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 ```
 
 Then set these configuration values to finish configuring LDAP:
 ```bash
-set -euo pipefail; export LDAP_ENABLED=True
-export LDAP_SERVER_URI="ldap://ldap.example.com:3389"
-export LDAP_BIND_DN="ou=archivebox,ou=services,dc=ldap.example.com"
-export LDAP_BIND_PASSWORD="secret-bind-user-password"
-export LDAP_USER_BASE="ou=users,ou=archivebox,ou=services,dc=ldap.example.com"
-export LDAP_USER_FILTER="(objectClass=user)"
-export LDAP_USERNAME_ATTR="uid"
-export LDAP_FIRSTNAME_ATTR="givenName"
-export LDAP_LASTNAME_ATTR="sn"
-export LDAP_EMAIL_ATTR="mail"
-test "$LDAP_ENABLED" = True; test "$LDAP_USERNAME_ATTR" = uid; test "$LDAP_EMAIL_ATTR" = mail
+LDAP_ENABLED=True
+LDAP_SERVER_URI="ldap://ldap.example.com:3389"
+LDAP_BIND_DN="ou=archivebox,ou=services,dc=ldap.example.com"
+LDAP_BIND_PASSWORD="secret-bind-user-password"
+LDAP_USER_BASE="ou=users,ou=archivebox,ou=services,dc=ldap.example.com"
+LDAP_USER_FILTER="(objectClass=user)"
+
+LDAP_USERNAME_ATTR="uid"
+LDAP_FIRSTNAME_ATTR="givenName"
+LDAP_LASTNAME_ATTR="sn"
+LDAP_EMAIL_ATTR="mail"
 ```
 
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#ldap
@@ -138,22 +135,21 @@ The IdP server can act as a middleman gateway to authenticate users using an ext
 The REST API (available starting in v0.8.0) supports several methods of authentication for convenience.  
   
 To see API docs, try endpoints interactively, and see how auth works, visit this URL on your ArchiveBox server:  
-[`http://127.0.0.1:8000/api/v1/docs`](http://127.0.0.1:8000/api/v1/docs)
+[`http://api.archivebox.localhost:8000/api/v1/docs`](http://api.archivebox.localhost:8000/api/v1/docs)
 
 <img width="500" alt="Screenshot of django-ninja Swagger API docs page" src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/ad914143-f48b-4d4e-aa8c-f89a2c70cee7">
 
 <br/><br/>
 
-To get started using the REST API, you can generate an API key for your user in the Admin Web UI:  
-[`http://127.0.0.1:8000/admin/api/apitoken/add/`](http://127.0.0.1:8000/admin/api/apitoken/add/)  
-  
-or by calling the `http://127.0.0.1:8000/api/v1/auth/get_api_token` endpoint with a username & password:
+To get started using the REST API, you can generate an API key for your user in the Admin Web UI:
+[`http://admin.archivebox.localhost:8000/admin/api/apitoken/add/`](http://admin.archivebox.localhost:8000/admin/api/apitoken/add/)
+
+or by calling the `http://api.archivebox.localhost:8000/api/v1/auth/get_api_token` endpoint with a username & password:
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; port="${ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT:-18000}"; cd "$archivebox_data"
-uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox server --daemonize "127.0.0.1:$port"; server_pid="$(uv run --project "$project_dir" --no-sync archivebox manage shell -c "from archivebox.machine.models import Process; print(Process.objects.filter(process_type='server', status='running').order_by('-started_at').values_list('pid', flat=True).first() or '')")"; test -n "$server_pid"; trap 'kill "$server_pid" 2>/dev/null || true' EXIT
-status="$(curl -sS -o response.json -w '%{http_code}' -X POST "http://127.0.0.1:$port/api/v1/auth/get_api_token" -H 'Content-Type: application/json' -d '{"username":"missing-user","password":"wrong-password"}')"
-test -s response.json; test "$status" -ge 400 || grep -q '"success": false' response.json
+curl -X 'POST' \
+  'http://api.archivebox.localhost:8000/api/v1/auth/get_api_token' \
+  -H 'Content-Type: application/json' \
+  -d '{"username": "YOURUSERNAMEHERE", "password": "YOURPASSWORDHERE"}'
 ```
 
 <br/>
@@ -167,10 +163,10 @@ test -s response.json; test "$status" -ge 400 || grep -q '"success": false' resp
 Pass `Authorization=Bearer YOURAPITOKENHERE` as a request header.
 
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; port="${ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT:-18000}"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox server --daemonize "127.0.0.1:$port"; server_pid="$(uv run --project "$project_dir" --no-sync archivebox manage shell -c "from archivebox.machine.models import Process; print(Process.objects.filter(process_type='server', status='running').order_by('-started_at').values_list('pid', flat=True).first() or '')")"; test -n "$server_pid"; trap 'kill "$server_pid" 2>/dev/null || true' EXIT
-status="$(curl -sS -o response.json -w '%{http_code}' "http://127.0.0.1:$port/api/v1/core/snapshots?limit=10" -H 'accept: application/json' -H 'Authorization: Bearer invalid-docs-token')"
-test "$status" -ge 400; test -s response.json
+curl -X 'GET' \
+  'http://api.archivebox.localhost:8000/api/v1/core/snapshots?limit=10' \
+  -H 'accept: application/json' \
+  -H 'Authorization: Bearer YOURAPITOKENHERE'
 ```
 
 ### API Request Header Authentication
@@ -180,10 +176,10 @@ test "$status" -ge 400; test -s response.json
 Pass `X-ArchiveBox-API-Key=YOURAPITOKENHERE` as a request header.
 
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; port="${ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT:-18000}"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox server --daemonize "127.0.0.1:$port"; server_pid="$(uv run --project "$project_dir" --no-sync archivebox manage shell -c "from archivebox.machine.models import Process; print(Process.objects.filter(process_type='server', status='running').order_by('-started_at').values_list('pid', flat=True).first() or '')")"; test -n "$server_pid"; trap 'kill "$server_pid" 2>/dev/null || true' EXIT
-status="$(curl -sS -o response.json -w '%{http_code}' "http://127.0.0.1:$port/api/v1/core/snapshots?limit=10" -H 'accept: application/json' -H 'X-ArchiveBox-API-Key: invalid-docs-token')"
-test "$status" -ge 400; test -s response.json
+curl -X 'GET' \
+  'http://api.archivebox.localhost:8000/api/v1/core/snapshots?limit=10' \
+  -H 'accept: application/json' \
+  -H 'X-ArchiveBox-API-Key: YOURAPITOKENHERE'
 ```
 
 <br/>
@@ -196,49 +192,9 @@ test "$status" -ge 400; test -s response.json
 Pass `api_key=YOURAPITOKENHERE` as a GET/POST query parameter.
 
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; port="${ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT:-18000}"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox server --daemonize "127.0.0.1:$port"; server_pid="$(uv run --project "$project_dir" --no-sync archivebox manage shell -c "from archivebox.machine.models import Process; print(Process.objects.filter(process_type='server', status='running').order_by('-started_at').values_list('pid', flat=True).first() or '')")"; test -n "$server_pid"; trap 'kill "$server_pid" 2>/dev/null || true' EXIT
-status="$(curl -sS -o response.json -w '%{http_code}' "http://127.0.0.1:$port/api/v1/core/snapshots?limit=10&api_key=invalid-docs-token" -H 'accept: application/json')"
-test "$status" -ge 400; test -s response.json
-```
-
-<br/>
-
-### API Session Cookie Authentication
-
-> [!CAUTION]
-> We recommend sticking to header-based authentication and not using this method unless you deeply understand the CSRF/CORS security risks.
-> This method is mostly useful when accessing the API from external apps where CSRF/CORS is not a concern (e.g. `curl`, mobile apps, other servers, etc.).
-
-> Browsers enforce that requests made to the ArchiveBox API from *other origins* will not include any session cookies by default. This is is a [foundational security principle of the web](https://docs.djangoproject.com/en/5.0/ref/csrf/) that protects you from API requests being initiated by JS on websites you don't control (aka CSRF/CORS attacks).
->
-> To allow incoming POST/PUT/DELETE requests from other domains **that you trust**, set [`BASE_URL`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#base_url) to the public URL of your instance — ArchiveBox derives Django's `ALLOWED_HOSTS` and `CSRF_TRUSTED_ORIGINS` from `BASE_URL` + [`SERVER_SECURITY_MODE`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode) automatically, including widening them to admit the admin/web/api subdomains. If your setup needs something the auto-derivation doesn't cover, [open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new/choose).
-
-Log in via the Admin Web UI: `/admin/login/`, you can then re-use your login session id (stored in the `sessionid` cookie) for REST API requests. By default, this only allows you to make requests from the same domain ArchiveBox is being served on (e.g. from browser devtools open on an ArchiveBox page or CLI tools).
-
-```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; port="${ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT:-18000}"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox server --daemonize "127.0.0.1:$port"; server_pid="$(uv run --project "$project_dir" --no-sync archivebox manage shell -c "from archivebox.machine.models import Process; print(Process.objects.filter(process_type='server', status='running').order_by('-started_at').values_list('pid', flat=True).first() or '')")"; test -n "$server_pid"; trap 'kill "$server_pid" 2>/dev/null || true' EXIT
-status="$(curl -sS -o response.json -w '%{http_code}' "http://127.0.0.1:$port/api/v1/core/snapshots?limit=10" -H 'accept: application/json' -H 'Cookie: sessionid=invalid-docs-session')"
-test "$status" -ge 400; test -s response.json
-```
-
-<br/>
-
-### API HTTP Basic Authentication
-
-> [!CAUTION]
-> This method is fairly uncommon and is only useful in a few niche situations where the other methods are not available.  
-> **We will likely remove this method in a future ArchiveBox release if nobody uses it.**  
-> *If you rely on this method and want us to keep it, please [open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new/choose) and explain your use-case!* 
-
-Pass your ArchiveBox admin username & password via HTTP Basic Authentication.
-
-```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; port="${ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT:-18000}"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox server --daemonize "127.0.0.1:$port"; server_pid="$(uv run --project "$project_dir" --no-sync archivebox manage shell -c "from archivebox.machine.models import Process; print(Process.objects.filter(process_type='server', status='running').order_by('-started_at').values_list('pid', flat=True).first() or '')")"; test -n "$server_pid"; trap 'kill "$server_pid" 2>/dev/null || true' EXIT
-status="$(curl -sS -o response.json -w '%{http_code}' "http://127.0.0.1:$port/api/v1/core/snapshots?limit=10" -u 'missing-user:wrong-password' -H 'accept: application/json')"
-test "$status" -ge 400; test -s response.json
+curl -X 'GET' \
+  'http://api.archivebox.localhost:8000/api/v1/core/snapshots?limit=10&api_key=YOURAPITOKENHERE' \
+  -H 'accept: application/json'
 ```
 
 <br/>

docs/Setting-up-Search.md

@@ -4,9 +4,9 @@
 
 You can search your ArchiveBox data in a number of ways:
 
-- using the CLI: `archivebox list --filter-type=search 'text to search'` (`archivebox list --help` for more)
+- using the CLI: `archivebox search 'text to search'` (`archivebox search --help` for more)
 - using the Web UI: both the `/public` index and `/admin/core/snapshot` pages provide a search box
-- using the REST API: `/api/v1/list?filter_type=search` provides the same search interface as the CLI  
+- using the REST API: `/api/v1/core/snapshots?search=text+to+search&search_mode=contents`
 - by searching the archive data folder directly with external tools (e.g. macOS Spotlight, [Cerebro](https://www.cerebroapp.com/), `ag`, [Yacy](https://yacy.net/), etc.)
 
 ![image](https://github.com/ArchiveBox/ArchiveBox/assets/511499/637675ee-bf4a-49f9-b936-c2da1bd64410)
@@ -30,11 +30,12 @@ ArchiveBox search works by doing substring matches in `Snapshot` metadata fields
 
 ArchiveBox provides a number of "Search Backend Engines" to tune its performance & behavior for different use-cases.
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"
-uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox config --set SEARCH_BACKEND_ENGINE=ripgrep
-uv run --project "$project_dir" --no-sync archivebox version
-uv run --project "$project_dir" --no-sync archivebox config --get SEARCH_BACKEND_ENGINE
+# this setting controls which search backend ArchiveBox uses
+archivebox config --set SEARCH_BACKEND_ENGINE=ripgrep
+
+# to see information about the backend you are currently using, run:
+archivebox version
+archivebox config --get SEARCH_BACKEND_ENGINE
 ```
 
 By default out-of-the-box, the selected engine is a simple but efficient tool similar to `grep -r` called [`ripgrep`](https://github.com/BurntSushi/ripgrep).
@@ -56,18 +57,17 @@ However, there are some fundamental limitations of scanning through every file o
 
 ### `ripgrep` *(the default)*
 
-If you do not already have `ripgrep` installed, follow the [instructions here](https://github.com/BurntSushi/ripgrep#installation) to get it.
-ArchiveBox will use `ripgrep` by default if it is found, however you can explicitly configure it to be used like so:
+ArchiveBox resolves `ripgrep` through `abxpkg`: a compatible host installation is used first, otherwise a managed copy is installed.
 
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"
-uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox install ripgrep
-uv run --project "$project_dir" --no-sync archivebox config --set SEARCH_BACKEND_ENGINE=ripgrep
-uv run --project "$project_dir" --no-sync archivebox version
-test -L "$ABXPKG_LIB_DIR/env/bin/rg"
-uv run --project "$project_dir" --no-sync archivebox add --plugins=wget "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
-search_result="$(uv run --project "$project_dir" --no-sync archivebox search --search contents:ripgrep 'ArchiveBox docs fixture')"; grep -q "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}" <<< "$search_result"
+archivebox install ripgrep
+archivebox config --set SEARCH_BACKEND_ENGINE=ripgrep
+
+# check the resolved provider, version, and projected binary:
+archivebox version
+
+# then try it out by searching via the Web UI or CLI:
+archivebox search 'text to search for'
 ```
 
 #### Pros
@@ -88,21 +88,7 @@ search_result="$(uv run --project "$project_dir" --no-sync archivebox search --s
 
 ### `ripgrep-all` (aka `rga`)
 
-The same as ripgrep except that it supports searching more binary filetypes like PDFs, eBooks, Office documents, zip, tar.gz, etc.
-
-To use it, follow the [install instruction for your OS](https://github.com/phiresky/ripgrep-all#installation), then configure ArchiveBox to use it like so:
-
-```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"; cd "$archivebox_data"
-uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync abxpkg env --install --lib="$ABXPKG_LIB_DIR" --binproviders env,brew --overrides '{"brew":{"install_args":["rga"]}}' rga >/dev/null
-uv run --project "$project_dir" --no-sync archivebox config --set SEARCH_BACKEND_ENGINE=ripgrep
-uv run --project "$project_dir" --no-sync archivebox config --set RIPGREP_BINARY="$ABXPKG_LIB_DIR/env/bin/rga"
-test -L "$ABXPKG_LIB_DIR/env/bin/rga"; "$ABXPKG_LIB_DIR/env/bin/rga" --version
-uv run --project "$project_dir" --no-sync archivebox add --plugins=wget "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
-search_result="$(uv run --project "$project_dir" --no-sync archivebox search --search contents:ripgrep 'ArchiveBox docs fixture')"; grep -q "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}" <<< "$search_result"
-```
+`ripgrep-all` supports more binary file types such as PDFs, eBooks, Office documents, zip, and tar files. It is useful as an external companion tool, but it is **not currently a supported drop-in binary for ArchiveBox's `ripgrep` backend**. The backend relies on `rg`'s command and output contract.
 
 <br/>
 
@@ -110,16 +96,12 @@ search_result="$(uv run --project "$project_dir" --no-sync archivebox search --s
 
 ### `ugrep`
 
-Not tested by the ArchiveBox team but it's very similar to `ripgrep` and may work as a drop-in replacement, with some caveats. (contributions welcome to improve support)
+`ugrep` is another capable external search tool, but it is **not a supported drop-in binary** for ArchiveBox's `ripgrep` backend. Contributions adding a dedicated integration are welcome.
 
 `ugrep` is similar to `ripgrep` and `ripgrep-all` in that it's an indexless disk-search tool, but it provides some more of the full-text search features without the performance overhead of maintaining a separate search backend worker with an independent index.
 
 https://github.com/Genivia/ugrep
 
-```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"; uv run --project "$project_dir" --no-sync archivebox init; uv run --project "$project_dir" --no-sync abxpkg env --install --lib="$ABXPKG_LIB_DIR" --binproviders env,apt,brew ugrep >/dev/null; test -L "$ABXPKG_LIB_DIR/env/bin/ugrep"; "$ABXPKG_LIB_DIR/env/bin/ugrep" --version; uv run --project "$project_dir" --no-sync archivebox config --set SEARCH_BACKEND_ENGINE=ripgrep; uv run --project "$project_dir" --no-sync archivebox config --set RIPGREP_BINARY="$ABXPKG_LIB_DIR/env/bin/ugrep"; uv run --project "$project_dir" --no-sync archivebox add --plugins=wget "${ARCHIVEBOX_DOCS_URL_TWO:-https://example.com/}"; search_result="$(uv run --project "$project_dir" --no-sync archivebox search --search contents:ripgrep 'ArchiveBox docs fixture')"; grep -q "${ARCHIVEBOX_DOCS_URL_TWO:-https://example.com/}" <<< "$search_result"
-```
-
 #### Pros
 
 - supports [boolean operators](https://github.com/Genivia/ugrep#bool) in search queries
@@ -144,30 +126,17 @@ Internally it functions as an index store, storing only the original IDs of the
 
 *ArchiveBox has supported Sonic for years, and it is the most thoroughly tested and recommended backend for ArchiveBox users that need to scale beyond `ripgrep`.*
 
-Using [sonic with ArchiveBox in Docker Compose](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docker-compose.yml) is the easiest way to get started, though you can also use it without Docker by [installing it manually](https://github.com/valeriansaliou/sonic#installation) and then running `uv tool install --python 3.13 --upgrade 'archivebox[sonic] @ git+https://github.com/ArchiveBox/ArchiveBox.git@dev'`.
+ArchiveBox resolves and starts Sonic through the same `abxpkg` lifecycle on both Docker and bare-metal installations.
 
 ```bash
-set -euo pipefail; compose_dir="$(mktemp -d)"; compose_file="$compose_dir/docker-compose.yml"; mkdir -p "$compose_dir/data" "$compose_dir/fixture" "$compose_dir/sonic/store/kv" "$compose_dir/sonic/store/fst"; printf '<title>ArchiveBox docs fixture</title> sonic indexed body\n' > "$compose_dir/fixture/sonic-docs"
-printf '%s\n' '[server]' 'log_level = "error"' '[channel]' 'inet = "0.0.0.0:1491"' 'tcp_timeout = 300' 'auth_password = "SecretPassword"' '[channel.search]' 'query_limit_default = 10' 'query_limit_maximum = 100' 'query_alternates_try = 4' 'suggest_limit_default = 5' 'suggest_limit_maximum = 20' 'list_limit_default = 100' 'list_limit_maximum = 500' '[store]' '[store.kv]' 'path = "/var/lib/sonic/store/kv/"' 'retain_word_objects = 1000' '[store.kv.pool]' 'inactive_after = 1800' '[store.kv.database]' 'flush_after = 1' 'compress = true' 'parallelism = 2' 'max_files = 100' 'max_compactions = 1' 'max_flushes = 1' 'write_buffer = 16384' 'write_ahead_log = true' '[store.fst]' 'path = "/var/lib/sonic/store/fst/"' '[store.fst.pool]' 'inactive_after = 300' '[store.fst.graph]' 'consolidate_after = 1' 'max_size = 2048' 'max_words = 250000' > "$compose_dir/sonic.cfg"; printf 'services:\n  archivebox:\n    image: archivebox-docs-ci\n    environment:\n      SEARCH_BACKEND_ENGINE: sonic\n      SEARCH_BACKEND_SONIC_HOST_NAME: sonic\n      SEARCH_BACKEND_SONIC_PORT: 1491\n      SEARCH_BACKEND_SONIC_PASSWORD: SecretPassword\n    depends_on:\n      sonic:\n        condition: service_started\n      fixture:\n        condition: service_started\n    volumes:\n      - %s:/data\n  sonic:\n    image: valeriansaliou/sonic:v1.4.9\n    volumes:\n      - %s:/etc/sonic.cfg:ro\n      - %s:/var/lib/sonic/store\n  fixture:\n    image: python:3.13-alpine\n    command: python -m http.server 8000 --directory /fixture\n    volumes:\n      - %s:/fixture:ro\n' "$compose_dir/data" "$compose_dir/sonic.cfg" "$compose_dir/sonic/store" "$compose_dir/fixture" > "$compose_file"
-trap 'docker compose -f "$compose_file" down --remove-orphans' EXIT
-docker compose -f "$compose_file" up -d sonic fixture
-docker compose -f "$compose_file" run --rm archivebox init
-docker compose -f "$compose_file" run --rm archivebox add --plugins=wget 'http://fixture:8000/sonic-docs'
-docker compose -f "$compose_file" run --rm archivebox update --index-only
-sonic_ids="$(docker compose -f "$compose_file" run --rm archivebox shell -c "from archivebox.search.query import iter_query_search_ids; print(*iter_query_search_ids('sonic-docs', search_mode='contents:sonic'))")"; test -n "$sonic_ids"
-search_result="$(docker compose -f "$compose_file" run --rm archivebox search --search contents:sonic 'sonic-docs')"; grep -q 'http://fixture:8000/sonic-docs' <<< "$search_result"
-docker compose -f "$compose_file" logs sonic
-test -f "$compose_dir/data/index.sqlite3"
-test -n "$(find "$compose_dir/sonic/store" -type f -print -quit)"
-running_services="$(docker compose -f "$compose_file" ps --status running --services)"; grep -qx sonic <<< "$running_services"; grep -qx fixture <<< "$running_services"
-docker compose -f "$compose_file" down --remove-orphans
-trap - EXIT
-test -f "$compose_file"
-test -d "$compose_dir/data/archive"
-docker image inspect archivebox-docs-ci >/dev/null
-docker image inspect valeriansaliou/sonic:v1.4.9 >/dev/null
+archivebox config --set SEARCH_BACKEND_ENGINE=sonic
+archivebox install sonic
+archivebox update --index-only
+archivebox search 'some text to search'
 ```
 
+Run the same commands as `docker compose run archivebox ...` when using Docker Compose.
+
 *Fore more detailed instructions [see here](https://github.com/ArchiveBox/ArchiveBox/issues/956#issuecomment-1320587158)...*
 
 #### Pros
@@ -192,26 +161,28 @@ docker image inspect valeriansaliou/sonic:v1.4.9 >/dev/null
 This is a [recently added](https://github.com/ArchiveBox/ArchiveBox/pull/1241) experimental option that uses a separate SQLite3 Database (similar to the one ArchiveBox already uses for Snapshot metadata) to provide full-text search.
 
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"; cd "$archivebox_data"
-uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox add --plugins=mercury "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
-uv run --project "$project_dir" --no-sync archivebox config --set SEARCH_BACKEND_ENGINE=sqlite
-uv run --project "$project_dir" --no-sync archivebox config --set SEARCH_BACKEND_SQLITE_SEPARATE_DATABASE=True
-uv run --project "$project_dir" --no-sync archivebox update --index-only
-search_result="$(uv run --project "$project_dir" --no-sync archivebox search --search contents:sqlite 'ArchiveBox docs fixture')"; grep -q "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}" <<< "$search_result"
-uv run --project "$project_dir" --no-sync abxpkg env --install --lib="$ABXPKG_LIB_DIR" --binproviders env,apt,brew sqlite3 >/dev/null
-test -x "$ABXPKG_LIB_DIR/env/bin/sqlite3"
-sqlite_tables="$("$ABXPKG_LIB_DIR/env/bin/sqlite3" ./search.sqlite3 '.tables')"; grep -q search_index <<< "$sqlite_tables"
-sqlite_count="$("$ABXPKG_LIB_DIR/env/bin/sqlite3" ./search.sqlite3 'SELECT COUNT(*) FROM search_index;')"; grep -Eq '^[1-9][0-9]*$' <<< "$sqlite_count"
+archivebox config --set SEARCH_BACKEND_ENGINE=sqlite
+
+# add existing data to index by running update:
+archivebox update --index-only
+
+# test it out using the archivebox Web UI or CLI:
+archivebox search 'some text to search'
+```
+
+You can also inspect the separate FTS database directly:
+
+```bash
+sqlite3 ./search.sqlite3
+
+> SELECT snapshot_id, url FROM search_index
+      WHERE search_index MATCH 'some text to search';
 ```
 
 ```bash
-set -euo pipefail; project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"; archivebox_data="$(mktemp -d)"; cd "$archivebox_data"
-uv run --project "$project_dir" --no-sync archivebox init
-uv run --project "$project_dir" --no-sync archivebox config --set SEARCH_BACKEND_SQLITE_SEPARATE_DATABASE=True
-uv run --project "$project_dir" --no-sync archivebox config --set SEARCH_BACKEND_SQLITE_TOKENIZERS="porter unicode61 remove_diacritics 2"
-uv run --project "$project_dir" --no-sync archivebox config --get SEARCH_BACKEND_SQLITE_DB
+# optional advanced tuning:
+archivebox config --set FTS_SEPARATE_DATABASE=True
+archivebox config --set FTS_TOKENIZERS="porter unicode61 remove_diacritics 2"
 ```
 
 - https://www.sqlite.org/fts5.html

docs/Troubleshooting.md

@@ -19,12 +19,12 @@ If using `archivebox` without Docker, make sure you've followed the full guide i
 
 Then make sure `archivebox` is installed available in your `$PATH`.
 ```bash
-set -Eeuo pipefail; if command -v apt >/dev/null; then apt show archivebox || test "$?" -eq 100; fi
-if command -v brew >/dev/null; then brew info archivebox/archivebox/archivebox || test "$?" -eq 1; fi
-uv tool list
+apt show archivebox      # show info about the apt-installed version of archivebox
+brew info archivebox     # show info about the brew-installed version of archivebox
+uv tool list             # show info about uv-installed tools
 
-printf '%s\n' "$PATH"
-type -a archivebox
+echo $PATH               # show the directories your system is searching for binaries
+type -a archivebox       # show all installed archivebox binaries available
 ```
 **⭐️ Show the full archivebox version info + info about all installed dependencies:**
 ```bash
@@ -35,95 +35,59 @@ archivebox version       # shows lots of useful info about installed dependencie
 ### macOS
 ArchiveBox can be installed with Homebrew or `uv` on macOS:
 ```bash
-set -Eeuo pipefail
 brew tap archivebox/archivebox
-brew install archivebox; archivebox_binary="$(brew --prefix archivebox/archivebox/archivebox)/bin/archivebox"; test -x "$archivebox_binary"
-data_dir="$(mktemp -d)"; trap 'rm -rf "$data_dir"' EXIT
-mkdir -p "$data_dir"
-cd "$data_dir"
-"$archivebox_binary" init
-"$archivebox_binary" install
+brew install archivebox
+
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data     # (for example, can be anywhere)
+
+archivebox init
+archivebox install       # finish installing runtime dependencies
 ```
 More info: https://github.com/ArchiveBox/homebrew-archivebox
 
-### Python
+### Python and uv
 
-Make sure you have at least Python 3.13 installed on your system.
+ArchiveBox's supported bare-metal install uses `uv`, which manages the Python 3.13 environment for the tool:
 
 ```bash
-set -Eeuo pipefail; uv --version
-uv python find 3.13
-uv run --no-project --python 3.13 python --version
+uv --version
+uv tool list
+archivebox version
 ```
 
-If you still need help getting Python installed, [the official Python docs](https://docs.python.org/3.9/using/unix.html) are a good place to start.
+If `archivebox` is missing, repeat the `uv tool install` command from the [[Install]] guide.
 
 ### Chromium/Google Chrome
 
 For more info, see the [[Chromium Install]] page.
 
-ArchiveBox depends on being able to access a `chromium`/`google-chrome` executable.  The executable used
-defaults to `chromium` but can be manually specified with the environment variable [`CHROME_BINARY`](https://archivebox.github.io/abx-plugins/#chrome):
+ArchiveBox resolves Chrome through `abxpkg`, preferring a compatible browser already installed on the host and otherwise installing a managed build:
 
 ```bash
-set -Eeuo pipefail; eval "$(uv run abxpkg env chromium --install --lib "$ABXPKG_LIB_DIR" --binproviders env,playwright,puppeteer --min-version 111 --postinstall-scripts)"; chrome_binary="$(command -v chromium)"; test -x "$chrome_binary"; env CHROME_BINARY="$chrome_binary" archivebox version
+archivebox install chrome
+archivebox version
 ```
 
-1. Test to make sure you have Chrome on your `$PATH` with:
-
-```bash
-set -Eeuo pipefail; eval "$(uv run abxpkg env chromium --install --lib "$ABXPKG_LIB_DIR" --binproviders env,playwright,puppeteer --min-version 111 --postinstall-scripts)"; chrome_binary="$(command -v chromium)"; test -x "$chrome_binary"; printf '%s\n' "$chrome_binary"
-```
-If no executable is displayed, follow the setup instructions to install and link one of them.
-
-2. If a path is displayed, the next step is to check that it's runnable:
-
-```bash
-set -Eeuo pipefail; eval "$(uv run abxpkg env chromium --install --lib "$ABXPKG_LIB_DIR" --binproviders env,playwright,puppeteer --min-version 111 --postinstall-scripts)"; chrome_binary="$(command -v chromium)"; test -x "$chrome_binary"; "$chrome_binary" --version
-```
-If no version is displayed, try the setup instructions again, or confirm that you have permission to access chrome.
-
-3. If a version is displayed and it's `<111`, upgrade it:
-
-```bash
-set -Eeuo pipefail; eval "$(uv run abxpkg env chromium --install --lib "$ABXPKG_LIB_DIR" --binproviders env,playwright,puppeteer --min-version 111 --postinstall-scripts)"
-chrome_binary="$(command -v chromium)"; test -x "$chrome_binary"
-chrome_major="$("$chrome_binary" --version | sed -E 's/[^0-9]*([0-9]+).*/\1/')"; test "$chrome_major" -ge 111
-```
-
-4. If a version is displayed and it's `>=111`, make sure ArchiveBox is running the right one:
-
-```bash
-set -Eeuo pipefail; eval "$(uv run abxpkg env chromium --install --lib "$ABXPKG_LIB_DIR" --binproviders env,playwright,puppeteer --min-version 111 --postinstall-scripts)"; chrome_binary="$(command -v chromium)"; test -x "$chrome_binary"; env CHROME_BINARY="$chrome_binary" archivebox version
-```
+The version output shows the selected provider, version, and projected path. If it reports an incompatible host browser, update that browser or let ArchiveBox install the managed fallback; do not bypass the resolver with an unrelated path.
 
 
 ### Wget & Curl
 
-If you're missing `wget` or `curl`, simply install them using `apt` or your package manager of choice.
-See the "Manual Setup" instructions for more details.
+Resolve or update both tools through the same installer:
 
-If wget times out or randomly fails to download some sites that you have confirmed are online,
-upgrade wget to the most recent version with `brew upgrade wget` or `apt upgrade wget`.  There is
-a bug in versions `<=1.19.1_1` that caused wget to fail for perfectly valid sites.
+```bash
+archivebox install wget curl
+archivebox version
+```
 
 ### NPM Dependencies
 
-NPM packages like `readability`, `singlefile`, etc. are auto-installed by `archivebox install`.
-
-Make sure you have installed NodeJS + NPM first, here are their [official install docs](https://nodejs.org/en/download/package-manager/).
+Node.js and JavaScript extractor packages such as `readability` and `singlefile` are resolved through `abxpkg`; they do not require a separate global npm setup.
 
 ```bash
-set -Eeuo pipefail
-test -f index.sqlite3
-archivebox install node
-uv run abxpkg env node npm --install --lib "$ABXPKG_LIB_DIR" --binproviders env,npm >/dev/null
-node_binary="$ABXPKG_LIB_DIR/env/bin/node"
-npm_binary="$ABXPKG_LIB_DIR/env/bin/npm"
-test -x "$node_binary"
-test -x "$npm_binary"
-"$node_binary" --version
-"$npm_binary" --version
+cd ~/archivebox/data   # go into your data directory
+archivebox install node singlefile readability
 archivebox version
 ```
 
@@ -142,7 +106,7 @@ If you ran the archiver once, it wont re-download sites subsequent times, it wil
 If you haven't already run it, make sure you have a working internet connection and that the parsed URLs look correct.
 You can check the ArchiveBox stdout logs or the Web UI to see what links it's downloading.
 
-If you're still having issues, try deleting or moving the `./archive` folder (back it up first!) and running `archivebox init` again.
+To intentionally capture an already indexed URL again, use `archivebox add --no-only-new URL`. Do not delete or move the `archive/` tree to work around `ONLY_NEW`; that separates database state from its Snapshot files.
 
 ### Lots of errors
 
@@ -169,23 +133,7 @@ if you have problem with a particular nginx config.
 
 #### Docker Permissions issues
 
-Make sure the mounted data directory is writable by the user that owns it. The `archivebox` username only exists inside the Docker container, so on the host you should check numeric ownership instead. For a new or root-owned Docker data directory, make sure it is writable by UID/GID `911:911`.
-
-Try using [`bindfs`](https://github.com/clecherbauer/docker-volume-bindfs) to work around issues by remapping permissions, for example to remap `uid:33 gid:33` on the host to `911:911` inside the container:
-`docker-compose.yml`:
-```yaml
-services:
-  archivebox:
-    volumes:
-      - archivebox-data:/data
-
-volumes:
-  archivebox-data:
-    driver: lebokus/bindfs:latest
-    driver_opts:
-      sourcePath: "${EXTERNAL_MOUNT_PARENT}/external-parent/external/archivebox"
-      map: "33/911:@33/@911"
-```
+Make sure the mounted data directory is writable by its intended non-root owner. The current Docker entrypoint detects the first non-root collection owner and runs ArchiveBox with matching numeric UID/GID; a new root-owned collection falls back to the image's `archivebox` user. Check the host directory's numeric ownership and the entrypoint's startup output before changing permissions.
 
 <br/>
 
@@ -199,14 +147,12 @@ Database and filesystem issues are uncommon but do come up from time to time (es
 
 *ℹ️ Generally, these commands can help you resolve most issues:*
 ```bash
-set -Eeuo pipefail
 archivebox init                 # upgrade the archivebox collection
-archivebox install wget         # upgrade a runtime dependency through the normal installer
+archivebox install              # upgrade the archivebox runtime dependencies
 archivebox update --index-only  # force an upgrade of some of the archivebox index/collection files
-archivebox server --debug --help
-archivebox shell --help
-uv run abxpkg env sqlite3 --install --lib "$ABXPKG_LIB_DIR" --binproviders env,apt,brew >/dev/null
-"$ABXPKG_LIB_DIR/env/bin/sqlite3" --version
+archivebox server --debug       # run the server with more verbose debug log output
+archivebox shell                # access the Python API / Django management shell
+sqlite3 index.sqlite3           # access the SQLite3 SQL database shell
 ```
 
 Don't be scared by the volume of content here. Almost all of these issues linked below are duplicates or old resolved bugs, but they contain valuable context and troubleshooting steps if you're trying to figure out the cause of a problem with your setup.
@@ -230,8 +176,7 @@ More info:
 
 ArchiveBox can sometimes struggle when archiving many links in parallel with multiple ArchiveBox processes trying to write to the database at the same time, leading to errors like this:
 ```bash
-error='Unable to create the django_migrations table (database is locked)'
-printf '%s\n' "$error" | grep -F 'database is locked'
+Unable to create the django_migrations table (database is locked)
 ```
 
 These errors can also be encountered when there are permissions, network, or filesystem issues preventing writes to `index.sqlite3`.
@@ -284,8 +229,7 @@ A corrupted database file can theoretically only happen if an external process o
 
 Note this is specific to this error, these steps do not apply to other migrations/db errors (see above/below for other issues):
 ```bash
-error='sqlite3.DatabaseError: database disk image is malformed'
-printf '%s\n' "$error" | grep -F 'database disk image is malformed'
+sqlite3.DatabaseError: database disk image is malformed
 ```
 
 Generally all index issues should be fixable by running `archivebox init`.  
@@ -293,7 +237,7 @@ You can see the status of Snapshots and find any invalid/orphan/missing snapshot
 
 **Error output:**
 
-```text
+```python3
 [i] [2022-03-24 20:37:27] ArchiveBox v0.6.2: archivebox init                                                                                                                                  
     > /data                                                                                                                                                                                   
                                                                                                                                                                                               
@@ -316,17 +260,10 @@ sqlite3.DatabaseError: database disk image is malformed
 **Steps to fix:**
 
 ```bash
-set -Eeuo pipefail
-test -s index.sqlite3
-test ! -e corrupt_index.sqlite3
-test ! -e repaired_index.sqlite3
-uv run abxpkg env sqlite3 --install --lib "$ABXPKG_LIB_DIR" --binproviders env,apt,brew >/dev/null
-sqlite3_binary="$ABXPKG_LIB_DIR/env/bin/sqlite3"; test -x "$sqlite3_binary"
-echo '.dump' | "$sqlite3_binary" index.sqlite3 | "$sqlite3_binary" repaired_index.sqlite3
-"$sqlite3_binary" repaired_index.sqlite3 'PRAGMA integrity_check;' | grep -Fx ok
+cd ~/archivebox/data
+echo '.dump' | sqlite3 index.sqlite3 | sqlite3 repaired_index.sqlite3
 mv index.sqlite3 corrupt_index.sqlite3
 mv repaired_index.sqlite3 index.sqlite3
-"$sqlite3_binary" index.sqlite3 'PRAGMA integrity_check;' | grep -Fx ok
 ```
 
 More info:

docs/Upgrading.md

@@ -1,15 +1,17 @@
 # Upgrading Versions
 
 ```bash
-set -Eeuo pipefail; cd "${ARCHIVEBOX_DATA_DIR:-$PWD}"
-test -f index.sqlite3
+# cd /path/to/your/archivebox/data
+cd ~/archivebox/data
 
-archivebox_source="${ARCHIVEBOX_PROJECT_DIR:-git+https://github.com/ArchiveBox/ArchiveBox.git@dev}"; if test -n "${RUNNER_TEMP:-}"; then export UV_TOOL_DIR="$RUNNER_TEMP/archivebox-upgrade-tool" UV_TOOL_BIN_DIR="$RUNNER_TEMP/archivebox-upgrade-tool/bin"; fi
-uv tool install --python 3.13 --upgrade "$archivebox_source"
-archivebox_binary="$(uv tool dir --bin)/archivebox"
-"$archivebox_binary" init
-"$archivebox_binary" status
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
+# or
+docker pull archivebox/archivebox:dev
 
+# upgrade the collection to a new version
+archivebox init
+archivebox install
+archivebox update
 ```
 
 
@@ -19,7 +21,7 @@ archivebox_binary="$(uv tool dir --bin)/archivebox"
 2. **Read the release notes carefully** for any instructions or extra steps around upgrading for each release you're skipping or installing
 3. **Make a full backup** of your `index.sqlite3` and `archive/` content before upgrading!  
 `gzip -9 < index.sqlite3 > "index.sqlite3.$(date +%s).bak"`
-4. Follow the steps below depending on your setup to run `archivebox init` (repeating as necessary for each major version if upgrading across multiple major versions)
+4. Follow the steps below for your installation method, then run `archivebox init`, `archivebox install`, and `archivebox update` inside the collection
 5. Confirm the upgrade succeeded and check for any orphan/corrupted snapshots with `archivebox status`
 
 💬 [Open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new/choose) in our bug tracker if you experience any problems with upgrading/merging/modifying collections.
@@ -34,14 +36,14 @@ You can specify exact versions with uv like so: `uv tool install --python 3.13 -
 
 **ℹ️ How it works internally:**
 
-The same command is used for initializing a new archive and upgrading an existing one. `archivebox init` is idempotent and safely be run multiple times. Running it will ensure your collection is on the latest version and all the files are in their correct locations. `archivebox status` can be used to check for orphan/corrupted snapshots or invalid index data.
+The same command is used for initializing a new archive and upgrading an existing database. `archivebox init` is idempotent and can safely be run multiple times; it applies database migrations and prepares collection-level state. `archivebox install` resolves runtime dependencies for the new version. `archivebox update` performs filesystem migrations and reconciles Snapshot metadata with the current layout. `archivebox status` checks collection health afterward.
 
 There are three main areas on disk that ArchiveBox modifies during upgrades:
 - `index.sqlite3` contains the SQLite3 DB index that gets upgraded automatically by Django based on the changes in [`archivebox/core/models.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/models.py).
-- `archive/*/index.json` these files are redundant json exports of the data for each Snapshot in `index.sqlite3`, these files are overwritten on every `archivebox update` run or anytime the Snapshot is modified from the GUI or CLI. These files will be [lazily updated](https://github.com/ArchiveBox/ArchiveBox/issues/962) to the latest schema versions as ArchiveBox accesses them, but are usually not modified in bulk during `archivebox init` when upgrading.
-- `archive/*` the Snapshot output files may be moved or renamed by future upgrades (so far they have remained unchanged since v0.1, but future versions reserve the right to change their locations)
+- `archive/users/<user>/snapshots/<date>/<domain>/<uuid>/index.jsonl` stores per-Snapshot metadata alongside plugin-namespaced output. `archivebox update` may rewrite metadata, migrate older layouts, and maintain legacy timestamp compatibility symlinks.
+- Snapshot output directories and plugin paths can move as filesystem schemas evolve, so the entire `archive/` tree must be backed up with the database.
 
-The `ArchiveBox.conf` file is not modified by upgrades and should remain forward-compatible across future versions (even when config options are renamed, we check the old names internally to maintain compatibility).
+`ArchiveBox.conf` is migrated through the normal config loader/writer when options are renamed or normalized. Back it up with the rest of the collection and review release notes for config changes.
 
 As of v0.4 and above, ArchiveBox uses the Django migrations system for deterministic, atomic, safe upgrades, so your DB should always be left in a consistent state in the event of a failure or power outage. If you need help fixing a corrupted collection, open an issue using the link above.
 
@@ -58,11 +60,10 @@ Using Docker Compose is recommended because it makes upgrading a breeze! ✨
 Pulling and running the latest version automatically upgrades the ArchiveBox collection and all of ArchiveBox's internal dependencies.
 
 ```bash
-set -Eeuo pipefail; compose_file="$(mktemp)"; docker_data="$(mktemp -d)"; if test -n "${CI:-}"; then docker tag archivebox-docs-ci archivebox/archivebox:dev; fi
-printf 'services:\n  archivebox:\n    image: archivebox/archivebox:dev\n    volumes:\n      - %s:/data\n' "$docker_data" > "$compose_file"
-docker compose -f "$compose_file" down; if test -n "${CI:-}"; then docker image inspect archivebox/archivebox:dev >/dev/null; else docker compose -f "$compose_file" pull; fi
-docker compose -f "$compose_file" run --rm archivebox init
-docker compose -f "$compose_file" up -d; container_id="$(docker compose -f "$compose_file" ps -q --status running archivebox)"; test -n "$container_id"; docker compose -f "$compose_file" down
+cd ~/archivebox        # or wherever your folder containing docker-compose.yml is
+docker compose down    # stop the currently running ArchiveBox containers
+docker compose pull    # pull the latest image version from Docker Hub
+docker compose up      # collection will be automatically upgraded as it starts
 ```
 
 More info:
@@ -75,14 +76,16 @@ More info:
 Upgrading with plain Docker is similar to the process with Docker Compose, but you have to run `archivebox init` manually at the end to finish the process.
 
 ```bash
-set -Eeuo pipefail; docker_data="$(mktemp -d)"; if test -n "${CI:-}"; then docker tag archivebox-docs-ci archivebox/archivebox:dev; fi
-docker image inspect archivebox/archivebox:dev >/dev/null
-docker run --rm -v "$docker_data:/data" archivebox/archivebox:dev init
-container_id="$(docker run --rm -d -v "$docker_data:/data" archivebox/archivebox:dev server 0.0.0.0:8000)"
-test -n "$container_id"; docker inspect --format '{{.State.Running}}' "$container_id" | grep -Fx true
-docker kill "$container_id"
-docker run --rm -v "$docker_data:/data" archivebox/archivebox:dev init
-docker run --rm -v "$docker_data:/data" archivebox/archivebox:dev server --help
+docker ps -a -q  --filter ancestor=archivebox/archivebox  # find any currently running archivebox containers
+docker stop CONTAINER_ID
+
+docker pull archivebox/archivebox:dev
+docker run -v $PWD:/data -it archivebox/archivebox:dev init
+docker run -v $PWD:/data -it archivebox/archivebox:dev install
+docker run -v $PWD:/data -it archivebox/archivebox:dev update
+
+# restart the archivebox server container if needed
+docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev server 0.0.0.0:8000
 ```
 
 More info:
@@ -94,32 +97,30 @@ More info:
 
 Package manager releases take a lot of effort to maintain ([contributions welcome!](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)) and sometimes lag behind the Docker releases. We make a best effort to have the latest release available through all channels within a reasonable timeframe.
 
-Use the same package manager you originally used to install ArchiveBox. For a `uv` installation:
-
 ```bash
-set -Eeuo pipefail
-cd "${ARCHIVEBOX_DATA_DIR:-$PWD}"
-test -f index.sqlite3
-archivebox_source="${ARCHIVEBOX_PROJECT_DIR:-git+https://github.com/ArchiveBox/ArchiveBox.git@dev}"
-if test -n "${RUNNER_TEMP:-}"; then export UV_TOOL_DIR="$RUNNER_TEMP/archivebox-upgrade-tool" UV_TOOL_BIN_DIR="$RUNNER_TEMP/archivebox-upgrade-tool/bin"; fi
-uv tool install --python 3.13 --upgrade "$archivebox_source"
-archivebox_binary="$(uv tool dir --bin)/archivebox"
-"$archivebox_binary" init
-"$archivebox_binary" install
-"$archivebox_binary" update --index-only
-"$archivebox_binary" status
-```
+cd ~/archivebox/data   # or wherever your data folder is
 
-For the Debian package, run `sudo apt update` followed by `sudo apt install --only-upgrade archivebox`. The optional auto-installer can be updated by running `curl -sSL 'https://get.archivebox.io' | sh`. Do not mix package managers for the same installation.
+# upgrade ArchiveBox using the package manager you originally used to install it
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
+# or
+sudo apt update
+sudo apt install --only-upgrade archivebox
+# or with the optional auto-installer script
+curl -sSL 'https://get.archivebox.io' | sh
+
+archivebox init        # run init to upgrade the collection to the latest version
+archivebox install     # refresh runtime dependencies if needed
+
+archivebox update       # migrate/reconcile Snapshot files and metadata
+
+archivebox status      # check that everything succeeded
+```
 
 More info:
 - https://github.com/ArchiveBox/ArchiveBox#-package-manager-setup
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
-- https://github.com/ArchiveBox/pip-archivebox
 - https://github.com/ArchiveBox/homebrew-archivebox
-- https://github.com/ArchiveBox/docker-archivebox
 - https://github.com/ArchiveBox/debian-archivebox
-- https://github.com/ArchiveBox/electron-archivebox
 - https://aur.archlinux.org/packages/archivebox
 - https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix
 

docs/Usage.md

@@ -29,7 +29,7 @@ All three of these ways of running ArchiveBox are equivalent and interchangeable
   *Using the Python package via `uv tool install archivebox`*
 - `docker run ... archivebox/archivebox [subcommand] [...args]`  
   *Using the official Docker image*
-- `docker-compose run archivebox [subcommand] [...args]`  
+- `docker compose run archivebox [subcommand] [...args]`
   *Using the official Docker image w/ Docker Compose*
 
 You can share a single archivebox data directory between Docker and non-Docker instances as well, allowing you to run the server in a container but still execute CLI commands on the host for example.
@@ -48,17 +48,17 @@ For more examples see [README: Usage](https://github.com/ArchiveBox/ArchiveBox#%
 You can set environment variables in your shell profile, a config file, or by using the `env` command.
 
 ```bash
-# Persist a setting in this collection and verify the effective value.
+# set config via the CLI
 archivebox config --set TIMEOUT=120
-config_output="$(archivebox config --get TIMEOUT)"
-case "$config_output" in *'TIMEOUT = 120'*) ;; *) exit 1 ;; esac
 
-# Environment variables override the persisted value for one command.
-config_output="$(TIMEOUT=121 archivebox config --get TIMEOUT)"
-case "$config_output" in *'TIMEOUT = 121'*) ;; *) exit 1 ;; esac
+# OR edit ArchiveBox.conf and add this under its existing [ARCHIVING_CONFIG] section:
+TIMEOUT=120
+
+# OR use environment variables
+env TIMEOUT=120 archivebox add 'https://example.com'
 ```
 
-See [[Configuration]] page for core ArchiveBox config options and the [abx-plugins config reference](https://archivebox.github.io/abx-plugins/) for per-plugin options (e.g. `MEDIA_MAX_SIZE`, `CHROME_USER_DATA_DIR`, `WGET_ARGS`, etc.).  
+See [[Configuration]] page for core ArchiveBox config options and the [abx-plugins config reference](https://archivebox.github.io/abx-plugins/) for per-plugin options (e.g. `YTDLP_MAX_SIZE`, `CHROME_USER_DATA_DIR`, `WGET_ARGS`, etc.).
 If you're using Docker, also make sure to read the Configuration section on the [[Docker]] page.
 
 > [!TIP]  
@@ -70,10 +70,9 @@ If you're using Docker, also make sure to read the Configuration section on the
 ### Import a single URL
 
 ```bash
-url="$ARCHIVEBOX_DOCS_URL_ONE/usage-single"
-archivebox add --index-only "$url"
-archivebox shell -c \
-    "from archivebox.crawls.models import Crawl; assert Crawl.objects.filter(urls__contains='$url').exists()"
+archivebox add 'https://example.com'
+# OR
+echo 'https://example.com' | archivebox add
 ```
 
 You can also add `--depth=1` to any of these commands if you want to recursively archive the URLs and all URLs one hop away. (e.g. all the outlinks on a page + the page).
@@ -81,34 +80,23 @@ You can also add `--depth=1` to any of these commands if you want to recursively
 ### Import a list of URLs from a text file
 
 ```bash
-urls_file="$(mktemp)"
-printf '%s\n%s\n' \
-    "$ARCHIVEBOX_DOCS_URL_ONE/usage-list-one" \
-    "$ARCHIVEBOX_DOCS_URL_TWO/usage-list-two" > "$urls_file"
-archivebox add --index-only < "$urls_file"
-
-feed_file="$(mktemp)"
-"$CURL_BINARY" --fail --silent --show-error \
-    "$ARCHIVEBOX_DOCS_URL_ONE/usage-feed" > "$feed_file"
-archivebox add --index-only < "$feed_file"
+cat urls_to_archive.txt | archivebox add
+# OR
+archivebox add < urls_to_archive.txt
+# OR
+curl 'https://example.com/some/rss/feed.xml' | archivebox add
+# OR
+archivebox add --depth=1 'https://example.com/some/rss/feed.xml'
 ```
 
 You can also pipe in RSS, XML, Netscape, or any of the other [supported import formats](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) via stdin.
 
 ```bash
-imports_dir="$(mktemp -d)"
-cat > "$imports_dir/bookmarks.html" <<EOF
-<!DOCTYPE NETSCAPE-Bookmark-file-1>
-<DL><p><DT><A HREF="$ARCHIVEBOX_DOCS_URL_ONE/usage-bookmark">Docs bookmark</A></DL>
-EOF
-printf '[{"href":"%s","description":"Pinboard fixture"}]\n' \
-    "$ARCHIVEBOX_DOCS_URL_TWO/usage-pinboard" > "$imports_dir/pinboard.json"
-printf 'Read %s next.\n' \
-    "$ARCHIVEBOX_DOCS_URL_ONE/usage-text" > "$imports_dir/urls.txt"
-
-archivebox add --index-only < "$imports_dir/bookmarks.html"
-archivebox add --index-only < "$imports_dir/pinboard.json"
-archivebox add --index-only < "$imports_dir/urls.txt"
+archivebox add < ~/Downloads/browser_bookmarks_export.html
+# OR
+archivebox add < ~/Downloads/pinboard_bookmarks.json
+# OR
+archivebox add < ~/Downloads/any_text_containing_urls.txt
 ```
 
 ---
@@ -119,40 +107,14 @@ Look in the `bin/` folder of this repo to find a script to parse your browser's
 Specify the type of the browser as the first argument, and optionally the path to the SQLite history file as the second argument.
 
 ```bash
-history_dir="$(mktemp -d)"
-uv run --project "$ARCHIVEBOX_PROJECT_DIR" --no-sync \
-    abxpkg install sqlite3 --lib "$ABXPKG_LIB_DIR" --binproviders env,apt,brew
-SQLITE3_BINARY="$ABXPKG_LIB_DIR/env/bin/sqlite3"
-test -L "$SQLITE3_BINARY" && test -x "$SQLITE3_BINARY"
-
-"$SQLITE3_BINARY" "$history_dir/History" \
-    "CREATE TABLE urls (last_visit_time INTEGER, title TEXT, url TEXT); INSERT INTO urls VALUES (1, 'Chrome fixture', '$ARCHIVEBOX_DOCS_URL_ONE/chrome-history');"
-printf '{"roots":{"other":{"children":[{"url":"%s","name":"Chrome bookmark","date_added":"2"}]}}}\n' \
-    "$ARCHIVEBOX_DOCS_URL_TWO/chrome-bookmark" > "$history_dir/Bookmarks"
-"$ARCHIVEBOX_PROJECT_DIR/bin/export_browser_history.sh" --chrome "$history_dir/History"
-"$JQ_BINARY" -e --arg url "$ARCHIVEBOX_DOCS_URL_ONE/chrome-history" \
-    '.[0].href == $url' chrome_history.json
-"$JQ_BINARY" -e --arg url "$ARCHIVEBOX_DOCS_URL_TWO/chrome-bookmark" \
-    '.[0].href == $url' chrome_bookmarks.json
-archivebox add --index-only < chrome_history.json
-archivebox add --index-only < chrome_bookmarks.json
-
-"$SQLITE3_BINARY" "$history_dir/places.sqlite" \
-    "CREATE TABLE moz_places (id INTEGER, last_visit_date INTEGER, title TEXT, url TEXT); CREATE TABLE moz_bookmarks (id INTEGER, parent INTEGER, fk INTEGER, dateAdded INTEGER, title TEXT); INSERT INTO moz_places VALUES (1, 3, 'Firefox fixture', '$ARCHIVEBOX_DOCS_URL_ONE/firefox-history'), (2, 4, 'Firefox bookmark', '$ARCHIVEBOX_DOCS_URL_TWO/firefox-bookmark'); INSERT INTO moz_bookmarks VALUES (1, 0, NULL, 0, 'root'), (2, 1, NULL, 0, 'docs'), (3, 2, 2, 4, 'Firefox bookmark');"
-"$ARCHIVEBOX_PROJECT_DIR/bin/export_browser_history.sh" --firefox "$history_dir/places.sqlite"
-"$JQ_BINARY" -e --arg url "$ARCHIVEBOX_DOCS_URL_ONE/firefox-history" \
-    '.[0].href == $url' firefox_history.json
-"$JQ_BINARY" -e --arg url "$ARCHIVEBOX_DOCS_URL_TWO/firefox-bookmark" \
-    '.[0].href == $url' firefox_bookmarks.json
-archivebox add --index-only < firefox_history.json
-archivebox add --index-only < firefox_bookmarks.json
-
-"$SQLITE3_BINARY" "$history_dir/History.db" \
-    "CREATE TABLE history_items (url TEXT); INSERT INTO history_items VALUES ('$ARCHIVEBOX_DOCS_URL_ONE/safari-history');"
-"$ARCHIVEBOX_PROJECT_DIR/bin/export_browser_history.sh" --safari "$history_dir/History.db"
-safari_history="$(< safari_history.json)"
-test "$safari_history" = "$ARCHIVEBOX_DOCS_URL_ONE/safari-history"
-archivebox add --index-only < safari_history.json
+bash ./bin/export_browser_history.sh --chrome
+archivebox add < chrome_history.json
+# or
+bash ./bin/export_browser_history.sh --firefox
+archivebox add < firefox_history.json
+# or
+bash ./bin/export_browser_history.sh --safari
+archivebox add < safari_history.json
 ```
 
 <br/>
@@ -163,57 +125,13 @@ archivebox add --index-only < safari_history.json
 
 ### Import browser cookies into a persona
 
-To archive logged-in sites, import a Chrome, Chromium, Brave, or Edge profile into a persona. Importing generates a `cookies.txt` file for wget/curl/yt-dlp and copies the profile so Chrome-based extractors can reuse it. Use `--profile='Profile 1'` when the browser profile is not named `Default`.
+To archive logged-in sites, you can import cookies from your browser into a persona. This generates a `cookies.txt` file in the persona directory (used by wget/curl/yt-dlp, etc.) and, for Chromium-based browsers, also copies the profile into the persona so Chrome-based extractors can reuse it.
 
 ```bash
-plugins_dir="$(
-    uv run --project "$ARCHIVEBOX_PROJECT_DIR" --no-sync python -c \
-        'from abx_plugins import get_plugins_dir; print(get_plugins_dir())'
-)"
-chrome_env="$(
-    uv run --project "$ARCHIVEBOX_PROJECT_DIR" --no-sync abxpkg env \
-        --install \
-        --json \
-        --lib "$ABXPKG_LIB_DIR" \
-        --deps-from "$plugins_dir/chrome/config.json:required_binaries"
-)"
-while IFS='=' read -r key value; do
-    export "$key=$value"
-done < <("$JQ_BINARY" -r 'to_entries[] | select(.key != "PATH") | "\(.key)=\(.value)"' <<< "$chrome_env")
-
-# This creates a real local Chromium profile for the executable example.
-# Normally, use the profile your desktop browser has already created.
-docs_home="$(mktemp -d)"
-host_profile_root="$(
-    HOME="$docs_home" uv run --project "$ARCHIVEBOX_PROJECT_DIR" --no-sync python -c \
-        'import platform; from pathlib import Path; home = Path.home(); print(home / "Library/Application Support/Chromium" if platform.system() == "Darwin" else home / ".config/chromium")'
-)"
-mkdir -p "$host_profile_root"
-HOST_PROFILE_ROOT="$host_profile_root" \
-HOST_PROFILE_URL="$ARCHIVEBOX_DOCS_URL_ONE/persona-profile" \
-"$NODE_BINARY" - <<'JS'
-const puppeteer = require('puppeteer');
-(async () => {
-  const browser = await puppeteer.launch({
-    executablePath: process.env.CHROME_BINARY,
-    headless: true,
-    userDataDir: process.env.HOST_PROFILE_ROOT,
-    args: ['--no-sandbox'],
-  });
-  const page = await browser.newPage();
-  await page.goto(process.env.HOST_PROFILE_URL);
-  await browser.close();
-})().catch(error => { console.error(error); process.exit(1); });
-JS
-test -s "$host_profile_root/Default/Preferences"
-
-persona_name="docs-personal-$$"
-persona_record="$(HOME="$docs_home" archivebox persona create --import=chromium --profile=Default "$persona_name")"
-archivebox shell -c \
-    "from archivebox.personas.models import Persona; p = Persona.objects.get(name='$persona_name'); assert (p.path / 'chrome_profile' / 'Default' / 'Preferences').is_file(); assert (p.path / 'cookies.txt').stat().st_size > 0"
-printf '%s\n' "$persona_record" | archivebox persona delete --yes
-archivebox shell -c \
-    "from archivebox.personas.models import Persona; assert not Persona.objects.filter(name='$persona_name').exists()"
+archivebox persona create --import=chrome personal
+# supported: chrome/chromium/brave/edge (Chromium-based only)
+# use --profile to target a specific profile (e.g. Default, Profile 1)
+# re-running import merges/dedupes cookies.txt (by domain/path/name) but replaces chrome_user_data
 ```
 
 If cookie extraction fails, you can still export a Netscape-format `cookies.txt` using a browser extension and place it at `data/personas/<NAME>/cookies.txt`.
@@ -232,12 +150,10 @@ archivebox config --set PUBLIC_INDEX=False
 archivebox config --set PUBLIC_ADD_VIEW=False
 archivebox config --set PERMISSIONS=private        # default visibility of newly created snapshots (was: PUBLIC_SNAPSHOTS=False)
 
-admin_username="docs-admin-$$"
-DJANGO_SUPERUSER_PASSWORD="$ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD" \
-    archivebox manage createsuperuser --noinput \
-        --username "$admin_username" --email "$admin_username@example.com"
-archivebox shell -c \
-    "from django.contrib.auth import get_user_model; assert get_user_model().objects.get(username='$admin_username').is_superuser"
+archivebox manage createsuperuser  # set an admin password to use for any areas requiring login
+archivebox server 0.0.0.0:8000     # start the archivebox web server
+
+open http://admin.archivebox.localhost:8000  # open the admin UI
 ```
 
 *See the [Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#permissions) and [Security Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#archiving-private-content) for more info...*
@@ -317,21 +233,27 @@ The `OUTPUT_DIR` folder (usually whatever folder you run the `archivebox` comman
 
 Simply back up the entire `data/` folder to back up your archive, e.g. `zip -r data.backup.zip data`.
 
-```text
+```yaml
  - data/
    - index.sqlite3        # Main index of all archived URLs
    - ArchiveBox.conf      # Main config file in ini format
 
    - archive/
-      - 155243135/        # Archived links are stored in folders by timestamp
-         - index.json     # Index/details page for individual archived link
-         - index.html
+      - 155243135 -> users/admin/snapshots/20210406/example.com/SNAPSHOT_UUID/
+      - users/
+         - admin/
+            - snapshots/
+               - 20210406/
+                  - example.com/
+                     - SNAPSHOT_UUID/
+                        - index.jsonl
+                        - index.html
 
-         # Archive method outputs:
-         - warc/
-         - media/
-         - git/
-         ...
+                        # Archive method outputs:
+                        - wget/warc/
+                        - ytdlp/media/
+                        - git/
+                        ...
 
    - sources/             # Each imported URL list is saved as a copy here
       - getpocket.com-1552432264.txt
@@ -353,16 +275,11 @@ Those numbers are from running it single-threaded on my i5 machine with 50mbps d
 
 Storage requirements go up immensely if you're using [`MEDIA_ENABLED=True`](https://archivebox.github.io/abx-plugins/#media) (or its [`FETCH_MEDIA`](https://archivebox.github.io/abx-plugins/#ytdlp) / `YTDLP_ENABLED` aliases) and are archiving many pages with audio & video.
 
-Import one combined list and let ArchiveBox's crawl runner manage concurrency. Starting multiple writers against the same collection can cause `database locked` errors on slower filesystems.
+ArchiveBox's unified crawl runner handles bounded concurrency without starting competing writers:
 ```bash
-combined_urls="$(mktemp)"
-printf '%s\n%s\n' \
-    "$ARCHIVEBOX_DOCS_URL_ONE/usage-batch-one" \
-    "$ARCHIVEBOX_DOCS_URL_TWO/usage-batch-two" > "$combined_urls"
-archivebox add --index-only < "$combined_urls"
-archivebox shell -c \
-    "from archivebox.crawls.models import Crawl; assert Crawl.objects.filter(urls__contains='usage-batch-one').filter(urls__contains='usage-batch-two').exists()"
+env CRAWL_MAX_CONCURRENT_SNAPSHOTS=4 archivebox add < urls_to_archive.txt
 ```
+Higher concurrency is not always faster on slow disks or network filesystems, so increase it gradually.
 
 Users have reported running it with 50k+ bookmarks with success (though it will take more RAM while running).
 
@@ -385,12 +302,13 @@ For more info about troubleshooting filesystem permissions, performance, or issu
 
 Explore the SQLite3 DB a bit to see what's available using the SQLite3 shell:
 ```bash
-uv run --project "$ARCHIVEBOX_PROJECT_DIR" --no-sync \
-    abxpkg install sqlite3 --lib "$ABXPKG_LIB_DIR" --binproviders env,apt,brew
-SQLITE3_BINARY="$ABXPKG_LIB_DIR/env/bin/sqlite3"
-test -L "$SQLITE3_BINARY" && test -x "$SQLITE3_BINARY"
-snapshot_count="$("$SQLITE3_BINARY" index.sqlite3 'SELECT COUNT(*) FROM core_snapshot;')"
-case "$snapshot_count" in ''|*[!0-9]*) exit 1 ;; esac
+cd ~/archivebox/data
+sqlite3 index.sqlite3
+
+# example usage:
+SELECT * FROM core_snapshot;
+UPDATE auth_user SET email = 'someNewEmail@example.com' WHERE username = 'someUsernameHere';
+...
 ```
 
 More info:
@@ -413,8 +331,23 @@ Explore the Python API a bit to see what's available using the archivebox shell:
 **Python API Documentation:** https://docs.archivebox.io/dev/apidocs/index.html
 
 ```bash
-archivebox shell -c \
-    "from archivebox.core.models import Snapshot; count = Snapshot.objects.count(); assert isinstance(count, int); print(f'{count} snapshots')"
+$ archivebox shell
+[i] ArchiveBox shell
+>>> from archivebox.core.models import Snapshot
+>>> from archivebox.cli.archivebox_add import add
+
+# count completed snapshots
+>>> print(Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED).count())
+24
+
+# inspect or add URLs through current APIs
+>>> Snapshot.objects.filter(url="https://example.com").first()
+<Snapshot: https://example.com>
+>>> crawl, snapshots = add(urls=["https://example.com/new"], index_only=True)
+
+# show raw SQL queries run
+>>> from django.db import connection
+>>> print(connection.queries)
 ```
 
 For more info and example usage:
@@ -436,26 +369,23 @@ You can interact with ArchiveBox as a Python library from external scripts or pr
 
 This API is a *local* API, designed to be used on the same machine as the ArchiveBox collection.
 
-For example, a local Python program can initialize Django and queue a URL like so:
+For example, you could create a script `add_archivebox_url.py` like so:
 ```python
 import os
 from pathlib import Path
 
-data_dir = Path(os.environ["ARCHIVEBOX_DOCS_DATA_DIR"])
-os.chdir(data_dir)
+DATA_DIR = Path("~/archivebox/data").expanduser()
+os.chdir(DATA_DIR)
 
 # you must import and setup django first to establish a DB connection
 from archivebox.config.django import setup_django
 setup_django(check_db=True)
 
-# then import and use the same implementation as the CLI
+# then import the specific API you need
 from archivebox.cli.archivebox_add import add
-from archivebox.crawls.models import Crawl
 
-url = f'{os.environ["ARCHIVEBOX_DOCS_URL_ONE"]}/usage-python-api'
-crawl, snapshots = add([url], index_only=True)
-assert snapshots.count() == 0
-assert Crawl.objects.get(pk=crawl.pk).get_urls_list() == [url]
+crawl, snapshots = add(urls=["https://example.com"], index_only=True)
+print(crawl.id, list(snapshots.values_list("id", flat=True)))
 ```
 
 For more information see:

docs/codeblocks.toml

@@ -1,105 +1,411 @@
-version = 1
-environments = ["core", "merge", "publishing", "docker", "root", "macos", "freebsd", "openbsd"]
+# Generated with: uv run --no-sync python docs/test_codeblocks_manifest.py render
+# Every authored occurrence is classified here without adding test plumbing to the rendered docs.
+version = 2
 
-[syntax]
-executed = ["bash", "sh", "python"]
-shell_syntax_only = ["console"]
-structured = ["json", "yaml", "ini", "sql", "nginx", "mermaid"]
-prose = ["text"]
-directive_prefixes = ["{"]
-console_blocks = [
-    "README.md::line 489",
-    "README.md::line 1400",
-    "README.md::line 1421",
-    "README.md::line 1436",
-    "README.md::line 1466",
-    "README.md::line 1490",
-    "README.md::line 1500",
-    "README.md::line 1524",
-    "README.md::line 1537",
-    "README.md::line 1601",
-]
+[snippets]
 
-[ci.standard]
-core = "ubuntu-24.04"
-merge = "ubuntu-24.04"
-publishing = "ubuntu-24.04"
-docker = "ubuntu-24.04"
-root = "ubuntu-24.04"
-macos = "macos-15"
+# README.md
+"7299ee2aeeaffe3a-1" = "illustration"
+"072d74f81af28e41-1" = "illustration"
+"347ba4a5f066d4d9-1" = "run"
+"e5efbef69f81e4ac-1" = "illustration"
+"8e0e2bcdf08b2ef5-1" = "illustration"
+"7e6bd7d1ecb5c1e7-1" = "illustration"
+"16f90085b2eb8756-1" = "illustration"
+"811eaaada5eb1861-1" = "run"
+"8e6f975ced292777-1" = "run"
+"588be84a1f0fda9a-1" = "illustration"
+"7669e806d194bf07-1" = "run"
+"15db87cd5a52c1c4-1" = "run"
+"588be84a1f0fda9a-2" = "illustration"
+"623dec26599ab9f2-1" = "run"
+"2b08779e4d24d305-1" = "run"
+"588be84a1f0fda9a-3" = "illustration"
+"c6e77a7ea47549e5-1" = "illustration"
+"6392c7b5f3593d07-1" = "run"
+"5d43b3d527ac69c6-1" = "run"
+"b5176e0b5ad1bcfc-1" = "illustration"
+"13fefe6d12cab753-1" = "illustration"
+"7cf50ffa6a169683-1" = "illustration"
+"35e12e491c22f48a-1" = "illustration"
+"eac57da2650e527c-1" = "illustration"
+"a7e2cb64628d3d38-1" = "illustration"
+"88fb0039056c161e-1" = "run"
+"715e94940898c09d-1" = "illustration"
+"d6cb543b1403f8ef-1" = "run"
+"e4a4f50f45b47ec8-1" = "run"
+"5604a7d87f7492cf-1" = "output"
+"59e14df6c51c6bb0-1" = "run"
+"0ddf9613e881e636-1" = "illustration"
+"285b452d2dce81a3-1" = "run"
+"fab7ffa7e26bc3e5-1" = "run"
+"06e963d9fad5b4f9-1" = "illustration"
+"834b852409be5697-1" = "illustration"
+"6cc5d78f24372d44-1" = "illustration"
+"747d7bb0b608c5ae-1" = "illustration"
+"efbeb534feb1eaf8-1" = "illustration"
+"511f47bb050d2921-1" = "illustration"
+"6f0ed916c341d2fc-1" = "illustration"
+"1e5a1e7eef7f2237-1" = "illustration"
+"3c4ecdabd2c9ba35-1" = "illustration"
 
-[ci.core_shards]
-metadata = [
-    "README.md",
-    "archivebox/mcp/README.md",
-    "docs/Changelog.md",
-    "docs/Chromium-Install.md",
-]
-agents = [
-    "AGENTS.md",
-    "skills/archivebox/SKILL.md",
-]
-guides = [
-    "docs/Quickstart.md",
-    "docs/Scheduled-Archiving.md",
-    "docs/Security-Overview.md",
-    "docs/Setting-Up-Storage.md",
-]
-configuration = ["docs/Configuration.md"]
-install = ["docs/Install.md"]
-authentication = ["docs/Setting-up-Authentication.md"]
-search = ["docs/Setting-up-Search.md"]
-troubleshooting = ["docs/Troubleshooting.md"]
-usage = [
-    "docs/Upgrading.md",
-    "docs/Usage.md",
-]
+# AGENTS.md
+"2695777473e3a217-1" = "run"
+"16f996904f0eaa96-1" = "run"
+"93dd34b605cd6775-1" = "run"
+"6a6234b63cd3d0ae-1" = "run"
+"396490664ce459e9-1" = "run"
 
-[ci.bsd]
-freebsd = "vmactions/freebsd-vm@v1.5.2"
-openbsd = "vmactions/openbsd-vm@v1.4.5"
+# archivebox/mcp/README.md
+"d9ff84abeb97d380-1" = "run"
+"aebc15190116f5cd-1" = "run"
+"49747d67b7e3ea8d-1" = "illustration"
+"fead0184493aab00-1" = "illustration"
+"256531ee2b730012-1" = "illustration"
+"399c370c70523428-1" = "run"
+"7d33d72cc96df65b-1" = "run"
 
-[files]
-"README.md" = "core"
-"AGENTS.md" = "core"
-"archivebox/mcp/README.md" = "core"
-"skills/archivebox/SKILL.md" = "core"
-"docs/Changelog.md" = "core"
-"docs/Chromium-Install.md" = "core"
-"docs/Configuration.md" = "core"
-"docs/Docker.md" = "docker"
-"docs/Install.md" = "core"
-"docs/Merging-Collections.md" = "merge"
-"docs/Publishing-Your-Archive.md" = "publishing"
-"docs/Quickstart.md" = "core"
-"docs/Scheduled-Archiving.md" = "core"
-"docs/Security-Overview.md" = "core"
-"docs/Setting-Up-Storage.md" = "core"
-"docs/Setting-up-Authentication.md" = "core"
-"docs/Setting-up-Search.md" = "core"
-"docs/Troubleshooting.md" = "core"
-"docs/Upgrading.md" = "core"
-"docs/Usage.md" = "core"
+# skills/archivebox/SKILL.md
+"2695777473e3a217-2" = "run"
+"93dd34b605cd6775-2" = "run"
+"46fe9d81940ea385-1" = "run"
+"396490664ce459e9-2" = "run"
 
-[blocks]
-"docs/Chromium-Install.md::line 100" = "docker"
-"docs/Chromium-Install.md::line 105" = "docker"
-"docs/Chromium-Install.md::line 114" = "docker"
-"docs/Configuration.md::line 137" = "macos"
-"docs/Install.md::line 126" = "macos"
-"docs/Install.md::line 146" = "root"
-"docs/Install.md::line 168" = "freebsd"
-"docs/Install.md::line 178" = "openbsd"
-"docs/Quickstart.md::line 64" = "docker"
-"docs/Scheduled-Archiving.md::line 53" = "docker"
-"docs/Security-Overview.md::line 140" = "root"
-"docs/Setting-Up-Storage.md::line 62" = "root"
-"docs/Setting-Up-Storage.md::line 174" = "root"
-"docs/Setting-Up-Storage.md::line 258" = "docker"
-"docs/Setting-Up-Storage.md::line 289" = "docker"
-"docs/Setting-up-Search.md::line 95" = "macos"
-"docs/Setting-up-Search.md::line 149" = "docker"
-"docs/Troubleshooting.md::line 37" = "macos"
-"docs/Troubleshooting.md::line 88" = "root"
-"docs/Upgrading.md::line 60" = "docker"
-"docs/Upgrading.md::line 77" = "docker"
+# docs/ArchiveBox-Architecture-Diagrams.md
+"7442dd337fd0a1dd-1" = "illustration"
+"9d6f8ac3043ff691-1" = "illustration"
+"8c8802244e05cfb7-1" = "illustration"
+"d0eb428affe5bb44-1" = "illustration"
+"7d1d068e1e754a75-1" = "illustration"
+
+# docs/Changelog.md
+"6fdf9ece2f8a3f7f-1" = "illustration"
+
+# docs/Chromium-Install.md
+"201988374bff6d14-1" = "run"
+"d85a11895d69713a-1" = "run"
+"d8061ea425363f8c-1" = "illustration"
+"bd5e6d3fc3193a25-1" = "illustration"
+"0ff6fc827fe34ab1-1" = "illustration"
+"3e1dabfc6426fef6-1" = "illustration"
+"ce2edd9be87280fc-1" = "run"
+"7890998169008ad0-1" = "illustration"
+
+# docs/Configuration.md
+"129eb9ca56f17500-1" = "illustration"
+"798793d9d6b2e12e-1" = "run"
+"3fecc495abdc20ef-1" = "illustration"
+"ce0de8ed0d7cd552-1" = "run"
+"83cb9be168d203ca-1" = "run"
+"2028f9a887105a13-1" = "run"
+"83c90c1a23a33f90-1" = "run"
+"00c51bd82b457d50-1" = "illustration"
+"c4266755c0748aa0-1" = "run"
+"f0a08cf932c513b3-1" = "illustration"
+"ea6d6196f0985d3c-1" = "run"
+
+# docs/Docker.md
+"03a42395e6560eba-1" = "illustration"
+"3e31436bc1fa4b6b-1" = "illustration"
+"ecb1424a53832ee7-1" = "run"
+"b0032c4cfcac6915-1" = "run"
+"c46afe2bbd31a239-1" = "run"
+"1165a871312ed99e-1" = "illustration"
+"c7b0be01cfea2d3a-1" = "illustration"
+"150ae2c7f6d012ab-1" = "illustration"
+"4c9a12d2d0a26431-1" = "run"
+"6bcb6c2e8770907a-1" = "illustration"
+"fcb2471eb154161b-1" = "illustration"
+"6353a7cf3014612e-1" = "illustration"
+"d66324df4c7f97c7-1" = "illustration"
+"17ed06e5b75aac1b-1" = "illustration"
+"b0b21bad724545d4-1" = "illustration"
+"148aae5565f0afd9-1" = "illustration"
+"638a9af2fdb6ce41-1" = "illustration"
+"0ac2cef38147537b-1" = "illustration"
+"7494fb04398b9fd8-1" = "illustration"
+
+# docs/Install.md
+"662be0691524555c-1" = "illustration"
+"fb630418e789bca8-1" = "run"
+"bf5409d061557508-1" = "run"
+"39d6e53c2e53ef77-1" = "run"
+"6a149b1a387f76cc-1" = "run"
+"73e5b1670430202c-1" = "run"
+"77b61fd52c4c3c32-1" = "run"
+"a8f16f6be4f2b3c0-1" = "run"
+"035ecb710f7fc2a8-1" = "illustration"
+"e4e22c56cb720344-1" = "illustration"
+"2905d7f9dece0b61-1" = "run"
+"e1c3e8b9016a71a9-1" = "illustration"
+"0dc947d99d081ca3-1" = "run"
+
+# docs/Merging-Collections.md
+"befadadb67c37c7b-1" = "illustration"
+"92e13fe91bc77af9-1" = "illustration"
+"f0ff0e950c4f04bd-1" = "illustration"
+"1343441ed4354bf6-1" = "illustration"
+"3d38133db42791b8-1" = "illustration"
+"25f5a83ce92e6b8d-1" = "transcript"
+"93f3695c9bcebd1e-1" = "illustration"
+"9bbf385bf2eb1160-1" = "run"
+"ef0e524c29534852-1" = "transcript"
+"25f5a83ce92e6b8d-2" = "transcript"
+"3ecf97d15d7e8ae9-1" = "illustration"
+
+# docs/Publishing-Your-Archive.md
+"a3e3e05317559025-1" = "illustration"
+"44fb1b5e4e4b8019-1" = "illustration"
+"e3fbec28244a58a0-1" = "illustration"
+
+# docs/Quickstart.md
+"9900ea0b79f00814-1" = "illustration"
+"4972b2ae112247f0-1" = "illustration"
+"c84835e713d1c625-1" = "run"
+
+# docs/Scheduled-Archiving.md
+"e6102e50b4491a0d-1" = "illustration"
+"e256179d1504c37d-1" = "illustration"
+"302de01a4a1490f0-1" = "run"
+"2bec913ab03ee911-1" = "run"
+"f96ae0f8a9b6f3a1-1" = "run"
+"244aee50bbfc11be-1" = "run"
+"5c9d291b47101962-1" = "run"
+
+# docs/Security-Overview.md
+"cf6f0d08c86314d6-1" = "illustration"
+"2b5b645b4047ba9c-1" = "run"
+"42d2d62ba382ad48-1" = "illustration"
+"226c4fb317fb9a60-1" = "illustration"
+
+# docs/Setting-Up-Storage.md
+"75084db745eabaf9-1" = "illustration"
+"499d97044307896f-1" = "illustration"
+"62b0ce987c89c1ef-1" = "illustration"
+"2945f0463a3da1ec-1" = "illustration"
+"2586f140447eb628-1" = "illustration"
+"f1b6a0d897e62225-1" = "run"
+"d8013405262d9dee-1" = "run"
+"5a399294a8709168-1" = "illustration"
+"f0923529c67db731-1" = "illustration"
+"48e05033c49e460c-1" = "illustration"
+"0fdcf228bb957933-1" = "run"
+"d76a2cce93150387-1" = "illustration"
+"acb33c3336d33ac5-1" = "illustration"
+"8bc58240c72e3eaa-1" = "illustration"
+"74e75b4a84bbb636-1" = "run"
+
+# docs/Setting-up-Authentication.md
+"c2de5f67fd56b58c-1" = "illustration"
+"9821835f15ad67b2-1" = "illustration"
+"83c90c1a23a33f90-2" = "run"
+"7caaf25736e20132-1" = "illustration"
+"4f35c142a9d6a1c6-1" = "illustration"
+"38551637f2a2b4ee-1" = "illustration"
+"eca3786ae3cadccf-1" = "illustration"
+"15b6f6ecd23a6736-1" = "illustration"
+
+# docs/Setting-up-Search.md
+"aeb19a9eb984d7dd-1" = "run"
+"9613aa41cb49f134-1" = "run"
+"ebebc2e7861a80e9-1" = "run"
+"23b079b81c0eaf87-1" = "run"
+"276ad19decf92bba-1" = "transcript"
+"20b7130b3bf05a00-1" = "run"
+
+# docs/Troubleshooting.md
+"f51f94b1347085a0-1" = "illustration"
+"24badf54daccd7fa-1" = "run"
+"f729b2ac542cbd20-1" = "run"
+"ad5f652a64306a33-1" = "run"
+"201988374bff6d14-2" = "run"
+"725a7a8c6f829ece-1" = "run"
+"7658fabf90ed3c2e-1" = "run"
+"bc29163a3bad12d6-1" = "illustration"
+"2f491912a15ecd93-1" = "output"
+"e0cbb64ddf1d7135-1" = "run"
+"0e66f4942ee1586a-1" = "output"
+"ca9f38c9c70b2d0b-1" = "output"
+"03803c44e54399c4-1" = "illustration"
+
+# docs/Upgrading.md
+"d4f106f7f176cc4f-1" = "illustration"
+"e566c21263033134-1" = "illustration"
+"5d39ef7a4a12e290-1" = "illustration"
+"03abe5364d9c713f-1" = "illustration"
+
+# docs/Usage.md
+"865eeba3c03d1e2d-1" = "run"
+"a336bc3a2b591f3b-1" = "run"
+"6611c2dee47b1092-1" = "illustration"
+"2560680733c8e908-1" = "illustration"
+"debc2548308dec0c-1" = "illustration"
+"d29b422c54745a7a-1" = "illustration"
+"463a437ec1fc9586-1" = "illustration"
+"5cd641727f4b04bb-1" = "output"
+"f0e895e5b375c198-1" = "illustration"
+"72a06916ecdabcca-1" = "transcript"
+"5036043d7f4b90f4-1" = "transcript"
+"85f531841ec3e169-1" = "run"
+
+[scenarios]
+"347ba4a5f066d4d9-1" = "docker"
+"811eaaada5eb1861-1" = "system"
+"8e6f975ced292777-1" = "collection"
+"7669e806d194bf07-1" = "system"
+"15db87cd5a52c1c4-1" = "collection"
+"623dec26599ab9f2-1" = "system"
+"2b08779e4d24d305-1" = "system"
+"6392c7b5f3593d07-1" = "collection"
+"5d43b3d527ac69c6-1" = "docker"
+"88fb0039056c161e-1" = "collection"
+"d6cb543b1403f8ef-1" = "collection"
+"e4a4f50f45b47ec8-1" = "collection"
+"59e14df6c51c6bb0-1" = "collection"
+"285b452d2dce81a3-1" = "collection"
+"fab7ffa7e26bc3e5-1" = "collection"
+"2695777473e3a217-1" = "project"
+"16f996904f0eaa96-1" = "project"
+"93dd34b605cd6775-1" = "system"
+"6a6234b63cd3d0ae-1" = "collection"
+"396490664ce459e9-1" = "project"
+"d9ff84abeb97d380-1" = "collection"
+"aebc15190116f5cd-1" = "collection"
+"399c370c70523428-1" = "collection"
+"7d33d72cc96df65b-1" = "collection"
+"2695777473e3a217-2" = "project"
+"93dd34b605cd6775-2" = "system"
+"46fe9d81940ea385-1" = "collection"
+"396490664ce459e9-2" = "project"
+"201988374bff6d14-1" = "collection"
+"d85a11895d69713a-1" = "collection"
+"ce2edd9be87280fc-1" = "collection"
+"798793d9d6b2e12e-1" = "collection"
+"ce0de8ed0d7cd552-1" = "collection"
+"83cb9be168d203ca-1" = "collection"
+"2028f9a887105a13-1" = "collection"
+"83c90c1a23a33f90-1" = "system"
+"c4266755c0748aa0-1" = "collection"
+"ea6d6196f0985d3c-1" = "collection"
+"ecb1424a53832ee7-1" = "docker"
+"b0032c4cfcac6915-1" = "docker"
+"c46afe2bbd31a239-1" = "docker"
+"4c9a12d2d0a26431-1" = "docker"
+"fb630418e789bca8-1" = "system"
+"bf5409d061557508-1" = "system"
+"39d6e53c2e53ef77-1" = "system"
+"6a149b1a387f76cc-1" = "system"
+"73e5b1670430202c-1" = "system"
+"77b61fd52c4c3c32-1" = "collection"
+"a8f16f6be4f2b3c0-1" = "collection"
+"2905d7f9dece0b61-1" = "collection"
+"0dc947d99d081ca3-1" = "system-data"
+"9bbf385bf2eb1160-1" = "collection"
+"c84835e713d1c625-1" = "collection"
+"302de01a4a1490f0-1" = "docker"
+"2bec913ab03ee911-1" = "collection"
+"f96ae0f8a9b6f3a1-1" = "collection"
+"244aee50bbfc11be-1" = "collection"
+"5c9d291b47101962-1" = "collection"
+"2b5b645b4047ba9c-1" = "collection"
+"f1b6a0d897e62225-1" = "system"
+"d8013405262d9dee-1" = "system"
+"0fdcf228bb957933-1" = "docker-data"
+"74e75b4a84bbb636-1" = "docker"
+"83c90c1a23a33f90-2" = "system"
+"aeb19a9eb984d7dd-1" = "collection"
+"9613aa41cb49f134-1" = "collection"
+"ebebc2e7861a80e9-1" = "collection"
+"23b079b81c0eaf87-1" = "collection"
+"20b7130b3bf05a00-1" = "collection"
+"24badf54daccd7fa-1" = "collection"
+"f729b2ac542cbd20-1" = "system"
+"ad5f652a64306a33-1" = "collection"
+"201988374bff6d14-2" = "collection"
+"725a7a8c6f829ece-1" = "collection"
+"7658fabf90ed3c2e-1" = "collection"
+"e0cbb64ddf1d7135-1" = "collection"
+"865eeba3c03d1e2d-1" = "collection"
+"a336bc3a2b591f3b-1" = "collection"
+"85f531841ec3e169-1" = "collection"
+
+[environments]
+"347ba4a5f066d4d9-1" = "docker"
+"811eaaada5eb1861-1" = "ubuntu"
+"8e6f975ced292777-1" = "ubuntu"
+"7669e806d194bf07-1" = "root"
+"15db87cd5a52c1c4-1" = "ubuntu"
+"623dec26599ab9f2-1" = "macos"
+"2b08779e4d24d305-1" = "macos"
+"6392c7b5f3593d07-1" = "ubuntu"
+"5d43b3d527ac69c6-1" = "docker"
+"88fb0039056c161e-1" = "ubuntu"
+"d6cb543b1403f8ef-1" = "ubuntu"
+"e4a4f50f45b47ec8-1" = "ubuntu"
+"59e14df6c51c6bb0-1" = "ubuntu"
+"285b452d2dce81a3-1" = "ubuntu"
+"fab7ffa7e26bc3e5-1" = "ubuntu"
+"2695777473e3a217-1" = "ubuntu"
+"16f996904f0eaa96-1" = "ubuntu"
+"93dd34b605cd6775-1" = "ubuntu"
+"6a6234b63cd3d0ae-1" = "ubuntu"
+"396490664ce459e9-1" = "ubuntu"
+"d9ff84abeb97d380-1" = "ubuntu"
+"aebc15190116f5cd-1" = "ubuntu"
+"399c370c70523428-1" = "ubuntu"
+"7d33d72cc96df65b-1" = "ubuntu"
+"2695777473e3a217-2" = "ubuntu"
+"93dd34b605cd6775-2" = "ubuntu"
+"46fe9d81940ea385-1" = "ubuntu"
+"396490664ce459e9-2" = "ubuntu"
+"201988374bff6d14-1" = "ubuntu"
+"d85a11895d69713a-1" = "ubuntu"
+"ce2edd9be87280fc-1" = "ubuntu"
+"798793d9d6b2e12e-1" = "ubuntu"
+"ce0de8ed0d7cd552-1" = "ubuntu"
+"83cb9be168d203ca-1" = "ubuntu"
+"2028f9a887105a13-1" = "ubuntu"
+"83c90c1a23a33f90-1" = "ubuntu"
+"c4266755c0748aa0-1" = "ubuntu"
+"ea6d6196f0985d3c-1" = "ubuntu"
+"ecb1424a53832ee7-1" = "docker"
+"b0032c4cfcac6915-1" = "docker"
+"c46afe2bbd31a239-1" = "docker"
+"4c9a12d2d0a26431-1" = "docker"
+"fb630418e789bca8-1" = "macos"
+"bf5409d061557508-1" = "root"
+"39d6e53c2e53ef77-1" = "freebsd"
+"6a149b1a387f76cc-1" = "openbsd"
+"73e5b1670430202c-1" = "ubuntu"
+"77b61fd52c4c3c32-1" = "ubuntu"
+"a8f16f6be4f2b3c0-1" = "ubuntu"
+"2905d7f9dece0b61-1" = "ubuntu"
+"0dc947d99d081ca3-1" = "ubuntu"
+"9bbf385bf2eb1160-1" = "ubuntu"
+"c84835e713d1c625-1" = "ubuntu"
+"302de01a4a1490f0-1" = "docker"
+"2bec913ab03ee911-1" = "ubuntu"
+"f96ae0f8a9b6f3a1-1" = "ubuntu"
+"244aee50bbfc11be-1" = "ubuntu"
+"5c9d291b47101962-1" = "ubuntu"
+"2b5b645b4047ba9c-1" = "ubuntu"
+"f1b6a0d897e62225-1" = "ubuntu"
+"d8013405262d9dee-1" = "root"
+"0fdcf228bb957933-1" = "docker"
+"74e75b4a84bbb636-1" = "docker"
+"83c90c1a23a33f90-2" = "ubuntu"
+"aeb19a9eb984d7dd-1" = "ubuntu"
+"9613aa41cb49f134-1" = "ubuntu"
+"ebebc2e7861a80e9-1" = "ubuntu"
+"23b079b81c0eaf87-1" = "ubuntu"
+"20b7130b3bf05a00-1" = "ubuntu"
+"24badf54daccd7fa-1" = "ubuntu"
+"f729b2ac542cbd20-1" = "macos"
+"ad5f652a64306a33-1" = "ubuntu"
+"201988374bff6d14-2" = "ubuntu"
+"725a7a8c6f829ece-1" = "ubuntu"
+"7658fabf90ed3c2e-1" = "ubuntu"
+"e0cbb64ddf1d7135-1" = "ubuntu"
+"865eeba3c03d1e2d-1" = "ubuntu"
+"a336bc3a2b591f3b-1" = "ubuntu"
+"85f531841ec3e169-1" = "ubuntu"

docs/test_codeblocks_manifest.py

@@ -1,188 +1,472 @@
+"""Inventory, validate, and run the code examples in the authored documentation.
+
+The docs stay written for people.  This scanner reads Markdown fences and the
+README's deliberately hand-authored ``<pre lang="bash">`` examples without
+requiring test directives in the prose.
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
 import configparser
-from collections import Counter
+from dataclasses import dataclass
+from hashlib import sha256
+from html import unescape
 import json
+import os
 from pathlib import Path
+import re
 import sqlite3
 import subprocess
+import sys
+import tempfile
 import tomllib
 
-from pytest_codeblocks.main import extract_from_file
-import yaml
+ROOT = Path(__file__).resolve().parent.parent
+MANIFEST = ROOT / "docs" / "codeblocks.toml"
+DISPOSITIONS = {"run", "illustration", "transcript", "output"}
+ENVIRONMENT_RUNNERS = {
+    "ubuntu": "ubuntu-24.04",
+    "root": "ubuntu-24.04",
+    "docker": "ubuntu-24.04",
+    "macos": "macos-15",
+    "freebsd": "ubuntu-24.04",
+    "openbsd": "ubuntu-24.04",
+}
+SCENARIOS = {"project", "collection", "system", "system-data", "docker", "docker-data"}
+FENCE_START = re.compile(r"^(?P<indent>[ \t]*)(?P<fence>`{3,}|~{3,})[ \t]*(?P<info>[^\n]*)$")
+HTML_PRE = re.compile(
+    r"""<pre\b(?=[^>]*\blang=(?:"(?:bash|sh|console)"|'(?:bash|sh|console)'|(?:bash|sh|console)\b))[^>]*>
+        \s*<code\b[^>]*>(?P<code>.*?)</code>\s*</pre>""",
+    re.IGNORECASE | re.DOTALL | re.VERBOSE,
+)
+HTML_COMMENT = re.compile(r"<!--.*?-->", re.DOTALL)
 
 
-REPO_ROOT = Path(__file__).parent.parent
-MANIFEST_PATH = REPO_ROOT / "docs" / "codeblocks.toml"
-WORKFLOW_PATH = REPO_ROOT / ".github" / "workflows" / "docs.yml"
+@dataclass(frozen=True)
+class Snippet:
+    id: str
+    path: str
+    line: int
+    style: str
+    syntax: str
+    code: str
 
 
-def markdown_paths() -> tuple[Path, ...]:
-    candidates = [
-        REPO_ROOT / "README.md",
-        REPO_ROOT / "AGENTS.md",
-        REPO_ROOT / "archivebox" / "mcp" / "README.md",
-    ]
-    candidates.extend(sorted((REPO_ROOT / "skills").rglob("*.md")))
-    candidates.extend(sorted((REPO_ROOT / "docs").rglob("*.md")))
+def authored_markdown_paths() -> tuple[Path, ...]:
+    candidates = [ROOT / "README.md", ROOT / "AGENTS.md", ROOT / "archivebox" / "mcp" / "README.md"]
+    candidates.extend(sorted((ROOT / "skills").rglob("*.md")))
+    candidates.extend(sorted((ROOT / "docs").rglob("*.md")))
 
-    unique_paths: dict[Path, Path] = {}
+    paths: list[Path] = []
+    seen: set[Path] = set()
     for path in candidates:
-        unique_paths.setdefault(path.resolve(), path)
-    return tuple(unique_paths.values())
+        if not path.is_file() or path.is_symlink() or "apidocs" in path.parts:
+            continue
+        resolved = path.resolve()
+        if resolved not in seen:
+            seen.add(resolved)
+            paths.append(path)
+    return tuple(paths)
 
 
-def docs_blocks() -> dict[str, object]:
-    return {
-        f"{path.relative_to(REPO_ROOT).as_posix()}::line {block.lineno}": block
-        for path in markdown_paths()
-        for block in extract_from_file(path)
-    }
+def _masked_html_comments(text: str) -> str:
+    chars = list(text)
+    for match in HTML_COMMENT.finditer(text):
+        for index in range(match.start(), match.end()):
+            if chars[index] != "\n":
+                chars[index] = " "
+    return "".join(chars)
 
 
-def test_every_executable_docs_block_has_exactly_one_ci_environment() -> None:
-    with MANIFEST_PATH.open("rb") as manifest_file:
-        manifest = tomllib.load(manifest_file)
-
-    environments = set(manifest["environments"])
-    executable_syntaxes = set(manifest["syntax"]["executed"])
-    file_environments = manifest["files"]
-    block_environments = manifest["blocks"]
-    discovered: dict[str, str] = {}
-
-    executable_files = {
-        path.relative_to(REPO_ROOT).as_posix()
-        for path in markdown_paths()
-        if any(block.syntax in executable_syntaxes for block in extract_from_file(path))
-    }
-
-    assert set(file_environments) == executable_files
-
-    for relative_path in sorted(executable_files):
-        file_environment = file_environments[relative_path]
-        assert file_environment in environments
-        path = REPO_ROOT / relative_path
-        assert path.is_file(), relative_path
-        for block in extract_from_file(path):
-            if block.syntax not in executable_syntaxes:
-                continue
-            nodeid = f"{relative_path}::line {block.lineno}"
-            environment = block_environments.get(nodeid, file_environment)
-            assert environment in environments, nodeid
-            assert nodeid not in discovered
-            discovered[nodeid] = environment
-
-    assert set(block_environments) <= set(discovered)
-    assert set(discovered.values()) == environments
+def _markdown_fences(text: str) -> list[tuple[int, str, str, str]]:
+    original_lines = text.splitlines(keepends=True)
+    masked_lines = _masked_html_comments(text).splitlines(keepends=True)
+    found: list[tuple[int, str, str, str]] = []
+    index = 0
+    while index < len(masked_lines):
+        start = FENCE_START.match(masked_lines[index].rstrip("\r\n"))
+        if start is None:
+            index += 1
+            continue
+        fence = start.group("fence")
+        syntax = start.group("info").strip().split(maxsplit=1)[0].lower()
+        if syntax.startswith("{.") and syntax.endswith("}"):
+            syntax = syntax[2:-1]
+        closing = re.compile(rf"^[ \t]*{re.escape(fence[0])}{{{len(fence)},}}[ \t]*$")
+        end = index + 1
+        while end < len(masked_lines) and closing.match(masked_lines[end].rstrip("\r\n")) is None:
+            end += 1
+        if end == len(masked_lines):
+            raise AssertionError(f"Unclosed Markdown fence at line {index + 1}")
+        found.append((index + 1, "fence", syntax, "".join(original_lines[index + 1 : end]).rstrip("\r\n")))
+        index = end + 1
+    return found
 
 
-def test_every_docs_fence_syntax_is_explicitly_classified() -> None:
-    with MANIFEST_PATH.open("rb") as manifest_file:
-        syntax_manifest = tomllib.load(manifest_file)["syntax"]
-
-    classified = set(syntax_manifest["executed"])
-    classified.update(syntax_manifest["shell_syntax_only"])
-    classified.update(syntax_manifest["structured"])
-    classified.update(syntax_manifest["prose"])
-    directive_prefixes = tuple(syntax_manifest["directive_prefixes"])
-
-    unknown = {
-        f"{nodeid} ({block.syntax!r})"
-        for nodeid, block in docs_blocks().items()
-        if block.syntax not in classified and not block.syntax.startswith(directive_prefixes)
-    }
-    assert not unknown
+def _html_pre_blocks(text: str) -> list[tuple[int, str, str, str]]:
+    found = []
+    for match in HTML_PRE.finditer(text):
+        opening = text[match.start() : match.start("code")]
+        language = re.search(r"""\blang=["']?(bash|sh|console)""", opening, re.IGNORECASE)
+        assert language is not None
+        code = re.sub(r"<br\s*/?>", "\n", match.group("code"), flags=re.IGNORECASE)
+        code = re.sub(r"</?[^>]+>", "", code)
+        found.append((text.count("\n", 0, match.start()) + 1, "html-pre", language.group(1).lower(), unescape(code).strip()))
+    return found
 
 
-def test_every_console_fence_is_inventoried_and_shell_parseable() -> None:
-    with MANIFEST_PATH.open("rb") as manifest_file:
-        expected_console_blocks = set(tomllib.load(manifest_file)["syntax"]["console_blocks"])
+def scan_snippets() -> tuple[Snippet, ...]:
+    raw: list[tuple[str, int, str, str, str]] = []
+    for path in authored_markdown_paths():
+        relative_path = path.relative_to(ROOT).as_posix()
+        text = path.read_text()
+        blocks = _markdown_fences(text)
+        blocks.extend(_html_pre_blocks(text))
+        for line, style, syntax, code in sorted(blocks):
+            raw.append((relative_path, line, style, syntax, code))
 
-    console_blocks = {nodeid: block for nodeid, block in docs_blocks().items() if block.syntax == "console"}
-    assert set(console_blocks) == expected_console_blocks
-    for nodeid, block in console_blocks.items():
-        result = subprocess.run(["bash", "-n"], input=block.code, text=True, capture_output=True, check=False)
-        assert result.returncode == 0, f"{nodeid}: {result.stderr}"
+    hash_counts: dict[str, int] = {}
+    snippets: list[Snippet] = []
+    for path, line, style, syntax, code in raw:
+        normalized = code.replace("\r\n", "\n").rstrip() + "\n"
+        digest = sha256(f"{syntax}\0{normalized}".encode()).hexdigest()[:16]
+        hash_counts[digest] = hash_counts.get(digest, 0) + 1
+        snippets.append(
+            Snippet(
+                id=f"{digest}-{hash_counts[digest]}",
+                path=path,
+                line=line,
+                style=style,
+                syntax=syntax,
+                code=code,
+            ),
+        )
+    return tuple(snippets)
 
 
-def test_structured_data_fences_parse() -> None:
-    blocks = docs_blocks()
-    sql_connection = sqlite3.connect(":memory:")
-    sql_connection.execute(
-        "CREATE TABLE auth_user (password, last_login, is_superuser, username, first_name, last_name, email, is_staff, is_active, date_joined)",
+def load_manifest() -> dict[str, dict[str, str]]:
+    with MANIFEST.open("rb") as manifest_file:
+        document = tomllib.load(manifest_file)
+    assert document.get("version") == 2
+    records = {snippet_id: {"disposition": disposition} for snippet_id, disposition in document.get("snippets", {}).items()}
+    for snippet_id, scenario in document.get("scenarios", {}).items():
+        assert snippet_id in records
+        records[snippet_id]["scenario"] = scenario
+    for snippet_id, environment in document.get("environments", {}).items():
+        assert snippet_id in records
+        records[snippet_id]["environment"] = environment
+    return records
+
+
+def check_inventory() -> tuple[Snippet, ...]:
+    snippets = scan_snippets()
+    records = load_manifest()
+    by_id = {snippet.id: snippet for snippet in snippets}
+    assert set(records) == set(by_id), (
+        f"Docs inventory is stale. Missing: {sorted(set(by_id) - set(records))}; removed: {sorted(set(records) - set(by_id))}"
     )
 
-    for nodeid, block in blocks.items():
-        if block.syntax == "json":
-            json.loads(block.code)
-        elif block.syntax == "yaml":
-            list(yaml.safe_load_all(block.code))
-        elif block.syntax == "ini":
-            parser = configparser.ConfigParser()
-            parser.read_string(block.code)
-        elif block.syntax == "sql":
+    for snippet in snippets:
+        record = records[snippet.id]
+        assert record.get("disposition") in DISPOSITIONS, f"{snippet.path}:{snippet.line}: missing disposition"
+        if record["disposition"] == "run":
+            assert record.get("scenario") in SCENARIOS, f"{snippet.id}: unknown run scenario"
+            assert record.get("environment") in ENVIRONMENT_RUNNERS, f"{snippet.id}: unknown run environment"
+        else:
+            assert "scenario" not in record, f"{snippet.id}: non-running examples must not declare runtime setup"
+            assert "environment" not in record, f"{snippet.id}: non-running examples must not declare a CI environment"
+    return snippets
+
+
+def validate_non_running(snippet: Snippet, disposition: str) -> None:
+    assert snippet.code.strip(), f"{snippet.path}:{snippet.line}: empty code block"
+    if disposition in {"transcript", "output"}:
+        return
+    if snippet.syntax in {"bash", "sh", "console"}:
+        illustrative_shell = re.sub(r"<[A-Za-z][A-Za-z0-9_-]*>", "PLACEHOLDER", snippet.code)
+        bash = Path(os.environ["BASH_BINARY"])
+        assert bash.is_file() and os.access(bash, os.X_OK)
+        result = subprocess.run([bash, "-n"], input=illustrative_shell, text=True, capture_output=True, check=False)
+        assert result.returncode == 0, f"{snippet.path}:{snippet.line}: {result.stderr}"
+    elif snippet.syntax in {"python", "python3"}:
+        ast.parse(snippet.code, filename=f"{snippet.path}:{snippet.line}")
+    elif snippet.syntax == "json":
+        json.loads(snippet.code)
+    elif snippet.syntax in {"yaml", "yml"}:
+        import yaml
+
+        illustrative_yaml = "\n".join(line for line in snippet.code.splitlines() if line.strip() != "...")
+        list(yaml.safe_load_all(illustrative_yaml))
+    elif snippet.syntax == "ini":
+        parser = configparser.ConfigParser()
+        parser.read_string(snippet.code)
+    elif snippet.syntax == "sql":
+        connection = sqlite3.connect(":memory:")
+        connection.execute(
+            "CREATE TABLE auth_user "
+            "(password, last_login, is_superuser, username, first_name, last_name, email, "
+            "is_staff, is_active, date_joined)",
+        )
+        connection.executescript(snippet.code)
+    elif snippet.syntax == "mermaid":
+        assert snippet.code.lstrip().startswith(("stateDiagram", "flowchart", "graph", "sequenceDiagram"))
+        mmdc = Path(os.environ["ABXPKG_LIB_DIR"]) / "env" / "bin" / "mmdc"
+        assert mmdc.is_file() and os.access(mmdc, os.X_OK)
+        with tempfile.TemporaryDirectory(prefix="archivebox-docs-mermaid-") as temp:
+            source = Path(temp) / "diagram.mmd"
+            output = Path(temp) / "diagram.svg"
+            source.write_text(snippet.code)
+            result = subprocess.run(
+                [mmdc, "--input", source, "--output", output],
+                text=True,
+                capture_output=True,
+                check=False,
+            )
+            assert result.returncode == 0, result.stderr or result.stdout
+            assert output.stat().st_size > 0
+    elif snippet.syntax == "nginx":
+        assert snippet.code.count("{") == snippet.code.count("}")
+        nginx = Path(os.environ["ABXPKG_LIB_DIR"]) / "env" / "bin" / "nginx"
+        assert nginx.is_file() and os.access(nginx, os.X_OK)
+        with tempfile.TemporaryDirectory(prefix="archivebox-docs-nginx-") as temp:
+            config = Path(temp) / "nginx.conf"
+            config.write_text(f"events {{}}\nhttp {{\nserver {{\nlisten 8080;\n{snippet.code}\n}}\n}}\n")
+            result = subprocess.run(
+                [nginx, "-t", "-c", config, "-p", temp],
+                text=True,
+                capture_output=True,
+                check=False,
+            )
+            assert result.returncode == 0, result.stderr or result.stdout
+
+
+def validate_all() -> tuple[Snippet, ...]:
+    snippets = check_inventory()
+    records = load_manifest()
+    for snippet in snippets:
+        disposition = records[snippet.id]["disposition"]
+        if disposition != "run":
             try:
-                sql_connection.executescript(block.code)
-            except sqlite3.Error as err:
-                raise AssertionError(nodeid) from err
+                validate_non_running(snippet, disposition)
+            except Exception as error:
+                raise AssertionError(
+                    f"{snippet.path}:{snippet.line} ({snippet.id}, {disposition}) failed validation",
+                ) from error
+    return snippets
 
 
-def test_mermaid_fences_render_with_resolved_mmdc(tmp_path: Path) -> None:
-    mermaid_blocks = [(nodeid, block) for nodeid, block in docs_blocks().items() if block.syntax == "mermaid"]
-    assert len(mermaid_blocks) == 4
-    for index, (nodeid, block) in enumerate(mermaid_blocks):
-        source = tmp_path / f"diagram-{index}.mmd"
-        output = tmp_path / f"diagram-{index}.svg"
-        source.write_text(block.code)
-        result = subprocess.run(["mmdc", "--input", source, "--output", output], text=True, capture_output=True, check=False)
-        assert result.returncode == 0, f"{nodeid}: {result.stderr}"
-        assert output.stat().st_size > 0
+def matrix() -> dict[str, list[dict[str, str]]]:
+    snippets = validate_all()
+    records = load_manifest()
+    environments = {records[snippet.id]["environment"] for snippet in snippets if records[snippet.id]["disposition"] == "run"}
+    include = [
+        {
+            "name": environment,
+            "environment": environment,
+            "runner": ENVIRONMENT_RUNNERS[environment],
+        }
+        for environment in ENVIRONMENT_RUNNERS
+        if environment in environments
+    ]
+    assigned_ids = [snippet.id for snippet in snippets if records[snippet.id]["disposition"] == "run"]
+    assert include, "At least one deterministic documentation example must run in CI"
+    assert len(set(assigned_ids)) == len(assigned_ids)
+    assert {entry["environment"] for entry in include} == environments, "Every runnable snippet environment must have exactly one CI lane"
+    return {"include": include}
 
 
-def test_nginx_fence_parses_with_resolved_nginx(tmp_path: Path) -> None:
-    nginx_blocks = [(nodeid, block) for nodeid, block in docs_blocks().items() if block.syntax == "nginx"]
-    assert len(nginx_blocks) == 1
-    nodeid, block = nginx_blocks[0]
-    config = tmp_path / "nginx.conf"
-    config.write_text(f"events {{}}\nhttp {{\nserver {{\nlisten 8080;\n{block.code}\n}}\n}}\n")
-    result = subprocess.run(
-        ["nginx", "-t", "-c", str(config), "-p", str(tmp_path)],
-        text=True,
-        capture_output=True,
-        check=False,
+def run_environment(environment: str) -> None:
+    assert environment in ENVIRONMENT_RUNNERS, f"Unknown documentation environment: {environment}"
+    snippets = check_inventory()
+    records = load_manifest()
+    snippet_ids = tuple(
+        snippet.id
+        for snippet in snippets
+        if records[snippet.id]["disposition"] == "run" and records[snippet.id]["environment"] == environment
     )
-    assert result.returncode == 0, f"{nodeid}: {result.stderr}"
+    assert snippet_ids, f"No documentation snippets are assigned to {environment}"
+    run_snippets(snippet_ids)
 
 
-def test_docs_ci_matrix_covers_every_manifest_environment() -> None:
-    with MANIFEST_PATH.open("rb") as manifest_file:
-        manifest = tomllib.load(manifest_file)
+def run_snippets(snippet_ids: tuple[str, ...]) -> None:
+    snippets = {snippet.id: snippet for snippet in check_inventory()}
+    records = load_manifest()
+    assert len(set(snippet_ids)) == len(snippet_ids), "Each documentation snippet must run exactly once"
+    for snippet_id in snippet_ids:
+        assert snippet_id in snippets, f"Unknown snippet ID: {snippet_id}"
+        assert records[snippet_id]["disposition"] == "run", f"{snippet_id} is classified as {records[snippet_id]['disposition']}"
 
-    environments = set(manifest["environments"])
-    standard_environments = set(manifest["ci"]["standard"])
-    bsd_environments = set(manifest["ci"]["bsd"])
-    assert standard_environments.isdisjoint(bsd_environments)
-    assert standard_environments | bsd_environments == environments
+    with tempfile.TemporaryDirectory(prefix="archivebox-docs-") as temp:
+        temp_dir = Path(temp)
+        env = os.environ.copy()
+        env.update(
+            {
+                "HOME": str(temp_dir / "home"),
+                "XDG_CACHE_HOME": str(temp_dir / "cache"),
+                "XDG_CONFIG_HOME": str(temp_dir / "config"),
+                "XDG_DATA_HOME": str(temp_dir / "share"),
+                "UV_TOOL_BIN_DIR": str(temp_dir / "home" / ".local" / "bin"),
+                "ABXPKG_LIB_DIR": str(temp_dir / "lib"),
+                "PATH": f"{Path(sys.executable).parent}:{env['PATH']}",
+            },
+        )
+        Path(env["HOME"]).mkdir()
+        system_home = temp_dir / "system-home"
+        system_home.mkdir()
+        system_env = {
+            "HOME": str(system_home),
+            "XDG_CACHE_HOME": str(system_home / ".cache"),
+            "XDG_CONFIG_HOME": str(system_home / ".config"),
+            "XDG_DATA_HOME": str(system_home / ".local" / "share"),
+            "UV_TOOL_BIN_DIR": str(system_home / ".local" / "bin"),
+            "ABXPKG_LIB_DIR": str(temp_dir / "system-lib"),
+        }
+        system_path = os.pathsep.join(
+            [
+                str(system_home / ".local" / "bin"),
+                str(system_home / ".cargo" / "bin"),
+                *(part for part in env["PATH"].split(os.pathsep) if Path(part).resolve() != Path(sys.executable).parent.resolve()),
+            ],
+        )
+        scenarios = {records[snippet_id]["scenario"] for snippet_id in snippet_ids}
+        workdirs = {
+            "project": ROOT,
+            "system": system_home,
+            "system-data": system_home / "archivebox" / "data",
+            "collection": Path(env["HOME"]) / "archivebox" / "data",
+            "docker": Path(env["HOME"]) / "archivebox",
+            "docker-data": temp_dir / "data",
+        }
 
-    workflow = WORKFLOW_PATH.read_text()
-    assert "pytest -q docs/test_codeblocks_manifest.py" in workflow
-    assert "--docs-environment=${{ matrix.environment }}" in workflow
-    assert "DOCS_CORE_SHARD: ${{ matrix.core_shard }}" in workflow
-    for environment in bsd_environments:
-        assert f"--docs-environment={environment}" in workflow
+        if "system-data" in scenarios:
+            workdirs["system-data"].mkdir(parents=True)
+        if "collection" in scenarios:
+            workdirs["collection"].mkdir(parents=True, exist_ok=True)
+            archivebox = Path(sys.executable).with_name("archivebox")
+            assert archivebox.is_file() and os.access(archivebox, os.X_OK)
+            subprocess.run(
+                [archivebox, "init"],
+                cwd=workdirs["collection"],
+                env=env,
+                check=True,
+            )
+        if scenarios.intersection({"docker", "docker-data"}):
+            env["ARCHIVEBOX_IMAGE"] = "archivebox/archivebox:dev"
+            docker = Path(env["DOCKER_BINARY"])
+            assert docker.is_file() and os.access(docker, os.X_OK)
+        if "docker" in scenarios:
+            workdirs["docker"].mkdir(parents=True)
+            (workdirs["docker"] / "docker-compose.yml").write_text((ROOT / "docker-compose.yml").read_text())
+            subprocess.run(
+                [docker, "compose", "run", "--rm", "archivebox", "init"],
+                cwd=workdirs["docker"],
+                env=env,
+                check=True,
+            )
+        if "docker-data" in scenarios:
+            workdirs["docker-data"].mkdir()
+            subprocess.run(
+                [docker, "run", "--rm", "-v", f"{workdirs['docker-data']}:/data", "archivebox/archivebox:dev", "init"],
+                env=env,
+                check=True,
+            )
+
+        for snippet_id in snippet_ids:
+            snippet = snippets[snippet_id]
+            record = records[snippet_id]
+            snippet_env = env.copy()
+            if record["scenario"] in {"system", "system-data"}:
+                snippet_env.update(system_env)
+                snippet_env["PATH"] = system_path
+            print(f"Running {snippet.id}: {snippet.path}:{snippet.line} ({record['scenario']})", flush=True)
+            if snippet.syntax in {"bash", "sh", "console"}:
+                bash = Path(snippet_env["BASH_BINARY"])
+                assert bash.is_file() and os.access(bash, os.X_OK)
+                subprocess.run(
+                    [bash, "-Eeuo", "pipefail", "-c", snippet.code],
+                    cwd=workdirs[record["scenario"]],
+                    env=snippet_env,
+                    check=True,
+                )
+            elif snippet.syntax in {"python", "python3"}:
+                subprocess.run(
+                    [sys.executable, "-c", snippet.code],
+                    cwd=workdirs[record["scenario"]],
+                    env=snippet_env,
+                    check=True,
+                )
+            else:
+                raise AssertionError(f"{snippet.id}: {snippet.syntax} cannot have run disposition")
 
 
-def test_every_core_file_belongs_to_exactly_one_explicit_shard() -> None:
-    with MANIFEST_PATH.open("rb") as manifest_file:
-        manifest = tomllib.load(manifest_file)
+def render_manifest() -> None:
+    existing: dict[str, dict[str, str | None]] = {}
+    if MANIFEST.exists():
+        with MANIFEST.open("rb") as manifest_file:
+            document = tomllib.load(manifest_file)
+        if document.get("version") == 2:
+            existing = {
+                snippet_id: {
+                    "disposition": disposition,
+                    "scenario": document.get("scenarios", {}).get(snippet_id),
+                    "environment": document.get("environments", {}).get(snippet_id),
+                }
+                for snippet_id, disposition in document.get("snippets", {}).items()
+            }
+    lines = [
+        "# Generated with: uv run --no-sync python docs/test_codeblocks_manifest.py render",
+        "# Every authored occurrence is classified here without adding test plumbing to the rendered docs.",
+        "version = 2",
+        "",
+        "[snippets]",
+    ]
+    snippets = scan_snippets()
+    last_path = ""
+    for snippet in snippets:
+        old = existing.get(snippet.id, {})
+        disposition = old.get("disposition", "illustration")
+        if snippet.syntax in {"python", "python3"} and any(line.lstrip().startswith((">>>", "...")) for line in snippet.code.splitlines()):
+            disposition = old.get("disposition", "transcript")
+        if snippet.path != last_path:
+            lines.extend(["", f"# {snippet.path}"])
+            last_path = snippet.path
+        lines.append(f"{json.dumps(snippet.id)} = {json.dumps(disposition)}")
 
-    file_environments = manifest["files"]
-    block_environments = manifest["blocks"]
-    core_files = {path for path, environment in file_environments.items() if environment == "core"}
-    core_files.update(nodeid.partition("::")[0] for nodeid, environment in block_environments.items() if environment == "core")
+    scenarios = {
+        snippet.id: existing[snippet.id]["scenario"] for snippet in snippets if existing.get(snippet.id, {}).get("disposition") == "run"
+    }
+    environments = {
+        snippet.id: existing[snippet.id]["environment"] for snippet in snippets if existing.get(snippet.id, {}).get("disposition") == "run"
+    }
+    lines.extend(["", "[scenarios]"])
+    lines.extend(f"{json.dumps(snippet_id)} = {json.dumps(scenario)}" for snippet_id, scenario in scenarios.items())
+    lines.extend(["", "[environments]"])
+    lines.extend(f"{json.dumps(snippet_id)} = {json.dumps(environment)}" for snippet_id, environment in environments.items())
+    print("\n".join(lines))
 
-    core_shards = manifest["ci"]["core_shards"]
-    assert core_shards
-    shard_members = [path for paths in core_shards.values() for path in paths]
-    member_counts = Counter(shard_members)
 
-    assert set(shard_members) == core_files
-    assert {path: count for path, count in member_counts.items() if count != 1} == {}
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("command", choices=("check", "matrix", "run", "run-environment", "render"))
+    parser.add_argument("snippet_id", nargs="?")
+    args = parser.parse_args()
+
+    if args.command == "check":
+        snippets = validate_all()
+        print(f"Validated {len(snippets)} authored documentation snippets.")
+    elif args.command == "matrix":
+        print(json.dumps(matrix(), separators=(",", ":")))
+    elif args.command == "run":
+        assert args.snippet_id, "run requires a snippet ID"
+        run_snippets((args.snippet_id,))
+    elif args.command == "run-environment":
+        assert args.snippet_id, "run-environment requires an environment"
+        run_environment(args.snippet_id)
+    else:
+        render_manifest()
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -71,6 +71,7 @@ dependencies = [
     "django-stubs>=5.0.4",   # for: vscode type hints on models and common django APIs
     ### API clients
     "requests>=2.32.3",      # for: fetching title, static files, headers (TODO: replace with httpx?)
+    "httpx>=0.28.1",         # for: streaming the admin-only OpenCode proxy
     "sonic-client>=1.0.0",
     ### Parsers
     "dateparser>=1.2.0",     # for: parsing pocket/pinboard/etc. RSS/bookmark import dates
@@ -80,9 +81,9 @@ dependencies = [
     ### Extractor dependencies (runtime binaries resolved through abxpkg)
     ### Binary/Package Management
     "abxbus==2.5.40",         # EventBus API
-    "abxpkg==1.11.288",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins==1.11.311",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl==1.11.268",       # shared ArchiveBox downloader package with blocking install preflight
+    "abxpkg==1.11.289",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
+    "abx-plugins==1.11.312",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl==1.11.269",       # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]
@@ -140,8 +141,7 @@ dev = [
     # "snakeviz",                                      # usage: python -m cProfile -o flamegraph.prof ../.venv/bin/archivebox manage check
     ### TESTING
     "pytest-django>=4.11.1",
-    "pytest>=8.3.3,<9",                               # pytest-codeblocks 0.17.0 still uses the pre-pytest-9 collect hook signature.
-    "pytest-codeblocks>=0.17.0",
+    "pytest>=8.3.3",
     "pytest-cov>=6.0.0",
     "pytest-github-actions-annotate-failures>=0.4.0",
     "pytest-httpserver>=1.1.0",
@@ -240,17 +240,13 @@ exclude = ["*.pyi", "*.html", "**/*.html", "typings/", "migrations/", "archivebo
 ignore = ["E731", "E303", "E266", "E241", "E222"]
 
 [tool.codespell]
-ignore-words-list = "abx,archivebox,adminsnapshots,bu,wit,dont,cant,wont,havent,thats,shouldnt,doesnt,doenst,succeded,re-use,re-used,re-using,re-usable,generat"
+ignore-words-list = "abx,archivebox,adminsnapshots,bu,wit,dont,cant,wont,havent,thats,shouldnt,doesnt,doenst,succeded,re-use,re-used,re-using,re-usable,generat,te"
 skip = "*.json,*.min.js,*.min.css,uv.lock,old/*,publicsite/*,docs/apidocs/*"
 
 [tool.pytest.ini_options]
-testpaths = [ "archivebox/tests", "README.md", "AGENTS.md", "skills", "docs" ]
-addopts = ["--codeblocks"]
-norecursedirs = ["archivebox/tests/data"]
+testpaths = [ "archivebox/tests" ]
 timeout = 600
 timeout_method = "thread"
-# Note: Plugin tests under abx_plugins/plugins/ must NOT load Django
-# They use a conftest.py to disable Django automatically
 
 [tool.coverage.run]
 # Enable branch coverage (tracks if/else branches)

skills/archivebox/SKILL.md

@@ -22,10 +22,10 @@ ArchiveBox is the full self-hosted web archiving app. Use this skill for collect
 ## Development Setup
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"
-uv sync --project "$project_dir" --dev --all-extras
-cd "$archivebox_data" && uv run --project "$project_dir" --no-sync archivebox init --install
+uv sync --dev --all-extras
+mkdir -p data
+cd data
+uv run --project .. archivebox init --install
 ```
 
 ## User-Facing Setup
@@ -33,12 +33,11 @@ cd "$archivebox_data" && uv run --project "$project_dir" --no-sync archivebox in
 Recommended CLI install:
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-tool_root="$(mktemp -d)"; export UV_TOOL_DIR="$tool_root/tools" UV_TOOL_BIN_DIR="$tool_root/bin"
-uv tool install --force "$project_dir"
-export PLUGINS=parse_txt_urls
-archivebox_data="$(mktemp -d)"
-cd "$archivebox_data" && "$UV_TOOL_BIN_DIR/archivebox" init --install && "$UV_TOOL_BIN_DIR/archivebox" add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
+uv tool install --python 3.13 --prerelease allow archivebox
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init --install
+archivebox add --plugins=parse_txt_urls 'https://example.com/'
 ```
 
 Alternative install methods:
@@ -52,25 +51,24 @@ Alternative install methods:
 
 Run from inside an initialized data dir:
 
-<!--pytest-codeblocks:cont-->
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
-archivebox_data="$(mktemp -d)"
-cd "$archivebox_data" && uv run --project "$project_dir" --no-sync archivebox init --install
-uv run --project "$project_dir" --no-sync archivebox version && uv run --project "$project_dir" --no-sync archivebox status && uv run --project "$project_dir" --no-sync archivebox install
-uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/docs-basic-usage}"
-uv run --project "$project_dir" --no-sync archivebox list --json --with-headers
-uv run --project "$project_dir" --no-sync archivebox search 'example' && uv run --project "$project_dir" --no-sync archivebox update --filter-type=domain example.com
-uv run --project "$project_dir" --no-sync archivebox remove --yes --delete --filter-type=exact "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/docs-basic-usage}"
-uv run --project "$project_dir" --no-sync archivebox run
+cd ~/archivebox/data
+archivebox version
+archivebox status
+archivebox install
+archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
+archivebox list --json --with-headers
+archivebox search 'example'
+archivebox update --filter-type=domain example.com
+archivebox remove --yes --delete --filter-type=exact 'https://example.com/docs-basic-usage'
+archivebox run
 ```
 
 ## Verification
 
 ```bash
-project_dir="${ARCHIVEBOX_PROJECT_DIR:-.}"
-uv run --project "$project_dir" --no-sync pytest "$project_dir/archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options" -q
-(cd "$project_dir" && uv run --no-sync prek run --all-files)
+uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
+uv run prek run --all-files
 ```
 
 Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

uv.lock

@@ -13,18 +13,18 @@ supported-markers = [
 ]
 
 [options]
-exclude-newer = "2026-07-19T07:19:43.036580449Z"
+exclude-newer = "2026-07-19T10:29:25.327679866Z"
 exclude-newer-span = "P5D"
 
 [options.exclude-newer-package]
-abxbus = { timestamp = "2026-07-24T07:19:42.03659601Z", span = "PT1S" }
+abxbus = { timestamp = "2026-07-24T10:29:24.328016284Z", span = "PT1S" }
 abx-plugins = "2100-01-01T00:00:00Z"
 abx-dl = "2100-01-01T00:00:00Z"
 abxpkg = "2100-01-01T00:00:00Z"
 
 [[package]]
 name = "abx-dl"
-version = "1.11.268"
+version = "1.11.269"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -39,14 +39,14 @@ dependencies = [
     { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/ea/b3/ba83c367c76c1fa82e8e94f468f093d092f52f13a31c468a56436f388442/abx_dl-1.11.268.tar.gz", hash = "sha256:e8cb9ce008f2d0dfdc36adbd54c692caf505b78f5ab8510fe77eda7c4a44699b", size = 86729, upload-time = "2026-07-23T12:12:52.806Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/38/41/2274b7c524deec3a68ce373c09580247dfe2179841e38b1f5bd67feea923/abx_dl-1.11.269.tar.gz", hash = "sha256:36c884ca009e26b9c030ed200fcdd2983994757d8b4ed663cac12d3a872a7fdb", size = 86735, upload-time = "2026-07-23T16:29:26.622Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/51/e3/fff903a6e9e3515bee7d8f06cc9d561e8f95881ab3bbdad7033b36008c22/abx_dl-1.11.268-py3-none-any.whl", hash = "sha256:429e9afa258dd714b3dc1e3b1cc0ada4f25b395d19db194752a2bf1e1bdc54c6", size = 90472, upload-time = "2026-07-23T12:12:53.915Z" },
+    { url = "https://files.pythonhosted.org/packages/36/1b/b6f7a114ab7a9ef2024cbe2c9fdbeab28e19b152aab700613066eafddf25/abx_dl-1.11.269-py3-none-any.whl", hash = "sha256:e20e6b307294eedb2c2c175d5eb495315b5672881e5db4d554091b2e54a19ce1", size = 90470, upload-time = "2026-07-23T16:29:27.852Z" },
 ]
 
 [[package]]
 name = "abx-plugins"
-version = "1.11.311"
+version = "1.11.312"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -57,9 +57,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/49/f8/ad69bed9751ccbdc892ac3fd6a63b9eace0bd7b0bbb254b855fd8351bc58/abx_plugins-1.11.311.tar.gz", hash = "sha256:acf3ad909f63f610b8981cf9796ff758d713585a380dd9404f42c8419f1ef020", size = 266481, upload-time = "2026-07-23T11:32:48.583Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/49/b9/7a803aff1f498a066cdd6d613e1ec0f44db9990b700fc30a5d7ade65e8e6/abx_plugins-1.11.312.tar.gz", hash = "sha256:ad38272b0b7da42f0cf349ae8e2aa2d2f8001e5c0e2a6323d812ac674aac2d00", size = 266483, upload-time = "2026-07-23T16:03:47.502Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/31/43/27f1582e2d6800cec0dd9f98957d5a02088072b0e48932c3405277c91f29/abx_plugins-1.11.311-py3-none-any.whl", hash = "sha256:67b289c4f2ed4eced783c5a86cf8065f517cbfe42202f54554f33b24411e53c9", size = 415643, upload-time = "2026-07-23T11:32:49.84Z" },
+    { url = "https://files.pythonhosted.org/packages/27/84/e5537184ff930e4f014056227abb5f32b2dd1969c24bc9edfe8ec0f55c70/abx_plugins-1.11.312-py3-none-any.whl", hash = "sha256:8580a372a1c82b5565aabe3b0d136b3d3c9827e29860f6fa7abf013f34ebfad4", size = 415641, upload-time = "2026-07-23T16:03:48.623Z" },
 ]
 
 [[package]]
@@ -78,7 +78,7 @@ wheels = [
 
 [[package]]
 name = "abxpkg"
-version = "1.11.288"
+version = "1.11.289"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "pip", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -87,9 +87,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/8d/e0/f98aab6246c6d328ee289ced80e95befd46d8046abbbfc317ccf696582b1/abxpkg-1.11.288.tar.gz", hash = "sha256:d802c34ee655cc17930572b07a4ed9f62c18022fe98f380b38fb043dc7ba26b1", size = 212909, upload-time = "2026-07-23T03:30:24.528Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/49/43/40eb370f6616b70d531ec6a8da2ee9cbd58eb16cc055de2a518f6ed89aad/abxpkg-1.11.289.tar.gz", hash = "sha256:d26c54ce0270ce3015c0804967492417a14218d903bd99688bc298cc355f3587", size = 213548, upload-time = "2026-07-23T15:09:48.197Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f8/c5/21478d9a246cacd51dc7d700bd0bce3ea08ac14b34ce09496640dea27d71/abxpkg-1.11.288-py3-none-any.whl", hash = "sha256:de7b0ed66a4d7ef06764634a1dae1b28f27d17c40f824159df4c2d2d89e3efa9", size = 227472, upload-time = "2026-07-23T03:30:22.856Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/76/5f7823d37d6c0361f88c027a520721e51f598b81921661c5c6096f2bc17e/abxpkg-1.11.289-py3-none-any.whl", hash = "sha256:edc82c990955d061848e618cf225fa8b50c4e903d1b66b4253f89094a1f17c2e", size = 228018, upload-time = "2026-07-23T15:09:46.586Z" },
 ]
 
 [[package]]
@@ -145,6 +145,7 @@ dependencies = [
     { name = "django-object-actions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "django-signal-webhooks", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "django-stubs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+    { name = "httpx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "ipython", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "psutil", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -203,7 +204,6 @@ dev = [
     { name = "prek", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "pyright", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "pytest", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-    { name = "pytest-codeblocks", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "pytest-cov", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "pytest-django", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "pytest-github-actions-annotate-failures", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -224,10 +224,10 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "abx-dl", specifier = "==1.11.268" },
-    { name = "abx-plugins", specifier = "==1.11.311" },
+    { name = "abx-dl", specifier = "==1.11.269" },
+    { name = "abx-plugins", specifier = "==1.11.312" },
     { name = "abxbus", specifier = "==2.5.40" },
-    { name = "abxpkg", specifier = "==1.11.288" },
+    { name = "abxpkg", specifier = "==1.11.289" },
     { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
     { name = "atomicwrites", specifier = "==1.4.1" },
     { name = "base32-crockford", specifier = ">=0.3.0" },
@@ -247,6 +247,7 @@ requires-dist = [
     { name = "django-signal-webhooks", specifier = ">=0.3.0" },
     { name = "django-stubs", specifier = ">=5.0.4" },
     { name = "djdt-flamegraph", marker = "extra == 'debug'", specifier = ">=0.2.13" },
+    { name = "httpx", specifier = ">=0.28.1" },
     { name = "ipdb", marker = "extra == 'debug'", specifier = ">=0.13.13" },
     { name = "ipython", specifier = ">=8.27.0" },
     { name = "platformdirs", specifier = ">=4.3.6" },
@@ -286,8 +287,7 @@ dev = [
     { name = "opentelemetry-instrumentation-sqlite3", specifier = ">=0.47b0" },
     { name = "prek", specifier = ">=0.3.6" },
     { name = "pyright", specifier = ">=1.1.406" },
-    { name = "pytest", specifier = ">=8.3.3,<9" },
-    { name = "pytest-codeblocks", specifier = ">=0.17.0" },
+    { name = "pytest", specifier = ">=8.3.3" },
     { name = "pytest-cov", specifier = ">=6.0.0" },
     { name = "pytest-django", specifier = ">=4.11.1" },
     { name = "pytest-github-actions-annotate-failures", specifier = ">=0.4.0" },
@@ -1941,17 +1941,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" },
 ]
 
-[[package]]
-name = "pytest-codeblocks"
-version = "0.18.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "pytest", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-]
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/3a/cb/5f40df7db75c0ac2555009287bb97b322f8d79f9f2bfdfb0bdb560834c28/pytest_codeblocks-0.18.0-py3-none-any.whl", hash = "sha256:3fe944dc505107421204c83e9232e0155eea1279b9425a10bee327079b272efb", size = 8009, upload-time = "2026-06-15T20:48:02.608Z" },
-]
-
 [[package]]
 name = "pytest-cov"
 version = "7.1.0"