The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into audit-pr1846

Browse Source
This commit is contained in:
Nick Sweeting 2026-09-01 11:38:24 -07:00
commit c817da7af6
No known key found for this signature in database
51 changed files with 1758 additions and 988 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.github/workflows/deploy-publicsite.yml vendored
View File

@ -27,7 +27,6 @@ concurrency:
jobs:
deploy:
if: github.event_name != 'push' || github.event.head_commit.author.email != 'release-bot@archivebox.io'
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
@ -51,6 +50,22 @@ jobs:
env:
BASE_URL: http://archivebox.localhost:8000
- name: Verify gallery was generated from this exact revision
run: |
uv run --no-cache --no-sync python - <<'PY'
import json
import os
import tomllib
from pathlib import Path
provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
assert provenance["version"] == version, provenance
assert provenance["capture_count"] >= 3 * 35, provenance
assert len(provenance["files"]) == provenance["capture_count"], provenance
PY
- name: Setup Pages
uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5

15
Dockerfile
View File

@ -10,10 +10,10 @@
# --build-context abx-plugins=../abx-plugins \
# -t archivebox/abx-dl:dev
# docker buildx build . -f Dockerfile \
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.188 \
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.225 \
# -t archivebox:multistage
ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.188
ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.225
FROM archivebox/sonic:1.4.9 AS sonic
FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
@ -156,19 +156,20 @@ PY
--no-install-project \
--no-install-workspace \
--no-sources
ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$ABXPKG_LIB_DIR" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
while IFS= read -r -d '' native_library; do
magic=''
if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
"$ABXPKG_LIB_DIR/env/bin/strip" --strip-unneeded "$native_library" || exit $?
"$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
fi
done < /tmp/archivebox-native-libraries
rm -f /tmp/archivebox-native-libraries
rm -f /venv/bin/uv /venv/bin/uvx
abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev
abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get autoremove -y
/usr/bin/find "$ABXPKG_LIB_DIR/env/bin" -maxdepth 1 -type l -name strip -delete
abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev
abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
rm -rf "$builder_abxpkg_lib_dir"
rm -rf /venv/lib/python3.*/site-packages/pip* \
/venv/lib/python3.*/site-packages/wheel* \
/venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel

51
README.md
View File

@ -80,8 +80,6 @@ docker compose up -d --wait # ini
<br/>
# Option B: Or use it as a plain Docker container:
mkdir -p ~/archivebox/data && cd ~/archivebox/data
docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev init
docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev install
docker run -d --name archivebox -v "$PWD:/data" -p 8000:8000 archivebox/archivebox:dev
# open http://admin.archivebox.localhost:8000 to finish setup
# docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
@ -187,17 +185,16 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
<br/>
<ol>
<li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
<li>Create a new empty directory and initialize your collection (can be anywhere).
<li>Create a new empty directory and start the server, which initializes the collection automatically (can be anywhere).
<pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
docker run --rm -v $PWD:/data -it archivebox/archivebox:dev init
docker run --rm -v $PWD:/data -it archivebox/archivebox:dev install
docker run -d --name archivebox -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
</code></pre>
</li>
<li>Optional: Start the server, then open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
<li>Open <code>/admin/</code> on the hostname or IP used to reach ArchiveBox (local example: <a href="http://admin.archivebox.localhost:8000/admin/">http://admin.archivebox.localhost:8000/admin/</a>) to create the first admin. If <code>BASE_URL</code> is not configured yet, continue through the web setup wizard.
<pre lang="bash"><code style="white-space: pre-line">
# completely optional, CLI can always be used without running a server
# docker run -v $PWD:/data -it archivebox/archivebox:dev [subcommand] [--help]
docker run -v $PWD:/data -it archivebox/archivebox:dev help
# docker exec archivebox archivebox [subcommand] [--help]
docker exec archivebox archivebox help
</code></pre>
<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
</li>
@ -497,12 +494,10 @@ archivebox help # get list of archivebox subcommands that can be ru
<pre lang="bash"><code style="white-space: pre-line">
# make sure you have `docker-compose.yml` from the Quickstart instructions first
<br/>
# docker compose run --rm archivebox [subcommand] [--help]
docker compose run --rm archivebox init
docker compose run --rm archivebox install
docker compose run --rm archivebox version
docker compose run --rm archivebox help
docker compose run --rm archivebox add 'https://example.com'
# docker compose exec archivebox archivebox [subcommand] [--help]
docker compose exec archivebox archivebox version
docker compose exec archivebox archivebox help
docker compose exec archivebox archivebox add 'https://example.com'
# to start webserver: docker compose up
</code></pre>
<i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@ -514,15 +509,12 @@ docker compose run --rm archivebox add 'https://example.com'
<summary><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="22px" align="top"/> <b>CLI Usage Examples: Docker</b></summary>
<br/>
<pre lang="bash"><code style="white-space: pre-line">
# make sure you create and cd into in a new empty directory first
# make sure the `archivebox` server container from the Quickstart is running first
<br/>
# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
docker run -v $PWD:/data -it archivebox/archivebox:dev init
docker run -v $PWD:/data -it archivebox/archivebox:dev install
docker run -v $PWD:/data -it archivebox/archivebox:dev version
docker run -v $PWD:/data -it archivebox/archivebox:dev help
docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
# docker exec archivebox archivebox [subcommand] [--help]
docker exec archivebox archivebox version
docker exec archivebox archivebox help
docker exec archivebox archivebox add 'https://example.com'
</code></pre>
<i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
</details>
@ -682,7 +674,7 @@ It uses all available methods out-of-the-box, but you can disable extractors and
<summary><i>Expand to see the full list of ways it saves each page...</i></summary>
<code>data/archive/{Snapshot.id}/</code><br/>
<code>data/archive/users/{username}/snapshots/{YYYYMMDD}/{domain}/{Snapshot.id}/</code><br/>
<ul>
<li><strong>Index:</strong> <code>index.html</code> &amp; <code>index.json</code> HTML and JSON index files containing metadata and details</li>
<li><strong>Title</strong>, <strong>Favicon</strong>, <strong>Headers</strong> Response headers, site favicon, and parsed site title</li>
@ -849,7 +841,7 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
...
</code></pre>
Each snapshot subfolder includes static metadata and plain extractor output files. ArchiveBox also maintains a backwards-compatible <code>data/archive/TIMESTAMP</code> symlink for each snapshot.
Each snapshot subfolder includes static metadata and plain extractor output files. Current releases do not create top-level <code>data/archive/TIMESTAMP</code> projections; legacy timestamp directories are migrated into the user-scoped tree by <code>archivebox update --migrate-only</code>.
<h4>Learn More</h4>
<ul>
@ -1345,14 +1337,11 @@ archivebox server 0.0.0.0:8000
# inside the container will reload and pick up your changes
./bin/build_docker.sh dev
docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
# Run the development server w/ autoreloading (but no bg workers)
docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init --debug --reload 0.0.0.0:8000
# Run the production server (with bg workers but no autoreloading)
docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server
docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --init
# (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
# When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@ -1413,7 +1402,7 @@ services:
# or with plain Docker:
docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
docker run -it -v $PWD:/data archivebox:dev init
docker run -it -v $PWD:/data -p 8000:8000 archivebox:dev
# or with uv:
uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'

6
archivebox/cli/archivebox_run.py
View File

@ -54,6 +54,11 @@ RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
# A supervised `archivebox run --daemon` is intentionally a disposable
# child. If it receives SIGINT/SIGTERM directly, exit with the conventional
# signal status so supervisord treats it as an unexpected worker death and
# restarts only the runner. The parent `archivebox server` owns supervisord
# shutdown and must not be pulled down by a killed daemon worker.
os._exit(128 + int(sig))
@ -292,6 +297,7 @@ def run_runner(
interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
if daemon:
os.environ[RUNNER_DAEMON_ENV] = "1"
try:
with (
foreground_shutdown_signals(

26
archivebox/cli/archivebox_update.py
View File

@ -561,21 +561,20 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
"""
Drain old archive/ directories (0.8.x → 0.9.x migration).
Only processes real directories (skips symlinks - those are already migrated).
Removes obsolete timestamp symlinks and processes real legacy directories.
For each old dir found in archive/:
1. Load or create DB snapshot
2. Trigger fs migration on save() to move to data/archive/users/{user}/...
3. Leave symlink in archive/ pointing to new location
3. Remove the old timestamp path after the verified migration commits
After this drains, archive/ should only contain symlinks and we can trust
1:1 mapping between DB and filesystem.
After this drains, current snapshot data exists only under archive/users/.
"""
from archivebox.core.models import Snapshot
from archivebox.config import CONSTANTS
from archivebox.crawls.models import Crawl
from django.utils import timezone
stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0}
stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0, "removed_symlinks": 0}
crawl_url_lines: dict[str, list[str]] = {}
crawl_url_sets: dict[str, set[str]] = {}
dirty_crawl_ids: set[str] = set()
@ -606,8 +605,23 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
if changed:
Crawl.objects.filter(pk=crawl.pk).update(urls="\n".join(lines), modified_at=timezone.now())
# Scan for real directories only (skip symlinks - they're already migrated)
# Compatibility timestamp projections are harmful in portable exports and
# duplicate the obsolete 0.7.x-looking namespace without containing data.
all_entries = list(os.scandir(archive_dir))
for entry in all_entries:
entry_path = Path(entry.path)
if entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry_path):
try:
target_path = entry_path.resolve(strict=True)
points_into_current_layout = target_path.is_relative_to(CONSTANTS.USERS_DIR.resolve(strict=True))
except OSError:
points_into_current_layout = False
if points_into_current_layout:
entry_path.unlink(missing_ok=True)
stats["removed_symlinks"] += 1
# Scan real legacy directories only; these still contain data to migrate.
entries = [
(e.stat().st_mtime, e.path)
for e in all_entries

15
archivebox/config/common.py
View File

@ -406,7 +406,11 @@ class DatabaseConfig(BaseConfigSet):
DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
SQLITE_JOURNAL_MODE: str = Field(
default="WAL",
# Docker collections commonly live on a host bind mount. WAL's -shm
# locking is only safe when every SQLite process is on the same host;
# Docker Desktop/OrbStack place the container and host in different
# locking domains and a host-side reader can corrupt the live DB.
default="DELETE" if IN_DOCKER else "WAL",
alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
pattern=r"(?i)^(DELETE|TRUNCATE|PERSIST|MEMORY|WAL|OFF)$",
)
@ -419,6 +423,15 @@ class DatabaseConfig(BaseConfigSet):
SQLITE_LOCK_RETRY_TIMEOUT: float = Field(default=60.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_TIMEOUT", ge=0)
SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)
@model_validator(mode="after")
def reject_docker_sqlite_wal(self):
if IN_DOCKER and self.DATABASE_ENGINE.lower() == "sqlite" and self.SQLITE_JOURNAL_MODE.upper() == "WAL":
raise ValueError(
"SQLITE_JOURNAL_MODE=WAL is unsafe for Docker collections because host bind mounts cross SQLite "
"locking domains; use DELETE (the Docker default) or PostgreSQL",
)
return self
class ArchivingConfig(BaseConfigSet):
toml_section_header: str = "ARCHIVING_CONFIG"

43
archivebox/core/admin_archiveresults.py
View File

@ -10,9 +10,12 @@ from pathlib import Path
from urllib.parse import quote
from django.contrib import admin
from django.core.exceptions import ValidationError
from django.contrib.admin.actions import delete_selected
from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME
from django.core.exceptions import PermissionDenied, SuspiciousOperation, ValidationError
from django.db.models import Count, Min, Prefetch, Q, Subquery, TextField, Window
from django.db.models.functions import Cast
from django.shortcuts import redirect
from django.urls import resolve, reverse
from django.utils import timezone
from django.utils.html import format_html
@ -341,7 +344,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None, can_del
class ArchiveResultInline(admin.TabularInline):
name = "Archive Results Log"
name = "Archive Results"
model = ArchiveResult
parent_model = Snapshot
extra = 0
@ -543,10 +546,44 @@ class ArchiveResultAdmin(BaseModelAdmin):
def changelist_view(self, request, extra_context=None):
self.request = request
selected = request.GET.getlist(ACTION_CHECKBOX_NAME)
if request.method == "GET" and request.GET.get("action") == "delete_selected" and selected:
if not request.user.is_superuser:
raise PermissionDenied
if len(selected) > 100:
raise SuspiciousOperation("Too many ArchiveResults selected for deletion")
try:
queryset = self.get_queryset(request).filter(pk__in=selected)
if not queryset.exists():
snapshot = Snapshot.objects.only("id").filter(pk=request.GET.get("snapshot")).first()
return redirect(build_snapshot_url(str(snapshot.id), "index.html", request=request) if snapshot else request.path)
except (ValidationError, ValueError):
return redirect(request.path)
return delete_selected(self, request, queryset)
handoff_snapshot = (
request.GET.get("snapshot") if request.method == "POST" and request.GET.get("action") == "delete_selected" else None
)
if handoff_snapshot:
request.GET = request.GET.copy()
request.GET.clear()
request.META["QUERY_STRING"] = ""
try:
handoff_snapshot = Snapshot.objects.only("id").filter(pk=handoff_snapshot).first()
except (ValidationError, ValueError):
handoff_snapshot = None
saved_list_per_page = self.list_per_page
self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
try:
return super().changelist_view(request, extra_context)
response = super().changelist_view(request, extra_context)
if (
handoff_snapshot
and response.status_code in (301, 302)
and not ArchiveResult.objects.filter(
pk__in=request.POST.getlist(ACTION_CHECKBOX_NAME),
).exists()
):
return redirect(build_snapshot_url(str(handoff_snapshot.id), "index.html", request=request))
return response
finally:
self.list_per_page = saved_list_per_page

2
archivebox/core/admin_users.py
View File

@ -96,7 +96,7 @@ class CustomUserAdmin(UserAdmin):
+ f'<br/><a href="/admin/core/snapshot/?created_by__id__exact={obj.pk}">{total_count} total records...<a>',
)
@admin.display(description="Archive Result Logs")
@admin.display(description="Archive Results")
def archiveresult_set(self, obj):
total_count = obj.archiveresult_set.count()
return mark_safe(

30
archivebox/core/middleware.py
View File

@ -39,8 +39,8 @@ ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
def _admin_login_hint_cookie_domain(config) -> str | None:
"""Resolve the parent domain to scope the cross-subdomain login hint.
NOTE: this cookie carries only the single bit "user is logged in on
admin somewhere"; it MUST NOT be confused with the session cookie,
NOTE: this cookie carries only the single bit "a superuser is logged in
on admin somewhere"; it MUST NOT be confused with the session cookie,
which stays admin-host-scoped (see core/settings.py
SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).
@ -88,6 +88,10 @@ def AdminCookieIsolationMiddleware(get_response):
def middleware(request):
response = get_response(request)
if request.path == "/admin" or request.path.startswith("/admin/"):
response.headers["X-Frame-Options"] = "DENY"
response.headers["Content-Security-Policy"] = "frame-ancestors 'none'"
config = request.__dict__.get("archivebox_config")
if config is None or config.SERVER_SECURITY_MODE == "auto":
from archivebox.config.common import get_request_config
@ -197,6 +201,21 @@ def ServerSecurityModeMiddleware(get_response):
config = get_request_config(request, resolve_plugins=False)
if config.USES_SUBDOMAIN_ROUTING and config.BASE_URL and request.method.upper() not in allowed_methods:
request_host, _request_port = split_host_port((request.get_host() or "").lower())
control_hosts = {
split_host_port(host)[0]
for host in (
get_base_host(config=config),
get_admin_host(config=config),
get_api_host(config=config),
get_web_host(config=config),
)
if host
}
if request_host not in control_hosts:
return HttpResponseForbidden("ArchiveBox is running with the control plane disabled on this host.")
if config.CONTROL_PLANE_ENABLED:
return get_response(request)
@ -319,7 +338,12 @@ def HostRoutingMiddleware(get_response):
return redirect(target)
response = get_response(request)
hint_cookie_domain = _admin_login_hint_cookie_domain(config)
if request.user.is_authenticated and not request.path.startswith("/admin/logout"):
if (
request.user.is_authenticated
and request.user.is_active
and request.user.is_superuser
and not request.path.startswith("/admin/logout")
):
response.set_cookie(
ADMIN_LOGIN_HINT_COOKIE,
"1",

16
archivebox/core/migrations/0053_alter_archiveresult_options.py Normal file
View File

@ -0,0 +1,16 @@
# Generated by Django 6.1 on 2026-08-30 18:20
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("core", "0052_unique_archiveresult_per_snapshot_plugin"),
]
operations = [
migrations.AlterModelOptions(
name="archiveresult",
options={"verbose_name": "Archive Result", "verbose_name_plural": "Archive Results"},
),
]

317
archivebox/core/models.py
View File

@ -46,8 +46,6 @@ from archivebox.misc.util import (
sanitize_html_text,
to_json,
ts_to_date_str,
urldecode,
urlencode,
validate_url,
)
from archivebox.plugins.discovery import (
@ -440,7 +438,7 @@ class SnapshotQuerySet(models.QuerySet):
else {}
)
snapshot_dicts = [s.to_dict(extended=True) for s in self.iterator(chunk_size=500)]
snapshot_dicts = [s.to_dict(extended=True, static_export=True) for s in self.iterator(chunk_size=500)]
if with_headers:
output = {
@ -474,6 +472,24 @@ class SnapshotQuerySet(models.QuerySet):
template = "static_index.html" if with_headers else "minimal_index.html"
snapshot_list = list(self.iterator(chunk_size=500))
manifest_records = []
for snapshot in snapshot_list:
outputs = snapshot.discover_outputs(include_filesystem_fallback=True)
output_paths = [str(output.get("path") or "") for output in outputs]
snapshot._public_preview_paths = [
path for preferred in ("screenshot/screenshot.png", "screenshot.png") for path in output_paths if path == preferred
]
snapshot._public_favicon_paths = [path for path in output_paths if path in ("favicon/favicon.ico", "favicon.ico")]
snapshot.write_html_details()
if with_headers:
# Use the same portable schema as the JSON export. Rendering
# above has already populated result-count caches, archive_size
# reuses the sealed output_size field, and tags are prefetched.
manifest_records.append(snapshot.to_dict(extended=True, static_export=True))
if with_headers:
manifest = "".join(f"{to_json(record, indent=None, sort_keys=True)}\n" for record in manifest_records)
atomic_write(str(CONSTANTS.DATA_DIR / CONSTANTS.JSONL_INDEX_FILENAME), manifest)
return render_to_string(
template,
@ -485,6 +501,8 @@ class SnapshotQuerySet(models.QuerySet):
"time_updated": datetime.now(UTC).strftime("%Y-%m-%d %H:%M"),
"links": snapshot_list,
"FOOTER_INFO": config.FOOTER_INFO,
"STATIC_EXPORT": True,
"STATIC_EXPORT_DIR": CONSTANTS.DATA_DIR,
},
)
@ -1009,7 +1027,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
from django.db import transaction
def finish_snapshot_save():
self.ensure_legacy_archive_symlink()
self.remove_legacy_archive_symlink()
self.ensure_crawl_symlink()
crawl = Crawl.objects.filter(pk=self.crawl_id).first()
if crawl is None:
@ -1286,9 +1304,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
return (old_dir, new_dir)
def _cleanup_old_migration_dir(self, old_dir: Path, new_dir: Path):
"""
Delete old directory and create symlink after successful migration.
"""
"""Delete the old directory after its contents are verified at the new path."""
import logging
import shutil
@ -1305,20 +1321,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
logging.getLogger("archivebox.migration").warning(
f"Could not remove old migration directory {old_dir}: {e}",
)
return # Don't create symlink if cleanup failed
return
# Create backwards-compat symlink (after old dir is deleted)
symlink_path = old_dir # Same path as old_dir
if symlink_path.is_symlink():
symlink_path.unlink()
if not symlink_path.exists():
try:
symlink_path.symlink_to(new_dir, target_is_directory=True)
except OSError as e:
logging.getLogger("archivebox.migration").warning(
f"Could not create symlink from {symlink_path} to {new_dir}: {e}",
)
# Older migration runs may already have left a timestamp projection.
if old_dir.is_symlink():
old_dir.unlink(missing_ok=True)
# =========================================================================
# Path Calculation and Migration Helpers
@ -2297,8 +2304,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
return calc_tags_str()
return calc_tags_str()
def icons(self, path: str | None = None) -> str:
def icons(self, path: str | None = None, prefix: str = "/", quote_paths: bool = False) -> str:
"""Generate HTML icons showing which extractor plugins have succeeded for this snapshot"""
from urllib.parse import quote
from django.utils.html import format_html
compact_icons = self.__dict__.get("_icons_compact", False)
@ -2384,7 +2393,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
archive_path = path or self.archive_path
output = ""
output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a>'
output_template = '<a href="{}{}/{}" class="exists-{}" title="{}">{}</a>'
# Get all plugins from hooks system (sorted by numeric prefix)
all_plugins = self.__dict__.get("_icons_plugin_names")
@ -2410,8 +2419,20 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
continue
embed_path = f"{plugin}/" if compact_icons else result.embed_path()
if not embed_path or str(embed_path).strip() in (".", "/", "./"):
continue
output_path = Path(str(embed_path))
if (
quote_paths
and not compact_icons
and (output_path.is_absolute() or ".." in output_path.parts or not (Path(self.output_dir) / output_path).exists())
):
continue
if quote_paths:
embed_path = quote(str(embed_path), safe="/@-._~!$&'()*+,;=")
output += format_html(
output_template,
prefix,
archive_path,
embed_path,
str(bool(existing)),
@ -2562,35 +2583,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
return current_path
def ensure_legacy_archive_symlink(self) -> None:
"""Ensure the legacy archive/<timestamp> path resolves to this snapshot."""
import os
legacy_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
target = Path(self.get_storage_path_for_version(self._fs_current_version()))
if target == legacy_path:
return
legacy_path.parent.mkdir(parents=True, exist_ok=True)
if legacy_path.exists() or legacy_path.is_symlink():
if legacy_path.is_symlink():
try:
if legacy_path.resolve() == target.resolve():
return
except OSError:
pass
legacy_path.unlink(missing_ok=True)
else:
return
rel_target = os.path.relpath(target, legacy_path.parent)
try:
legacy_path.symlink_to(rel_target, target_is_directory=True)
except OSError:
return
def ensure_crawl_symlink(self, *, crawl_dir: Path | None = None, snapshot_dir: Path | None = None) -> None:
"""Ensure snapshot is symlinked under its crawl output directory."""
import os
@ -2630,6 +2622,21 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
except OSError:
return
def remove_legacy_archive_symlink(self) -> None:
"""Remove a stale archive/<timestamp> compatibility projection."""
legacy_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
current_path = self.get_storage_path_for_version(self._fs_current_version())
if not legacy_path.is_symlink() or not current_path.exists():
return
try:
points_to_current_path = legacy_path.resolve(strict=True) == current_path.resolve(strict=True)
except OSError:
points_to_current_path = False
if points_to_current_path:
legacy_path.unlink(missing_ok=True)
@cached_property
def legacy_archive_path(self) -> str:
return f"{CONSTANTS.ARCHIVE_DIR_NAME}/{self.timestamp}"
@ -3460,7 +3467,15 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
# Serialization Methods
# =========================================================================
def to_dict(self, extended: bool = False) -> dict[str, Any]:
@property
def static_archive_path(self) -> str:
"""Snapshot output path relative to the data root, for portable exports."""
try:
return Path(self.output_dir).relative_to(CONSTANTS.DATA_DIR).as_posix()
except ValueError:
return Path(self.output_dir).as_posix()
def to_dict(self, extended: bool = False, static_export: bool = False) -> dict[str, Any]:
"""Convert Snapshot to a dictionary (replacement for Link._asdict())"""
from archivebox.core.routes_util import build_snapshot_url
@ -3491,8 +3506,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
"extension": self.extension,
"is_static": self.is_static,
"is_archived": self.is_archived,
"archive_path": self.archive_path,
"archive_url": build_snapshot_url(str(self.id), "index.html"),
"archive_path": self.static_archive_path if static_export else self.archive_path,
"archive_url": f"./{self.static_archive_path}/index.html" if static_export else build_snapshot_url(str(self.id), "index.html"),
"output_dir": self.output_dir,
"link_dir": self.output_dir, # backwards compatibility alias
"archive_size": archive_size,
@ -3522,67 +3537,173 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
"""Write JSON index file for this snapshot to its output directory"""
output_dir = Path(out_dir) if out_dir is not None else self.output_dir
path = output_dir / CONSTANTS.JSON_INDEX_FILENAME
atomic_write(str(path), self.to_dict(extended=True))
def write_html_details(self, out_dir: Path | str | None = None) -> None:
"""Write HTML detail page for this snapshot to its output directory"""
from django.template.loader import render_to_string
atomic_write(str(path), self.to_dict(extended=True, static_export=True))
def get_html_details_context(self, request=None, *, static_export_dir: Path | None = None) -> dict[str, Any]:
"""Build the one context used by both served and on-disk snapshot pages."""
from archivebox.config.common import get_request_config
from archivebox.core.permissions import get_snapshot_permissions
from archivebox.core.widgets import TagEditorWidget
from archivebox.misc.logging_util import printable_filesize
from archivebox.progressmonitor.views import progress_endpoint
output_dir = Path(out_dir) if out_dir is not None else self.output_dir
TITLE_LOADING_MSG = "Not yet archived..."
runtime_config = get_request_config(request) if request is not None else get_config()
self._runtime_config = runtime_config
snapshot_permissions = get_snapshot_permissions(self)
archive_results = list(self.archiveresult_set.all().order_by("start_ts"))
tags = list(self.tags.all())
self.__dict__["_admin_archiveresults"] = archive_results
self.__dict__["_tags_str_cached"] = ",".join(sorted(tag.name for tag in tags))
self.__dict__["num_outputs_cached"] = sum(result.status == ArchiveResult.StatusChoices.SUCCEEDED for result in archive_results)
self.__dict__["num_failures_cached"] = sum(result.status == ArchiveResult.StatusChoices.FAILED for result in archive_results)
preview_priority = [
"singlefile",
"screenshot",
"wget",
"dom",
"pdf",
"readability",
hidden_card_plugins = {"archivedotorg", "favicon", "title"}
outputs = [
output
for output in self.discover_outputs(include_filesystem_fallback=True, archive_results=archive_results)
if (output.get("size") or 0) > 0 and output.get("name") not in hidden_card_plugins
]
if static_export_dir is not None:
outputs = self.discover_outputs(include_filesystem_fallback=True)
loose_items, failed_items = self.get_detail_page_auxiliary_items(outputs)
outputs_by_plugin = {out["name"]: out for out in outputs}
output_size = sum(int(out.get("size") or 0) for out in outputs)
is_archived = bool(outputs or self.downloaded_at or self.status == self.StatusChoices.SEALED)
def static_output_exists(output: dict[str, Any]) -> bool:
raw_path = str(output.get("path") or "")
if not raw_path:
return False
output_path = Path(raw_path)
return not output_path.is_absolute() and ".." not in output_path.parts and (static_export_dir / output_path).exists()
best_preview_path = "about:blank"
outputs = [output for output in outputs if static_output_exists(output)]
outputs_by_name: dict[str, dict[str, Any]] = {}
result_ids_by_name: dict[str, list[str]] = {}
for output in outputs:
if output.get("result"):
result_ids_by_name.setdefault(output["name"], []).append(str(output["result"].id))
current = outputs_by_name.get(output["name"])
if current is None or (output.get("size") or 0) > (current.get("size") or 0):
outputs_by_name[output["name"]] = output
for name, output in outputs_by_name.items():
output["result_ids"] = ",".join(result_ids_by_name.get(name, ()))
hash_index = self.hashes_index
loose_items, failed_items = self.get_detail_page_auxiliary_items(
outputs,
hidden_card_plugins=hidden_card_plugins,
archive_results=archive_results,
)
preview_priority = ("singlefile", "screenshot", "wget", "dom", "pdf", "readability")
output_order = {result_type: index for index, result_type in enumerate(outputs_by_name)}
ordered_outputs = sorted(
outputs_by_name.values(),
key=lambda output: (
preview_priority.index(output["name"]) if output["name"] in preview_priority else len(preview_priority),
output_order.get(output["name"], len(output_order)),
),
)
best_result = {"path": "about:blank", "result": None}
for plugin in preview_priority:
out = outputs_by_plugin.get(plugin)
if out and out.get("path"):
best_preview_path = str(out["path"])
best_result = out
for result_type in preview_priority:
if result_type in outputs_by_name:
best_result = outputs_by_name[result_type]
break
if best_result["path"] == "about:blank" and ordered_outputs:
best_result = ordered_outputs[0]
if best_preview_path == "about:blank" and outputs:
best_preview_path = str(outputs[0].get("path") or "about:blank")
best_result = outputs[0]
non_compact_outputs = [output for output in ordered_outputs if not output.get("is_compact") and not output.get("is_metadata")]
compact_outputs = [output for output in ordered_outputs if output.get("is_compact") or output.get("is_metadata")]
archive_dates = [result.start_ts for result in archive_results if result.start_ts]
output_size = sum(int(output.get("size") or 0) for output in ordered_outputs)
has_outputs = bool(ordered_outputs)
is_archived = has_outputs or self.status == self.StatusChoices.SEALED
snapshot_status = str(self.status or "").lower()
status_label_by_state = {
"queued": ("queued", "info"),
"started": ("running", "warning"),
"paused": ("paused", "default"),
"sealed": ("archived", "success"),
}
if has_outputs:
status_label, status_color = ("archived", "success") if is_archived else ("partial", "warning")
else:
status_label, status_color = status_label_by_state.get(snapshot_status, ("not yet archived", "danger"))
related_snapshots = list(
type(self)
.objects.filter(url=self.url)
.exclude(id=self.id)
.only("id", "url", "bookmarked_at", "created_at", "downloaded_at", "output_size")
.order_by("-bookmarked_at", "-created_at", "-timestamp")[:25],
)
related_years_map: dict[int, list[Snapshot]] = {}
for snapshot in [self, *related_snapshots]:
snapshot_date = snapshot.bookmarked_at or snapshot.created_at or snapshot.downloaded_at
if snapshot_date:
related_years_map.setdefault(snapshot_date.year, []).append(snapshot)
related_years = []
for year, snapshots in related_years_map.items():
snapshots.sort(
key=lambda snapshot: snapshot.bookmarked_at or snapshot.created_at or snapshot.downloaded_at or timezone.now(),
reverse=True,
)
related_years.append({"year": year, "latest": snapshots[0], "snapshots": snapshots})
related_years.sort(key=lambda item: item["year"], reverse=True)
warc_path = next(
(rel_path for rel_path in hash_index if rel_path.startswith("warc/") and ".warc" in Path(rel_path).name),
"warc/",
)
user = getattr(request, "user", None)
can_delete_outputs = bool(
static_export_dir is None
and request is not None
and (
(user and user.is_authenticated and user.is_active and user.is_superuser)
or request.COOKIES.get("archivebox_admin_logged_in") == "1"
),
)
tag_widget = TagEditorWidget()
context = {
**self.to_dict(extended=True),
"snapshot": self,
"title": htmldecode(self.resolved_title or (self.base_url if is_archived else TITLE_LOADING_MSG)),
"url_str": htmldecode(urldecode(self.base_url)),
"archive_url": urlencode(f"warc/{self.timestamp}") or "about:blank",
return {
"id": str(self.id),
"snapshot_id": str(self.id),
"progress_endpoint": progress_endpoint("snapshot", self.id) if request is not None else "",
"progress_auto_expand": snapshot_status in {"queued", "started", "paused"},
"url": self.url,
"archive_path": self.archive_path_from_db,
"title": htmldecode(self.resolved_title or (self.base_url if is_archived else "Not yet archived...")),
"extension": self.extension or "html",
"tags": self.tags_str() or "untagged",
"size": printable_filesize(output_size) if output_size else "pending",
"status": "archived" if is_archived else "not yet archived",
"status_color": "success" if is_archived else "danger",
"oldest_archive_date": ts_to_date_str(self.oldest_archive_date),
"best_preview_path": best_preview_path,
"size": printable_filesize(output_size) if output_size else "—",
"status": status_label,
"status_color": status_color,
"snapshot_state": snapshot_status,
"has_outputs": has_outputs,
"snapshot_permissions": snapshot_permissions,
"snapshot_permissions_icon": {"public": "👥", "unlisted": "🔗", "private": "🔒"}.get(snapshot_permissions, "👥"),
"bookmarked_date": self.bookmarked_date,
"downloaded_datestr": self.downloaded_datestr,
"num_outputs": self.num_outputs,
"num_failures": self.num_failures,
"oldest_archive_date": ts_to_date_str(min(archive_dates) if archive_dates else None),
"warc_path": warc_path,
"archiveresults": [*non_compact_outputs, *compact_outputs],
"best_result": best_result,
"archiveresults": outputs,
"snapshot": self,
"CONFIG": runtime_config,
"related_snapshots": related_snapshots,
"related_years": related_years,
"loose_items": loose_items,
"failed_items": failed_items,
"related_snapshots": [],
"related_years": [],
"title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in self.tags.all().order_by("name")],
"can_delete_outputs": can_delete_outputs,
"title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in sorted(tags, key=lambda tag: tag.name)],
"STATIC_EXPORT": static_export_dir is not None,
"STATIC_EXPORT_DIR": static_export_dir,
}
def write_html_details(self, out_dir: Path | str | None = None) -> None:
"""Write the unified snapshot detail page with portable filesystem URLs."""
from django.template.loader import render_to_string
output_dir = Path(out_dir) if out_dir is not None else self.output_dir
output_dir.mkdir(parents=True, exist_ok=True)
context = self.get_html_details_context(static_export_dir=output_dir)
rendered_html = render_to_string("core/snapshot.html", context)
atomic_write(str(output_dir / CONSTANTS.HTML_INDEX_FILENAME), rendered_html)
@ -4003,7 +4124,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
):
app_label = "core"
verbose_name = "Archive Result"
verbose_name_plural = "Archive Results Log"
verbose_name_plural = "Archive Results"
indexes: ClassVar[list[models.Index]] = [
models.Index(fields=["snapshot", "status"], name="archiveresult_snap_status_idx"),
models.Index(fields=["status", "snapshot"], name="archiveresult_status_snap_idx"),

196
archivebox/core/takeover_util.py
View File

@ -1,3 +1,17 @@
"""Coordinate local foreground processes without pretending to provide a distributed lock.
ArchiveBox wants one orchestrator per collection, but Process rows can only prove
liveness for PIDs visible on the current machine/PID namespace. We therefore
enforce one active runner per ``(Machine, DATA_DIR)`` locally, retire stale rows
from sequential containers on that machine, and only warn about rows owned by a
different machine. Foreign-machine rows must never block progress or be killed:
multi-machine coordination belongs in the Crawl/Snapshot CAS claim layer, not in
process takeover.
These helpers only hand local supervisord/runner ownership between CLI parents.
They must not hold database transactions or filesystem locks while work runs.
"""
from __future__ import annotations
import time
@ -15,25 +29,6 @@ RUNNER_WAITING_WORKER_TYPE = "runner_waiting"
RUNNER_GATE_WORKER_TYPES = (RUNNER_ACTIVE_WORKER_TYPE, RUNNER_WAITING_WORKER_TYPE, "")
def runtime_stack_owner_types():
from archivebox.machine.models import Process
return (
Process.TypeChoices.SERVER,
Process.TypeChoices.ORCHESTRATOR,
)
def foreground_runner_owner_types():
from archivebox.machine.models import Process
return (
Process.TypeChoices.SERVER,
Process.TypeChoices.ADD,
Process.TypeChoices.UPDATE,
)
def current_command(process_type: str, *, data_dir: str | Path, url: str | None = None):
from archivebox.machine.models import Process
@ -42,31 +37,8 @@ def current_command(process_type: str, *, data_dir: str | Path, url: str | None
return proc
def live_processes(*, process_type: str, data_dir: str | Path, url: str | None = None):
from archivebox.machine.models import Machine, Process
qs = Process.objects.filter(
machine=Machine.current(),
process_type=process_type,
status=Process.StatusChoices.RUNNING,
pwd=str(data_dir),
)
if url is not None:
qs = qs.filter(url=url)
return [proc for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50) if proc.is_running]
def newest_live_process(*, process_type: str, data_dir: str | Path, url: str | None = None):
processes = live_processes(process_type=process_type, data_dir=data_dir, url=url)
return processes[0] if processes else None
def command_is_newest(command, *, process_type: str, data_dir: str | Path, url: str | None = None) -> bool:
leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
return bool(leader and leader.id == command.id)
def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
"""Return the live local parent allowed to own the server runtime stack."""
from archivebox.machine.models import Machine, Process
machine = Machine.current()
@ -74,7 +46,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
machine=machine,
status=Process.StatusChoices.RUNNING,
pwd=str(data_dir),
process_type__in=runtime_stack_owner_types(),
process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ORCHESTRATOR),
)
if exclude_id is not None:
base_qs = base_qs.exclude(id=exclude_id)
@ -103,6 +75,7 @@ def command_owns_runtime_stack(command, *, data_dir: str | Path) -> bool:
def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
"""Return the newest live local parent allowed to borrow runner/sonic."""
from archivebox.machine.models import Machine, Process
machine = Machine.current()
@ -110,7 +83,7 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
machine=machine,
status=Process.StatusChoices.RUNNING,
pwd=str(data_dir),
process_type__in=foreground_runner_owner_types(),
process_type__in=(Process.TypeChoices.SERVER, Process.TypeChoices.ADD, Process.TypeChoices.UPDATE),
)
if exclude_id is not None:
qs = qs.exclude(id=exclude_id)
@ -126,26 +99,6 @@ def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
return bool(owner and owner.id == command.id)
def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
try:
from archivebox.workers.supervisord_util import active_supervisord_runtime_components
components = active_supervisord_runtime_components()
except Exception:
components = []
names = list(components)
if not names and owner is not None:
from archivebox.machine.models import Process
if owner.process_type == Process.TypeChoices.SERVER:
names = ["orchestrator", "server"]
elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
names = ["orchestrator"]
return ", ".join(dict.fromkeys(names)) or "runtime stack"
def ensure_daemon_stack(*, reason: str = ""):
from archivebox.config.common import get_config
from archivebox.workers.supervisord_util import (
@ -181,56 +134,65 @@ def ensure_daemon_stack(*, reason: str = ""):
return start_worker(supervisor, sonic_worker)
def healthy_orchestrator(*, data_dir: str | Path):
from archivebox.machine.models import Machine, Process
from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
def live_runner_processes(*, data_dir: str | Path):
"""Return locally verifiable runners and warn about unsupported overlap.
supervisor = get_existing_supervisord_process()
worker = get_worker(supervisor, "worker_runner") if supervisor else None
if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
return worker
for proc in Process.objects.filter(
machine=Machine.current(),
process_type=Process.TypeChoices.ORCHESTRATOR,
status=Process.StatusChoices.RUNNING,
pwd=str(data_dir),
).order_by("-created_at"):
if proc.is_running:
return proc
return None
def _runner_sort_key(process):
return (process.started_at or process.created_at, process.created_at, str(process.id))
def live_runner_processes(*, data_dir: str | Path, exclude_id=None):
A Process row from another machine is observability only: its PID cannot be
checked or signalled here, so it neither joins the local election nor gets
mutated. A row for this same Machine from another PID namespace represents
a previous sequential container under the supported model; warn, retire the
unreachable row, and let the new container continue.
"""
from archivebox.machine.models import Machine, Process
machine = Machine.current()
Process.cleanup_stale_running(machine=machine)
qs = Process.objects.filter(
machine=machine,
status=Process.StatusChoices.RUNNING,
process_type=Process.TypeChoices.ORCHESTRATOR,
worker_type__in=RUNNER_GATE_WORKER_TYPES,
pwd=str(data_dir),
)
if exclude_id is not None:
qs = qs.exclude(id=exclude_id)
return [process for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20) if process.is_running]
foreign_machine_exists = qs.exclude(machine=machine).exists()
qs = qs.filter(machine=machine)
live = []
foreign_namespace_ids = []
for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
if not process.shares_pid_namespace:
foreign_namespace_ids.append(process.id)
continue
if process.is_running:
live.append(process)
if foreign_machine_exists or foreign_namespace_ids:
rprint(
"[bold yellow]WARNING: Multiple orchestrators sharing a single collection is not officially supported! "
"Corruption may occur if you run two ArchiveBox workers on the same collection at once.[/bold yellow]",
file=sys.stderr,
soft_wrap=True,
)
if foreign_namespace_ids:
now = timezone.now()
Process.objects.filter(id__in=foreign_namespace_ids, status=Process.StatusChoices.RUNNING).update(
status=Process.StatusChoices.EXITED,
exit_code=0,
ended_at=now,
retry_at=None,
modified_at=now,
)
return live
def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout: float = 5.0) -> bool:
"""
Admit exactly one active runner for this DATA_DIR using Process rows.
Admit one active runner for this Machine and DATA_DIR using Process rows.
The current process is a real OS process while it waits, so we keep its
Process row RUNNING but mark worker_type=runner_waiting. Only the process
that wins takeover is promoted to worker_type=worker_runner, which is
protected by a partial unique DB constraint. Older runners are terminated
and fully waited out before promotion, so the runner work loop never overlaps.
protected by a partial unique DB constraint scoped to (Machine, DATA_DIR).
Older locally verifiable runners are terminated and fully waited out before
promotion, so runner work never overlaps on one machine. Foreign machines
are intentionally outside this gate and only produce a warning above.
"""
from archivebox.machine.models import Process
@ -252,7 +214,7 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
)
runners = live_runner_processes(data_dir=data_dir)
newest = max(runners, key=_runner_sort_key)
newest = max(runners, key=lambda process: (process.started_at or process.created_at, process.created_at, str(process.id)))
if newest.id != command.id:
rprint(
f"[yellow][*] Newer ArchiveBox runner pid={newest.pid} is taking over; exiting this runner.[/yellow]",
@ -263,14 +225,8 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
older_runners = [process for process in runners if process.id != command.id]
if older_runners:
for process in older_runners:
if process.shares_pid_namespace:
rprint(f"[yellow][*] Stopping older ArchiveBox runner process (pid={process.pid})...[/yellow]", file=sys.stderr)
process.kill_tree(graceful_timeout=graceful_timeout)
else:
rprint(
"[yellow][*] Waiting for older ArchiveBox runner in another PID namespace to stop...[/yellow]",
file=sys.stderr,
)
rprint(f"[yellow][*] Stopping older ArchiveBox runner process (pid={process.pid})...[/yellow]", file=sys.stderr)
process.kill_tree(graceful_timeout=graceful_timeout)
time.sleep(0.1)
continue
@ -290,24 +246,9 @@ def enter_single_runner_gate(command, *, data_dir: str | Path, graceful_timeout:
time.sleep(0.1)
def standby_until_leader_needed(command, *, process_type: str, data_dir: str | Path, url: str | None = None, interval: float = 2.0) -> None:
from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
announced = False
while not command_is_newest(command, process_type=process_type, data_dir=data_dir, url=url):
reap_foreground_supervisord_process()
if not announced:
leader = newest_live_process(process_type=process_type, data_dir=data_dir, url=url)
leader_pid = leader.pid if leader else "unknown"
rprint(f"[yellow][*] Standing by; newer ArchiveBox process pid={leader_pid} is running the orchestrator and server.[/yellow]")
announced = True
time.sleep(interval)
command.modified_at = timezone.now()
command.save(update_fields=["modified_at"])
def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interval: float = 2.0) -> dict[str, object]:
from archivebox.workers.supervisord_util import reap_foreground_supervisord_process
from archivebox.machine.models import Process
from archivebox.workers.supervisord_util import active_supervisord_runtime_components, reap_foreground_supervisord_process
announced = False
previous_owner_pid = None
@ -316,7 +257,16 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
if not announced:
owner = runtime_stack_owner(data_dir=data_dir)
owner_pid = owner.pid if owner else "unknown"
components = runtime_stack_component_label(owner=owner, data_dir=data_dir)
try:
component_names = list(active_supervisord_runtime_components())
except Exception:
component_names = []
if not component_names and owner is not None:
if owner.process_type == Process.TypeChoices.SERVER:
component_names = ["orchestrator", "server"]
elif owner.process_type == Process.TypeChoices.ORCHESTRATOR:
component_names = ["orchestrator"]
components = ", ".join(dict.fromkeys(component_names)) or "runtime stack"
previous_owner_pid = owner_pid
rprint(
f"[yellow][*] A newer archivebox process took over the {components} "

150
archivebox/core/templatetags/core_tags.py
View File

@ -2,6 +2,7 @@ import os
from html import unescape
from pathlib import Path
from typing import Any
from urllib.parse import quote
from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
from django import template
@ -11,6 +12,7 @@ from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.text import Truncator
from archivebox.config import CONSTANTS
from archivebox.core.routes_util import (
build_snapshot_url,
get_admin_base_url,
@ -28,6 +30,7 @@ register = template.Library()
_TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log")
_IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
_STATIC_URL_SAFE = "/@-._~!$&'()*+,;="
_MEDIA_FILE_EXTS = {
".mp4",
@ -202,16 +205,70 @@ def _is_root_snapshot_output_path(raw_output_path: str | None) -> bool:
return normalized in ("", ".", "./", "/", "index.html", "index.json")
def _build_snapshot_files_url(snapshot_id: str, request=None, config=None) -> str:
return build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config)
def _static_snapshot_base_url(context, snapshot) -> str:
start_dir = Path(context["STATIC_EXPORT_DIR"])
relative_path = os.path.relpath(Path(snapshot.output_dir), start=start_dir).replace(os.sep, "/")
if relative_path == ".":
return "."
return f"./{quote(relative_path, safe=_STATIC_URL_SAFE)}"
def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None, config=None, plugin: str = "") -> str:
def _snapshot_base_url_for_context(context, snapshot) -> str:
if context.get("STATIC_EXPORT"):
return _static_snapshot_base_url(context, snapshot)
return get_snapshot_base_url(
str(_snapshot_id(snapshot)),
request=context.get("request"),
config=context.get("CONFIG"),
)
def _snapshot_url_for_context(context, snapshot, path: str = "") -> str:
if not context.get("STATIC_EXPORT"):
return build_snapshot_url(
str(_snapshot_id(snapshot)),
path,
request=context.get("request"),
config=context.get("CONFIG"),
)
base_url = _static_snapshot_base_url(context, snapshot)
raw_path = str(path or "")
if not raw_path:
return base_url
path_part, separator, query = raw_path.lstrip("/").partition("?")
quoted_path = quote(path_part, safe=_STATIC_URL_SAFE)
suffix = f"?{query}" if separator else ""
return f"{base_url.rstrip('/')}/{quoted_path}{suffix}"
def _build_snapshot_files_url(snapshot_id: str, request=None, config=None, base_url: str | None = None) -> str:
return (
f"{base_url.rstrip('/')}/index.jsonl"
if base_url
else build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config)
)
def _build_snapshot_preview_url(
snapshot_id: str,
path: str = "",
request=None,
config=None,
plugin: str = "",
base_url: str | None = None,
) -> str:
if path == "about:blank":
return path
if _is_root_snapshot_output_path(path):
return _build_snapshot_files_url(snapshot_id, request=request, config=config)
url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
return _build_snapshot_files_url(snapshot_id, request=request, config=config, base_url=base_url)
if base_url:
path_part, separator, query = str(path).lstrip("/").partition("?")
url = f"{base_url.rstrip('/')}/{quote(path_part, safe=_STATIC_URL_SAFE)}"
if separator:
url = f"{url}?{query}"
else:
url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
path_parts = Path(path).parts
plugin = get_plugin_name(plugin) if plugin else (path_parts[0] if len(path_parts) > 1 else "")
has_plugin_preview = bool(plugin and get_plugin_template(plugin, "full", fallback=False))
@ -441,23 +498,35 @@ def web_base_url(context) -> str:
return get_web_base_url(request=context.get("request"), config=context.get("CONFIG"))
@register.simple_tag(takes_context=True)
def static_export_root_url(context) -> str:
if not context.get("STATIC_EXPORT"):
return ""
relative_path = os.path.relpath(CONSTANTS.DATA_DIR, start=Path(context["STATIC_EXPORT_DIR"]))
relative_path = relative_path.replace(os.sep, "/")
return "./" if relative_path == "." else f"{quote(relative_path, safe=_STATIC_URL_SAFE)}/"
@register.simple_tag(takes_context=True)
def snapshot_base_url(context, snapshot) -> str:
snapshot_id = _snapshot_id(snapshot)
return get_snapshot_base_url(str(snapshot_id), request=context.get("request"), config=context.get("CONFIG"))
return _snapshot_base_url_for_context(context, snapshot)
@register.simple_tag(takes_context=True)
def snapshot_url(context, snapshot, path: str = "") -> str:
snapshot_id = _snapshot_id(snapshot)
return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
return _snapshot_url_for_context(context, snapshot, path)
@register.simple_tag(takes_context=True)
def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
snapshot_id = str(_snapshot_id(snapshot))
url_cache = snapshot.__dict__.setdefault("_snapshot_archiveresult_url_cache", {})
cache_key = (plugin, filename)
cache_key = (
plugin,
filename,
bool(context.get("STATIC_EXPORT")),
str(context.get("STATIC_EXPORT_DIR") or ""),
)
if cache_key in url_cache:
return url_cache[cache_key]
@ -493,7 +562,7 @@ def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) ->
if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
continue
output_path = filename if file_info.get("root_relative") else f"{plugin}/{filename}"
url_cache[cache_key] = build_snapshot_url(snapshot_id, output_path, request=context.get("request"), config=context.get("CONFIG"))
url_cache[cache_key] = _snapshot_url_for_context(context, snapshot, output_path)
return url_cache[cache_key]
url_cache[cache_key] = ""
@ -502,10 +571,7 @@ def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) ->
@register.simple_tag(takes_context=True)
def snapshot_index_row(context, link) -> str:
snapshot_id = str(_snapshot_id(link))
request = context.get("request")
config = context.get("CONFIG")
snapshot_base = get_snapshot_base_url(snapshot_id, request=request, config=config)
snapshot_base = _snapshot_base_url_for_context(context, link)
status = getattr(link, "status", None) or "unknown"
bookmarked_at = getattr(link, "bookmarked_at", None)
@ -534,22 +600,28 @@ def snapshot_index_row(context, link) -> str:
tag_cell = '<span class="empty-value">...</span>'
num_outputs = int(getattr(link, "num_outputs", 0) or 0)
icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
if context.get("STATIC_EXPORT") and callable(getattr(link, "icons", None)):
icons = link.icons(path=quote(link.static_archive_path, safe=_STATIC_URL_SAFE), prefix="./", quote_paths=True)
else:
icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
icons_cell = str(icons) if icons else '<span class="empty-value">...</span>'
archive_size = int(getattr(link, "archive_size", 0) or 0)
size_cell = file_size(archive_size) if archive_size else '<span class="empty-value">...</span>'
output_plural = "" if num_outputs == 1 else "s"
files_url = _snapshot_url_for_context(context, link, "index.jsonl") if context.get("STATIC_EXPORT") else f"{snapshot_base}/?files=1"
if is_pending:
preview_html = (
'<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
"</span>"
)
preview_html = '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress"></span>'
if not context.get("STATIC_EXPORT"):
preview_html = (
'<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
"</span>"
)
elif "_public_preview_paths" in link.__dict__:
preview_paths = list(getattr(link, "_public_preview_paths", []) or [])
if preview_paths:
preview_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in preview_paths]
preview_urls = [_snapshot_url_for_context(context, link, path) for path in preview_paths]
preview_html = (
f'<img src="{escape(preview_urls[0])}" '
f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
@ -579,7 +651,7 @@ def snapshot_index_row(context, link) -> str:
if "_public_favicon_paths" in link.__dict__:
favicon_paths = list(getattr(link, "_public_favicon_paths", []) or [])
if favicon_paths:
favicon_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in favicon_paths]
favicon_urls = [_snapshot_url_for_context(context, link, path) for path in favicon_paths]
favicon_html = (
f'<img src="{escape(favicon_urls[0])}" '
f'data-fallbacks="{escape(",".join(favicon_urls[1:]))}" '
@ -640,7 +712,7 @@ def snapshot_index_row(context, link) -> str:
</span>
</td>
<td class="snapshot-size-cell">
<a href="{escape(snapshot_base)}/?files=1" title="View archived files">
<a href="{escape(files_url)}" title="View archived file manifest">
{size_cell}
</a>
<small>{num_outputs} output{output_plural}</small>
@ -660,6 +732,7 @@ def snapshot_preview_url(context, snapshot, path: str = "", result=None) -> str:
request=context.get("request"),
config=context.get("CONFIG"),
plugin=plugin,
base_url=_snapshot_base_url_for_context(context, snapshot) if context.get("STATIC_EXPORT") else None,
)
@ -699,24 +772,26 @@ def plugin_card(context, result) -> str:
# Use embed_path() for the display path
raw_output_path = result.embed_path() or ""
output_url = build_snapshot_url(
str(result.snapshot_id),
raw_output_path or "",
request=context.get("request"),
config=context.get("CONFIG"),
)
output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path or "")
icon_html = get_plugin_icon(plugin)
plugin_lower = (plugin or "").lower()
media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
if context.get("STATIC_EXPORT") and media_files:
media_files = [
item
for item in media_files
if (media_path := Path(str(item.get("path") or "")))
and not media_path.is_absolute()
and ".." not in media_path.parts
and (Path(result.snapshot_dir) / media_path).is_file()
]
media_file_count = len(media_files)
if media_files:
snapshot_id = str(result.snapshot_id)
request = context.get("request")
config = context.get("CONFIG")
for item in media_files:
path = item.get("path") or ""
item["url"] = build_snapshot_url(snapshot_id, path, request=request, config=config) if path else ""
item["url"] = _snapshot_url_for_context(context, result.snapshot, path) if path else ""
output_lower = (raw_output_path or "").lower()
force_text_preview = output_lower.endswith(_TEXT_PREVIEW_EXTS)
@ -794,12 +869,7 @@ def plugin_full(context, result) -> str:
raw_output_path = result.embed_path() or ""
if _is_root_snapshot_output_path(raw_output_path):
return ""
output_url = build_snapshot_url(
str(result.snapshot_id),
raw_output_path,
request=context.get("request"),
config=context.get("CONFIG"),
)
output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path)
try:
tpl = template.Template(template_str)

166
archivebox/core/views.py
View File

@ -21,7 +21,6 @@ from django.core.paginator import InvalidPage
from django.db.models import Case, IntegerField, Q, Value, When
from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidden, QueryDict
from django.shortcuts import redirect, render
from django.utils import timezone
from django.utils.decorators import method_decorator
from django.utils.html import format_html, format_html_join
from django.utils.safestring import mark_safe
@ -54,7 +53,6 @@ from archivebox.core.permissions import (
can_view_snapshot,
direct_snapshots_queryset,
filter_personas_by_permissions,
get_snapshot_permissions,
is_admin_user,
public_snapshots_queryset,
)
@ -69,15 +67,12 @@ from archivebox.core.routes_util import (
host_matches,
)
from archivebox.crawls.models import Crawl
from archivebox.misc.logging_util import printable_filesize
from archivebox.misc.paginators import AcceleratedPaginator
from archivebox.misc.serve_static import serve_static_with_byterange_support
from archivebox.misc.util import (
base_url,
filter_queryset_by_uuid_substring,
htmldecode,
sanitize_html_text,
ts_to_date_str,
urldecode,
validate_url,
without_fragment,
@ -85,7 +80,7 @@ from archivebox.misc.util import (
from archivebox.plugins.discovery import get_plugin_name, get_plugin_template
from archivebox.plugins.forms import get_plugin_config_binary_urls
from archivebox.plugins.views import get_config_definition_link
from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
from archivebox.progressmonitor.views import live_progress_view
from archivebox.search.config import (
get_search_mode,
get_search_mode_backend,
@ -332,162 +327,11 @@ class SnapshotView(View):
@staticmethod
def render_live_index(request, snapshot):
TITLE_LOADING_MSG = "Not yet archived..."
from archivebox.core.widgets import TagEditorWidget
# Reuse the middleware-attached config; never re-bootstrap from env + plugin
# schemas just to render a snapshot page (that pays ~30ms for no reason).
runtime_config = get_request_config(request)
snapshot._runtime_config = runtime_config
snapshot_permissions = get_snapshot_permissions(snapshot)
archive_results = list(snapshot.archiveresult_set.all().order_by("start_ts"))
tags = list(snapshot.tags.all())
snapshot.__dict__["_admin_archiveresults"] = archive_results
snapshot.__dict__["_tags_str_cached"] = ",".join(sorted(tag.name for tag in tags))
snapshot.__dict__["num_outputs_cached"] = sum(result.status == ArchiveResult.StatusChoices.SUCCEEDED for result in archive_results)
snapshot.__dict__["num_failures_cached"] = sum(result.status == ArchiveResult.StatusChoices.FAILED for result in archive_results)
hidden_card_plugins = {"archivedotorg", "favicon", "title"}
outputs = [
out
for out in snapshot.discover_outputs(include_filesystem_fallback=True, archive_results=archive_results)
if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins
]
archiveresults = {}
result_ids_by_name = {}
for output in outputs:
if output.get("result"):
result_ids_by_name.setdefault(output["name"], []).append(str(output["result"].id))
current = archiveresults.get(output["name"])
if current is None or (output.get("size") or 0) > (current.get("size") or 0):
archiveresults[output["name"]] = output
for name, output in archiveresults.items():
output["result_ids"] = ",".join(result_ids_by_name.get(name, ()))
hash_index = snapshot.hashes_index
loose_items, failed_items = snapshot.get_detail_page_auxiliary_items(
outputs,
hidden_card_plugins=hidden_card_plugins,
archive_results=archive_results,
return render(
template_name="core/snapshot.html",
request=request,
context=snapshot.get_html_details_context(request=request),
)
preview_priority = [
"singlefile",
"screenshot",
"wget",
"dom",
"pdf",
"readability",
]
preferred_types = tuple(preview_priority)
output_order = {result_type: index for index, result_type in enumerate(archiveresults.keys())}
best_result = {"path": "about:blank", "result": None}
for result_type in preferred_types:
if result_type in archiveresults:
best_result = archiveresults[result_type]
break
related_snapshots_qs = SnapshotView.find_snapshots_for_url(
snapshot.url,
allow_fallback=False,
).only("id", "url", "bookmarked_at", "created_at", "downloaded_at", "output_size")
related_snapshots = list(
related_snapshots_qs.exclude(id=snapshot.id).order_by("-bookmarked_at", "-created_at", "-timestamp")[:25],
)
related_years_map: dict[int, list[Snapshot]] = {}
for snap in [snapshot, *related_snapshots]:
snap_dt = snap.bookmarked_at or snap.created_at or snap.downloaded_at
if not snap_dt:
continue
related_years_map.setdefault(snap_dt.year, []).append(snap)
related_years = []
for year, snaps in related_years_map.items():
snaps_sorted = sorted(
snaps,
key=lambda s: s.bookmarked_at or s.created_at or s.downloaded_at or timezone.now(),
reverse=True,
)
related_years.append(
{
"year": year,
"latest": snaps_sorted[0],
"snapshots": snaps_sorted,
},
)
related_years.sort(key=lambda item: item["year"], reverse=True)
warc_path = next(
(rel_path for rel_path in hash_index if rel_path.startswith("warc/") and ".warc" in Path(rel_path).name),
"warc/",
)
ordered_outputs = sorted(
archiveresults.values(),
key=lambda r: (
preferred_types.index(r["name"]) if r["name"] in preferred_types else len(preferred_types),
output_order.get(r["name"], len(output_order)),
),
)
if best_result["path"] == "about:blank" and ordered_outputs:
best_result = ordered_outputs[0]
non_compact_outputs = [out for out in ordered_outputs if not out.get("is_compact") and not out.get("is_metadata")]
compact_outputs = [out for out in ordered_outputs if out.get("is_compact") or out.get("is_metadata")]
tag_widget = TagEditorWidget()
output_size = sum(int(out.get("size") or 0) for out in ordered_outputs)
archive_dates = [result.start_ts for result in archive_results if result.start_ts]
has_outputs = bool(ordered_outputs)
is_archived = has_outputs or snapshot.status == Snapshot.StatusChoices.SEALED
snapshot_status = str(snapshot.status or "").lower()
status_label_by_state = {
"queued": ("queued", "info"),
"started": ("running", "warning"),
"paused": ("paused", "default"),
"sealed": ("archived", "success"),
}
if has_outputs and not is_archived:
status_label, status_color = ("partial", "warning")
elif has_outputs:
status_label, status_color = ("archived", "success")
else:
status_label, status_color = status_label_by_state.get(snapshot_status, ("not yet archived", "danger"))
context = {
"id": str(snapshot.id),
"snapshot_id": str(snapshot.id),
"progress_endpoint": progress_endpoint("snapshot", snapshot.id),
"progress_auto_expand": snapshot_status in {"queued", "started", "paused"},
"url": snapshot.url,
"archive_path": snapshot.archive_path_from_db,
"title": htmldecode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)),
"extension": snapshot.extension or "html",
"tags": snapshot.tags_str() or "untagged",
"size": printable_filesize(output_size) if output_size else "—",
"status": status_label,
"status_color": status_color,
"snapshot_state": snapshot_status,
"has_outputs": has_outputs,
"snapshot_permissions": snapshot_permissions,
"snapshot_permissions_icon": {
"public": "👥",
"unlisted": "🔗",
"private": "🔒",
}.get(snapshot_permissions, "👥"),
"bookmarked_date": snapshot.bookmarked_date,
"downloaded_datestr": snapshot.downloaded_datestr,
"num_outputs": snapshot.num_outputs,
"num_failures": snapshot.num_failures,
"oldest_archive_date": ts_to_date_str(min(archive_dates) if archive_dates else None),
"warc_path": warc_path,
"archiveresults": [*non_compact_outputs, *compact_outputs],
"best_result": best_result,
"snapshot": snapshot, # Pass the snapshot object for template tags
"CONFIG": runtime_config,
"related_snapshots": related_snapshots,
"related_years": related_years,
"loose_items": loose_items,
"failed_items": failed_items,
"can_delete_outputs": bool(request.user.is_authenticated and request.user.is_active and request.user.is_superuser),
"title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in sorted(tags, key=lambda tag: tag.name)],
}
return render(template_name="core/snapshot.html", request=request, context=context)
def get(self, request, path):
snapshot = None

7
archivebox/machine/models.py
View File

@ -1156,6 +1156,11 @@ class Process(ModelWithDeleteAfter, models.Model):
models.Index(fields=["machine", "status", "process_type"], name="mach_proc_running_idx"),
]
constraints = [
# This is deliberately machine-scoped. It prevents two locally
# verifiable runners from working the same collection, while not
# claiming to coordinate independent hosts that share a database.
# Cross-machine work ownership belongs to short Crawl/Snapshot CAS
# claims so PostgreSQL deployments can support that model later.
models.UniqueConstraint(
fields=["machine", "pwd"],
condition=Q(status="running", process_type="orchestrator", worker_type="worker_runner"),
@ -1362,7 +1367,7 @@ class Process(ModelWithDeleteAfter, models.Model):
self.save(update_fields=updates)
def heartbeat(self) -> None:
"""Touch modified_at so standby/leader selection can see this parent is alive."""
"""Keep a long-lived watcher visible in recent-process monitoring."""
self.save(update_fields=["modified_at"])
def mark_exited(self, *, exit_code: int = 0) -> None:

5
archivebox/misc/checks.py
View File

@ -48,6 +48,11 @@ def _exit_on_migration_interrupt():
os.write(sys.stderr.fileno(), _migration_interrupt_message().encode())
except Exception:
pass
# Django's migration executor can catch or delay normal exceptions while
# unwinding transactions. Use the real process exit path after printing
# the recovery command so Ctrl+C/SIGTERM during auto-migrations does not
# leave `archivebox server` apparently hung after the user asked it to
# stop. Migrations are atomic, so this does not record partial progress.
os._exit(130)
try:

19
archivebox/services/runner.py
View File

@ -261,6 +261,10 @@ class CrawlRunner:
def _request_abort_from_signal(self, _sig: signal.Signals) -> None:
if os.environ.get("ARCHIVEBOX_RUNNER_DAEMON") == "1":
# The daemon runner is owned by supervisord, not by the interactive
# CLI foreground flow. A direct signal to this child should be short
# and unambiguous: exit non-zero immediately so supervisord restarts
# the runner, while the parent server and supervisord stay alive.
os._exit(128 + int(_sig))
already_requested = self._signal_abort_requested
self._signal_abort_requested = True
@ -479,6 +483,11 @@ class CrawlRunner:
await self.enqueue_pending_snapshots_from_projection()
async def heartbeat_active_leases(self) -> None:
# These are resumable work-item leases, not orchestrator-election
# heartbeats. Each update is a short autocommit statement; network and
# filesystem work continues outside a database transaction. A future
# PostgreSQL multi-machine runner uses these Crawl/Snapshot claims as
# its coordination boundary while SQLite keeps one local orchestrator.
if self._run_task is None:
return
now_monotonic = time.monotonic()
@ -717,10 +726,8 @@ class CrawlRunner:
stdout_is_tty = sys.stdout.isatty()
stderr_is_tty = sys.stderr.isatty()
interactive_tty = stdout_is_tty or stderr_is_tty
if not interactive_tty:
return None
stream = sys.stderr if stderr_is_tty else sys.stdout
if os.path.exists("/dev/tty"):
stream = sys.stderr if stderr_is_tty or not stdout_is_tty else sys.stdout
if interactive_tty and os.path.exists("/dev/tty"):
try:
self._live_stream = open("/dev/tty", "w", buffering=1, encoding=stream.encoding or "utf-8")
stream = self._live_stream
@ -736,7 +743,7 @@ class CrawlRunner:
terminal_height = terminal_size.lines
ui_console = Console(
file=stream,
force_terminal=True,
force_terminal=interactive_tty,
width=terminal_width,
height=terminal_height,
_environ={
@ -750,7 +757,7 @@ class CrawlRunner:
total_hooks=_count_selected_hooks(self.plugins, self.selected_plugins),
timeout_seconds=self.base_config["TIMEOUT"],
ui_console=ui_console,
interactive_tty=True,
interactive_tty=interactive_tty,
)
live_ui.print_intro(
url=self.primary_url or "crawl",

98
archivebox/templates/core/snapshot.html
View File

@ -319,9 +319,9 @@
user-select: all;
}
.header-toggle {
line-height: 12px;
font-size: 70px;
margin-top: -13px;
line-height: 1;
font-size: 24px;
margin-top: 0;
margin-left: 4px;
}
@container snapshot-header (max-width: 900px) {
@ -1056,8 +1056,9 @@
.thumb-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(clamp(180px, 14vw, 250px), 1fr));
grid-auto-flow: row dense;
grid-auto-rows: 42px;
gap: 6px;
align-content: start;
width: 100%;
max-width: 100%;
margin-left: 0;
@ -1077,17 +1078,23 @@
display: inline-flex;
flex-direction: column;
align-items: stretch;
grid-row: span 3;
break-inside: avoid;
margin-bottom: 6px;
margin-bottom: 0;
vertical-align: top;
content-visibility: auto;
contain-intrinsic-size: 138px;
}
.thumb-card:not([data-plugin-name]) {
order: 1;
}
.thumb-card:has([data-compact]) {
height: 46px;
min-height: 46px;
max-height: 46px;
contain-intrinsic-size: 46px;
height: 42px;
min-height: 42px;
max-height: 42px;
grid-row: span 1;
order: 2;
contain-intrinsic-size: 42px;
}
.thumb-card .thumb-body {
display: grid;
@ -1247,7 +1254,7 @@
}
.thumb-card:has([data-compact]) .thumbnail-wrapper,
.thumb-card:has([data-compact]) .thumbnail-wrapper.compact {
height: 24px;
height: 20px;
flex: 0 0 auto;
}
.thumb-card:has([data-compact]) .thumb-body {
@ -1468,17 +1475,19 @@
</head>
<body>
<div id="main-frame-wrapper" class="full-page-wrapper" data-has-outputs="{{ has_outputs|yesno:'1,0' }}" data-snapshot-state="{{ snapshot_state }}">
{% if not STATIC_EXPORT %}
{% if snapshot_state == 'queued' or snapshot_state == 'started' or snapshot_state == 'paused' %}
<div id="snapshot-progress-wrapper">
{% include "progressmonitor/progress_monitor.html" with progress_endpoint=progress_endpoint progress_scope="snapshot" %}
</div>
{% endif %}
{% endif %}
{% if has_outputs %}
<iframe id="main-frame"
sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms"
class="full-page-iframe"
src="about:blank"
data-default-src="{% if best_result.path %}{% snapshot_preview_url snapshot best_result.path %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}"
data-default-src="{% if best_result.path %}{% snapshot_preview_url snapshot best_result.path %}{% elif STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}"
name="preview"
loading="eager"
fetchpriority="high"></iframe>
@ -1506,7 +1515,9 @@
<div class="snapshot-empty-actions">
<button type="button" id="snapshot-empty-refresh" class="snapshot-empty-btn">🔄 Refresh page</button>
<a href="{{ url }}" target="_blank" rel="noopener noreferrer" class="snapshot-empty-btn">🌐 Open original URL</a>
{% if not STATIC_EXPORT %}
<a href="{% admin_base_url %}/admin/core/snapshot/{{ snapshot_id }}/change/" class="snapshot-empty-btn">✏️ Edit in admin</a>
{% endif %}
</div>
</div>
</div>
@ -1528,7 +1539,7 @@
const frame = document.getElementById('main-frame')
if (!frame) return
const snapshotBaseUrlEarly = "{% snapshot_base_url snapshot %}"
const snapshotFilesUrlEarly = `${snapshotBaseUrlEarly}/?files=1`
const snapshotFilesUrlEarly = "{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}"
const defaultSrc = frame.dataset.defaultSrc || snapshotFilesUrlEarly
const rawHash = window.location.hash ? window.location.hash.slice(1) : ''
@ -1560,8 +1571,9 @@
<div class="header-nav">
<div class="header-col header-left" style="line-height: 58px; vertical-align: middle">
{% web_base_url as web_base %}
<a href="{% if web_base %}{{ web_base }}/public/{% else %}/{% endif %}" class="header-archivebox" title="Go to Public Index...">
<img src="{% if web_base %}{{ web_base }}/static/archive.png{% else %}{% static 'archive.png' %}{% endif %}" alt="Archive Icon">
{% static_export_root_url as static_root %}
<a href="{% if STATIC_EXPORT %}{{ static_root }}index.html{% elif web_base %}{{ web_base }}/public/{% else %}/{% endif %}" class="header-archivebox" title="Go to Public Index...">
<img src="{% if STATIC_EXPORT %}data:image/svg+xml;utf8,&lt;svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 64 64'&gt;&lt;rect x='8' y='5' width='48' height='54' rx='7' fill='%23fff' opacity='.95'/&gt;&lt;path d='M18 16h28M18 26h28M18 36h20M18 46h24' stroke='%23aa1e55' stroke-width='5' stroke-linecap='round'/&gt;&lt;/svg&gt;{% elif web_base %}{{ web_base }}/static/archive.png{% else %}{% static 'archive.png' %}{% endif %}" alt="Archive Icon">
ArchiveBox
</a>
</div>
@ -1607,34 +1619,44 @@
</div>
<div class="badge badge-info">
<span class="badge-label">Size</span>
{% if STATIC_EXPORT %}
<span class="badge-value">{{size}}</span>
{% else %}
<a class="badge-value" href="{% admin_base_url %}/admin/core/snapshot/{{snapshot_id|default:id}}/change/" title="Click to edit this Snapshot in the Admin UI">
{{size}}
</a>
{% endif %}
</div>
{% if not STATIC_EXPORT %}
<div class="badge badge-default">
<span class="badge-label">Manage</span>
<a class="badge-value" href="{% admin_base_url %}/admin/core/snapshot/{{snapshot_id|default:id}}/change/" title="Click to edit this Snapshot in the Admin UI">
<span class="badge-desktop-icon">✏️</span><span class="badge-mobile-text">Edit</span>
</a>
</div>
{% endif %}
<div class="badge badge-{{status_color}}">
<span class="badge-label">Status</span>
{% if STATIC_EXPORT %}
<span class="badge-value">{{status|upper}}</span>
{% else %}
<a class="badge-value" href="{% admin_base_url %}/admin/core/snapshot/?q={{snapshot_id|default:id}}" title="Click to see options to pull, re-snapshot, or delete this Snapshot">
{{status|upper}}
</a>
{% endif %}
</div>
</div>
<div class="header-mobile-badges">
<a class="header-mobile-badge" href="{% snapshot_base_url snapshot %}/?files=1" title="Browse all files for this snapshot">
<a class="header-mobile-badge" href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" title="Browse all files for this snapshot">
<span>{{num_outputs}} Files</span>
<span class="header-mobile-badge-separator">|</span>
<span>{{size}}</span>
</a>
<a class="header-mobile-badge" href="{% admin_base_url %}/admin/core/snapshot/{{snapshot_id|default:id}}/change/" title="Edit this snapshot in the Admin UI">
{% if not STATIC_EXPORT %}<a class="header-mobile-badge" href="{% admin_base_url %}/admin/core/snapshot/{{snapshot_id|default:id}}/change/" title="Edit this snapshot in the Admin UI">
<span>{{status|capfirst}}</span>
<span class="header-mobile-badge-separator">|</span>
<span>Edit</span>
</a>
</a>{% endif %}
</div>
{% if related_years %}
<div class="header-year-badges">
@ -1680,14 +1702,14 @@
</div>
</details>
{% else %}
<a class="header-date" href="{% web_base_url %}/{{archive_path}}/index.html" title="Date Added: {{bookmarked_date}} | First Archived: {{oldest_archive_date|default:downloaded_datestr}} | Last Checked: {{downloaded_datestr}} (UTC)">
<a class="header-date" href="{% snapshot_base_url snapshot %}/index.html" title="Date Added: {{bookmarked_date}} | First Archived: {{oldest_archive_date|default:downloaded_datestr}} | Last Checked: {{downloaded_datestr}} (UTC)">
<span class="header-date-label">Captures</span>
{{oldest_archive_date|default:downloaded_datestr|default:bookmarked_date|slice:":10"}}
</a>
{% endif %}
<br/>
<div class="external-links">
<a href="{% snapshot_base_url snapshot %}/?files=1" title="Browse the full SNAP_DIR for this snapshot" target="_blank">📁 See all files...</a>
<a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" title="Browse the full SNAP_DIR for this snapshot" target="_blank">📁 {% if STATIC_EXPORT %}Snapshot manifest{% else %}See all files...{% endif %}</a>
<span class="external-links-separator">|</span>
<a href="https://web.archive.org/web/{{url}}" title="Search for a copy of the URL saved in Archive.org" target="_blank" rel="noreferrer">🏛️ Archive.org</a>
<!--<a href="https://archive.md/{{url}}" title="Search for a copy of the URL saved in Archive.today" target="_blank" rel="noreferrer">Archive.today</a> &nbsp;|&nbsp; -->
@ -1707,12 +1729,12 @@
<div class="thumb-card{% if forloop.first %} selected-card{% endif %}" data-plugin-name="{{result.name|plugin_name}}"{% if preview_url %} data-preview-url="{{preview_url}}"{% endif %}{% if display_path %} data-output-path="{{display_path}}"{% endif %}>
<div class="thumb-body">
<div class="thumb-actions">
<a href="{% snapshot_url snapshot result.name %}/?files=1" data-no-preview="1" title="Open output folder" target="_blank" rel="noopener">📁</a>
{% if not STATIC_EXPORT %}<a href="{% snapshot_url snapshot result.name %}/?files=1" data-no-preview="1" title="Open output folder" target="_blank" rel="noopener">📁</a>{% endif %}
{% if display_path %}
<a href="{{display_url}}" data-no-preview="1" title="Download output file" download>⬇️</a>
{% endif %}
{% if can_delete_outputs and result.result %}
<button type="button" data-no-preview="1" data-archive-result-ids="{{result.result_ids}}" data-delete-url="{% admin_base_url %}/admin/core/archiveresult/" title="Delete this output">×</button>
<button type="button" data-no-preview="1" data-archive-result-ids="{{result.result_ids}}" data-delete-url="{% admin_base_url %}/admin/core/archiveresult/" data-delete-handoff="1" data-delete-snapshot-id="{{snapshot.id}}" title="Delete this output">×</button>
{% endif %}
</div>
{% if display_path %}
@ -1761,13 +1783,13 @@
<div class="thumb-card">
<div class="thumb-body">
<div class="thumb-actions">
<a href="{% snapshot_base_url snapshot %}/?files=1" data-no-preview="1" title="Browse all snapshot files" target="_blank" rel="noopener">📁</a>
<a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" data-no-preview="1" title="{% if STATIC_EXPORT %}Open snapshot manifest{% else %}Browse all snapshot files{% endif %}" target="_blank" rel="noopener">📁</a>
</div>
<h4>📦 Other files</h4>
<div class="loose-items">
{% for item in loose_items %}
{% if item.is_dir %}
<a href="{% snapshot_url snapshot item.path %}/?files=1" data-no-preview="1" target="_blank" rel="noopener">📁 {{item.name}}</a>
<a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_url snapshot item.path %}/?files=1{% endif %}" data-no-preview="1" target="_blank" rel="noopener">📁 {{item.name}}</a>
{% else %}
<a href="{% snapshot_url snapshot item.path %}" data-no-preview="1" target="_blank" rel="noopener">📄 {{item.name}}</a>
{% endif %}
@ -1780,13 +1802,13 @@
<div class="thumb-card">
<div class="thumb-body">
<div class="thumb-actions">
<a href="{% snapshot_base_url snapshot %}/?files=1" data-no-preview="1" title="Browse all snapshot files" target="_blank" rel="noopener">📁</a>
<a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" data-no-preview="1" title="{% if STATIC_EXPORT %}Open snapshot manifest{% else %}Browse all snapshot files{% endif %}" target="_blank" rel="noopener">📁</a>
</div>
<h4>⚠️ Failed</h4>
<div class="loose-items failed-items">
{% for item in failed_items %}
{% if item.is_dir %}
<a href="{% snapshot_url snapshot item.path %}/?files=1" data-no-preview="1" target="_blank" rel="noopener">📁 {{item.name}}</a>
<a href="{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_url snapshot item.path %}/?files=1{% endif %}" data-no-preview="1" target="_blank" rel="noopener">📁 {{item.name}}</a>
{% else %}
<a href="{% snapshot_url snapshot item.path %}" data-no-preview="1" target="_blank" rel="noopener">📄 {{item.name}}</a>
{% endif %}
@ -1799,13 +1821,11 @@
</div>
</header>
{% if can_delete_outputs %}<input type="hidden" id="delete-output-csrf" value="{{csrf_token}}">{% endif %}
<script src="{% static 'jquery.min.js' %}" type="text/javascript"></script>
{% if can_delete_outputs %}{% include "includes/output_delete_controls.html" %}{% endif %}
<script>
const snapshotBaseUrl = "{% snapshot_base_url snapshot %}";
const snapshotFilesUrl = `${snapshotBaseUrl}/?files=1`;
const snapshotFilesUrl = "{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}";
function tryCenterImageFrame(frame) {
try {
@ -1925,9 +1945,7 @@
}
// un-sandbox iframes showing pdfs (required to display pdf viewer)
jQuery('iframe').map(function() {
attachPreviewFrameHandlers(this)
})
document.querySelectorAll('iframe').forEach((frame) => attachPreviewFrameHandlers(frame))
function getPreviewHashValueFromHref(href, normalize=false) {
if (href == './') {
@ -1987,7 +2005,7 @@
const basePath = (base.pathname || '').replace(/\/+$/, '')
const targetPath = (target.pathname || '').replace(/\/+$/, '')
return (
target.search !== '?files=1'
target.search !== "{% if STATIC_EXPORT %}{% else %}?files=1{% endif %}"
&& (
targetPath === basePath
|| targetPath === `${basePath}/index.html`
@ -2051,8 +2069,8 @@
return false
}
jQuery('.selected-card').removeClass('selected-card')
jQuery(card).closest('.thumb-card').addClass('selected-card')
document.querySelectorAll('.selected-card').forEach((selected) => selected.classList.remove('selected-card'))
card.closest('.thumb-card').classList.add('selected-card')
const nextSrc = isSnapshotRootPreview(target) ? snapshotFilesUrl : target
const existingFrame = document.getElementById('main-frame')
@ -2151,8 +2169,8 @@
function hideSnapshotHeader() {
console.log('Collapsing Snapshot header...')
jQuery('.header-toggle').text('▸')
jQuery('.header-bottom').hide()
document.querySelectorAll('.header-toggle').forEach((toggle) => { toggle.textContent = '▸' })
document.querySelectorAll('.header-bottom').forEach((header) => { header.hidden = true })
try {
localStorage.setItem("archivebox-snapshot-header-visible", "false")
} catch (e) {
@ -2161,8 +2179,8 @@
}
function showSnapshotHeader() {
console.log('Expanding Snapshot header...')
jQuery('.header-toggle').text('▾')
jQuery('.header-bottom').show()
document.querySelectorAll('.header-toggle').forEach((toggle) => { toggle.textContent = '▾' })
document.querySelectorAll('.header-bottom').forEach((header) => { header.hidden = false })
try {
localStorage.setItem("archivebox-snapshot-header-visible", "true")
} catch (e) {
@ -2183,7 +2201,7 @@
}
function handleSnapshotHeaderToggle(event) {
event.preventDefault()
if (jQuery('.header-toggle').text().includes('▾')) {
if ([...document.querySelectorAll('.header-toggle')].some((toggle) => toggle.textContent.includes('▾'))) {
hideSnapshotHeader()
} else {
showSnapshotHeader()
@ -2192,7 +2210,7 @@
}
// Hide or show the header once when its title row or collapse icon is clicked.
jQuery('.header-toggle-trigger').on('click', handleSnapshotHeaderToggle)
document.querySelectorAll('.header-toggle-trigger').forEach((trigger) => trigger.addEventListener('click', handleSnapshotHeaderToggle))
// check URL for hash e.g. #git and load relevant preview
selectInitialPreview()

379
archivebox/templates/core/static_index.html
View File

@ -1,258 +1,129 @@
{% load static core_tags %}
{% load core_tags %}
<!DOCTYPE html>
<html lang="en">
<head>
<title>Archived Sites</title>
<meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
<style>
:root {
--bg-main: #efefef;
--accent-1: #aa1e55;
--accent-2: #ffebeb;
--accent-3: #efefef;
--text-1: #1c1c1c;
--text-2: #eaeaea;
--text-main: #1a1a1a;
--font-main: "Gill Sans", Helvetica, sans-serif;
<head>
<title>Archived Sites</title>
<meta charset="utf-8" name="viewport" content="width=device-width, initial-scale=1">
<style>
* { box-sizing: border-box; }
body { margin: 0; background: #f8fafc; color: #0f172a; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
a { color: inherit; }
.static-header { display: flex; align-items: center; justify-content: space-between; gap: 16px; padding: 14px 20px; background: #aa1e55; color: #fff; }
.static-brand { display: flex; align-items: center; gap: 10px; font-size: 20px; font-weight: 700; text-decoration: none; }
.static-brand-mark { display: grid; place-items: center; width: 30px; height: 30px; border-radius: 7px; background: #fff; color: #aa1e55; font-size: 19px; }
.static-meta { color: rgba(255,255,255,.82); font-size: 12px; }
main { width: min(1500px, 100%); margin: 0 auto; padding: 18px; }
.toolbar { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-bottom: 14px; padding: 10px 12px; border: 1px solid #e2e8f0; border-radius: 10px; background: #fff; box-shadow: 0 1px 2px rgba(15,23,42,.04); }
#static-search { width: min(560px, 100%); height: 36px; padding: 7px 11px; border: 1px solid #cbd5e1; border-radius: 7px; font: inherit; font-size: 13px; }
.count { color: #64748b; font-size: 12px; white-space: nowrap; }
.table-wrap { overflow-x: auto; border: 1px solid #e2e8f0; border-radius: 10px; background: #fff; box-shadow: 0 1px 2px rgba(15,23,42,.04); }
table { width: 100%; min-width: 980px; table-layout: fixed; border-collapse: collapse; }
th { padding: 9px 10px; border-bottom: 1px solid #e2e8f0; background: #f8fafc; color: #475569; font-size: 11px; letter-spacing: .04em; text-align: left; text-transform: uppercase; }
td { padding: 8px 10px; border-bottom: 1px solid #edf2f7; vertical-align: middle; }
tbody tr:hover { background: #f8fafc; }
tbody tr:last-child td { border-bottom: 0; }
.snapshot-time { width: 102px; white-space: nowrap; }
.snapshot-time a { display: inline-flex; flex-direction: column; color: #334155; line-height: 1.2; text-decoration: none; }
.snapshot-time small { color: #94a3b8; font-size: 11px; }
.snapshot-preview-cell { width: 116px; }
.snapshot-preview-cell a { display: inline-flex; width: 102px; height: 102px; align-items: center; justify-content: center; }
.snapshot-preview { display: block; width: 100px; height: 100px; border: 1px solid #e2e8f0; border-radius: 8px; background: #f8fafc; object-fit: cover; object-position: top; }
.snapshot-preview-spinner { display: grid; place-items: center; }
.snapshot-preview-spinner::after { width: 20px; height: 20px; border: 2px solid #cbd5e1; border-top-color: #aa1e55; border-radius: 50%; content: ""; animation: spin .8s linear infinite; }
.snapshot-title-line { display: flex; align-items: center; gap: 8px; min-width: 0; margin-bottom: 2px; }
.snapshot-favicon-link { display: inline-flex; width: 22px; min-width: 22px; height: 22px; align-items: center; justify-content: center; }
.link-favicon { display: block; width: 18px; height: 18px; border-radius: 4px; object-fit: contain; }
.snapshot-title { overflow: hidden; color: #0f172a; font-size: 14px; font-weight: 650; line-height: 1.25; text-overflow: ellipsis; white-space: nowrap; text-decoration: none; }
.snapshot-url { display: block; margin-left: 30px; overflow: hidden; color: #64748b; font-size: 12px; text-overflow: ellipsis; white-space: nowrap; text-decoration: none; }
.snapshot-mobile-saved { display: none; }
.snapshot-tags-cell { width: 180px; }
.snapshot-tags { display: flex; flex-wrap: wrap; gap: 4px; max-height: 46px; overflow: hidden; }
.snapshot-tag { max-width: 150px; padding: 2px 7px; overflow: hidden; border: 1px solid #bfdbfe; border-radius: 999px; background: #eff6ff; color: #1d4ed8; font-size: 11px; font-weight: 600; text-overflow: ellipsis; white-space: nowrap; }
.snapshot-status-cell { width: 96px; }
.snapshot-status { display: inline-flex; min-width: 68px; justify-content: center; padding: 3px 8px; border-radius: 999px; background: #f1f5f9; color: #475569; font-size: 11px; font-weight: 700; text-transform: uppercase; }
.status-sealed .snapshot-status, .status-succeeded .snapshot-status { background: #d1fae5; color: #047857; }
.status-started .snapshot-status { background: #dbeafe; color: #1d4ed8; }
.status-queued .snapshot-status, .status-backoff .snapshot-status { background: #fef3c7; color: #b45309; }
.status-failed .snapshot-status { background: #fee2e2; color: #b91c1c; }
.snapshot-files-cell { width: 220px; overflow: hidden; }
.snapshot-files-cell .files-icons { max-width: 200px; }
.snapshot-size-cell { width: 102px; white-space: nowrap; }
.snapshot-size-cell a { display: block; color: #334155; font-size: 12px; font-weight: 700; text-decoration: none; }
.snapshot-size-cell small, .empty-value { color: #94a3b8; font-size: 10px; }
footer { padding: 24px 16px; color: #64748b; font-size: 12px; text-align: center; }
@keyframes spin { to { transform: rotate(360deg); } }
@media (max-width: 700px) {
.static-header, .toolbar { align-items: stretch; flex-direction: column; }
main { padding: 10px; }
table { min-width: 0; }
th:nth-child(n+4), td:nth-child(n+4), th.snapshot-time, td.snapshot-time { display: none; }
.snapshot-preview-cell { width: 88px; }
.snapshot-preview-cell a { width: 76px; height: 76px; }
.snapshot-preview { width: 74px; height: 74px; }
.snapshot-mobile-saved { display: block; margin: 4px 0 0 30px; color: #64748b; font-size: 11px; }
}
</style>
<script>
function nextPublicSnapshotPreview(img) {
const fallbacks = (img.dataset.fallbacks || '').split(',').filter(Boolean)
if (fallbacks.length) {
img.src = fallbacks.shift()
img.dataset.fallbacks = fallbacks.join(',')
return
}
/* Dark Mode (WIP) */
/*
@media (prefers-color-scheme: dark) {
:root {
--accent-2: hsl(160, 100%, 96%);
--text-1: #eaeaea;
--text-2: #1a1a1a;
--bg-main: #101010;
}
#table-bookmarks_wrapper,
#table-bookmarks_wrapper img,
tbody td:nth-child(3),
tbody td:nth-child(3) span,
footer {
filter: invert(100%);
}
}*/
html, body {
width: 100%;
height: 100%;
font-size: 18px;
font-weight: 200;
text-align: center;
margin: 0px;
padding: 0px;
font-family: var(--font-main);
}
.header-top small {
font-weight: 200;
color: var(--accent-3);
}
.header-top {
width: 100%;
height: auto;
min-height: 40px;
margin: 0px;
text-align: center;
color: white;
font-size: calc(11px + 0.84vw);
font-weight: 200;
padding: 4px 4px;
border-bottom: 3px solid var(--accent-1);
background-color: var(--accent-1);
}
input[type=search] {
width: 22vw;
border-radius: 4px;
border: 1px solid #aeaeae;
padding: 3px 5px;
}
.nav > div {
min-height: 30px;
}
.header-top a {
text-decoration: none;
color: rgba(0,0,0,0.6);
}
.header-top a:hover {
text-decoration: none;
color: rgba(0,0,0,0.9);
}
.header-top .col-lg-4 {
text-align: center;
padding-top: 4px;
padding-bottom: 4px;
}
.header-archivebox img {
display: inline-block;
margin-right: 3px;
height: 30px;
margin-left: 12px;
margin-top: -4px;
margin-bottom: 2px;
}
.header-archivebox img:hover {
opacity: 0.5;
}
#table-bookmarks_length, #table-bookmarks_filter {
padding-top: 12px;
opacity: 0.8;
padding-left: 24px;
padding-right: 22px;
margin-bottom: -16px;
}
table {
padding: 6px;
width: 100%;
}
table thead th {
font-weight: 400;
}
table tr {
height: 35px;
}
tbody tr:nth-child(odd) {
background-color: var(--accent-2) !important;
}
table tr td {
white-space: nowrap;
overflow: hidden;
/*padding-bottom: 0.4em;*/
/*padding-top: 0.4em;*/
padding-left: 2px;
text-align: center;
}
table tr td a {
text-decoration: none;
}
table tr td img, table tr td object {
display: inline-block;
margin: auto;
height: 24px;
width: 24px;
padding: 0px;
padding-right: 5px;
vertical-align: middle;
margin-left: 4px;
}
#table-bookmarks {
width: 100%;
overflow-y: scroll;
table-layout: fixed;
}
.dataTables_wrapper {
background-color: #fafafa;
}
table tr a span[data-archived~=False] {
opacity: 0.4;
}
.files-spinner {
height: 15px;
width: auto;
opacity: 0.5;
vertical-align: -2px;
}
.in-progress {
display: none;
}
tr td a.favicon img {
padding-left: 6px;
padding-right: 12px;
vertical-align: -4px;
}
tr td a.title {
font-size: 1.4em;
text-decoration:none;
color:black;
}
tr td a.title small {
background-color: var(--accent-3);
border-radius: 4px;
float:right
}
input[type=search]::-webkit-search-cancel-button {
-webkit-appearance: searchfield-cancel-button;
}
.title-col {
text-align: left;
}
.title-col a {
color: black;
}
</style>
<link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
<link rel="stylesheet" href="{% static 'jquery.dataTables.min.css' %}"/>
<script src="{% static 'jquery.min.js' %}"></script>
<script src="{% static 'jquery.dataTables.min.js' %}"></script>
<script>
document.addEventListener('error', function(e) {
e.target.style.opacity = 0;
}, true)
jQuery(document).ready(function() {
jQuery('#table-bookmarks').DataTable({
stateSave: true, // save state (filtered input, number of entries shown, etc) in localStorage
dom: '<lf<t>ip>', // how to show the table and its helpers (filter, etc) in the DOM
order: [[0, 'desc']],
iDisplayLength: 100,
});
});
</script>
</head>
<body>
<header>
<div class="header-top container-fluid">
<div class="row nav">
<div class="col-sm-2">
<a href="/" class="header-archivebox">
<img src="{% static 'archive.png' %}" alt="Logo"/>
ArchiveBox: Index
</a>
</div>
<div class="col-sm-10" style="text-align: right">
<a href="/add/">Add Links</a> &nbsp; | &nbsp;
<a href="/admin/core/snapshot/">Admin</a> &nbsp; | &nbsp;
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
</div>
</div>
</div>
</header>
<table id="table-bookmarks">
<thead>
<tr>
<th class="snapshot-time">Saved</th>
<th class="snapshot-preview-cell">Preview</th>
<th>Snapshot ({{num_links}})</th>
<th class="snapshot-tags-cell">Tags</th>
<th class="snapshot-status-cell">Status</th>
<th class="snapshot-files-cell">Files</th>
<th class="snapshot-size-cell">Size</th>
</tr>
</thead>
<tbody>
{% for link in links %}
{% snapshot_index_row link %}
{% endfor %}
</tbody>
</table>
<footer>
<br/>
<center>
<small>
Archive created using <a href="https://github.com/ArchiveBox/ArchiveBox" title="Github">ArchiveBox</a>
version <a href="https://github.com/ArchiveBox/ArchiveBox/releases/tag/v{{version}}" title="View source code and release info">v{{version}}</a> &nbsp; | &nbsp;
Download index as <a href="index.json" title="JSON summary of archived links.">JSON</a>
<br/><br/>
{{FOOTER_INFO}}
</small>
</center>
<br/>
</footer>
</body>
img.style.opacity = 0
}
</script>
</head>
<body>
<header class="static-header">
<a class="static-brand" href="./index.html"><span class="static-brand-mark">▤</span> ArchiveBox</a>
<span class="static-meta">Portable static archive · updated {{ time_updated }}</span>
</header>
<main>
<div class="toolbar">
<input id="static-search" type="search" placeholder="Filter title, URL, tag, or status…" autocomplete="off">
<span class="count"><span id="visible-count">{{ num_links }}</span> of {{ num_links }} snapshots</span>
</div>
<div class="table-wrap">
<table id="table-bookmarks">
<thead>
<tr>
<th class="snapshot-time">Saved</th>
<th class="snapshot-preview-cell">Preview</th>
<th>Snapshot ({{ num_links }})</th>
<th class="snapshot-tags-cell">Tags</th>
<th class="snapshot-status-cell">Status</th>
<th class="snapshot-files-cell">Files</th>
<th class="snapshot-size-cell">Size</th>
</tr>
</thead>
<tbody>
{% for link in links %}{% snapshot_index_row link %}{% endfor %}
</tbody>
</table>
</div>
</main>
<footer>
Generated by <a href="https://github.com/ArchiveBox/ArchiveBox">ArchiveBox v{{ version }}</a>.
<a href="./index.jsonl">Download JSONL metadata</a>.<br><br>{{ FOOTER_INFO }}
</footer>
<script>
(() => {
const search = document.getElementById('static-search')
const rows = [...document.querySelectorAll('#table-bookmarks tbody tr')]
const count = document.getElementById('visible-count')
search.addEventListener('input', () => {
const query = search.value.trim().toLowerCase()
let visible = 0
rows.forEach((row) => {
const show = !query || row.textContent.toLowerCase().includes(query)
row.hidden = !show
if (show) visible += 1
})
count.textContent = visible
})
})()
</script>
</body>
</html>

16
archivebox/templates/includes/output_delete_controls.html
View File

@ -55,10 +55,20 @@
}
function deleteQueuedOutputs() {
const csrf = document.querySelector('#delete-output-csrf, input[name="csrfmiddlewaretoken"]')?.value
const deleteUrl = deleteOutputButtons.find((button) => button.dataset.deleteUrl)?.dataset.deleteUrl
if (!csrf || !deleteUrl) return window.alert('Delete failed: missing admin request data')
const button = deleteOutputButtons.find((candidate) => candidate.dataset.deleteUrl)
let deleteUrl = button?.dataset.deleteUrl
if (!deleteUrl) return window.alert('Delete failed: missing admin request data')
deleteOutputButtons.forEach((button) => button.disabled = true)
if (button.dataset.deleteHandoff) {
deleteUrl = new URL(deleteUrl, window.location.href)
deleteUrl.searchParams.set('action', 'delete_selected')
deleteUrl.searchParams.set('snapshot', button.dataset.deleteSnapshotId)
queuedOutputIds.forEach((id) => deleteUrl.searchParams.append('_selected_action', id))
window.location.assign(deleteUrl)
return
}
const csrf = document.querySelector('input[name="csrfmiddlewaretoken"]')?.value
if (!csrf) return window.alert('Delete failed: missing admin request data')
const body = new URLSearchParams({action: 'delete_selected', post: 'yes', csrfmiddlewaretoken: csrf})
queuedOutputIds.forEach((id) => body.append('_selected_action', id))
fetch(deleteUrl, {

19
archivebox/tests/migrations_helpers.py
View File

@ -13,6 +13,7 @@ import hashlib
import sqlite3
from pathlib import Path
from datetime import datetime, timezone
from urllib.parse import urlparse
from archivebox.tests.conftest import cli_env, run_archivebox_cmd
from archivebox.uuid_compat import uuid7
@ -1272,6 +1273,24 @@ def filesystem_manifest(root: Path) -> dict[str, tuple[str, str | int]]:
return manifest
def current_snapshot_dir(data_dir: Path, db_path: Path, timestamp: str) -> Path:
"""Resolve a migrated snapshot's canonical archive/users/... directory."""
with sqlite3.connect(db_path) as connection:
username, bookmarked_at, snapshot_id, url = connection.execute(
"""
SELECT u.username, s.bookmarked_at, s.id, s.url
FROM core_snapshot s
JOIN crawls_crawl c ON c.id = s.crawl_id
JOIN auth_user u ON u.id = c.created_by_id
WHERE s.timestamp = ?
""",
(timestamp,),
).fetchone()
date_bucket = datetime.fromisoformat(bookmarked_at).strftime("%Y%m%d")
domain = urlparse(url).hostname or "unknown"
return data_dir / "archive" / "users" / username / "snapshots" / date_bucket / domain / snapshot_id
def verify_snapshot_count(db_path: Path, expected: int) -> tuple[bool, str]:
"""Verify the number of snapshots in the database."""
conn = sqlite3.connect(str(db_path))

10
archivebox/tests/test_cli_archiveresult.py
View File

@ -61,7 +61,11 @@ class TestArchiveResultCreate:
ar = next(r for r in records if r["type"] == "ArchiveResult")
assert ar["plugin"] == "title"
assert ar["hook_name"].startswith("on_Snapshot__")
# Queue projection is one row per plugin, while a plugin can contain
# several ordered hooks. The runner records the concrete hook only
# after execution; inventing one here would make the pending row claim
# work that has not run and reintroduce hook-level duplicate results.
assert ar["hook_name"] == ""
assert "id" not in ar
def test_create_with_specific_plugin(self, initialized_archive):
@ -90,7 +94,9 @@ class TestArchiveResultCreate:
ar_records = [r for r in records if r.get("type") == "ArchiveResult"]
assert len(ar_records) >= 1
assert all(record["plugin"] == "screenshot" for record in ar_records)
assert all(record["hook_name"].startswith("on_Snapshot__") for record in ar_records)
# A requested plugin is the schedulable unit; its concrete hook is an
# execution result, not input metadata on this pre-execution request.
assert all(record["hook_name"] == "" for record in ar_records)
def test_create_pass_through_crawl(self, initialized_archive):
"""Pass-through Crawl records unchanged."""

142
archivebox/tests/test_cli_list.py
View File

@ -4,6 +4,7 @@ Verify list emits snapshot JSONL and applies the documented filters.
"""
import json
from pathlib import Path
import pytest
from django.contrib.auth import get_user_model
@ -19,6 +20,147 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
pytestmark = pytest.mark.django_db(transaction=True)
def test_static_export_creates_detail_page_for_unarchived_snapshot(snapshot):
from archivebox.config import CONSTANTS
snapshot_dir = Path(snapshot.output_dir)
assert snapshot_dir.is_dir()
assert not any(snapshot_dir.iterdir())
snapshot_dir.rmdir()
assert not snapshot_dir.exists()
html = Snapshot.objects.filter(pk=snapshot.pk).to_html(with_headers=True)
static_path = snapshot_dir.relative_to(CONSTANTS.DATA_DIR).as_posix()
detail_path = snapshot_dir / "index.html"
assert f"./{static_path}/index.html" in html
# The portable export command emits JSONL records; index.json is a legacy
# filename that is not created and leaves a broken footer link offline.
assert 'href="./index.jsonl"' in html
assert 'href="./index.json"' not in html
root_manifest = CONSTANTS.DATA_DIR / "index.jsonl"
assert root_manifest.exists()
manifest_records = [json.loads(line) for line in root_manifest.read_text().splitlines() if line.strip()]
assert [record["id"] for record in manifest_records] == [str(snapshot.id)]
# JSON and JSONL are alternate containers for one static-export schema;
# consumers must not see TYPE/tags/archive paths change by file format.
assert manifest_records[0]["TYPE"] == "core.models.Snapshot"
assert "type" not in manifest_records[0]
assert isinstance(manifest_records[0]["tags"], list)
assert manifest_records[0]["archive_path"] == static_path
assert manifest_records[0]["archive_url"] == f"./{static_path}/index.html"
assert detail_path.exists()
detail_html = detail_path.read_text()
assert f"/snapshot/{snapshot.id.hex}" not in detail_html
assert "/admin/" not in detail_html
def test_static_exports_use_filesystem_paths_not_live_django_routes(snapshot):
from archivebox.config import CONSTANTS
from archivebox.core.models import ArchiveResult
snapshot_dir = Path(snapshot.output_dir)
screenshot_dir = snapshot_dir / "screenshot"
screenshot_dir.mkdir(parents=True, exist_ok=True)
screenshot_file = screenshot_dir / "screenshot.png"
screenshot_file.write_bytes(b"real screenshot")
ArchiveResult.objects.create(
snapshot=snapshot,
plugin="screenshot",
hook_name="on_Snapshot__50_screenshot.py",
status=ArchiveResult.StatusChoices.SUCCEEDED,
output_str="screenshot.png",
output_files={"screenshot.png": {"size": screenshot_file.stat().st_size}},
output_size=screenshot_file.stat().st_size,
)
ArchiveResult.objects.create(
snapshot=snapshot,
plugin="chrome_screencast",
hook_name="on_Snapshot__02_chrome_screencast.py",
status=ArchiveResult.StatusChoices.SUCCEEDED,
output_str="2 screencast frames (0 kept)",
output_files={"hook.stderr.log": {"size": 12}},
output_size=12,
)
wget_dir = snapshot_dir / "wget"
wget_dir.mkdir()
wget_file = wget_dir / "index%3A.html"
wget_file.write_text("archived page")
ArchiveResult.objects.create(
snapshot=snapshot,
plugin="wget",
hook_name="on_Snapshot__35_wget.py",
status=ArchiveResult.StatusChoices.SUCCEEDED,
output_str="index%3A.html",
output_files={"index%3A.html": {"size": wget_file.stat().st_size}},
output_size=wget_file.stat().st_size,
)
ArchiveResult.objects.create(
snapshot=snapshot,
plugin="staticfile",
hook_name="on_Snapshot__26_staticfile.py",
status=ArchiveResult.StatusChoices.SUCCEEDED,
output_str="prenav.json",
output_files={"prenav.json": {"size": 66}},
output_size=66,
)
ytdlp_dir = snapshot_dir / "ytdlp"
ytdlp_dir.mkdir()
media_file = ytdlp_dir / "saved.m4a"
media_file.write_bytes(b"audio")
ArchiveResult.objects.create(
snapshot=snapshot,
plugin="ytdlp",
hook_name="on_Snapshot__60_ytdlp.py",
status=ArchiveResult.StatusChoices.SUCCEEDED,
output_str="saved.m4a",
output_files={
"saved.m4a": {"size": media_file.stat().st_size},
"deleted.temp.m4a": {"size": 123},
},
output_size=media_file.stat().st_size,
)
hashes_dir = snapshot_dir / "hashes"
hashes_dir.mkdir()
(hashes_dir / "hashes.json").write_text(
json.dumps(
{
"screenshot/screenshot.png": {"size": screenshot_file.stat().st_size},
"wget/index%3A.html": {"size": wget_file.stat().st_size},
"staticfile/prenav.json": {"size": 66},
"ytdlp/saved.m4a": {"size": media_file.stat().st_size},
"ytdlp/deleted.temp.m4a": {"size": 123},
},
),
)
static_path = snapshot_dir.relative_to(CONSTANTS.DATA_DIR).as_posix()
queryset = Snapshot.objects.filter(pk=snapshot.pk).prefetch_related("tags")
html = queryset.to_html(with_headers=True)
[record] = json.loads(queryset.to_json(with_headers=False))
detail_html = snapshot_dir / "index.html"
assert f"./{static_path}/index.html" in html
assert f"./{static_path}/screenshot/screenshot.png" in html
assert f"./{static_path}/wget/index%253A.html" in html
assert f"./{static_path}/index.jsonl" in html
assert f"/snapshot/{snapshot.id.hex}" not in html
assert "/web/" not in html
assert "/static/" not in html
assert "/None" not in html
assert "staticfile/prenav.json" not in html
assert record["archive_path"] == static_path
assert record["archive_url"] == f"./{static_path}/index.html"
assert detail_html.exists()
rendered_detail = detail_html.read_text()
assert "core/snapshot.html" not in rendered_detail
assert "screenshot/screenshot.png" in rendered_detail
assert "staticfile/prenav.json" not in rendered_detail
assert "ytdlp/saved.m4a" in rendered_detail
assert "ytdlp/deleted.temp.m4a" not in rendered_detail
assert f"/snapshot/{snapshot.id.hex}" not in rendered_detail
def test_streaming_json_matches_snapshot_serializer(initialized_archive):
from archivebox.crawls.models import Crawl

160
archivebox/tests/test_cli_mcp.py
View File

@ -1,11 +1,18 @@
#!/usr/bin/env python3
"""
Tests for archivebox mcp command.
"""
"""Tests for the ArchiveBox MCP server and CLI entry point."""
import json
import os
import pytest
from archivebox.mcp.server import MCPServer
from archivebox.tests.conftest import run_archivebox_cmd
pytestmark = pytest.mark.django_db(transaction=True)
def test_mcp_help_runs_successfully(tmp_path):
"""The mcp command should be registered and expose help."""
@ -13,3 +20,150 @@ def test_mcp_help_runs_successfully(tmp_path):
assert result.returncode == 0
assert "mcp" in result.stdout.lower()
def test_mcp_stdio_handles_handshake_notification_and_ping(initialized_archive):
requests = [
{
"jsonrpc": "2.0",
"id": 1,
"method": "initialize",
"params": {
"protocolVersion": "2025-11-25",
"capabilities": {},
"clientInfo": {"name": "archivebox-test", "version": "1"},
},
},
{"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}},
{"jsonrpc": "2.0", "id": 2, "method": "ping", "params": {}},
]
result = run_archivebox_cmd(
["mcp"],
cwd=initialized_archive,
input="".join(json.dumps(request) + "\n" for request in requests),
default_cli_env=True,
)
responses = [json.loads(line) for line in result.stdout.splitlines()]
assert result.returncode == 0
assert [response["id"] for response in responses] == [1, 2]
assert responses[0]["result"]["protocolVersion"] == "2025-11-25"
assert responses[1]["result"] == {}
def test_mcp_exposes_six_focused_tools():
tools = MCPServer().handle_tools_list({})["tools"]
tools_by_name = {tool["name"]: tool for tool in tools}
assert set(tools_by_name) == {"add", "search", "crawl", "snapshot", "archiveresult", "shell"}
assert len(tools) == 6
assert all("outputSchema" in tool for tool in tools)
crawl_schema = tools_by_name["crawl"]["inputSchema"]
assert crawl_schema["properties"]["action"]["enum"] == ["create", "delete", "list", "update"]
assert crawl_schema["properties"]["urls"]["type"] == "array"
assert crawl_schema["properties"]["records"]["type"] == "array"
assert tools_by_name["search"]["annotations"]["readOnlyHint"] is True
assert tools_by_name["shell"]["inputSchema"]["required"] == ["code"]
assert tools_by_name["shell"]["annotations"]["destructiveHint"] is True
def test_mcp_crawl_create_returns_structured_json(initialized_archive):
os.chdir(initialized_archive)
result = MCPServer().handle_tools_call(
{
"name": "crawl",
"arguments": {
"action": "create",
"urls": ["https://mcp-test.example.com/"],
"depth": 1,
"tag": "mcp-test",
},
},
)
assert result["isError"] is False, result
assert result["structuredContent"]["success"] is True
assert result["structuredContent"]["error"] is None
assert result["structuredContent"]["command"] == "archivebox crawl create"
assert result["structuredContent"]["exitCode"] == 0
assert result["structuredContent"]["records"][0]["urls"] == "https://mcp-test.example.com/"
assert result["structuredContent"]["records"][0]["max_depth"] == 1
assert json.loads(result["content"][0]["text"]) == result["structuredContent"]
def test_mcp_snapshot_update_accepts_records_without_a_jsonl_pipeline(initialized_archive):
os.chdir(initialized_archive)
server = MCPServer()
created = server.handle_tools_call(
{
"name": "snapshot",
"arguments": {
"action": "create",
"urls": ["https://mcp-update.example.com/"],
},
},
)
assert created["isError"] is False, created
snapshot = created["structuredContent"]["records"][0]
updated = server.handle_tools_call(
{
"name": "snapshot",
"arguments": {
"action": "update",
"records": [{"id": snapshot["id"]}],
"tag": "updated-through-mcp",
},
},
)
assert updated["isError"] is False
assert updated["structuredContent"]["records"][0]["id"] == snapshot["id"]
assert "updated-through-mcp" in updated["structuredContent"]["records"][0]["tags"]
def test_mcp_cli_errors_are_structured_for_agents(initialized_archive):
os.chdir(initialized_archive)
result = MCPServer().handle_tools_call(
{
"name": "crawl",
"arguments": {"action": "create"},
},
)
assert result["isError"] is True
assert result["structuredContent"]["success"] is False
assert result["structuredContent"]["exitCode"] == 1
assert "No URLs provided" in result["structuredContent"]["error"]
assert json.loads(result["content"][0]["text"]) == result["structuredContent"]
def test_mcp_invalid_action_is_a_protocol_error():
response = MCPServer().handle_request(
{
"jsonrpc": "2.0",
"id": 1,
"method": "tools/call",
"params": {"name": "crawl", "arguments": {"action": "bogus"}},
},
)
assert response["error"]["code"] == -32602
assert "Choose one of: create, delete, list, update" in response["error"]["message"]
def test_mcp_shell_runs_python_through_archivebox_shell(initialized_archive):
os.chdir(initialized_archive)
result = MCPServer().handle_tools_call(
{
"name": "shell",
"arguments": {
"code": "from archivebox.core.models import Snapshot; print(f'shell_ok={Snapshot.objects.count() >= 0}')",
},
},
)
assert result["isError"] is False, result
assert result["structuredContent"]["command"] == "archivebox shell"
assert result["structuredContent"]["stdout"] == "shell_ok=True\n"

57
archivebox/tests/test_cli_run.py
View File

@ -737,6 +737,59 @@ class TestRunDaemonMode:
cleanup_process_group(proc.pid)
proc.wait(timeout=15)
def test_run_daemon_retires_runner_from_previous_pid_namespace(self, initialized_archive, db):
from django.utils import timezone
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE
from archivebox.machine.models import Machine, PROCESS_PID_NAMESPACE_KEY, Process, get_current_pid_namespace
from archivebox.tests.test_orm_helpers import use_archivebox_db
env = cli_env(PLUGINS="__archivebox_test_no_plugins__")
with use_archivebox_db(initialized_archive):
stopped_runner = Process.objects.create(
machine=Machine.current(),
process_type=Process.TypeChoices.ORCHESTRATOR,
worker_type=RUNNER_ACTIVE_WORKER_TYPE,
status=Process.StatusChoices.RUNNING,
pwd=str(initialized_archive),
pid=1,
started_at=timezone.now(),
env={PROCESS_PID_NAMESPACE_KEY: f"{get_current_pid_namespace()}-stopped-container"},
)
queued = run_archivebox_cmd(["crawl", "create", create_test_url()], cwd=initialized_archive, env=env, timeout=60)
assert queued.returncode == 0, queued.stderr or queued.stdout
daemon_log = initialized_archive / "run-daemon-stopped-container.log"
daemon_log_handle = daemon_log.open("w", encoding="utf-8")
replacement = run_archivebox_cmd(
["run", "--daemon"],
cwd=initialized_archive,
env=env,
stdin=subprocess.DEVNULL,
stdout=daemon_log_handle,
stderr=subprocess.STDOUT,
start_new_session=True,
wait=False,
)
daemon_log_handle.close()
try:
wait_for_log(daemon_log, "[Crawl#", timeout=10)
assert "Multiple orchestrators sharing a single collection is not officially supported" in daemon_log.read_text()
with use_archivebox_db(initialized_archive):
stopped_runner.refresh_from_db()
assert stopped_runner.status == Process.StatusChoices.EXITED
active_runner = Process.objects.get(
process_type=Process.TypeChoices.ORCHESTRATOR,
worker_type=RUNNER_ACTIVE_WORKER_TYPE,
status=Process.StatusChoices.RUNNING,
pwd=str(initialized_archive),
)
assert active_runner.pid == replacement.pid
finally:
cleanup_process_group(replacement.pid)
replacement.wait(timeout=15)
@pytest.mark.django_db
class TestRecoverOrchestratorState:
@ -1574,7 +1627,7 @@ class TestRecoverOrchestratorState:
assert snapshot.status == Snapshot.StatusChoices.SEALED
assert snapshot.fs_version == Snapshot._fs_current_version()
assert snapshot.output_dir.joinpath("index.html").read_text(encoding="utf-8") == "legacy archive"
assert legacy_dir.is_symlink()
assert not legacy_dir.exists()
@pytest.mark.django_db(transaction=True)
def test_run_due_snapshot_migrates_filesystem_after_sealed_parent_reconciliation(self):
@ -1616,7 +1669,7 @@ class TestRecoverOrchestratorState:
assert snapshot.status == Snapshot.StatusChoices.SEALED
assert snapshot.fs_version == Snapshot._fs_current_version()
assert snapshot.output_dir.joinpath("index.html").read_text(encoding="utf-8") == "legacy archive"
assert legacy_dir.is_symlink()
assert not legacy_dir.exists()
@pytest.mark.django_db(transaction=True)
def test_run_due_snapshot_runs_queued_plugin_after_fs_migration(self):

93
archivebox/tests/test_cli_server.py
View File

@ -206,6 +206,99 @@ def test_sqlite_connections_use_explicit_busy_timeout():
assert "PRAGMA journal_mode = WAL;" in SQLITE_CONNECTION_OPTIONS["OPTIONS"]["init_command"]
def test_docker_sqlite_never_uses_wal_on_host_shared_collection(tmp_path):
"""Docker collections must not expose a WAL database through /data.
Docker Desktop/OrbStack bind mounts cross the Linux VM/host locking
boundary. A container WAL writer plus a host-side sqlite reader can make
the writer SIGBUS and leave index.sqlite3 malformed; the reader does not
need to write. Run a fresh real settings process with Docker identity so a
host test cannot accidentally pass by reusing this process's non-Docker
config imports.
"""
env = os.environ.copy()
env["IN_DOCKER"] = "True"
env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
result = subprocess.run(
[
sys.executable,
"-c",
(
"import django;"
"django.setup();"
"from django.db import connection;"
"cursor=connection.cursor();"
"cursor.execute('CREATE TABLE journal_probe (value INTEGER)');"
"cursor.execute('INSERT INTO journal_probe VALUES (1)');"
"cursor.execute('PRAGMA journal_mode');"
"print(cursor.fetchone()[0]);"
"cursor.close();"
"connection.close()"
),
],
cwd=tmp_path,
env=env,
capture_output=True,
text=True,
check=True,
)
assert result.stdout.strip().lower() == "delete"
assert not tmp_path.joinpath("index.sqlite3-wal").exists()
assert not tmp_path.joinpath("index.sqlite3-shm").exists()
def test_docker_rejects_explicit_wal_override(tmp_path):
"""An old config or environment override must not bypass the invariant."""
env = os.environ.copy()
env["IN_DOCKER"] = "True"
result = subprocess.run(
[
sys.executable,
"-c",
("from archivebox.config.common import DatabaseConfig;DatabaseConfig(SQLITE_JOURNAL_MODE='WAL')"),
],
cwd=tmp_path,
env=env,
capture_output=True,
text=True,
)
assert result.returncode != 0
assert "WAL is unsafe for Docker collections" in result.stderr
def test_docker_postgres_ignores_irrelevant_sqlite_wal_override(tmp_path):
"""The Docker SQLite safety invariant must not reject PostgreSQL.
Operators can switch an existing deployment to PostgreSQL while an old
SQLITE_JOURNAL_MODE setting remains in ArchiveBox.conf or the environment.
PostgreSQL never consumes that SQLite pragma, so rejecting the otherwise
valid configuration would prevent ArchiveBox from starting without making
any database safer.
"""
env = os.environ.copy()
env["IN_DOCKER"] = "True"
result = subprocess.run(
[
sys.executable,
"-c",
(
"from archivebox.config.common import DatabaseConfig;"
"config=DatabaseConfig(DATABASE_ENGINE='postgres',SQLITE_JOURNAL_MODE='WAL');"
"print(config.DATABASE_ENGINE)"
),
],
cwd=tmp_path,
env=env,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
assert result.stdout.strip() == "postgres"
def test_server_shows_usage_info(initialized_archive):
"""Test that server command shows usage or starts."""

12
archivebox/tests/test_cli_update_reindex_snapshots.py
View File

@ -1,6 +1,8 @@
import json
import os
from datetime import datetime, timedelta
from pathlib import Path
from archivebox.tests.conftest import cli_env, run_archivebox_cmd
import pytest
@ -120,12 +122,12 @@ def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive):
assert update_process.returncode == 0, update_process.stderr
with use_archivebox_db(tmp_path):
row = Snapshot.objects.values_list("url", "fs_version").get()
migrated_snapshot = Snapshot.objects.get()
row = (migrated_snapshot.url, migrated_snapshot.fs_version)
migrated_dir = Path(migrated_snapshot.output_dir)
assert row == ("https://example.com", Snapshot._fs_current_version())
assert legacy_dir.is_symlink()
migrated_dir = legacy_dir.resolve()
assert not legacy_dir.exists()
assert migrated_dir.exists()
assert '{"type":"Process","id":"incomplete"}\n' in (migrated_dir / "index.jsonl").read_text()
assert (migrated_dir / "singlefile.html").exists()
@ -201,7 +203,7 @@ def test_update_migrates_every_declared_filesystem_version(tmp_path, initialized
assert {path: migrated_tree.get(path) for path in original_tree} == original_tree
if legacy_layout:
assert source_dir.is_symlink()
assert not source_dir.exists()
update_process = run_archivebox_cmd(["update", "--migrate-only"], env=env, timeout=90)
assert update_process.returncode == 0, f"Idempotency update failed: {update_process.stderr}"

14
archivebox/tests/test_crawl_runner.py
View File

@ -11,6 +11,20 @@ from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary
pytestmark = pytest.mark.django_db
@pytest.mark.django_db(transaction=True)
def test_crawl_runner_creates_progress_reporter_without_a_tty(crawl):
from archivebox.services.runner import CrawlRunner
runner = CrawlRunner(crawl)
runner.load_run_state()
live_ui = runner._create_live_ui()
assert live_ui is not None
assert live_ui.interactive_tty is False
asyncio.run(runner.bus.destroy(clear=False))
@pytest.mark.django_db(transaction=True)
def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus():
from archivebox.base_models.models import get_or_create_system_user_pk

5
archivebox/tests/test_migrations_04_to_09.py
View File

@ -6,6 +6,7 @@ import sqlite3
from .migrations_helpers import (
SCHEMA_0_4,
create_data_dir_structure,
current_snapshot_dir,
filesystem_manifest,
run_archivebox_migration_cmd,
seed_0_4_data,
@ -94,8 +95,8 @@ def test_oldest_django_collection_migrates_end_to_end_without_data_loss(tmp_path
for timestamp, expected_tree in original_trees.items():
legacy_dir = tmp_path / "archive" / timestamp
assert legacy_dir.is_symlink()
migrated_tree = filesystem_manifest(legacy_dir.resolve())
assert not legacy_dir.exists()
migrated_tree = filesystem_manifest(current_snapshot_dir(tmp_path, db_path, timestamp))
assert {path: migrated_tree.get(path) for path in expected_tree} == expected_tree
with sqlite3.connect(db_path) as connection:

16
archivebox/tests/test_migrations_08_to_09.py
View File

@ -21,6 +21,7 @@ import pytest
from .migrations_helpers import (
SCHEMA_0_7,
SCHEMA_0_8,
current_snapshot_dir,
filesystem_manifest,
seed_0_8_data,
seed_0_7_data,
@ -800,7 +801,7 @@ def test_update_preserves_legacy_folder_timestamp_over_index_float_variant(tmp_p
conn.close()
assert row == (timestamp,)
assert (work_dir / "archive" / timestamp).is_symlink()
assert not (work_dir / "archive" / timestamp).exists()
assert not (work_dir / "archive" / f"{timestamp}.0").exists()
assert not (work_dir / "invalid").exists()
@ -842,8 +843,8 @@ def test_update_preserves_distinct_legacy_dirs_with_integer_and_float_timestamps
conn.close()
assert rows == [("1508259732",), ("1508259732.0",)]
assert (work_dir / "archive" / "1508259732").is_symlink()
assert (work_dir / "archive" / "1508259732.0").is_symlink()
assert not (work_dir / "archive" / "1508259732").exists()
assert not (work_dir / "archive" / "1508259732.0").exists()
assert not (work_dir / "invalid").exists()
@ -991,8 +992,8 @@ def test_07_filesystem_hop_preserves_complete_output_tree(tmp_path):
for timestamp, expected_tree in original_trees.items():
legacy_dir = tmp_path / "archive" / timestamp
assert legacy_dir.is_symlink()
migrated_tree = filesystem_manifest(legacy_dir.resolve())
assert not legacy_dir.exists()
migrated_tree = filesystem_manifest(current_snapshot_dir(tmp_path, db_path, timestamp))
assert {path: migrated_tree.get(path) for path in expected_tree} == expected_tree
assert (destination / "preexisting-output.bin").read_bytes() == b"destination-only output"
@ -1065,7 +1066,10 @@ def test_each_declared_filesystem_hop_preserves_outputs(migration_08_data, fs_ve
result = run_archivebox_migration_cmd(work_dir, ["update", "--migrate-only"], timeout=180)
assert result.returncode == 0, result.stderr
migrated_dir = source_dir.resolve()
migrated_dir = current_snapshot_dir(work_dir, db_path, snapshot["timestamp"])
assert {path: filesystem_manifest(migrated_dir).get(path) for path in expected_tree} == expected_tree
if fs_version in ("0.7.0", "0.8.0", "0.8.5"):
assert not source_dir.exists()
assert not source_dir.is_symlink()
with sqlite3.connect(db_path) as connection:
assert connection.execute("SELECT fs_version FROM core_snapshot WHERE id = ?", (snapshot["id"],)).fetchone() == ("0.9.4",)

42
archivebox/tests/test_setup_script.py
View File

@ -5,27 +5,37 @@ from pathlib import Path
SETUP_SCRIPT = Path(__file__).parents[2] / "bin" / "setup.sh"
def test_setup_script_is_a_uv_install_shortcut():
def test_setup_script_initializes_then_links_container_dependencies():
script = SETUP_SCRIPT.read_text()
install_flow = script.partition("cat <<EOF")[0]
compose_flow = script.partition('if [ "$DOCKER_IMAGE_READY" = "true" ] && "$DOCKER_BINARY" compose version')[2].partition(
'elif [ "$DOCKER_IMAGE_READY" = "true" ]',
)[0]
docker_flow = script.partition('elif [ "$DOCKER_IMAGE_READY" = "true" ]')[2].partition("\nfi")[0]
docker_init = script.partition("docker_run_archivebox_init() {")[2].partition("\n}")[0]
# Docker images already contain every runtime dependency. Initialization
# must only create collection state; the following install command merely
# projects the preloaded image cache into that collection.
assert "docker_run_archivebox init" in docker_init
assert "--install" not in docker_init
assert compose_flow.index("docker_compose_run_archivebox init") < compose_flow.index("docker_compose_run_archivebox install")
assert docker_flow.index("docker_run_archivebox_init") < docker_flow.index("docker_run_archivebox_install")
assert "init --install" not in script
def test_setup_script_keeps_uv_as_the_native_fallback():
script = SETUP_SCRIPT.read_text()
native_flow = script.partition("install_archivebox_with_uv\n")[2]
assert 'ARCHIVEBOX_PYTHON="${ARCHIVEBOX_PYTHON:-3.13}"' in script
assert 'ARCHIVEBOX_PACKAGE="${ARCHIVEBOX_PACKAGE:-archivebox>=0.9.0rc0,<0.10}"' in script
assert '"$UV_BINARY" tool install --python "$ARCHIVEBOX_PYTHON" --prerelease explicit --upgrade "$ARCHIVEBOX_PACKAGE"' in install_flow
assert (
'run_as_archivebox_user "$UV_BINARY" --no-config tool install --python "$ARCHIVEBOX_PYTHON" '
'--prerelease explicit --upgrade "$ARCHIVEBOX_PACKAGE"'
) in script
assert "https://astral.sh/uv/install.sh" in script
assert "archivebox init" not in install_flow
assert "archivebox install" not in install_flow
def test_setup_script_does_not_select_a_system_installer_or_runtime():
script = SETUP_SCRIPT.read_text()
assert "docker" not in script.lower()
assert "debian-archivebox" not in script
assert "apt-get" not in script
assert "launchpad" not in script
assert "archivebox server" not in script
assert "useradd" not in script
assert ': | "$ARCHIVEBOX_BINARY" init' in native_flow
assert native_flow.index(': | "$ARCHIVEBOX_BINARY" init') < native_flow.index('"$ARCHIVEBOX_BINARY" install')
def test_setup_script_has_valid_bash_syntax():

38
archivebox/tests/test_takeover_util.py
View File

@ -416,7 +416,12 @@ def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tm
encoding="utf-8",
errors="replace",
)
assert "Stopping older ArchiveBox runner process" in update_stdout
worker_name_match = re.search(r"Worker (worker_runner_update_\d+):", update_stdout)
assert worker_name_match, update_stdout
wait_for_log(
tmp_path / "logs" / f"{worker_name_match.group(1)}.log",
"Stopping older ArchiveBox runner process",
)
supervisord_text = wait_for_log_count(supervisord_log, runner_spawn_text, runner_spawn_count + 1, timeout=30)
runner_pid_after = int(re.findall(r"spawned: 'worker_runner' with pid (\d+)", supervisord_text)[-1])
@ -955,3 +960,34 @@ def test_runtime_stack_owner_allows_top_level_runner_when_no_parent_command_exis
assert owner.id == runner_row.id
finally:
_stop_archivebox_shells([proc])
def test_foreign_machine_runner_only_warns(tmp_path, initialized_archive, capsys):
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, live_runner_processes
from archivebox.machine.models import Machine
foreign_machine = Machine.objects.create(
guid="foreign-machine",
hostname="foreign-host",
hw_manufacturer="Test",
hw_product="Test",
hw_uuid="foreign-hardware",
os_arch="x86_64",
os_family="linux",
os_platform="linux",
os_release="test",
os_kernel="test",
)
foreign_runner = Process.objects.create(
machine=foreign_machine,
process_type=Process.TypeChoices.ORCHESTRATOR,
worker_type=RUNNER_ACTIVE_WORKER_TYPE,
pwd=str(tmp_path),
pid=1,
status=Process.StatusChoices.RUNNING,
)
assert live_runner_processes(data_dir=tmp_path) == []
foreign_runner.refresh_from_db()
assert foreign_runner.status == Process.StatusChoices.RUNNING
assert "Multiple orchestrators sharing a single collection is not officially supported" in capsys.readouterr().err

138
archivebox/tests/test_ui_admin_snapshot.py
View File

@ -1,6 +1,8 @@
"""Snapshot model and admin UI tests."""
import json
import os
import re
import shutil
import warnings
from pathlib import Path
@ -14,6 +16,7 @@ from django.core.paginator import UnorderedObjectListWarning
from django.test import RequestFactory
from django.urls import reverse
from archivebox.core.middleware import ADMIN_LOGIN_HINT_COOKIE
from archivebox.tests.conftest import ADMIN_TEST_HOST
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook, _snapshot_hook_name
@ -21,6 +24,16 @@ pytestmark = pytest.mark.django_db(transaction=True)
REPO_ROOT = Path(__file__).resolve().parents[2]
def test_current_snapshot_layout_has_no_top_level_timestamp_projection(snapshot):
from archivebox.config import CONSTANTS
legacy_path = CONSTANTS.ARCHIVE_DIR / snapshot.timestamp
assert Path(snapshot.output_dir).is_relative_to(CONSTANTS.ARCHIVE_DIR / "users")
assert not legacy_path.exists()
assert not legacy_path.is_symlink()
@pytest.fixture
def real_hash_projection(snapshot, cached_abxpkg_lib_dir):
snapshot.output_dir.mkdir(parents=True, exist_ok=True)
@ -772,7 +785,77 @@ class TestSnapshotProgressStats:
assert "wrapper.style.height = `${contentHeight}px`" in rendered
assert 'class="header-toggle header-toggle-trigger"' not in rendered
assert "event.preventDefault()" in rendered
assert rendered.count(".on('click', handleSnapshotHeaderToggle)") == 1
assert rendered.count("addEventListener('click', handleSnapshotHeaderToggle)") == 1
def test_static_snapshot_detail_uses_same_output_cards_with_relative_files(self, snapshot):
from archivebox.config import CONSTANTS
from archivebox.core.models import ArchiveResult
from archivebox.core.views import SnapshotView
output_dir = Path(snapshot.output_dir)
singlefile_dir = output_dir / "singlefile"
singlefile_dir.mkdir(parents=True, exist_ok=True)
output_file = singlefile_dir / "singlefile.html"
output_file.write_text("<html><body>real static output</body></html>", encoding="utf-8")
ArchiveResult.objects.create(
snapshot=snapshot,
plugin="singlefile",
hook_name="on_Snapshot__50_singlefile.py",
status=ArchiveResult.StatusChoices.SUCCEEDED,
output_str="singlefile.html",
output_files={"singlefile.html": {"size": output_file.stat().st_size}},
output_size=output_file.stat().st_size,
)
favicon_dir = output_dir / "favicon"
favicon_dir.mkdir(parents=True, exist_ok=True)
favicon_file = favicon_dir / "favicon.ico"
favicon_file.write_bytes(b"real favicon")
ArchiveResult.objects.create(
snapshot=snapshot,
plugin="favicon",
hook_name="on_Snapshot__50_favicon.py",
status=ArchiveResult.StatusChoices.SUCCEEDED,
output_str="favicon.ico",
output_files={"favicon.ico": {"size": favicon_file.stat().st_size}},
output_size=favicon_file.stat().st_size,
)
request = RequestFactory().get(f"/{snapshot.url_path}/index.html", HTTP_HOST=ADMIN_TEST_HOST)
request.user = AnonymousUser()
live_html = SnapshotView.render_live_index(request, snapshot).content.decode()
snapshot.write_html_details()
snapshot.write_json_details()
static_html = (output_dir / "index.html").read_text(encoding="utf-8")
static_json = json.loads((output_dir / "index.json").read_text(encoding="utf-8"))
assert re.findall(r'data-plugin-name="([^"]+)"', static_html) == re.findall(r'data-plugin-name="([^"]+)"', live_html)
assert 'data-plugin-name="singlefile"' in static_html
assert 'href="./singlefile/singlefile.html"' in static_html
assert 'data-default-src="./singlefile/singlefile.html"' in static_html
assert 'src="./favicon/favicon.ico"' in static_html
root_href = os.path.relpath(CONSTANTS.DATA_DIR, start=output_dir).replace(os.sep, "/")
assert f'href="{root_href}/index.html" class="header-archivebox"' in static_html
assert f"/snapshot/{snapshot.id.hex}" not in static_html
assert "/static/jquery.min.js" not in static_html
assert static_json["archive_path"].startswith("archive/users/")
assert static_json["archive_url"] == f"./{static_json['archive_path']}/index.html"
def test_compact_output_cards_pack_into_dense_grid_rows(self):
template = (REPO_ROOT / "archivebox" / "templates" / "core" / "snapshot.html").read_text()
thumb_grid_css = template.split(".thumb-grid {", 1)[1].split("}", 1)[0]
thumb_card_css = template.split(".thumb-card {", 1)[1].split("}", 1)[0]
auxiliary_card_css = template.split(".thumb-card:not([data-plugin-name]) {", 1)[1].split("}", 1)[0]
compact_card_css = template.split(".thumb-card:has([data-compact]) {", 1)[1].split("}", 1)[0]
assert "display: grid;" in thumb_grid_css
assert "grid-template-columns: repeat(auto-fit, minmax(clamp(180px, 14vw, 250px), 1fr));" in thumb_grid_css
assert "grid-auto-flow: row dense;" in thumb_grid_css
assert "grid-auto-rows: 42px;" in thumb_grid_css
assert "grid-row: span 3;" in thumb_card_css
assert "order: 1;" in auxiliary_card_css
assert "grid-row: span 1;" in compact_card_css
assert "order: 2;" in compact_card_css
class TestSnapshotOutputDeletion:
@ -805,11 +888,13 @@ class TestSnapshotOutputDeletion:
assert f'data-archive-result-ids="{result.id}"' in html
assert 'title="Delete this output"' in html
assert 'data-delete-handoff="1"' in html
assert "const queuedOutputIds = new Set()" in html
assert "action: 'delete_selected'" in html
assert "window.location.assign(deleteUrl)" in html
assert "/admin/core/archiveresult/" in html
assert "[deleting]" in html
assert ">×</button>" in html
assert "delete-output-csrf" not in html
assert "[data-archive-result-ids]:hover" in html
assert "[data-archive-result-ids].delete-pending" in html
assert "button.classList.toggle('delete-pending', queued)" in html
@ -819,6 +904,55 @@ class TestSnapshotOutputDeletion:
assert "data-archive-result-ids" not in anonymous_html
assert 'title="Delete this output"' not in anonymous_html
request.COOKIES[ADMIN_LOGIN_HINT_COOKIE] = "1"
hinted_html = SnapshotView.render_live_index(request, snapshot).content.decode()
assert f'data-archive-result-ids="{result.id}"' in hinted_html
assert "delete-output-csrf" not in hinted_html
def test_snapshot_delete_handoff_requires_superuser_confirmation_then_uses_standard_admin_action(self, client, snapshot, admin_user):
from archivebox.core.models import ArchiveResult
result = self._create_output(snapshot)
delete_url = reverse("admin:core_archiveresult_changelist")
handoff_query = {
"action": "delete_selected",
ACTION_CHECKBOX_NAME: str(result.id),
"snapshot": str(snapshot.id),
}
logged_out = client.get(delete_url, handoff_query, HTTP_HOST=ADMIN_TEST_HOST)
assert logged_out.status_code == 302
assert ArchiveResult.objects.filter(pk=result.pk).exists()
staff_user = admin_user.__class__.objects.create_user(username="output-reviewer", password="testpassword", is_staff=True)
client.force_login(staff_user)
denied = client.get(delete_url, handoff_query, HTTP_HOST=ADMIN_TEST_HOST)
assert denied.status_code == 403
assert ArchiveResult.objects.filter(pk=result.pk).exists()
client.force_login(admin_user)
confirmation = client.get(delete_url, handoff_query, HTTP_HOST=ADMIN_TEST_HOST)
confirmation_html = confirmation.content.decode()
assert confirmation.status_code == 200
assert "Yes, I’m sure" in confirmation_html
assert f'name="{ACTION_CHECKBOX_NAME}" value="{result.id}"' in confirmation_html
assert confirmation["X-Frame-Options"] == "DENY"
assert "frame-ancestors 'none'" in confirmation["Content-Security-Policy"]
assert ArchiveResult.objects.filter(pk=result.pk).exists()
confirmed = client.post(
f"{delete_url}?action=delete_selected&{ACTION_CHECKBOX_NAME}={result.id}&snapshot={snapshot.id}",
{
"action": "delete_selected",
"post": "yes",
ACTION_CHECKBOX_NAME: str(result.id),
},
HTTP_HOST=ADMIN_TEST_HOST,
)
assert confirmed.status_code == 302
assert not ArchiveResult.objects.filter(pk=result.pk).exists()
assert str(snapshot.id).replace("-", "")[-12:] in confirmed["Location"]
def test_batch_delete_removes_plugin_rows_files_and_refreshes_snapshot_size(self, client, snapshot, admin_user):
from archivebox.core.models import ArchiveResult

95
archivebox/tests/test_urls.py
View File

@ -389,6 +389,90 @@ class TestUrlRouting:
assert (installed[0] / "sw.js").is_file()
return installed[0]
@pytest.mark.parametrize(
"mode",
["auto", "safe-subdomains-fullreplay", "safe-onedomain-nojsreplay", "unsafe-onedomain-noadmin", "danger-onedomain-fullreplay"],
)
def test_snapshot_output_delete_handoff_is_non_mutating_in_every_security_mode(self, mode: str) -> None:
self._run(
"""
ensure_admin_user()
snapshot = get_snapshot()
snapshot.config = {**snapshot.config, "PERMISSIONS": "public"}
snapshot.save(update_fields=["config"])
plugin = "security_delete_test"
output_path = Path(snapshot.output_dir) / plugin / "output.txt"
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text("security test", encoding="utf-8")
result, _ = ArchiveResult.objects.update_or_create(
snapshot=snapshot,
plugin=plugin,
defaults={
"hook_name": "on_Snapshot__99_security_delete_test.py",
"status": ArchiveResult.StatusChoices.SUCCEEDED,
"output_str": "output.txt",
"output_files": {"output.txt": {"size": 13, "mimetype": "text/plain"}},
"output_size": 13,
},
)
client = Client(enforce_csrf_checks=True)
assert client.login(username="testadmin", password="testpassword")
admin_host = get_admin_host()
admin_page = client.get("/admin/", HTTP_HOST=admin_host)
if SERVER_CONFIG.CONTROL_PLANE_ENABLED:
assert admin_page.status_code == 200
else:
assert admin_page.status_code == 403
assert admin_page["X-Frame-Options"] == "DENY"
assert "frame-ancestors 'none'" in admin_page["Content-Security-Policy"]
snapshot_host = get_snapshot_host(str(snapshot.id)) if SERVER_CONFIG.USES_SUBDOMAIN_ROUTING else get_base_host()
snapshot_path = "/index.html" if SERVER_CONFIG.USES_SUBDOMAIN_ROUTING else f"/snapshot/{snapshot.id}/index.html"
detail = client.get(snapshot_path, HTTP_HOST=snapshot_host)
html = response_body(detail).decode("utf-8", "ignore")
assert detail.status_code == 200, (detail.status_code, detail.headers.get("Location"), html[:200])
assert "delete-output-csrf" not in html
if SERVER_CONFIG.CONTROL_PLANE_ENABLED:
assert f'data-archive-result-ids="{result.id}"' in html, html[-1000:]
assert 'data-delete-handoff="1"' in html, html[-1000:]
else:
assert f'data-archive-result-ids="{result.id}"' not in html
rejected = client.post(
"/admin/core/archiveresult/",
data={"action": "delete_selected", "post": "yes", "_selected_action": str(result.id)},
HTTP_HOST=snapshot_host,
)
assert rejected.status_code == 403, (rejected.status_code, rejected.headers.get("Location"), response_body(rejected)[:200])
assert ArchiveResult.objects.filter(pk=result.pk).exists()
result.delete()
print("OK")
""",
mode=mode,
)
def test_cross_domain_admin_hint_only_marks_superusers(self) -> None:
self._run(
"""
User = get_user_model()
staff = User.objects.create_user(username="staff-hint-test", password="testpassword", is_staff=True)
client = Client()
client.force_login(staff)
response = client.get("/admin/", HTTP_HOST=get_admin_host())
assert response.status_code == 200
assert client.cookies.get(ADMIN_LOGIN_HINT_COOKIE) is None or client.cookies[ADMIN_LOGIN_HINT_COOKIE].value != "1"
client.force_login(ensure_admin_user())
response = client.get("/admin/", HTTP_HOST=get_admin_host())
assert response.status_code == 200
assert client.cookies[ADMIN_LOGIN_HINT_COOKIE].value == "1"
print("OK")
""",
mode="safe-subdomains-fullreplay",
)
def test_routes_util_and_web_public_redirect(self) -> None:
self._run(
"""
@ -1286,8 +1370,15 @@ class TestUrlRouting:
assert ">Git<" not in live_html
static_html = Path(snapshot.output_dir, "index.html").read_text(encoding="utf-8", errors="ignore")
assert f"http://{snapshot_host}/" in static_html
assert f"http://{web_host}/static/archive.png" in static_html
assert f"http://{snapshot_host}/" not in static_html
assert f"http://{web_host}/static/archive.png" not in static_html
# Static pages are opened directly from disk or a plain HTTP server,
# where Django's live-only ?files=1 directory browser does not exist.
# Even hidden controls and JavaScript fallbacks must therefore use
# portable files, or an offline click can silently navigate nowhere.
assert "?files=1" not in static_html
assert "data:image/svg+xml" in static_html
assert 'href="./' in static_html
assert "?preview=1" in static_html
assert "function createMainFrame(previousFrame)" in static_html
assert "function activateCardPreview(card, link, updateHash=true)" in static_html

15
archivebox/workers/supervisord_util.py
View File

@ -275,6 +275,10 @@ def RUNNER_WORKER():
"command": _shell_join(archivebox_cmd("run", "--daemon")),
"autostart": "false",
"autorestart": "true",
# Mark the long-lived runner child so its own SIGINT/SIGTERM path exits
# with a signal code instead of running foreground server cleanup. That
# keeps "kill just archivebox run --daemon" as a worker restart event;
# only killing the parent server or supervisord should stop the stack.
"environment": 'PYTHONUNBUFFERED="1",COLUMNS="200",ARCHIVEBOX_RUNNER_DAEMON="1"',
"stopasgroup": "true",
"killasgroup": "true",
@ -288,6 +292,9 @@ RUNNER_ONCE_WORKER = lambda args, name="worker_runner_once": {
**RUNNER_WORKER(),
"name": name,
"command": _shell_join(archivebox_cmd("run", "--no-stdin", *args)),
# One-shot foreground jobs are awaited by the command that launched them,
# so they keep the normal cooperative shutdown path instead of the daemon
# marker that tells supervisord to restart an independently killed worker.
"environment": 'PYTHONUNBUFFERED="1",COLUMNS="200"',
"autorestart": "false",
"stopwaitsecs": "1",
@ -1067,8 +1074,8 @@ def run_runner_worker(
line = log_handle.readline()
if not line:
break
sys.stdout.write(line)
sys.stdout.flush()
sys.stderr.write(line)
sys.stderr.flush()
proc = get_worker(supervisor, name)
if proc is None:
return 1
@ -1077,8 +1084,8 @@ def run_runner_worker(
line = log_handle.readline()
if not line:
break
sys.stdout.write(line)
sys.stdout.flush()
sys.stderr.write(line)
sys.stderr.flush()
if proc["statename"] in {"EXITED", "STOPPED"}:
return int(proc.get("exitstatus") or 0)
return 1

22
bin/collect_ui_screenshots.sh
View File

@ -18,6 +18,8 @@ ARCHIVE_PID=""
CREATED_TEMP_USER=0
CREATE_API_TOKEN=0
CREATE_WEBHOOK=0
ABXPKG_LIB_DIR=""
SCREENSHOT_CHROME_BINARY=""
CAPTURE_ROOT="$(mktemp -d)"
MANIFEST_FILE="$CAPTURE_ROOT/manifest.jsonl"
PERSONAS_DIR="$CAPTURE_ROOT/personas"
@ -203,6 +205,13 @@ fi
# the Sweeting.me example below runs through its own real foreground runner.
stop_background_runner
ABXPKG_LIB_DIR="$(uv run --no-cache --project "$REPO_DIR" abx-dl config --get ABXPKG_LIB_DIR | sed 's/^[^=]*=//; s/^"//; s/"$//')"
SCREENSHOT_CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
if [[ ! -x "$SCREENSHOT_CHROME_BINARY" ]]; then
echo "[!] abx-dl projected Chromium was not found at $SCREENSHOT_CHROME_BINARY" >&2
exit 1
fi
VIEWS=(
"Login|$ADMIN_BASE_URL/admin/login/|/admin/login/|archivebox/templates/admin/login.html"
"Public snapshot list|$PUBLIC_BASE_URL/public/|/public/|archivebox/core/views.py"
@ -225,9 +234,11 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
SCREENSHOT_HEIGHT=1000 \
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$CAPTURE_ROOT/snapshot-header-state.png" >/dev/null
fi
view_timing_report=""
while IFS='|' read -r profile viewport_width viewport_height; do
filename="$(printf '%02d' "$capture_index")-$slug-$profile.png"
capture_dir="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/$profile"
timing_report_path="$view_timing_report"
capture_env=(
"RESOLUTION=$viewport_width,$viewport_height"
"CHROME_RESOLUTION=$viewport_width,$viewport_height"
@ -272,6 +283,8 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
SCREENSHOT_SNAPSHOT_HEADER=expanded \
SCREENSHOT_EXPECT_LIVE_PROGRESS=1 \
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
view_timing_report="$capture_dir/report.json"
timing_report_path="$view_timing_report"
fi
elif [[ "$profile" == "desktop" || "$capture_mode" == wait-replay:* || -z "${ABXPKG_LIB_DIR:-}" ]]; then
capture_log="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")-$profile-abx-dl.log"
@ -311,6 +324,8 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
SCREENSHOT_VARIANTS_JSON="$responsive_variants" \
SCREENSHOT_COLLAPSE_FILTERS=1 \
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
view_timing_report="$capture_dir/report.json"
timing_report_path="$view_timing_report"
uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" validate \
"$capture_dir/report.json" "$expected_path"
fi
@ -324,6 +339,7 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
cp "$screenshot_path" "$PUBLIC_OUTPUT_DIR/$filename"
UI_SCREENSHOT_NAME="$name" UI_SCREENSHOT_URL="$url" UI_SCREENSHOT_SOURCE="$source" \
UI_SCREENSHOT_FILENAME="$filename" UI_SCREENSHOT_PROFILE="$profile" \
UI_SCREENSHOT_TIMING_REPORT="$timing_report_path" \
uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" append \
"$MANIFEST_FILE" "$OUTPUT_DIR/$filename"
done <<<"$CAPTURE_PROFILES"
@ -331,12 +347,6 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
# Capture the public views before login because the real admin-login hint
# intentionally redirects authenticated personas away from /public/.
if [[ "$capture_index" == "2" ]]; then
ABXPKG_LIB_DIR="$(uv run --no-cache --project "$REPO_DIR" abx-dl config --get ABXPKG_LIB_DIR | sed 's/^[^=]*=//; s/^"//; s/"$//')"
SCREENSHOT_CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
if [[ ! -x "$SCREENSHOT_CHROME_BINARY" ]]; then
echo "[!] abx-dl projected Chromium was not found at $SCREENSHOT_CHROME_BINARY" >&2
exit 1
fi
echo "[*] Logging in through the real $ACTIVE_PERSONA browser persona"
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \

63
bin/generate_ui_screenshot_gallery.py
View File

@ -4,12 +4,15 @@ import hashlib
import html
import json
import os
import subprocess
import struct
import sys
import tomllib
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlparse
SOURCE_BASE_URL = "https://github.com/ArchiveBox/ArchiveBox/blob/dev/"
REPO_DIR = Path(__file__).resolve().parents[1]
CAPTURE_PROFILES = {
"desktop": (1600, 1000),
"tablet": (1024, 1366),
@ -55,6 +58,24 @@ REQUIRED_VIEW_NAMES = {
}
def build_provenance() -> dict[str, str]:
version = tomllib.loads((REPO_DIR / "pyproject.toml").read_text(encoding="utf-8"))["project"]["version"]
revision = os.environ.get("GITHUB_SHA", "").strip()
if not revision:
revision = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=REPO_DIR,
check=True,
capture_output=True,
text=True,
).stdout.strip()
return {
"version": version,
"revision": revision,
"generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
}
def validate_navigation(metadata_path: Path, expected_path: str) -> None:
navigation = json.loads(metadata_path.read_text(encoding="utf-8"))
navigation = navigation.get("checks", navigation)
@ -91,11 +112,19 @@ def append_manifest(manifest_path: Path, screenshot_path: Path) -> None:
"width": dimensions[0],
"height": dimensions[1],
}
timing_report_path = os.environ.get("UI_SCREENSHOT_TIMING_REPORT", "").strip()
if timing_report_path:
timing_report = json.loads(Path(timing_report_path).read_text(encoding="utf-8"))
ttfb_ms = timing_report.get("checks", {}).get("ttfbMs")
if isinstance(ttfb_ms, int | float):
item["ttfb_ms"] = round(ttfb_ms)
with manifest_path.open("a", encoding="utf-8") as manifest:
manifest.write(json.dumps(item) + "\n")
def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -> None:
provenance = build_provenance()
source_base_url = f"https://github.com/ArchiveBox/ArchiveBox/blob/{provenance['revision']}/"
captures = [json.loads(line) for line in manifest_path.read_text(encoding="utf-8").splitlines() if line]
allow_partial = os.environ.get("UI_SCREENSHOT_ALLOW_PARTIAL") == "1"
grouped_captures: list[dict[str, object]] = []
@ -155,7 +184,9 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
route = parsed_url.path or "/"
if parsed_url.fragment:
route = f"{route}#{parsed_url.fragment}"
source_url = f"{SOURCE_BASE_URL}{capture['source']}"
source_url = f"{source_base_url}{capture['source']}"
ttfb_values = [variant["ttfb_ms"] for variant in variants.values() if isinstance(variant.get("ttfb_ms"), int | float)]
ttfb_text = f" · ~{round(sum(ttfb_values) / len(ttfb_values))}ms TTFB" if ttfb_values else ""
markdown_cells = []
html_figures = []
for profile, (width, height) in CAPTURE_PROFILES.items():
@ -183,7 +214,7 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
(
f"## {capture['name']}",
"",
f"View: [`{route}`]({capture['url']}) · [View code]({source_url})",
f"View: [`{route}`]({capture['url']}) · [View code]({source_url}){ttfb_text}",
"",
"<table><thead><tr>",
"".join(f"<th>{profile.title()}</th>" for profile in CAPTURE_PROFILES),
@ -196,7 +227,7 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
html_sections.append(
f"<article><h2>{html.escape(capture['name'])}</h2>"
f'<p><a href="{html.escape(capture["url"])}"><code>{html.escape(route)}</code></a> · '
f'<a href="{html.escape(source_url)}">View code</a></p>'
f'<a href="{html.escape(source_url)}">View code</a>{html.escape(ttfb_text)}</p>'
f'<div class="shots">{"".join(html_figures)}</div></article>',
)
@ -211,6 +242,7 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
"These desktop, tablet, and mobile screenshots cover ArchiveBox's major public and authenticated UI views. "
"Raw API endpoints, API documentation, health-check, and error routes are intentionally excluded."
),
f"Generated from ArchiveBox `{provenance['version']}` at revision `{provenance['revision']}`.",
"",
"\n\n".join(markdown_sections),
"",
@ -229,12 +261,33 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
"border:1px solid #cbd5e1;border-radius:8px;background:white;box-shadow:0 8px 24px #0f172a18}code{overflow-wrap:anywhere}"
"@media(max-width:900px){.shots{grid-template-columns:1fr}body{padding:12px}}"
'</style></head><body><header><p><a href="../">← ArchiveBox</a></p><h1>ArchiveBox UI Screenshots</h1>'
"<p>Generated from the current <code>dev</code> UI at desktop, tablet, and mobile viewports.</p></header><main>"
f"<p>Generated from ArchiveBox <code>{html.escape(provenance['version'])}</code> at "
f'<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{html.escape(provenance["revision"])}">'
f"<code>{html.escape(provenance['revision'][:12])}</code></a> on "
f'<time datetime="{html.escape(provenance["generated_at"])}">{html.escape(provenance["generated_at"])}</time>. '
"Desktop, tablet, and mobile viewports are captured from the same build.</p></header><main>"
+ "".join(html_sections)
+ "</main></body></html>\n",
encoding="utf-8",
)
file_hashes = {
capture["filename"]: hashlib.sha256((html_path.parent / capture["filename"]).read_bytes()).hexdigest() for capture in captures
}
(html_path.parent / "build.json").write_text(
json.dumps(
{
**provenance,
"capture_count": len(captures),
"files": file_hashes,
},
indent=2,
sort_keys=True,
)
+ "\n",
encoding="utf-8",
)
if not allow_partial:
expected_filenames = {capture["filename"] for capture in captures}
for screenshot_dir in (markdown_path.parent / "screenshots", html_path.parent):

34
bin/take_screenshot.js
View File

@ -116,6 +116,28 @@ async function main() {
await Promise.all(restoredPages.map((restoredPage) => restoredPage.close()));
page.setDefaultTimeout(45000);
const client = await page.createCDPSession();
const documentRequests = new Map();
await client.send('Network.enable');
client.on('Network.requestWillBeSent', (event) => {
if (event.type !== 'Document') return;
documentRequests.set(event.requestId, {
url: event.request.url,
requestTimestamp: event.timestamp,
});
});
client.on('Network.responseReceived', (event) => {
if (event.type !== 'Document') return;
const record = documentRequests.get(event.requestId);
if (!record) return;
record.responseTimestamp = event.timestamp;
record.status = event.response.status;
record.responseUrl = event.response.url;
if (Number.isFinite(record.requestTimestamp) && Number.isFinite(record.responseTimestamp)) {
record.ttfbMs = Math.round((record.responseTimestamp - record.requestTimestamp) * 1000);
}
});
if (process.env.SCREENSHOT_SNAPSHOT_VIEW || process.env.SCREENSHOT_SNAPSHOT_HEADER || process.env.SCREENSHOT_COLLAPSE_FILTERS === '1' || process.env.SCREENSHOT_RESET_FILTERS === '1') {
await page.evaluateOnNewDocument((snapshotView, snapshotHeader, collapseFilters, resetFilters) => {
if (snapshotView) localStorage.setItem('preferred_snapshot_view_mode', snapshotView);
@ -281,6 +303,18 @@ async function main() {
}));
checks.status = navigationResponse ? navigationResponse.status() : null;
checks.finalUrl = page.url();
const navigationUrl = navigationResponse?.url() || page.url();
const matchingDocumentRequests = [...documentRequests.values()]
.filter((record) => record.status === checks.status && Number.isFinite(record.ttfbMs))
.filter((record) => {
const responseUrl = record.responseUrl || record.url || '';
return responseUrl === navigationUrl || responseUrl === checks.finalUrl || responseUrl.split('#')[0] === checks.finalUrl.split('#')[0];
});
const timingRecord = matchingDocumentRequests.at(-1)
|| [...documentRequests.values()].filter((record) => Number.isFinite(record.ttfbMs)).at(-1);
if (timingRecord) {
checks.ttfbMs = timingRecord.ttfbMs;
}
let frameChecks = null;
const embeddedFrameHandle = await page.$('.crawl-snapshots-embed iframe');

6
docs/Configuration.md
View File

@ -717,13 +717,13 @@ With the default [`DATABASE_ENGINE`](#database_engine)`=sqlite`, this is the pat
---
#### `SQLITE_JOURNAL_MODE`
**Possible Values:** [`WAL`]/`DELETE`/`TRUNCATE`/`PERSIST`/`MEMORY`/`OFF`
**Possible Values:** [`WAL` on native installs]/[`DELETE` in Docker]/`TRUNCATE`/`PERSIST`/`MEMORY`/`OFF`
SQLite [journal mode](https://www.sqlite.org/pragma.html#pragma_journal_mode), applied via `PRAGMA journal_mode = ...` on every new connection. Settable as `ARCHIVEBOX_SQLITE_JOURNAL_MODE`.
The default `WAL` (Write-Ahead Logging) lets readers and a single writer operate concurrently without blocking each other — readers see a stable snapshot while a write is in progress, instead of being serialized behind it. This is a substantial win for ArchiveBox, where the web UI, admin, and CLI workers frequently read the index while an extractor is writing.
The default is `WAL` (Write-Ahead Logging) on native installs and `DELETE` in Docker. WAL lets readers and a single writer operate concurrently without blocking each other, but its shared-memory locking is unsafe when a Docker bind mount exposes the same live database to host-side SQLite processes. Docker therefore rejects an explicit WAL override for the SQLite backend; use PostgreSQL for safe cross-runtime concurrency.
> [!WARNING]
> Do not change this unless you have a specific reason. `DELETE` and `TRUNCATE` serialize all readers against any writer (much worse concurrency). `MEMORY` and `OFF` disable durable journaling and can corrupt the database on crash or power loss. `WAL` requires the database to live on a real local filesystem — it does not work correctly over network filesystems like NFS or SMB.
> Do not change this unless you have a specific reason. `DELETE` and `TRUNCATE` serialize readers against writers. `MEMORY` and `OFF` disable durable journaling and can corrupt the database on crash or power loss. WAL requires one local filesystem and one locking domain; it is unsafe across network filesystems and Docker host bind mounts.
---
#### `SQLITE_MMAP_SIZE`

3
docs/Docker.md
View File

@ -192,13 +192,12 @@ Never enable on-demand TLS or request individual certificates for `snap-*` hostn
### Setup
Fetch and run the ArchiveBox Docker image to create your initial archive.
Fetch and run the ArchiveBox Docker image. Starting the server creates the initial archive automatically.
```bash
docker pull archivebox/archivebox:dev
mkdir -p ~/archivebox/data && cd ~/archivebox/data
docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev init
docker run -d --name archivebox -v "$PWD:/data" -p 8000:8000 archivebox/archivebox:dev
```

2
docs/Merging-Collections.md
View File

@ -5,7 +5,7 @@ Current ArchiveBox collections cannot be merged safely by copying their `archive
The workflow below is retained for **legacy collections whose real Snapshot directories are `archive/<timestamp>/`**. `archivebox update` can import those legacy directories into a fresh index.
> [!WARNING]
> Back up every collection before merging. Confirm that the source entries are real legacy timestamp directories, not compatibility symlinks into `archive/users/...`, and inspect path conflicts instead of allowing one collection to overwrite another.
> Back up every collection before merging. Confirm that the source entries are real legacy timestamp directories containing data, and inspect path conflicts instead of allowing one collection to overwrite another.
1. Upgrade both old collections to the most recent ArchiveBox version (following instructions above)
```bash

8
docs/Publishing-Your-Archive.md
View File

@ -56,7 +56,11 @@ location / {
Make sure you're not running any content as CGI or PHP, you only want to serve static files!
Legacy timestamp URLs remain available through compatibility symlinks, for example: `https://demo.archivebox.io/archive/1493350273/wget/en.wikipedia.org/wiki/Dining_philosophers_problem.html`
The generated links are relative, so the export works at a domain root or a project subpath such as GitHub Pages. Snapshot pages and outputs remain under their real filesystem paths, for example:
`archive/users/alice/snapshots/20260829/example.com/SNAPSHOT_UUID/index.html`
ArchiveBox does not create top-level timestamp symlinks for current snapshots. Run `archivebox update --migrate-only` to move real legacy `archive/<timestamp>/` directories into the user-scoped layout and remove obsolete timestamp projections.
<br/>
@ -71,7 +75,7 @@ Legacy timestamp URLs remain available through compatibility symlinks, for examp
Make sure you understand the dangers of [hosting untrusted HTML/JS/CSS](https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy). The default `SERVER_SECURITY_MODE=auto` uses isolated subdomains with full replay on `*.localhost`, and a one-domain no-JS replay policy on ordinary public or LAN hostnames. Choose `safe-subdomains-fullreplay` only when wildcard DNS and TLS for `*.archive.example.com` are configured; it separates the admin, web, and API control planes from replay content and gives each Snapshot its own replay subdomain.
Do not serve ArchiveBox from a shared subdirectory such as `myapps.example.com/archivebox/`; it cannot provide the required origin isolation. If you do not need JavaScript-capable replay, you can also disable the relevant extractors with `WGET_ENABLED=False` and `DOM_ENABLED=False`.
Do not serve the authenticated ArchiveBox application from a shared subdirectory such as `myapps.example.com/archivebox/`; it cannot provide the required origin isolation. A standalone static export can be hosted at a project subpath because it contains no authenticated control plane. If you do not need JavaScript-capable replay, you can also disable the relevant extractors with `WGET_ENABLED=False` and `DOM_ENABLED=False`.
More info:
- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview

2
docs/Roadmap.md
View File

@ -17,7 +17,7 @@
- right now, the paths of the extractor output are scattered all over the codebase, e.g. `output.pdf` (should be moved to constants at the top of the plugin config file)
- make out_dir, link_dir, extractor_dir, naming consistent across codebase
- remove `timestamps` as primary keys in favor of hashes, UUIDs, or some other slug https://github.com/ArchiveBox/ArchiveBox/issues/74
- create a migration system for folder layout independent of the index (`mv` is atomic at the FS level, so we just need a `transaction.atomic(): move(oldpath, newpath); snap.data_dir = newpath; snap.save()`)
- create a migration system for folder layout independent of the index
- make `Tag` a real model `ManyToMany` with Snapshots
- allow multiple Snapshots of the same site over time + CLI / UI to manage those, + migration from old style `#2020-01-01` hack to proper versioned snapshots
- upgrade from Django 3 to Django 5 https://github.com/ArchiveBox/ArchiveBox/issues/988

11
docs/Upgrading.md
View File

@ -18,12 +18,11 @@ archivebox install
archivebox update --migrate-only
archivebox status
# Docker Compose install
# Docker Compose upgrade
cd ~/archivebox
docker compose down
docker compose pull
docker compose run --rm archivebox init
docker compose run --rm archivebox install
docker compose run --rm archivebox update --migrate-only
docker compose up -d
```
@ -35,7 +34,7 @@ docker compose up -d
2. **Read the release notes carefully** for any instructions or extra steps around upgrading for each release you're skipping or installing
3. **Stop any running ArchiveBox server, scheduler, and worker processes**, then back up the entire collection data directory before upgrading. `archivebox config --get ...` and a database-only backup do not include archived outputs.
`cd ~/archivebox && tar -czf "archivebox-data-$(date +%s).tar.gz" data/`
4. Follow the steps below for your installation method, then run `archivebox init`, `archivebox install`, and `archivebox update --migrate-only` inside the collection
4. Follow the steps below for your installation method. Bare-metal installs run `archivebox init`, `archivebox install`, and `archivebox update --migrate-only` inside the collection; Docker images already include runtime dependencies.
5. Confirm the upgrade succeeded and check for any orphan/corrupted snapshots with `archivebox status`
💬 [Open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new/choose) in our bug tracker if you experience any problems with upgrading/merging/modifying collections.
@ -49,11 +48,11 @@ docker compose up -d
**ℹ️ How it works internally:**
The same command is used for initializing a new archive and upgrading an existing database. `archivebox init` is idempotent and can safely be run multiple times; it applies database migrations and prepares collection-level state. `archivebox install` resolves runtime dependencies for the new version. `archivebox update --migrate-only` performs filesystem migrations and reconciles Snapshot metadata with the current layout without scheduling normal archive maintenance jobs. `archivebox status` checks collection health afterward.
The same command is used for initializing a new archive and upgrading an existing database. `archivebox init` is idempotent and can safely be run multiple times; it applies database migrations and prepares collection-level state. For bare-metal installs, `archivebox install` resolves runtime dependencies for the new version; Docker images include those dependencies at build time. `archivebox update --migrate-only` performs filesystem migrations and reconciles Snapshot metadata with the current layout without scheduling normal archive maintenance jobs. `archivebox status` checks collection health afterward.
There are three main areas on disk that ArchiveBox modifies during upgrades:
- `index.sqlite3` contains the SQLite3 DB index that gets upgraded automatically by Django based on the changes in [`archivebox/core/models.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/models.py).
- `archive/users/<user>/snapshots/<date>/<domain>/<uuid>/index.jsonl` stores per-Snapshot metadata alongside plugin-namespaced output. `archivebox update --migrate-only` may rewrite metadata, migrate older layouts, and maintain legacy timestamp compatibility symlinks.
- `archive/users/<user>/snapshots/<date>/<domain>/<uuid>/index.jsonl` stores per-Snapshot metadata alongside plugin-namespaced output. `archivebox update --migrate-only` may rewrite metadata, migrate older layouts, and remove obsolete timestamp projections after verified migration.
- Snapshot output directories and plugin paths can move as filesystem schemas evolve, so the entire `archive/` tree must be backed up with the database.
`ArchiveBox.conf` is migrated through the normal config loader/writer when options are renamed or normalized. Back it up with the rest of the collection and review release notes for config changes.
@ -76,7 +75,6 @@ cd ~/archivebox # or wherever your folder containing docker-compose.yml i
docker compose down # stop the currently running ArchiveBox containers
docker compose pull # pull the latest image version from Docker Hub
docker compose run --rm archivebox init
docker compose run --rm archivebox install
docker compose run --rm archivebox update --migrate-only
docker compose up -d
```
@ -97,7 +95,6 @@ docker stop CONTAINER_ID
cd ~/archivebox/data # or wherever your existing collection is stored
docker pull archivebox/archivebox:dev
docker run --rm -v $PWD:/data -it archivebox/archivebox:dev init
docker run --rm -v $PWD:/data -it archivebox/archivebox:dev install
docker run --rm -v $PWD:/data -it archivebox/archivebox:dev update --migrate-only
# restart the archivebox server container if needed

8
docs/apidocs/archivebox/archivebox.core.models.md
View File

@ -1396,14 +1396,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
````
````{py:method} ensure_legacy_archive_symlink() -> None
:canonical: archivebox.core.models.Snapshot.ensure_legacy_archive_symlink
```{autodoc2-docstring} archivebox.core.models.Snapshot.ensure_legacy_archive_symlink
```
````
````{py:method} ensure_crawl_symlink(*, crawl_dir: pathlib.Path | None = None, snapshot_dir: pathlib.Path | None = None) -> None
:canonical: archivebox.core.models.Snapshot.ensure_crawl_symlink

101
docs/apidocs/archivebox/archivebox.core.takeover_util.md
View File

@ -15,30 +15,10 @@
:class: autosummary longtable
:align: left
* - {py:obj}`runtime_stack_owner_types <archivebox.core.takeover_util.runtime_stack_owner_types>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner_types
:summary:
```
* - {py:obj}`foreground_runner_owner_types <archivebox.core.takeover_util.foreground_runner_owner_types>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.foreground_runner_owner_types
:summary:
```
* - {py:obj}`current_command <archivebox.core.takeover_util.current_command>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.current_command
:summary:
```
* - {py:obj}`live_processes <archivebox.core.takeover_util.live_processes>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.live_processes
:summary:
```
* - {py:obj}`newest_live_process <archivebox.core.takeover_util.newest_live_process>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.newest_live_process
:summary:
```
* - {py:obj}`command_is_newest <archivebox.core.takeover_util.command_is_newest>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.command_is_newest
:summary:
```
* - {py:obj}`runtime_stack_owner <archivebox.core.takeover_util.runtime_stack_owner>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner
:summary:
@ -55,22 +35,10 @@
- ```{autodoc2-docstring} archivebox.core.takeover_util.command_owns_foreground_runner
:summary:
```
* - {py:obj}`runtime_stack_component_label <archivebox.core.takeover_util.runtime_stack_component_label>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_component_label
:summary:
```
* - {py:obj}`ensure_daemon_stack <archivebox.core.takeover_util.ensure_daemon_stack>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.ensure_daemon_stack
:summary:
```
* - {py:obj}`healthy_orchestrator <archivebox.core.takeover_util.healthy_orchestrator>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.healthy_orchestrator
:summary:
```
* - {py:obj}`_runner_sort_key <archivebox.core.takeover_util._runner_sort_key>`
- ```{autodoc2-docstring} archivebox.core.takeover_util._runner_sort_key
:summary:
```
* - {py:obj}`live_runner_processes <archivebox.core.takeover_util.live_runner_processes>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.live_runner_processes
:summary:
@ -79,10 +47,6 @@
- ```{autodoc2-docstring} archivebox.core.takeover_util.enter_single_runner_gate
:summary:
```
* - {py:obj}`standby_until_leader_needed <archivebox.core.takeover_util.standby_until_leader_needed>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_leader_needed
:summary:
```
* - {py:obj}`standby_until_runtime_stack_needed <archivebox.core.takeover_util.standby_until_runtime_stack_needed>`
- ```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_runtime_stack_needed
:summary:
@ -145,20 +109,6 @@
````
````{py:function} runtime_stack_owner_types()
:canonical: archivebox.core.takeover_util.runtime_stack_owner_types
```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_owner_types
```
````
````{py:function} foreground_runner_owner_types()
:canonical: archivebox.core.takeover_util.foreground_runner_owner_types
```{autodoc2-docstring} archivebox.core.takeover_util.foreground_runner_owner_types
```
````
````{py:function} current_command(process_type: str, *, data_dir: str | pathlib.Path, url: str | None = None)
:canonical: archivebox.core.takeover_util.current_command
@ -166,27 +116,6 @@
```
````
````{py:function} live_processes(*, process_type: str, data_dir: str | pathlib.Path, url: str | None = None)
:canonical: archivebox.core.takeover_util.live_processes
```{autodoc2-docstring} archivebox.core.takeover_util.live_processes
```
````
````{py:function} newest_live_process(*, process_type: str, data_dir: str | pathlib.Path, url: str | None = None)
:canonical: archivebox.core.takeover_util.newest_live_process
```{autodoc2-docstring} archivebox.core.takeover_util.newest_live_process
```
````
````{py:function} command_is_newest(command, *, process_type: str, data_dir: str | pathlib.Path, url: str | None = None) -> bool
:canonical: archivebox.core.takeover_util.command_is_newest
```{autodoc2-docstring} archivebox.core.takeover_util.command_is_newest
```
````
````{py:function} runtime_stack_owner(*, data_dir: str | pathlib.Path, exclude_id=None)
:canonical: archivebox.core.takeover_util.runtime_stack_owner
@ -215,13 +144,6 @@
```
````
````{py:function} runtime_stack_component_label(*, owner=None, data_dir: str | pathlib.Path) -> str
:canonical: archivebox.core.takeover_util.runtime_stack_component_label
```{autodoc2-docstring} archivebox.core.takeover_util.runtime_stack_component_label
```
````
````{py:function} ensure_daemon_stack(*, reason: str = '')
:canonical: archivebox.core.takeover_util.ensure_daemon_stack
@ -229,21 +151,7 @@
```
````
````{py:function} healthy_orchestrator(*, data_dir: str | pathlib.Path)
:canonical: archivebox.core.takeover_util.healthy_orchestrator
```{autodoc2-docstring} archivebox.core.takeover_util.healthy_orchestrator
```
````
````{py:function} _runner_sort_key(process)
:canonical: archivebox.core.takeover_util._runner_sort_key
```{autodoc2-docstring} archivebox.core.takeover_util._runner_sort_key
```
````
````{py:function} live_runner_processes(*, data_dir: str | pathlib.Path, exclude_id=None)
````{py:function} live_runner_processes(*, data_dir: str | pathlib.Path)
:canonical: archivebox.core.takeover_util.live_runner_processes
```{autodoc2-docstring} archivebox.core.takeover_util.live_runner_processes
@ -257,13 +165,6 @@
```
````
````{py:function} standby_until_leader_needed(command, *, process_type: str, data_dir: str | pathlib.Path, url: str | None = None, interval: float = 2.0) -> None
:canonical: archivebox.core.takeover_util.standby_until_leader_needed
```{autodoc2-docstring} archivebox.core.takeover_util.standby_until_leader_needed
```
````
````{py:function} standby_until_runtime_stack_needed(command, *, data_dir: str | pathlib.Path, interval: float = 2.0) -> dict[str, object]
:canonical: archivebox.core.takeover_util.standby_until_runtime_stack_needed

2
docs/codeblocks.toml
View File

@ -5,7 +5,7 @@ version = 2
[snippets]
# README.md
"c718250c7a8decb0-1" = "illustration"
"7829a1f6ac113348-1" = "illustration"
"fe1f06a0a6b02c01-1" = "illustration"
"dbbcf99c3829e104-1" = "run"
"698f2b0f88796c9b-1" = "illustration"

2
etc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.35rc344",
"version": "0.9.35rc365",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {

12
publicsite/index.html
View File

@ -99,9 +99,8 @@
<pre><span class="ab-dim"># Docker Compose is the recommended setup</span>
<span class="ab-prompt">$</span> <span class="ab-cmd">mkdir -p ~/archivebox/data &amp;&amp; cd ~/archivebox</span>
<span class="ab-prompt">$</span> <span class="ab-cmd">curl -fsSL 'https://docker-compose.archivebox.io' &gt; docker-compose.yml</span>
<span class="ab-prompt">$</span> <span class="ab-cmd">docker compose run archivebox init --install</span>
<span class="ab-out">-> created ./data/index.sqlite3</span>
<span class="ab-out">-> installed Chrome, wget, yt-dlp, SingleFile, readability</span>
<span class="ab-prompt">$</span> <span class="ab-cmd">docker compose up -d --wait</span>
<span class="ab-out">-> initialized ./data/index.sqlite3</span>
<span class="ab-ok">ok listening on http://127.0.0.1:8000</span>
<span class="ab-prompt">$</span> <span class="ab-cursor" aria-hidden="true">&nbsp;</span></pre>
</div>
@ -209,16 +208,15 @@
<div class="step">
<span>3</span>
<div>
<h3>Initialize and start</h3>
<div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">docker compose run archivebox init --install</span>
<span class="ab-prompt">$</span> <span class="ab-cmd">docker compose up</span></pre></div></div>
<h3>Start ArchiveBox (initializes automatically)</h3>
<div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">docker compose up -d --wait</span></pre></div></div>
</div>
</div>
<div class="step">
<span>4</span>
<div>
<h3>Add your first URL</h3>
<div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">docker compose run archivebox add 'https://example.com'</span></pre></div></div>
<div class="ab-terminal ab-terminal--inline"><div class="ab-terminal__body"><pre><span class="ab-prompt">$</span> <span class="ab-cmd">docker compose exec archivebox archivebox add 'https://example.com'</span></pre></div></div>
</div>
</div>
</div>

10
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.35rc344"
version = "0.9.35rc365"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -81,9 +81,9 @@ dependencies = [
### Extractor dependencies (runtime binaries resolved through abxpkg)
### Binary/Package Management
"abxbus==2.5.56", # direct imports only; version constrained by abx-dl -> abx-plugins
"abxpkg==1.12.109", # direct imports only; version constrained by abx-dl -> abx-plugins
"abx-plugins==1.12.192", # direct imports only; version constrained by abx-dl
"abx-dl==1.12.218", # shared ArchiveBox downloader package
"abxpkg==1.12.110", # direct imports only; version constrained by abx-dl -> abx-plugins
"abx-plugins==1.12.197", # direct imports only; version constrained by abx-dl
"abx-dl==1.12.225", # shared ArchiveBox downloader package
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]
@ -344,7 +344,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
[tool.bumpver]
current_version = "v0.9.35rc344"
current_version = "v0.9.35rc365"
version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
commit_message = "bump version {old_version} -> {new_version}"
tag_message = "{new_version}"

30
uv.lock
View File

@ -13,18 +13,18 @@ supported-markers = [
]
[options]
exclude-newer = "2026-08-24T02:15:26.974926946Z"
exclude-newer = "2026-08-27T06:11:53.268167371Z"
exclude-newer-span = "P5D"
[options.exclude-newer-package]
abxbus = { timestamp = "2026-08-29T02:15:25.974939359Z", span = "PT1S" }
abxbus = { timestamp = "2026-09-01T06:11:52.268183486Z", span = "PT1S" }
abx-plugins = "2100-01-01T00:00:00Z"
abx-dl = "2100-01-01T00:00:00Z"
abxpkg = "2100-01-01T00:00:00Z"
[[package]]
name = "abx-dl"
version = "1.12.218"
version = "1.12.225"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@ -38,14 +38,14 @@ dependencies = [
{ name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/80/71/7f316e610cf7f8421cc5b5fe3887ec2eb60ef52fe21741721681c9cda910/abx_dl-1.12.218.tar.gz", hash = "sha256:064c543943da3a5185184e9b261d9d909783d83ccbd141f74a404369b63045bc", size = 84041 }
sdist = { url = "https://files.pythonhosted.org/packages/5b/21/976fa0fdfdd6989b05cd328cd3d837e5638cdce5ea3644a13f07507fd13a/abx_dl-1.12.225.tar.gz", hash = "sha256:a886f51bfe9981cf6165ab6c65a4ec710756fc8a834e62b42ae3500ca578868c", size = 84223 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/26/2e/233a861d901356b5220ab543356ec8ca2f3d63388327bded4cac5f59c487/abx_dl-1.12.218-py3-none-any.whl", hash = "sha256:1ef7a7fb2f0204d6fd9dcaec45f48f0cfaee03502175d209498b381752c6bb70", size = 87728 },
{ url = "https://files.pythonhosted.org/packages/af/94/8d91e0b97789ea4541ffc0241341903b36ec4a7b018e46a28baf48d9d737/abx_dl-1.12.225-py3-none-any.whl", hash = "sha256:62b6a6ae9e8087249ea1fc726fe5e747ad02340e59ca791a1d3590f613ec9e8c", size = 87900 },
]
[[package]]
name = "abx-plugins"
version = "1.12.192"
version = "1.12.197"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@ -54,9 +54,9 @@ dependencies = [
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7f/13/412ee8b46d7fbb977c1d8607d521ba3ea0dae61feb5bf4fb8b176830b65c/abx_plugins-1.12.192.tar.gz", hash = "sha256:4edcc841a85f231bb2ab01c89cdaab6e7873b40af04f459449907348bcb19513", size = 255242, upload-time = "2026-08-28T22:24:40.73Z" }
sdist = { url = "https://files.pythonhosted.org/packages/11/ab/09ca5de37ea1505de4d7295ba2adbc4b83256a332a71079893153afa6f02/abx_plugins-1.12.197.tar.gz", hash = "sha256:d8ebbf8b47a9d5fb456717f414b12c84e09284a5e5fc6a91ffd4729b84b73c85", size = 259108, upload-time = "2026-09-01T06:03:06.121Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/17/3e/2504344111dd0a769d2430c76c131d4a6cc5f58418f296c547660e9a7f1f/abx_plugins-1.12.192-py3-none-any.whl", hash = "sha256:0b3c6ee3fc6600e08266206ae7eca3c5a98df2f6855ec772120703cfc3d6ff4f", size = 406098, upload-time = "2026-08-28T22:24:42.035Z" },
{ url = "https://files.pythonhosted.org/packages/30/a7/5b93c9bc8adb294df1f4415402bf5c7818d3f3a4a5c7196e3e3cdd043331/abx_plugins-1.12.197-py3-none-any.whl", hash = "sha256:e1533bddfc2f076e13c5a9c6b7124320a2863986d1a5e384dbc78da1e372f791", size = 412155, upload-time = "2026-09-01T06:03:08.224Z" },
]
[[package]]
@ -75,7 +75,7 @@ wheels = [
[[package]]
name = "abxpkg"
version = "1.12.109"
version = "1.12.110"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@ -83,9 +83,9 @@ dependencies = [
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/a9/70/1cabf0920b425191f91cbefde3b144ceb478c647dfecc49676f01a239f30/abxpkg-1.12.109.tar.gz", hash = "sha256:c20f3b28bff82a83984e940e9165d2f7b2fccd1dbe84ba04ff5414db2ade29e0", size = 239416, upload-time = "2026-08-28T22:19:19.002Z" }
sdist = { url = "https://files.pythonhosted.org/packages/dc/25/152a0e2dd37494cab5116629b4a3f26232dda6641e7b79f8eb7d1e9bfd75/abxpkg-1.12.110.tar.gz", hash = "sha256:94777e6ea76944b6577c46aaf4d845cdc68cfbdfe4c58ca24cd02071e367a660", size = 239950, upload-time = "2026-08-31T01:25:16.393Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/71/90/37a9df601259ce300bdfa840b5cab82d1566a39d8ddddde6283623caa17e/abxpkg-1.12.109-py3-none-any.whl", hash = "sha256:c8cbf3b6ea66e7ed7dfc4bdf2c277a99bea4728293c39840cf6be8fc349c01dd", size = 253436, upload-time = "2026-08-28T22:19:17.472Z" },
{ url = "https://files.pythonhosted.org/packages/05/95/49a113ceccf1d1ac33a90a0f8735abd43919f6cf2543917542acbdd61ea3/abxpkg-1.12.110-py3-none-any.whl", hash = "sha256:01e3e95c0af1a8fe775dee07eb16fa540a4e7007c3961a52e11d57bc2eae2430", size = 254023, upload-time = "2026-08-31T01:25:14.833Z" },
]
[[package]]
@ -120,7 +120,7 @@ wheels = [
[[package]]
name = "archivebox"
version = "0.9.35rc344"
version = "0.9.35rc365"
source = { editable = "." }
dependencies = [
{ name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@ -220,10 +220,10 @@ dev = [
[package.metadata]
requires-dist = [
{ name = "abx-dl", specifier = "==1.12.218" },
{ name = "abx-plugins", specifier = "==1.12.192" },
{ name = "abx-dl", specifier = "==1.12.225" },
{ name = "abx-plugins", specifier = "==1.12.197" },
{ name = "abxbus", specifier = "==2.5.56" },
{ name = "abxpkg", specifier = "==1.12.109" },
{ name = "abxpkg", specifier = "==1.12.110" },
{ name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
{ name = "atomicwrites", specifier = "==1.4.1" },
{ name = "base32-crockford", specifier = ">=0.3.0" },