The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

wip: snapshot live progress, universal --init, runner perms, supervisord SIGINT

Browse Source 
- Snapshot detail page: embed scoped live-progress monitor (same-origin
  /progress.json on whichever host the page is served from); hide admin
  action buttons when scoped; per-snapshot perms via can_view_snapshot.
- crawl_file API: respect crawl-level permissions; PUBLIC/UNLISTED served
  to guests, PRIVATE returns 404 for non-admin/non-owner.
- CrawlRunner: replace allow_paused_snapshot_maintenance with
  allow_maintenance_on_inactive_crawl so SEALED crawls don't short-circuit
  the cancellation guard for legitimate maintenance hooks (search backend
  backfill, fs migration, etc.). Fixes infinite STARTED loop on snapshots
  with queued search_backend results.
- Universal `--init` flag: works on any subcommand (server, update, add,
  shell, install, ...). Detected at module load, stripped from argv, and
  consumed in the dispatcher so subprocesses inherit a clean env.
- supervisord_util.run_runner_worker: route Ctrl+C through
  supervisor.signalProcess(name, "SIGINT") instead of raw os.kill on a
  cached pid, gated on statename=RUNNING. Prevents killing unrelated
  processes when the worker's pid has been reused by the OS.
- Login page: remove non-functional password-reset links; add
  has_real_admin_users template tag to gate the bootstrap hint.
- Add page: hide underline on the "Get the extension" link.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Sweeting 2026-05-30 04:45:15 -07:00
parent 2c2215f84d
commit b0a47e8bf5
No known key found for this signature in database
59 changed files with 1762 additions and 631 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
archivebox/api/v1_cli.py
View File

@ -8,10 +8,8 @@ from enum import Enum
from django.http import HttpRequest
from ninja import Router, Schema
from pydantic import Field
from archivebox.misc.util import ansi_to_html
from archivebox.config.common import get_config
# from .auth import API_AUTH_METHODS
@ -67,8 +65,7 @@ class AddCommandSchema(Schema):
snapshot_max_size: int = 0
parser: str = "auto"
plugins: str = ""
update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
overwrite: bool = False
only_new: bool | None = None
index_only: bool = False
@ -92,8 +89,7 @@ class ScheduleCommandSchema(Schema):
every: str | None = None
tag: str = ""
depth: int = 0
overwrite: bool = False
update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
only_new: bool | None = None
clear: bool = False
@ -121,6 +117,9 @@ class RemoveCommandSchema(Schema):
def cli_add(request: HttpRequest, args: AddCommandSchema):
from archivebox.cli.archivebox_add import add
config_overrides: dict[str, object] = {}
if args.only_new is not None:
config_overrides["ONLY_NEW"] = bool(args.only_new)
crawl, snapshots = add(
urls=args.urls,
snapshot_ids=args.snapshot_ids,
@ -130,13 +129,12 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
crawl_max_size=args.crawl_max_size,
crawl_timeout=args.crawl_timeout,
snapshot_max_size=args.snapshot_max_size,
update=args.update,
index_only=args.index_only,
overwrite=args.overwrite,
plugins=args.plugins,
parser=args.parser,
bg=True, # Always run in background for API calls
created_by_id=request.user.pk,
config=config_overrides or None,
)
snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
@ -188,6 +186,9 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
from archivebox.cli.archivebox_schedule import schedule
config_overrides: dict[str, object] = {}
if args.only_new is not None:
config_overrides["ONLY_NEW"] = bool(args.only_new)
result = schedule(
import_path=args.import_path,
add=args.add,
@ -199,8 +200,7 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
every=args.every,
tag=args.tag,
depth=args.depth,
overwrite=args.overwrite,
update=args.update,
config=config_overrides or None,
)
stdout = getattr(request, "stdout", None)

16
archivebox/api/v1_core.py
View File

@ -32,7 +32,7 @@ from archivebox.core.permissions import public_snapshots_queryset
from archivebox.api.auth import auth_using_token
from archivebox.config.common import get_config
from archivebox.core.host_utils import build_web_url
from archivebox.misc.util import validate_url_length
from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
from archivebox.core.tag_utils import (
add_snapshot_counts,
build_tag_cards,
@ -786,7 +786,8 @@ def _filter_snapshots_for_rss(
)
crawl_id = crawl_id.strip()
if crawl_id:
queryset = queryset.filter(crawl__id__icontains=crawl_id)
matching_crawl_pks = list(filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).values_list("pk", flat=True)[:100])
queryset = queryset.filter(crawl_id__in=matching_crawl_pks)
created_by = created_by.strip()
if created_by:
@ -845,7 +846,7 @@ def _snapshots_rss_response(
class SnapshotFilterSchema(FilterSchema):
id: Annotated[str | None, FilterLookup(["id__icontains", "timestamp__startswith"])] = None
id: Annotated[str | None, FilterLookup(["id__istartswith", "id__iendswith", "timestamp__startswith"])] = None
created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None
created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None
created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
@ -856,7 +857,9 @@ class SnapshotFilterSchema(FilterSchema):
modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
search: Annotated[
str | None,
FilterLookup(["url__icontains", "title__icontains", "tags__name__icontains", "id__icontains", "timestamp__startswith"]),
FilterLookup(
["url__icontains", "title__icontains", "tags__name__icontains", "id__istartswith", "id__iendswith", "timestamp__startswith"],
),
] = None
url: Annotated[str | None, FilterLookup("url")] = None
tag: Annotated[str | None, FilterLookup("tags__name")] = None
@ -919,7 +922,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
raise HttpError(400, "depth must be between 0 and 4")
if data.crawl_id:
crawl = Crawl.objects.get(id__icontains=data.crawl_id)
crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), data.crawl_id).get()
crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
tags = tags or crawl_tags
else:
@ -976,7 +979,7 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
try:
snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
except Snapshot.DoesNotExist:
snapshot = Snapshot.objects.get(Q(id__icontains=snapshot_id))
snapshot = filter_queryset_by_uuid_substring(Snapshot.objects.all(), snapshot_id).get()
payload = data.dict(exclude_unset=True)
update_fields = ["modified_at"]
@ -1252,6 +1255,7 @@ def search_tags(
"tags": build_tag_cards(
query=q,
request=request,
preview_limit=0,
sort=normalized_sort,
created_by=normalized_created_by,
year=normalized_year,

46
archivebox/api/v1_crawls.py
View File

@ -14,8 +14,15 @@ from ninja import Router, Schema
from ninja.errors import HttpError
from archivebox.core.models import Snapshot
from archivebox.core.permissions import (
PERMISSIONS_PUBLIC,
PERMISSIONS_UNLISTED,
is_admin_user,
normalize_permissions,
)
from archivebox.config.common import get_config
from archivebox.crawls.models import Crawl
from archivebox.misc.util import filter_queryset_by_uuid_substring
from .auth import API_AUTH_METHODS, auth_using_token
@ -127,7 +134,7 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
"""Get a specific Crawl by id."""
setattr(request, "with_snapshots", with_snapshots)
setattr(request, "with_archiveresults", with_archiveresults)
crawl = Crawl.objects.get(id__icontains=crawl_id)
crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
if crawl and as_rss:
query = request.GET.copy()
@ -139,21 +146,38 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
def crawl_file(request: HttpRequest, crawl_id: str, path: str):
# Try to resolve the crawl first; if it doesn't exist, return 404.
try:
crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
except Crawl.DoesNotExist:
raise HttpError(404, "Crawl not found")
# Determine the effective viewer: session user takes precedence, otherwise
# fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or
# Authorization: Bearer ... (so that programmatic clients still work).
user = getattr(request, "user", None)
is_superuser = bool(
getattr(user, "is_authenticated", False) and getattr(user, "is_active", False) and getattr(user, "is_superuser", False),
)
if not is_superuser:
is_authenticated = bool(getattr(user, "is_authenticated", False) and getattr(user, "is_active", False))
if not is_authenticated:
token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
auth_header = request.headers.get("Authorization", "")
if not token and auth_header.lower().startswith("bearer "):
token = auth_header.split(None, 1)[1].strip()
token_user = auth_using_token(token=token, request=request) if token else None
is_superuser = bool(token_user and token_user.is_active and token_user.is_superuser)
if not is_superuser:
raise HttpError(403, "Permission denied")
if token_user and token_user.is_active:
user = token_user
is_authenticated = True
# Re-bind so is_admin_user() / ownership checks below see the token user.
setattr(request, "user", token_user)
# Gate access using the same model as SnapshotView/can_view_snapshot:
# admins always pass; owners can see their own crawls; otherwise the crawl
# must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
if not is_admin_user(request):
permissions = normalize_permissions(crawl.permissions)
is_owner = bool(is_authenticated and getattr(crawl, "created_by_id", None) == getattr(user, "id", None))
if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
raise HttpError(404, "Crawl not found")
crawl = Crawl.objects.get(id__icontains=crawl_id)
crawl_root = Path(crawl.output_dir).resolve()
file_path = (crawl_root / path).resolve()
if not file_path.is_file() or crawl_root not in file_path.parents:
@ -185,7 +209,7 @@ def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfol
@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
"""Update a crawl (e.g., set status=sealed to cancel queued work)."""
crawl = Crawl.objects.get(id__icontains=crawl_id)
crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
payload = data.dict(exclude_unset=True)
update_fields = ["modified_at"]
@ -227,7 +251,7 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
def delete_crawl(request: HttpRequest, crawl_id: str):
crawl = Crawl.objects.get(id__icontains=crawl_id)
crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
crawl_id_str = str(crawl.id)
snapshot_count = crawl.snapshot_set.count()
deleted_count, _ = crawl.delete()

64
archivebox/base_models/admin.py
View File

@ -48,7 +48,7 @@ class KeyValueWidget(forms.Widget):
from archivebox.hooks import discover_plugin_configs
options: dict[str, ConfigOption] = {}
skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "CRAWL_OUTPUT_DIR", "SNAP_DIR"}
skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "SNAP_DIR"}
for key, field in ArchiveBoxConfig.model_fields.items():
if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
continue
@ -227,9 +227,6 @@ class KeyValueWidget(forms.Widget):
return 'Example: ["value"]';
}}
if (types.includes('object')) {{
if (key === 'SAVE_ALLOWLIST' || key === 'SAVE_DENYLIST') {{
return 'Example: {{"^https://example\\\\.com": ["wget"]}}';
}}
return 'Example: {{"key": "value"}}';
}}
return '';
@ -238,8 +235,6 @@ class KeyValueWidget(forms.Widget):
function isRegexConfigKey_{widget_id}(key) {{
return key === 'URL_ALLOWLIST' ||
key === 'URL_DENYLIST' ||
key === 'SAVE_ALLOWLIST' ||
key === 'SAVE_DENYLIST' ||
key.endsWith('_PATTERN') ||
key.includes('REGEX');
}}
@ -633,17 +628,74 @@ class KeyValueWidget(forms.Widget):
window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
function focusConfigKeyFromHash_{widget_id}() {{
// Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
// to (or creates) the matching row in this editor. Used by
// the in-banner "pin via admin" link and the
// ``→ Edit <KEY> in Machine.config`` shortcut on the live
// config detail page.
var hash = (window.location.hash || '').replace(/^#/, '').trim();
if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
return;
}}
var container = document.getElementById('{widget_id}_rows');
if (!container) {{
return;
}}
var match = null;
container.querySelectorAll('.key-value-row').forEach(function(row) {{
if (match) {{ return; }}
var keyInput = row.querySelector('.kv-key');
if (keyInput && keyInput.value.trim() === hash) {{
match = row;
}}
}});
if (!match) {{
// No existing row for this key — prepopulate one with the
// key filled in but value left blank so the operator just
// types/pastes the value and hits save.
window.addKeyValueRow_{widget_id}();
var rows = container.querySelectorAll('.key-value-row');
match = rows[rows.length - 1];
var keyInput = match.querySelector('.kv-key');
if (keyInput) {{
keyInput.value = hash;
keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
}}
}}
if (!match) {{
return;
}}
match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
var prevOutline = match.style.outline;
match.style.outline = '2px solid #f59e0b';
match.style.outlineOffset = '2px';
match.style.transition = 'outline 1.2s ease-out';
setTimeout(function() {{
match.style.outline = prevOutline || 'none';
}}, 1400);
var valueInput = match.querySelector('.kv-value');
if (valueInput) {{
valueInput.focus();
try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
}}
}}
// Initialize on load
document.addEventListener('DOMContentLoaded', function() {{
initializeRows_{widget_id}();
updateHiddenField_{widget_id}();
focusConfigKeyFromHash_{widget_id}();
}});
// Also run immediately in case DOM is already ready
if (document.readyState !== 'loading') {{
initializeRows_{widget_id}();
updateHiddenField_{widget_id}();
focusConfigKeyFromHash_{widget_id}();
}}
window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
// Update on any input change
var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');

18
archivebox/cli/__init__.py
View File

@ -16,6 +16,14 @@ if "--debug" in sys.argv:
os.environ["DEBUG"] = "True"
sys.argv.remove("--debug")
# Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
# `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
# the subcommand executes. Strip it from argv here so each subcommand's own click parser
# never sees it. Ignored for `help` and `init` themselves.
if "--init" in sys.argv:
sys.argv = [arg for arg in sys.argv if arg != "--init"]
os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
class ArchiveBoxGroup(click.Group):
"""lazy loading click group for archivebox commands"""
@ -172,6 +180,16 @@ def cli(ctx, help=False):
from archivebox.misc.checks import check_data_folder, check_migrations
setup_django()
if os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" and subcommand not in ("init", "help"):
# Universal `--init` was passed: build/upgrade the data folder before
# the regular preflight runs, so it succeeds on a fresh dir and an
# out-of-date schema both. Drop the env var afterwards so spawned
# subprocesses (supervisord workers, daphne, runner, etc.) inherit
# a clean env and don't re-trigger init in every child.
from archivebox.cli.archivebox_init import init as archivebox_init
archivebox_init(quick=True)
os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
check_data_folder()
if subcommand != "update":
check_migrations(auto_apply=True)

34
archivebox/cli/archivebox_add.py
View File

@ -9,7 +9,7 @@ import json
import os
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any, TYPE_CHECKING
import rich_click as click
@ -58,11 +58,10 @@ def add(
parser: str = "auto",
plugins: str = "",
persona: str = "Default",
overwrite: bool = False,
update: bool | None = None,
index_only: bool = False,
bg: bool = False,
created_by_id: int | None = None,
config: dict[str, Any] | None = None,
) -> tuple[Crawl, QuerySet[Snapshot]]:
"""Add a new URL or list of URLs to your archive.
@ -87,10 +86,11 @@ def add(
from archivebox.config.permissions import USER, HOSTNAME
from archivebox.config.common import get_config
config = get_config()
config_overrides = dict(config or {})
runtime_config = get_config()
crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
if crawl_max_concurrent_snapshots is None:
crawl_max_concurrent_snapshots = config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
crawl_max_concurrent_snapshots = runtime_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
if depth not in (0, 1, 2, 3, 4):
@ -119,8 +119,6 @@ def add(
created_by_id = created_by_id or get_or_create_system_user_pk()
created_by = get_user_model().objects.filter(pk=created_by_id).first()
started_at = timezone.now()
if update is None:
update = not config.ONLY_NEW
if isinstance(urls, str):
url_list = [line.strip() for line in urls.splitlines() if line.strip()]
@ -157,9 +155,7 @@ def add(
crawl_config = {
"PERMISSIONS": str(effective_persona_config.PERMISSIONS),
**({"ONLY_NEW": not update} if bool(not update) != bool(effective_persona_config.ONLY_NEW) else {}),
**({"INDEX_ONLY": True} if index_only else {}),
**({"OVERWRITE": True} if overwrite else {}),
**({"PLUGINS": plugins} if plugins else {}),
**(
{"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
@ -175,6 +171,11 @@ def add(
**({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
**({"URL_DENYLIST": url_denylist} if url_denylist else {}),
}
# Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
# priority — they win over persona/plugin/env defaults and get stamped
# directly onto crawl.config so the runtime resolution and admin UI both
# reflect them faithfully.
crawl_config.update(config_overrides)
crawl = Crawl.objects.create(
urls=urls_content,
@ -314,8 +315,13 @@ def add(
@click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
@click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
@click.option("--persona", default="Default", help="Authentication profile to use when archiving")
@click.option("--overwrite", "-F", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
@click.option("--update", is_flag=True, default=None, help="Retry any previously skipped/failed URLs when re-adding them")
@click.option(
"--only-new/--no-only-new",
"only_new",
default=None,
help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
"Pass --no-only-new to force re-archive of URLs that already exist.",
)
@click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
@click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
@click.argument("urls", nargs=-1, type=click.Path())
@ -345,6 +351,12 @@ def main(**kwargs):
if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
# Translate --only-new/--no-only-new into a crawl config override.
# add() takes config overrides as a dict; no per-flag kwargs.
only_new = kwargs.pop("only_new", None)
if only_new is not None:
kwargs["config"] = {"ONLY_NEW": bool(only_new)}
add(urls=urls, **kwargs)

43
archivebox/cli/archivebox_remove.py
View File

@ -3,11 +3,13 @@
__package__ = "archivebox.cli"
__command__ = "archivebox remove"
import time
from pathlib import Path
from collections.abc import Iterable
import rich_click as click
from django.db import OperationalError
from django.db.models import QuerySet
from archivebox.config import DATA_DIR
@ -62,13 +64,44 @@ def remove(
log_list_finished(snapshots)
log_removal_started(snapshots, yes=yes)
to_remove = snapshots.count()
from archivebox.search import flush_search_index
from archivebox.core.models import Snapshot
from archivebox.search import flush_search_index
# Freeze the target set up-front so a concurrent daemon writing new
# snapshots can't extend the deletion under us, and so the cursor isn't
# held open across the per-row deletes below.
snapshot_pks = list(snapshots.values_list("pk", flat=True))
to_remove = len(snapshot_pks)
# Search-index flush touches a separate backend (FTS / sonic), not the
# main index.sqlite3 writer lock, so it's safe to do once up front.
flush_search_index(snapshots=Snapshot.objects.filter(pk__in=snapshot_pks))
# Delete one snapshot at a time. Each ``.delete()`` is its own short
# Django-atomic block, so the writer lock is released between rows and
# an in-flight daemon transaction can interleave instead of deadlocking.
# Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
# in ``base_models/models.py`` and runs AFTER its row's tx commits — so
# rmtree doesn't hold the lock either.
#
# The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock
# errors when called inside an atomic block (because it can't safely
# release+reacquire a transaction), so we wrap each row's delete in our
# own retry loop at this outer (non-atomic) level. Each attempt is a
# fresh atomic; an exception cleanly rolls it back before we sleep.
retry_interval = 1.0
retry_timeout = 60.0
for pk in snapshot_pks:
deadline = time.monotonic() + retry_timeout
while True:
try:
Snapshot.objects.filter(pk=pk).delete()
break
except OperationalError as err:
if "database is locked" not in str(err) or time.monotonic() >= deadline:
raise
time.sleep(retry_interval)
flush_search_index(snapshots=snapshots)
snapshots.delete()
all_snapshots = Snapshot.objects.all()
log_removal_finished(all_snapshots.count(), to_remove)

26
archivebox/cli/archivebox_schedule.py
View File

@ -6,7 +6,6 @@ import rich_click as click
from rich import print
from archivebox.misc.util import enforce_types, docstring
from archivebox.config.common import get_config
@enforce_types
@ -20,9 +19,8 @@ def schedule(
every: str | None = None,
tag: str = "",
depth: int | str = 0,
overwrite: bool = False,
update: bool | None = None,
import_path: str | None = None,
config: dict[str, object] | None = None,
):
"""Manage database-backed scheduled crawls processed by the crawl runner."""
@ -33,9 +31,7 @@ def schedule(
from archivebox.crawls.schedule_utils import validate_schedule
from archivebox.services.runner import run_pending_crawls
if update is None:
update = not get_config().ONLY_NEW
config_overrides = dict(config or {})
depth = int(depth)
result: dict[str, object] = {
"created_schedule_ids": [],
@ -79,10 +75,12 @@ def schedule(
status=Crawl.StatusChoices.SEALED,
retry_at=None,
config={
"ONLY_NEW": not update,
"OVERWRITE": overwrite,
"DEPTH": 0 if is_update_schedule else depth,
"SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
# Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
# template defaults. Anything left unset falls through to the
# standard config stack at crawl-resolution time.
**config_overrides,
},
)
crawl_schedule = CrawlSchedule.objects.create(
@ -163,8 +161,13 @@ def schedule(
default="0",
help="Recursively archive linked pages up to N hops away",
)
@click.option("--overwrite", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
@click.option("--update", is_flag=True, help="Retry previously failed/skipped URLs when scheduled crawls run")
@click.option(
"--only-new/--no-only-new",
"only_new",
default=None,
help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
"Pass --no-only-new to force re-archive on each scheduled run.",
)
@click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
@click.option("--show", is_flag=True, help="Print all currently enabled schedules")
@click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
@ -173,6 +176,9 @@ def schedule(
@docstring(schedule.__doc__)
def main(**kwargs):
"""Manage database-backed scheduled crawls processed by the crawl runner."""
only_new = kwargs.pop("only_new", None)
if only_new is not None:
kwargs["config"] = {"ONLY_NEW": bool(only_new)}
schedule(**kwargs)

220
archivebox/cli/archivebox_server.py
View File

@ -15,11 +15,193 @@ from rich import print
from archivebox.misc.util import docstring, enforce_types
import re as _re
_IPV4_RE = _re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
_IPV6_CHARS_RE = _re.compile(r"^[0-9a-fA-F:.]+$")
_LOCAL_BIND_HOSTS = frozenset({"0.0.0.0", "::", "::0", "127.0.0.1", "::1"})
def _is_ipv4_literal(host: str) -> bool:
return bool(_IPV4_RE.match(host))
def _is_ipv6_literal(host: str) -> bool:
# Bracketed (e.g. ``[2001:db8::1]``) or bare form. Require at least two
# colons so we don't catch random strings with one ``:``.
stripped = host.strip("[]")
return stripped.count(":") >= 2 and bool(_IPV6_CHARS_RE.match(stripped))
def _bind_host_looks_like_ip(host: str) -> bool:
if not host or host in _LOCAL_BIND_HOSTS:
return False
return _is_ipv4_literal(host) or _is_ipv6_literal(host)
def _split_bind_spec(spec: str) -> tuple[str, str]:
"""Split a ``host:port`` / ``host`` / ``port`` spec into ``(host, port)``.
The empty strings stand in for "not provided"; the caller fills in
defaults. Bracketed IPv6 literals like ``[::1]:8000`` are handled.
"""
spec = (spec or "").strip()
if not spec:
return "", ""
if spec.startswith("["):
# Bracketed IPv6: ``[::1]`` or ``[::1]:8000``
end = spec.find("]")
if end == -1:
return spec, "" # malformed; let validator reject it
host = spec[: end + 1]
rest = spec[end + 1 :]
if rest.startswith(":"):
return host, rest[1:]
return host, ""
if ":" in spec:
host, _, port = spec.rpartition(":")
return host, port
# Bare token: digits = port, anything else = host
if spec.isdigit():
return "", spec
return spec, ""
def _parse_and_validate_bind_spec(spec: str) -> tuple[str, str]:
"""Resolve a CLI/config bind spec to ``(host, port)`` or hard-error.
Accepts only IP literals (v4 or v6) or the special string ``localhost``
(normalized to ``127.0.0.1``). Bare hostnames are rejected because the
bind address feeds Daphne, which has to listen on a numeric address;
public hostnames belong in ``BASE_URL`` instead. Empty values fall back
to ``127.0.0.1`` / ``8000``.
"""
raw_host, raw_port = _split_bind_spec(spec)
host = raw_host.strip()
port = (raw_port or "").strip() or "8000"
if host == "" or host.lower() == "localhost":
host = "127.0.0.1"
elif _is_ipv4_literal(host) or _is_ipv6_literal(host):
pass
else:
print(
f"[red][X] Invalid BIND_ADDR host {host!r}: must be an IP literal or 'localhost'.[/red]",
)
print(
"[red] Hostnames like archive.example.com are not valid bind addresses — Daphne[/red]",
)
print(
"[red] listens on numeric addresses only. Bind to 0.0.0.0 and set BASE_URL instead:[/red]",
)
print(
f"[red] BASE_URL=https://{host} archivebox server 0.0.0.0:{port}[/red]",
)
sys.exit(1)
try:
port_int = int(port)
except ValueError:
print(f"[red][X] Invalid BIND_ADDR port {port!r}: must be an integer 1-65535.[/red]")
sys.exit(1)
if not (0 < port_int < 65536):
print(f"[red][X] Invalid BIND_ADDR port {port_int}: must be 1-65535.[/red]")
sys.exit(1)
return host, port
def _print_server_startup_warnings(config, host: str, *, base_url_explicit: bool) -> None:
"""Print startup-time security / routing warnings for the server command.
Runs only from ``archivebox server`` so other entry points (manage shell,
plugin lookups, etc.) don't repeat this banner on every config load.
"""
if config.IS_LOWER_SECURITY_MODE:
print(
f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={config.SERVER_SECURITY_MODE}[/yellow]",
)
print("[yellow] Archived pages may share an origin with privileged app routes in this mode.[/yellow]")
print("[yellow] To switch to the safer isolated setup:[/yellow]")
print("[yellow] 1. Set SERVER_SECURITY_MODE=safe-subdomains-fullreplay[/yellow]")
print("[yellow] 2. Point *.archivebox.localhost (or your chosen base domain) at this server[/yellow]")
print(
"[yellow] 3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]",
)
print()
if base_url_explicit:
return
# If the user is upgrading from 0.7.3 and already had
# CSRF_TRUSTED_ORIGINS set, get_base_url() will silently use that as the
# implicit BASE_URL. Surface what we picked so the user knows where their
# links / redirects are going — and tell them how to make it explicit.
from archivebox.core.host_utils import derive_base_url_from_csrf
csrf_derived = derive_base_url_from_csrf(config)
if csrf_derived:
print(
f"[yellow][!] BASE_URL is not set; auto-derived [bold]{csrf_derived}[/bold] from a single CSRF_TRUSTED_ORIGINS entry.[/yellow]",
)
print(
"[yellow] Links / redirects / cookies will use that origin. To silence this hint, set BASE_URL[/yellow]",
)
print(
f"[yellow] explicitly: [bold]BASE_URL={csrf_derived}[/bold] (matches your existing CSRF_TRUSTED_ORIGINS).[/yellow]",
)
print()
return
# BASE_URL was not set explicitly. The host_utils derivation gives one of
# three results, with very different risk profiles — show a tailored hint
# so new users coming from the 0.7.x single-domain world know whether the
# default is fine for them or needs attention.
if _bind_host_looks_like_ip(host):
# Real IP literal: subdomain routing can't work, URLs leak the IP.
# This is the most urgent case.
print(
f"[yellow][!] WARNING: BASE_URL is not set and BIND_ADDR resolves to an IP literal ({host}).[/yellow]",
)
print(
"[yellow] Snapshot / admin / api URLs will be generated with the IP, and subdomain[/yellow]",
)
print(
"[yellow] routing cannot work against an IP address. Set BASE_URL explicitly, e.g.[/yellow]",
)
print(
"[yellow] BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
)
if config.USES_SUBDOMAIN_ROUTING:
print(
"[yellow] Or switch SERVER_SECURITY_MODE to a one-domain mode if you can't run a hostname.[/yellow]",
)
print()
else:
# Loopback / wildcard bind. The host_utils default of
# http://archivebox.localhost:PORT works in a browser on the same
# machine, but anything else (reverse proxy, k8s ingress, LAN client)
# needs BASE_URL set. (Real hostnames can't reach this branch — the
# bind validator rejects them upfront.)
print(
"[yellow][!] BASE_URL is not set. Generated URLs will fall back to http://archivebox.localhost:<port>.[/yellow]",
)
print(
"[yellow] That's fine for local browsing on this machine. Set BASE_URL when running behind[/yellow]",
)
print(
"[yellow] a reverse proxy / ingress / public hostname, e.g.[/yellow]",
)
print(
"[yellow] BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
)
print()
@enforce_types
def server(
runserver_args: Iterable[str] | None = None,
reload: bool = False,
init: bool = False,
debug: bool = False,
daemonize: bool = False,
nothreading: bool = False,
@ -30,12 +212,6 @@ def server(
config = get_config()
runserver_args = list(runserver_args or (config.BIND_ADDR,))
if init:
from archivebox.cli.archivebox_init import init as archivebox_init
archivebox_init(quick=True)
print()
run_in_debug = config.DEBUG or debug or reload
if debug or reload:
os.environ["DEBUG"] = "True"
@ -50,20 +226,12 @@ def server(
print(" [green]archivebox manage createsuperuser[/green]")
print()
host = "127.0.0.1"
port = "8000"
try:
host_and_port = [arg for arg in runserver_args if arg.replace(".", "").replace(":", "").isdigit()][0]
if ":" in host_and_port:
host, port = host_and_port.split(":")
else:
if "." in host_and_port:
host = host_and_port
else:
port = host_and_port
except IndexError:
pass
# First non-empty positional arg is the bind spec; otherwise inherit from
# config (which defaults to "127.0.0.1:8000"). _parse_and_validate_bind_spec
# hard-errors on hostnames so the rest of the server can assume a numeric
# bind host.
bind_spec = next((arg for arg in runserver_args if arg), "")
host, port = _parse_and_validate_bind_spec(bind_spec)
if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
from archivebox.config import CONSTANTS
@ -137,6 +305,15 @@ def server(
)
print(" > Writing ArchiveBox error log to ./logs/errors.log")
print()
# Reload config after we've set os.environ["BIND_ADDR"] above so the
# security-mode + base-url warnings see the effective values.
runtime_config = get_config()
_print_server_startup_warnings(
runtime_config,
host,
base_url_explicit=bool(os.environ.get("BASE_URL", "").strip()),
)
bind_url = f"http://{host}:{port}"
command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url)
@ -196,7 +373,6 @@ def server(
@click.option("--reload", is_flag=True, help="Enable auto-reloading when code or templates change")
@click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
@click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
@click.option("--init", is_flag=True, help="Run a full archivebox init/upgrade before starting the server")
@click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
@docstring(server.__doc__)
def main(**kwargs):

86
archivebox/config/common.py
View File

@ -33,7 +33,6 @@ PluginSchemaDocuments = dict[str, dict[str, Any]]
_STDOUT_CONSOLE = Console()
_STDERR_CONSOLE = Console(stderr=True)
_WARNED_SERVER_SECURITY_MODES: set[str] = set()
_WARNED_ARCHIVING_CONFIGS: set[tuple[int, bool]] = set()
@ -126,13 +125,9 @@ class StorageConfig(BaseConfigSet):
CUSTOM_TEMPLATES_DIR: Path = Field(default=CONSTANTS.CUSTOM_TEMPLATES_DIR)
OUTPUT_PERMISSIONS: str = Field(default="644")
RESTRICT_FILE_NAMES: str = Field(default="windows")
ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
ALLOW_NO_UNIX_SOCKETS: bool = Field(default=False, alias="ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS")
# not supposed to be user settable:
DIR_OUTPUT_PERMISSIONS: str = Field(default="755") # computed from OUTPUT_PERMISSIONS
class GeneralConfig(BaseConfigSet):
toml_section_header: str = "GENERAL_CONFIG"
@ -158,7 +153,6 @@ class ServerConfig(BaseConfigSet):
SERVER_SECURITY_MODE: str = Field(default="safe-subdomains-fullreplay")
SNAPSHOTS_PER_PAGE: int = Field(default=40)
PREVIEW_ORIGINALS: bool = Field(default=True)
FOOTER_INFO: str = Field(
default="Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.",
)
@ -241,39 +235,6 @@ class DatabaseConfig(BaseConfigSet):
SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)
def _print_server_security_mode_warning(config: ServerConfig) -> None:
if not config.IS_LOWER_SECURITY_MODE:
return
if config.SERVER_SECURITY_MODE in _WARNED_SERVER_SECURITY_MODES:
return
rprint(
f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={config.SERVER_SECURITY_MODE}[/yellow]",
file=sys.stderr,
)
rprint(
"[yellow] Archived pages may share an origin with privileged app routes in this mode.[/yellow]",
file=sys.stderr,
)
rprint(
"[yellow] To switch to the safer isolated setup:[/yellow]",
file=sys.stderr,
)
rprint(
"[yellow] 1. Set SERVER_SECURITY_MODE=safe-subdomains-fullreplay[/yellow]",
file=sys.stderr,
)
rprint(
"[yellow] 2. Point *.archivebox.localhost (or your chosen base domain) at this server[/yellow]",
file=sys.stderr,
)
rprint(
"[yellow] 3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]",
file=sys.stderr,
)
_WARNED_SERVER_SECURITY_MODES.add(config.SERVER_SECURITY_MODE)
class ArchivingConfig(BaseConfigSet):
toml_section_header: str = "ARCHIVING_CONFIG"
@ -285,17 +246,10 @@ class ArchivingConfig(BaseConfigSet):
default="",
description="Comma-separated plugin selection override used by the UI and API.",
)
ENABLED_EXTRACTORS: str = Field(
default="",
description="Legacy comma-separated plugin selection override.",
)
ONLY_NEW: bool = Field(default=True)
OVERWRITE: bool = Field(default=False)
TIMEOUT: int = Field(default=60)
MAX_URL_ATTEMPTS: int = Field(default=50)
MAX_DEPTH: int = Field(default=0)
CRAWL_MAX_URLS: int = Field(default=0)
CRAWL_MAX_SIZE: int = Field(default=0)
CRAWL_TIMEOUT: int = Field(default=0, description="Maximum total crawl runtime in seconds (0 = unlimited).")
@ -315,9 +269,6 @@ class ArchivingConfig(BaseConfigSet):
URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
SAVE_ALLOWLIST: dict[str, list[str]] = Field(default={}) # mapping of regex patterns to list of archive methods
SAVE_DENYLIST: dict[str, list[str]] = Field(default={})
DEFAULT_PERSONA: str = Field(default="Default")
PERMISSIONS: str = Field(
default="public",
@ -375,30 +326,6 @@ class ArchivingConfig(BaseConfigSet):
def URL_DENYLIST_PTN(self) -> re.Pattern:
return re.compile(self.URL_DENYLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS)
@property
def SAVE_ALLOWLIST_PTNS(self) -> dict[re.Pattern, list[str]]:
return (
{
# regexp: methods list
re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
for key, val in self.SAVE_ALLOWLIST.items()
}
if self.SAVE_ALLOWLIST
else {}
)
@property
def SAVE_DENYLIST_PTNS(self) -> dict[re.Pattern, list[str]]:
return (
{
# regexp: methods list
re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
for key, val in self.SAVE_DENYLIST.items()
}
if self.SAVE_DENYLIST
else {}
)
def parse_delete_after(value) -> timedelta | None:
if value is None:
@ -435,11 +362,7 @@ def parse_delete_after(value) -> timedelta | None:
class SearchBackendConfig(BaseConfigSet):
toml_section_header: str = "SEARCH_BACKEND_CONFIG"
USE_INDEXING_BACKEND: bool = Field(default=True)
USE_SEARCHING_BACKEND: bool = Field(default=True)
SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep")
SEARCH_PROCESS_HTML: bool = Field(default=True)
def _plugin_user_config_value(value: Any) -> str:
@ -517,7 +440,6 @@ class ArchiveBoxBaseConfig(
DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR)
ABX_RUNTIME: str = Field(default="archivebox")
CRAWL_DIR: Path | None = Field(default=None)
CRAWL_OUTPUT_DIR: Path | None = Field(default=None)
SNAP_DIR: Path | None = Field(default=None)
computed_config_keys: ClassVar[tuple[str, ...]] = COMPUTED_CONFIG_KEYS
@ -653,13 +575,8 @@ def get_config(
scope_overrides.update(crawl.config)
if crawl is not None:
crawl_output_dir = None
if not overrides or "CRAWL_OUTPUT_DIR" not in overrides or "CRAWL_DIR" not in overrides:
crawl_output_dir = crawl.output_dir
if not overrides or "CRAWL_OUTPUT_DIR" not in overrides:
scope_overrides["CRAWL_OUTPUT_DIR"] = crawl_output_dir
if not overrides or "CRAWL_DIR" not in overrides:
scope_overrides["CRAWL_DIR"] = crawl_output_dir
scope_overrides["CRAWL_DIR"] = crawl.output_dir
if snapshot is not None and snapshot.config:
scope_overrides.update(snapshot.config)
@ -704,7 +621,6 @@ def get_config(
if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:
config.warn_if_invalid()
_WARNED_ARCHIVING_CONFIGS.add(archiving_warning_key)
_print_server_security_mode_warning(config)
return config

2
archivebox/config/configset.py
View File

@ -22,8 +22,6 @@ COMPUTED_CONFIG_KEYS = (
"IS_LOWER_SECURITY_MODE",
"URL_ALLOWLIST_PTN",
"URL_DENYLIST_PTN",
"SAVE_ALLOWLIST_PTNS",
"SAVE_DENYLIST_PTNS",
)

6
archivebox/core/admin_snapshots.py
View File

@ -1605,7 +1605,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
from archivebox.cli.archivebox_add import add
add(urls=urls, bg=True)
# "Archive Now" is an explicit user re-archive — force ONLY_NEW=False
# on the resulting crawl so existing snapshots don't cause the crawl to
# seal immediately with zero new snapshots (the default ONLY_NEW=True
# would skip any URLs that have ever been archived before).
add(urls=urls, bg=True, config={"ONLY_NEW": False})
messages.success(
request,

9
archivebox/core/context_processors.py Normal file
View File

@ -0,0 +1,9 @@
from archivebox.config import VERSION
from archivebox.config.version import get_COMMIT_HASH
def archivebox_globals(request):
return {
"VERSION": VERSION,
"STATIC_CACHE_KEY": (get_COMMIT_HASH() or VERSION or "dev").strip(),
}

105
archivebox/core/forms.py
View File

@ -44,6 +44,7 @@ PLUGIN_GROUP_DEFINITIONS = (
"wget",
"archivedotorg",
"chrome_mhtml",
"archivewebpage",
),
),
(
@ -166,11 +167,9 @@ def get_plugin_choice_label(plugin_name: str, plugin_configs: dict[str, dict]) -
icon_html = get_plugin_icon(plugin_name)
return format_html(
'<span class="plugin-choice-icon">{}</span><span class="plugin-choice-name">{}</span><a class="plugin-choice-description" href="https://archivebox.github.io/abx-plugins/#{}" target="_blank" rel="noopener noreferrer">{}</a>',
'<span class="plugin-choice-icon">{}</span><span class="plugin-choice-name">{}</span>',
icon_html,
plugin_name,
plugin_name,
description,
)
@ -288,6 +287,7 @@ class PluginConfigFormMixin:
*PLUGIN_GROUP_DEFINITIONS,
("other_plugins", "Other", "", "", "", other_plugins),
)
binary_url_lookup = _build_required_binary_url_lookup(plugin_configs, runtime_config)
self.plugin_groups = [
{
"field_name": field_name,
@ -296,7 +296,7 @@ class PluginConfigFormMixin:
"dom_id": dom_id,
"select_all_group": select_all_group,
"show_selectors": field_name in self.fields,
"plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config),
"plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config, binary_url_lookup),
}
for field_name, title, note, dom_id, select_all_group, plugin_names in group_specs
if any(plugin in all_plugins for plugin in plugin_names)
@ -308,6 +308,7 @@ class PluginConfigFormMixin:
plugin_names: Iterable[str],
plugin_configs: dict[str, dict[str, Any]],
runtime_config: Mapping[str, Any],
binary_url_lookup: Mapping[str, str] | None = None,
) -> list[dict[str, Any]]:
if field_name in self.fields:
choices = list(get_choice_field(self, field_name).choices)
@ -341,7 +342,11 @@ class PluginConfigFormMixin:
"source_url": f"https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/{plugin_name}",
"docs_url": f"https://archivebox.github.io/abx-plugins/#{plugin_name}",
"required_plugins": [str(item) for item in schema.get("required_plugins") or []],
"required_binaries_count": len(schema.get("required_binaries") or []),
"required_binary_links": _build_required_binary_links(
schema.get("required_binaries") or [],
runtime_config,
binary_url_lookup,
),
"config_fields": config_fields,
"config_count": len(config_fields),
},
@ -476,6 +481,96 @@ class PluginConfigFormMixin:
}
_BINARY_TEMPLATE_PATTERN = re.compile(r"\{([A-Z_][A-Z0-9_]*)\}")
def _resolve_required_binary_name(template_name: str, runtime_config: Mapping[str, Any]) -> str:
if "{" not in template_name:
return template_name
def _replace(match: re.Match[str]) -> str:
key = match.group(1)
try:
value = runtime_config.get(key)
except Exception:
value = None
if value is None or value == "":
return match.group(0)
return str(value)
resolved = _BINARY_TEMPLATE_PATTERN.sub(_replace, template_name).strip()
if not resolved:
return template_name
return Path(resolved).name if "/" in resolved else resolved
def _iter_required_binary_names(
required_binaries: Iterable[Any],
runtime_config: Mapping[str, Any],
) -> Iterable[str]:
for item in required_binaries or []:
if not isinstance(item, dict):
continue
raw_name = str(item.get("name") or "").strip()
if not raw_name:
continue
resolved = _resolve_required_binary_name(raw_name, runtime_config)
if resolved:
yield resolved
def _build_required_binary_url_lookup(
plugin_configs: Mapping[str, dict[str, Any]],
runtime_config: Mapping[str, Any],
) -> dict[str, str]:
"""Resolve admin URLs for every required binary across all plugin schemas in a single DB query."""
from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url
from archivebox.machine.models import Binary, Machine
resolved_names: set[str] = set()
for schema in plugin_configs.values():
for name in _iter_required_binary_names(schema.get("required_binaries") or [], runtime_config):
resolved_names.add(name)
if not resolved_names:
return {}
machine = Machine.current()
name_to_binary: dict[str, Binary] = {}
for binary in (
Binary.objects.filter(machine=machine, name__in=resolved_names)
.exclude(abspath="")
.exclude(abspath__isnull=True)
.order_by("-modified_at")
):
key = binary.name.lower()
if key not in name_to_binary:
name_to_binary[key] = binary
return {
name: (get_installed_binary_change_url(name, name_to_binary.get(name.lower())) or get_environment_binary_url(name))
for name in resolved_names
}
def _build_required_binary_links(
required_binaries: list[dict[str, Any]],
runtime_config: Mapping[str, Any],
binary_url_lookup: Mapping[str, str] | None = None,
) -> list[dict[str, str]]:
from archivebox.config.views import get_environment_binary_url
links: list[dict[str, str]] = []
seen: set[str] = set()
for resolved in _iter_required_binary_names(required_binaries, runtime_config):
if resolved in seen:
continue
seen.add(resolved)
url = (binary_url_lookup or {}).get(resolved) or get_environment_binary_url(resolved)
links.append({"name": resolved, "url": url})
return links
def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str, str]:
from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url
from archivebox.machine.models import Binary, Machine

130
archivebox/core/host_utils.py
View File

@ -9,6 +9,7 @@ from archivebox.config.common import get_config
_SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
_SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api", "public")
def split_host_port(host: str) -> tuple[str, str | None]:
@ -29,13 +30,84 @@ def _normalize_base_url(value: str | None) -> str:
parsed = urlparse(base)
if not parsed.netloc:
return ""
return f"{parsed.scheme}://{parsed.netloc}"
# Accept ``*.<host>`` as a synonym for ``<host>`` so users can paste the
# wildcard-friendly form (e.g. from the banner suggestion) without it
# leaking ``*.`` into every downstream URL. Subdomain routing already
# prepends the appropriate role label (admin/web/api/snap-*) at build
# time, so the bare base host is what we want to store.
netloc = parsed.netloc
while netloc.startswith("*."):
netloc = netloc[2:]
if not netloc:
return ""
return f"{parsed.scheme}://{netloc}"
def normalize_base_url(value: str | None) -> str:
return _normalize_base_url(value)
def _csrf_trusted_origins(config) -> list[str]:
raw = (config.CSRF_TRUSTED_ORIGINS or "").strip()
if not raw:
return []
seen: list[str] = []
for entry in raw.split(","):
normalized = _normalize_base_url(entry.strip())
if normalized and normalized not in seen:
seen.append(normalized)
return seen
def _allowed_hosts(config) -> set[str]:
raw = (config.ALLOWED_HOSTS or "").strip()
if not raw:
return set()
return {entry.strip().lower() for entry in raw.split(",") if entry.strip() and entry.strip() != "*"}
def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
"""Pick a single CSRF_TRUSTED_ORIGINS entry to act as the implicit BASE_URL.
0.7.3 → 0.9.0 upgrade path: any reverse-proxied 0.7.3 deployment already
had ``CSRF_TRUSTED_ORIGINS=https://archive.example.com`` set (required for
admin login to work). On upgrade, ``BASE_URL`` is the new knob — but it
defaults to empty, and falling through to ``BIND_ADDR`` produces an
unreachable URL like ``http://0.0.0.0:8000``. If the user has exactly one
CSRF origin we treat it as the implicit BASE_URL so links/redirects keep
pointing at the public hostname they already configured.
Returns ``""`` when the inference is ambiguous (multiple origins) or
impossible (none set) so callers fall through to their next strategy.
"""
config = config or get_config(**config_kwargs)
origins = _csrf_trusted_origins(config)
if len(origins) == 1:
return origins[0]
return ""
def request_host_is_explicitly_allowed(request_host: str, config) -> bool:
"""True if the incoming Host is in ALLOWED_HOSTS or matches a CSRF origin.
Used by ``get_base_url`` to honour the request's own Host header when the
operator hasn't pinned ``BASE_URL`` explicitly. The check is intentionally
strict — we won't trust arbitrary Host headers, only ones the user has
already opted into via ALLOWED_HOSTS / CSRF_TRUSTED_ORIGINS.
"""
if not request_host:
return False
host, _ = split_host_port(request_host)
allowed = _allowed_hosts(config)
if host in allowed:
return True
for origin in _csrf_trusted_origins(config):
origin_host, _ = split_host_port(urlparse(origin).netloc)
if origin_host == host:
return True
return False
def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
return (config.BIND_ADDR or "").strip()
@ -50,10 +122,48 @@ def _with_port(host: str, port: str | None) -> str:
return f"{host}:{port}" if port else host
def strip_role_subdomain(host: str) -> str:
"""Strip leading ``admin.`` / ``web.`` / ``api.`` / ``public.`` / ``snap-*.``
labels from a host (preserving the port). Strips repeatedly so an
already-compounded host like ``snap-X.snap-X.<base>`` reduces all the
way down to ``<base>``.
Used when we want to recover the canonical base host from a request that
arrived on a role subdomain — otherwise builders that prepend their own
role label (e.g. ``snap-X.``) compound onto the existing prefix and you
get ``snap-X.snap-X.snap-X.<base>`` on every click.
"""
if not host:
return ""
hostname, port = split_host_port(host)
while hostname and "." in hostname:
head, _sep, rest = hostname.partition(".")
if head in _ROLE_SUBDOMAIN_LABELS or _SNAPSHOT_SUBDOMAIN_RE.match(head):
hostname = rest
continue
break
return _with_port(hostname, port)
def _is_local_bind_host(host: str) -> bool:
return host in {"", "0.0.0.0", "::", "127.0.0.1", "::1", "localhost"}
def canonical_base_host_for_request(request_host: str) -> str:
"""Strip role subdomains and remap loopback hostnames to ``archivebox.localhost``.
Used by the banner suggestion and the in-browser pin endpoint: when the
user is hitting the server on raw ``localhost:9292`` or ``127.0.0.1:9292``
we want to suggest the wildcard-friendly ``archivebox.localhost`` family
instead, so the eventual pinned ``BASE_URL`` plays nicely with subdomain
routing without forcing the user to add a /etc/hosts entry.
"""
hostname, port = split_host_port(strip_role_subdomain(request_host or ""))
if _is_local_bind_host(hostname):
hostname = "archivebox.localhost"
return _with_port(hostname, port)
def _root_host_from_listen(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
listen_host, listen_port = get_listen_parts(config=config)
@ -67,6 +177,13 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw
if override:
return override
# A) Implicit BASE_URL from a single CSRF_TRUSTED_ORIGINS entry. Catches
# 0.7.3 → 0.9.0 upgrades where users already set CSRF_TRUSTED_ORIGINS
# for their reverse-proxy login but never set BASE_URL.
csrf_derived = derive_base_url_from_csrf(config)
if csrf_derived:
return csrf_derived
scheme = request.scheme if request else "http"
if request:
req_host, req_port = split_host_port(request.get_host())
@ -74,6 +191,17 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw
return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
if _is_local_bind_host(req_host):
return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
# C) Per-request fallback: when ``BASE_URL`` is unset and CSRF didn't
# give us a single origin, trust the request's Host header — but first
# peel off any ``admin.`` / ``web.`` / ``api.`` / ``public.`` /
# ``snap-*.`` label. Otherwise the URL builders below prepend their own
# role label onto a host that already carries one, producing the
# ``snap-X.snap-X.snap-X.<base>`` compounding bug. Django has already
# admitted the host via ALLOWED_HOSTS; the misconfig banner surfaces
# the case where the resulting URL doesn't match what the operator
# probably intended.
canonical_host = strip_role_subdomain(request.get_host())
return f"{scheme}://{canonical_host}"
root_host = _root_host_from_listen(config=config)
return f"{scheme}://{root_host}" if root_host else ""

55
archivebox/core/middleware.py
View File

@ -21,6 +21,7 @@ from archivebox.core.host_utils import (
build_web_url,
get_api_host,
get_admin_host,
get_base_host,
get_listen_host,
get_listen_subdomain,
get_public_host,
@ -36,14 +37,32 @@ ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
def _admin_login_hint_cookie_domain(config) -> str | None:
"""Resolve the parent domain to scope the cross-subdomain login hint.
NOTE: this cookie carries only the single bit "user is logged in on
admin somewhere"; it MUST NOT be confused with the session cookie,
which stays admin-host-scoped (see core/settings.py
SESSION_COOKIE_DOMAIN comment — admin/public is a security boundary).
Returns the hostname portion of ``get_base_host`` (which respects
``BASE_URL`` and falls back to the local-bind mapping). Strips the
port — cookie ``Domain=`` attributes don't include ports. Returns
``None`` when subdomain routing is off, the base host is empty, or
the base host is an IP / bare ``localhost`` (browsers reject
cross-host cookies for those).
"""
if not config.USES_SUBDOMAIN_ROUTING:
return None
listen_host, _listen_port = split_host_port(get_listen_host(config=config))
base_host = get_base_host(config=config)
if not base_host:
return None
hostname, _port = split_host_port(base_host)
if not hostname or hostname == "localhost":
return None
try:
ipaddress.ip_address(listen_host)
ipaddress.ip_address(hostname)
except ValueError:
if listen_host and listen_host != "localhost":
return listen_host
return hostname
return None
@ -166,12 +185,32 @@ def HostRoutingMiddleware(get_response):
if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}:
return get_response(request)
# In subdomain mode with no explicit BASE_URL we can't safely emit
# ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
# request's own Host (via the request-host fallback in get_base_url),
# so prepending ``admin.`` to whatever the client sent produces a
# redirect chain of ``admin.admin.admin.<host>``. Pass the request
# through; the misconfig banner on the rendered page tells the user
# to pin BASE_URL so the redirects can resume.
if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
return get_response(request)
if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
# ``/add`` is admin-only unless ``PUBLIC_ADD_VIEW`` is on. Without
# this redirect, hitting it on public.* falls into AddView's
# auth check, bounces through ``/accounts/login/?next=/add/`` →
# ``/admin/login/?next=/add/``, and Django admin's LoginView
# silently drops ``next`` when the user already has an admin
# session — dumping the user on the admin homepage instead of
# the add form. Routing the request to admin.* directly lets
# AddView run on the host where the session lives.
add_should_redirect = not config.PUBLIC_ADD_VIEW and (request.path == "/add" or request.path.startswith("/add/"))
if (
request.path == "/admin"
or request.path.startswith("/admin/")
or request.path == "/accounts"
or request.path.startswith("/accounts/")
or add_should_redirect
):
target = build_admin_url(request.path, request=request)
if request.META.get("QUERY_STRING"):
@ -262,7 +301,13 @@ def HostRoutingMiddleware(get_response):
target = f"{target}?{request.META['QUERY_STRING']}"
return redirect(target)
if admin_host or web_host:
if (admin_host or web_host) and config.BASE_URL:
# Only force a canonical-host redirect when BASE_URL was set
# explicitly. If BASE_URL is empty (e.g. 0.7.3 → 0.9.0 upgrade
# where the user has CSRF_TRUSTED_ORIGINS but never set BASE_URL),
# the subdomain we'd redirect to may not actually resolve in the
# user's reverse proxy — serve the request as-is instead and let
# the misconfig banner surface the problem in the page.
target = build_web_url(request.path, request=request)
if target:
if request.META.get("QUERY_STRING"):

10
archivebox/core/models.py
View File

@ -278,12 +278,6 @@ class SnapshotQuerySet(models.QuerySet):
def search(self, patterns: list[str]) -> "SnapshotQuerySet":
"""Search snapshots using the configured search backend"""
from archivebox.search import query_search_index
from archivebox.misc.logging import stderr
if not get_config().USE_SEARCHING_BACKEND:
stderr()
stderr("[X] The search backend is not enabled, set config.USE_SEARCHING_BACKEND = True", color="red")
raise SystemExit(2)
qsearch = self.none()
for pattern in patterns:
@ -3060,8 +3054,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
from archivebox.misc.logging_util import printable_filesize
output_dir = Path(out_dir) if out_dir is not None else self.output_dir
config = get_config()
SAVE_ARCHIVE_DOT_ORG = config.get("SAVE_ARCHIVE_DOT_ORG", True)
TITLE_LOADING_MSG = "Not yet archived..."
preview_priority = [
@ -3104,8 +3096,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
"status": "archived" if is_archived else "not yet archived",
"status_color": "success" if is_archived else "danger",
"oldest_archive_date": ts_to_date_str(self.oldest_archive_date),
"SAVE_ARCHIVE_DOT_ORG": SAVE_ARCHIVE_DOT_ORG,
"PREVIEW_ORIGINALS": config.PREVIEW_ORIGINALS,
"best_preview_path": best_preview_path,
"best_result": best_result,
"archiveresults": outputs,

20
archivebox/core/settings.py
View File

@ -210,6 +210,7 @@ TEMPLATES = [
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
"archivebox.core.context_processors.archivebox_globals",
],
},
},
@ -371,12 +372,17 @@ api_base_url = normalize_base_url(get_api_base_url())
if api_base_url and api_base_url not in CSRF_TRUSTED_ORIGINS:
CSRF_TRUSTED_ORIGINS.append(api_base_url)
# automatically fix case when user sets ALLOWED_HOSTS (e.g. to archivebox.example.com)
# but forgets to add https://archivebox.example.com to CSRF_TRUSTED_ORIGINS
# Auto-extend CSRF_TRUSTED_ORIGINS with the explicit ALLOWED_HOSTS entries so
# users who set ALLOWED_HOSTS=archivebox.example.com don't have to also list
# https://archivebox.example.com under CSRF_TRUSTED_ORIGINS. (The previous
# per-host WARNING print was just noise — the auto-append below is the actual
# fix, and the effective CSRF_TRUSTED_ORIGINS gets surfaced once at startup
# from archivebox_server.py.)
for hostname in ALLOWED_HOSTS:
if hostname == "*":
continue
https_endpoint = f"https://{hostname}"
if hostname != "*" and https_endpoint not in CSRF_TRUSTED_ORIGINS:
print(f"[!] WARNING: {https_endpoint} from ALLOWED_HOSTS should be added to CSRF_TRUSTED_ORIGINS")
if https_endpoint not in CSRF_TRUSTED_ORIGINS:
CSRF_TRUSTED_ORIGINS.append(https_endpoint)
SECURE_BROWSER_XSS_FILTER = True
@ -386,6 +392,12 @@ SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
CSRF_COOKIE_SECURE = False
SESSION_COOKIE_SECURE = False
SESSION_COOKIE_HTTPONLY = True
# Auth cookies are intentionally scoped to the exact host that set them so
# the admin session is NOT readable from public.* / web.* / api.* — that
# split is a security boundary, not a UX choice. Subdomains that need to
# render an "is the user logged in?" indicator must use the single-bit
# `archivebox_admin_logged_in` hint cookie set by core/middleware.py
# (which IS scoped to the listen-host parent), never widen these.
SESSION_COOKIE_DOMAIN = None
CSRF_COOKIE_DOMAIN = None
SESSION_COOKIE_AGE = 1209600 # 2 weeks

57
archivebox/core/settings_logging.py
View File

@ -4,7 +4,6 @@ import re
import os
import tempfile
import logging
from pathlib import Path
from archivebox.config import CONSTANTS
@ -21,39 +20,6 @@ IGNORABLE_URL_PATTERNS = [
]
SENSITIVE_QUERY_PARAM_RE = re.compile(r"(?i)([?&](?:api_key|token|access_token|password|secret)=)([^&#\s]+)")
WEBREQUEST_RE = re.compile(r"<WebRequest\b[^>]*\bmethod=(?P<method>[A-Z]+)\s+uri=(?P<uri>\S+)")
RUNNING_AT_RE = re.compile(r"running at\s+([^>]+:\d+)")
def _redact_url(url: str) -> str:
return SENSITIVE_QUERY_PARAM_RE.sub(r"\1[REDACTED]", url)
def _short_code_path(path: str) -> str:
try:
return str(Path(path).resolve().relative_to(Path.cwd().resolve()))
except (OSError, ValueError):
parts = Path(path).parts
return "/".join(parts[-4:]) if len(parts) > 4 else path
def _resolve_view_name(url: str) -> str:
try:
from django.urls import resolve
match = resolve(url.split("?", 1)[0])
if match.view_name:
return match.view_name
view_func = match.func
view_class = getattr(view_func, "view_class", None)
if view_class is not None:
return f"{view_class.__module__}.{view_class.__name__}"
return f"{view_func.__module__}.{view_func.__name__}"
except Exception:
return "unknown"
class NoisyRequestsFilter(logging.Filter):
def filter(self, record) -> bool:
logline = record.getMessage()
@ -77,26 +43,27 @@ class NoisyRequestsFilter(logging.Filter):
class DaphneCloseTimeoutFilter(logging.Filter):
"""Drop daphne's noisy "killed slow response after client disconnect" warning.
Daphne emits this whenever a request handler is still running when the
client disconnects (e.g. iframe gets navigated away mid-response while
fetching favicon / screenshot / preview html). For our use case these are
always benign — the disconnect is the browser cancelling a request, not a
server-side fault — so we suppress them outright rather than spamming
WARNING. Other daphne.server lines pass through unchanged.
"""
def filter(self, record) -> bool:
if record.name != "daphne.server":
return True
logline = record.getMessage()
if not (
if (
"Application instance" in logline
and "for connection <WebRequest" in logline
and "took too long to shut down" in logline
and "was killed" in logline
):
return True
match = WEBREQUEST_RE.search(logline)
method = match.group("method") if match else "-"
uri = _redact_url(match.group("uri")) if match else "-"
view = _resolve_view_name(uri) if uri != "-" else "unknown"
code_paths = [_short_code_path(path) for path in RUNNING_AT_RE.findall(logline)]
code = code_paths[-1] if code_paths else "unknown"
record.msg = f"Daphne killed slow response after client disconnect: {method} {uri} view={view} code={code}"
record.args = ()
return False
return True

25
archivebox/core/tag_utils.py
View File

@ -6,7 +6,7 @@ from typing import Any
from urllib.parse import unquote
from django.contrib.auth.models import User
from django.db.models import Count, F, QuerySet
from django.db.models import Count, Exists, F, OuterRef, QuerySet
from django.db.models.functions import Lower
from django.http import HttpRequest
from django.urls import reverse
@ -65,7 +65,7 @@ def get_matching_tags(
) -> QuerySet[Tag]:
sort = normalize_tag_sort(sort)
has_snapshots = normalize_has_snapshots_filter(has_snapshots)
needs_snapshot_counts = with_snapshot_counts or sort.startswith("snapshots_") or has_snapshots != "all"
needs_snapshot_counts = sort.startswith("snapshots_")
queryset = Tag.objects.select_related("created_by")
if needs_snapshot_counts:
@ -83,10 +83,13 @@ def get_matching_tags(
if year:
queryset = queryset.filter(created_at__year=int(year))
if has_snapshots != "all" and not needs_snapshot_counts:
queryset = queryset.annotate(has_snapshot=Exists(SnapshotTag.objects.filter(tag_id=OuterRef("pk"))))
if has_snapshots == "yes":
queryset = queryset.filter(num_snapshots__gt=0)
queryset = queryset.filter(num_snapshots__gt=0) if needs_snapshot_counts else queryset.filter(has_snapshot=True)
elif has_snapshots == "no":
queryset = queryset.filter(num_snapshots=0)
queryset = queryset.filter(num_snapshots=0) if needs_snapshot_counts else queryset.filter(has_snapshot=False)
if sort == "name_asc":
queryset = queryset.order_by(Lower("name"), "id")
@ -252,12 +255,12 @@ def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = No
"name": tag.name,
"slug": tag.slug,
"num_snapshots": count,
"filter_url": f"{reverse('admin:core_snapshot_changelist')}?tags__id__exact={tag.pk}",
"edit_url": reverse("admin:core_tag_change", args=[tag.pk]),
"export_urls_url": reverse("api-1:tag_urls_export", args=[tag.pk]),
"export_jsonl_url": reverse("api-1:tag_snapshots_export", args=[tag.pk]),
"rename_url": reverse("api-1:rename_tag", args=[tag.pk]),
"delete_url": reverse("api-1:delete_tag", args=[tag.pk]),
"filter_url": f"/admin/core/snapshot/?tags__id__exact={tag.pk}",
"edit_url": f"/admin/core/tag/{tag.pk}/change/",
"export_urls_url": f"/api/v1/core/tag/{tag.pk}/urls.txt",
"export_jsonl_url": f"/api/v1/core/tag/{tag.pk}/snapshots.jsonl",
"rename_url": f"/api/v1/core/tag/{tag.pk}/rename",
"delete_url": f"/api/v1/core/tag/{tag.pk}/",
"snapshots": snapshot_previews or [],
}
@ -274,7 +277,7 @@ def build_tag_cards(
) -> list[dict[str, Any]]:
sort = normalize_tag_sort(sort)
has_snapshots = normalize_has_snapshots_filter(has_snapshots)
needs_snapshot_count_annotation = sort.startswith("snapshots_") or has_snapshots != "all"
needs_snapshot_count_annotation = sort.startswith("snapshots_")
queryset = get_matching_tags(
query=query,
sort=sort,

90
archivebox/core/templatetags/core_tags.py
View File

@ -13,6 +13,7 @@ from archivebox.hooks import (
get_plugin_name,
)
from archivebox.core.host_utils import (
canonical_base_host_for_request,
get_admin_base_url,
get_public_base_url,
get_web_base_url,
@ -26,6 +27,7 @@ register = template.Library()
_TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log")
_IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
_MHTML_PREVIEW_EXTS = (".mhtml", ".mht")
_WACZ_PREVIEW_EXTS = (".wacz", ".warc", ".warc.gz")
_MEDIA_FILE_EXTS = {
".mp4",
@ -204,7 +206,12 @@ def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None,
if _is_root_snapshot_output_path(path):
return _build_snapshot_files_url(snapshot_id, request=request, config=config)
url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
if not (_is_text_preview_path(path) or _is_image_preview_path(path) or (path or "").lower().endswith(_MHTML_PREVIEW_EXTS)):
if not (
_is_text_preview_path(path)
or _is_image_preview_path(path)
or (path or "").lower().endswith(_MHTML_PREVIEW_EXTS)
or (path or "").lower().endswith(_WACZ_PREVIEW_EXTS)
):
return url
separator = "&" if "?" in url else "?"
return f"{url}{separator}preview=1"
@ -317,6 +324,77 @@ def result_list_tag(parser, token):
)
@register.inclusion_tag("security_mode_banner.html", takes_context=True)
def security_mode_banner(context):
"""Render the top-of-page warning banner for one of two conditions:
1. ``mode="unconfigured"`` — ``BASE_URL`` is empty. The server is running
on whatever host the operator happens to be hitting; CSRF auto-derive
and the request-host fallback in ``get_base_url`` keep things working,
but the operator should pin ``BASE_URL`` explicitly so links stay
stable across hosts (and the misconfig banner goes away).
2. ``mode="unsafe"`` — ``SERVER_SECURITY_MODE`` is a non-subdomain mode.
Archived pages share an origin with privileged routes.
Both conditions can hold; we show the ``unconfigured`` banner first
because pinning ``BASE_URL`` is the more immediately actionable fix.
"""
config = context.get("CONFIG")
if config is None:
from archivebox.config.common import get_config
config = get_config(resolve_plugins=False)
if not config.BASE_URL:
return _unconfigured_banner_context(context.get("request"))
if not config.USES_SUBDOMAIN_ROUTING:
return {"mode": "unsafe"}
return {"mode": ""}
def _unconfigured_banner_context(request) -> dict:
"""Build the banner payload for the unset-BASE_URL case.
Always returns ``mode="unconfigured"`` — the user explicitly asked for
the banner to render whenever ``BASE_URL`` is empty, regardless of
whether the request host happens to match a CSRF-derived value. The
``suggested_base_url`` is derived from the current request when one is
available so the user can copy/paste the right value straight into
their config.
"""
if request is None:
return {
"mode": "unconfigured",
"actual_host": "",
"suggested_base_url": "",
"machine_admin_url": "",
}
scheme = request.scheme or "http"
actual_full_host = request.get_host() or ""
canonical_host = canonical_base_host_for_request(actual_full_host)
# Suggest the wildcard form ``http://*.<host>`` so the value lands in the
# operator's clipboard already aligned with subdomain routing. The config
# parser strips the leading ``*.`` so users can paste it verbatim.
suggested_base_url = f"{scheme}://*.{canonical_host}" if canonical_host else ""
user = getattr(request, "user", None)
is_superuser = bool(user and user.is_authenticated and user.is_superuser)
machine_admin_url = ""
if is_superuser:
try:
from archivebox.machine.models import Machine
machine = Machine.current()
machine_admin_url = f"/admin/machine/machine/{machine.id}/change/"
except Exception:
machine_admin_url = ""
return {
"mode": "unconfigured",
"actual_host": actual_full_host,
"suggested_base_url": suggested_base_url,
"machine_admin_url": machine_admin_url,
}
@register.simple_tag(takes_context=True)
def url_replace(context, **kwargs):
dict_ = context["request"].GET.copy()
@ -324,6 +402,16 @@ def url_replace(context, **kwargs):
return dict_.urlencode()
@register.simple_tag
def has_real_admin_users() -> bool:
"""True if any non-``system`` superuser exists. Used by the login page to
only show the bootstrap hint (createsuperuser / ADMIN_USERNAME env vars)
when the collection still has no real admin."""
from django.contrib.auth.models import User
return User.objects.filter(is_superuser=True).exclude(username="system").exists()
@register.simple_tag(takes_context=True)
def admin_base_url(context) -> str:
return get_admin_base_url(request=context.get("request"), config=context.get("CONFIG"))

11
archivebox/core/urls.py
View File

@ -66,10 +66,15 @@ urlpatterns = [
),
path("admin/core/snapshot/add/", RedirectView.as_view(url="/add/")),
path("add/", AddView.as_view(), name="add"),
path("accounts/login/", RedirectView.as_view(url="/admin/login/")),
path("accounts/logout/", RedirectView.as_view(url="/admin/logout/")),
# ``query_string=True`` preserves the ``?next=…`` param that Django's
# auth/login mixins append, so e.g. ``UserPassesTestMixin`` redirecting
# an unauthenticated ``/add`` visitor to ``/accounts/login/?next=/add/``
# carries the ``next`` through to ``/admin/login/`` and lands them at
# ``/add/`` after login instead of the admin homepage.
path("accounts/login/", RedirectView.as_view(url="/admin/login/", query_string=True)),
path("accounts/logout/", RedirectView.as_view(url="/admin/logout/", query_string=True)),
path("accounts/", include("django.contrib.auth.urls")),
path("admin/live-progress/", archivebox_admin.admin_view(live_progress_view), name="live_progress"),
path("progress.json", live_progress_view, name="live_progress"),
path("admin/", archivebox_admin.urls),
path("api/", include("archivebox.api.urls"), name="api"),
path("health/", HealthCheckView.as_view(), name="healthcheck"),

210
archivebox/core/views.py
View File

@ -36,7 +36,14 @@ from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, VERSION
from archivebox.config.common import get_config, get_all_configs
from archivebox.config.configset import BaseConfigSet
from archivebox.misc.paginators import CountlessPaginator
from archivebox.misc.util import base_url, htmlencode, ts_to_date_str, urldecode, without_fragment
from archivebox.misc.util import (
base_url,
filter_queryset_by_uuid_substring,
htmlencode,
ts_to_date_str,
urldecode,
without_fragment,
)
from archivebox.misc.serve_static import serve_static_with_byterange_support
from archivebox.misc.logging_util import printable_filesize
from archivebox.search import (
@ -55,6 +62,7 @@ from archivebox.core.permissions import (
can_view_snapshot,
direct_snapshots_queryset,
filter_personas_by_permissions,
get_snapshot_permissions,
is_admin_user,
public_snapshots_queryset,
)
@ -63,6 +71,7 @@ from archivebox.core.host_utils import (
build_snapshot_url,
build_web_url,
get_admin_host,
get_api_base_url,
get_snapshot_host,
get_snapshot_lookup_key,
get_web_host,
@ -159,25 +168,30 @@ class SnapshotView(View):
@staticmethod
def find_snapshots_for_url(path: str):
"""Return a queryset of snapshots matching a URL-ish path."""
"""Return a queryset of snapshots matching a URL-ish path. URL only — never tries ID matching.
Use ``find_snapshots_for_id`` separately if you also want to match by snapshot UUID.
"""
def _fragmentless_url_query(url: str) -> Q:
# Use a range comparison (url >= 'canonical#' AND url < 'canonical#\U0010ffff')
# instead of LIKE/__startswith — SQLite's case-insensitive LIKE bypasses the
# url index and forces a full-table scan over ~1M rows (~250ms). The range
# form lets SQLite use a MULTI-INDEX OR and stays under 1ms.
canonical = without_fragment(url)
return Q(url=canonical) | Q(url__startswith=f"{canonical}#")
return Q(url=canonical) | (Q(url__gte=f"{canonical}#") & Q(url__lt=f"{canonical}#\U0010ffff"))
normalized = without_fragment(path)
if path.startswith(("http://", "https://")):
# try exact match on full url / ID first
qs = Snapshot.objects.filter(_fragmentless_url_query(path) | Q(id__icontains=path) | Q(id__icontains=normalized))
# exact url match (indexed) — fastest path
qs = Snapshot.objects.filter(_fragmentless_url_query(path))
if qs.exists():
return qs
normalized = normalized.split("://", 1)[1]
# try exact match on full url / ID (without scheme)
# try exact match on full url (without scheme)
qs = Snapshot.objects.filter(
_fragmentless_url_query("http://" + normalized)
| _fragmentless_url_query("https://" + normalized)
| Q(id__icontains=normalized),
_fragmentless_url_query("http://" + normalized) | _fragmentless_url_query("https://" + normalized),
)
if qs.exists():
return qs
@ -193,28 +207,26 @@ class SnapshotView(View):
# fall back to matching base_url as prefix
return Snapshot.objects.filter(Q(url__startswith="http://" + base) | Q(url__startswith="https://" + base))
@staticmethod
def find_snapshots_for_id(slug: str):
"""Return a queryset of snapshots matching a (possibly truncated) UUID via prefix or suffix.
Strips non-hex characters from ``slug`` (so input with or without hyphens both work).
Requires at least 8 hex chars — shorter inputs return an empty queryset to avoid
scanning the entire snapshots table on too-broad matches.
"""
return filter_queryset_by_uuid_substring(Snapshot.objects.all(), slug)
@staticmethod
def render_live_index(request, snapshot):
TITLE_LOADING_MSG = "Not yet archived..."
from archivebox.core.widgets import TagEditorWidget
crawl = getattr(snapshot, "crawl", None)
runtime_config = getattr(request, "archivebox_config", None)
page_config_keys = {
"PREVIEW_ORIGINALS",
"BIND_ADDR",
"USES_SUBDOMAIN_ROUTING",
"BASE_URL",
"PERMISSIONS",
"SERVER_SECURITY_MODE",
}
scoped_config_keys = set((getattr(snapshot, "config", None) or {}).keys())
scoped_config_keys.update((getattr(crawl, "config", None) or {}).keys())
needs_scoped_config = bool(scoped_config_keys & page_config_keys)
if runtime_config is None or needs_scoped_config:
runtime_config = get_config(snapshot=snapshot, resolve_plugins=False)
request.archivebox_config = runtime_config
# Reuse the middleware-attached config; never re-bootstrap from env + plugin
# schemas just to render a snapshot page (that pays ~30ms for no reason).
runtime_config = _get_request_config(request)
snapshot._runtime_config = runtime_config
snapshot_permissions = get_snapshot_permissions(snapshot)
hidden_card_plugins = {"archivedotorg", "favicon", "title"}
outputs = [
out
@ -253,8 +265,17 @@ class SnapshotView(View):
best_result = archiveresults[result_type]
break
related_snapshots_qs = SnapshotView.find_snapshots_for_url(snapshot.url)
related_snapshots = list(related_snapshots_qs.exclude(id=snapshot.id).order_by("-bookmarked_at", "-created_at", "-timestamp")[:25])
related_snapshots_qs = (
SnapshotView.find_snapshots_for_url(snapshot.url)
.select_related("crawl", "crawl__created_by")
.annotate(
num_outputs_cached=Count("archiveresult", filter=Q(archiveresult__status="succeeded")),
num_failures_cached=Count("archiveresult", filter=Q(archiveresult__status="failed")),
)
)
related_snapshots = list(
related_snapshots_qs.exclude(id=snapshot.id).order_by("-bookmarked_at", "-created_at", "-timestamp")[:25],
)
related_years_map: dict[int, list[Snapshot]] = {}
for snap in [snapshot, *related_snapshots]:
snap_dt = snap.bookmarked_at or snap.created_at or snap.downloaded_at
@ -295,32 +316,53 @@ class SnapshotView(View):
compact_outputs = [out for out in ordered_outputs if out.get("is_compact") or out.get("is_metadata")]
tag_widget = TagEditorWidget()
output_size = sum(int(out.get("size") or 0) for out in ordered_outputs)
is_archived = bool(ordered_outputs or snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
has_outputs = bool(ordered_outputs)
is_archived = has_outputs or snapshot.status == Snapshot.StatusChoices.SEALED
snapshot_status = str(snapshot.status or "").lower()
status_label_by_state = {
"queued": ("queued", "info"),
"started": ("running", "warning"),
"paused": ("paused", "default"),
"sealed": ("archived", "success"),
}
if has_outputs and not is_archived:
status_label, status_color = ("partial", "warning")
elif has_outputs:
status_label, status_color = ("archived", "success")
else:
status_label, status_color = status_label_by_state.get(snapshot_status, ("not yet archived", "danger"))
# One canonical progress endpoint, same-origin to whichever host the page is on.
# The id is always carried explicitly in the query string (derived from the page
# context, never the host) so this works in every routing/security mode.
progress_endpoint = f"/progress.json?snapshot_id={snapshot.id}"
context = {
"id": str(snapshot.id),
"snapshot_id": str(snapshot.id),
"progress_endpoint": progress_endpoint,
"url": snapshot.url,
"archive_path": snapshot.archive_path_from_db,
"title": htmlencode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)),
"extension": snapshot.extension or "html",
"tags": snapshot.tags_str() or "untagged",
"size": printable_filesize(output_size) if output_size else "pending",
"status": "archived" if is_archived else "not yet archived",
"status_color": "success" if is_archived else "danger",
"snapshot_permissions": str(runtime_config.PERMISSIONS).strip().lower(),
"size": printable_filesize(output_size) if output_size else "—",
"status": status_label,
"status_color": status_color,
"snapshot_state": snapshot_status,
"has_outputs": has_outputs,
"snapshot_permissions": snapshot_permissions,
"snapshot_permissions_icon": {
"public": "👥",
"unlisted": "🔗",
"private": "🔒",
}[str(runtime_config.PERMISSIONS).strip().lower()],
}.get(snapshot_permissions, "👥"),
"bookmarked_date": snapshot.bookmarked_date,
"downloaded_datestr": snapshot.downloaded_datestr,
"num_outputs": snapshot.num_outputs,
"num_failures": snapshot.num_failures,
"oldest_archive_date": ts_to_date_str(snapshot.oldest_archive_date),
"warc_path": warc_path,
"PREVIEW_ORIGINALS": runtime_config.PREVIEW_ORIGINALS,
"archiveresults": [*non_compact_outputs, *compact_outputs],
"best_result": best_result,
"snapshot": snapshot, # Pass the snapshot object for template tags
@ -467,10 +509,20 @@ class SnapshotView(View):
status=404,
)
# slug is a URL
# slug is either a URL or a (possibly truncated) snapshot UUID
def _resolve_snapshots_for_slug(slug: str):
# full URLs go straight to the url-only path (fast, indexed)
if "://" in slug:
return SnapshotView.find_snapshots_for_url(slug)
# short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first
id_qs = SnapshotView.find_snapshots_for_id(slug)
if id_qs.exists():
return id_qs
return SnapshotView.find_snapshots_for_url(slug)
try:
try:
snapshot = direct_snapshots_queryset(request, SnapshotView.find_snapshots_for_url(path)).get()
snapshot = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path)).get()
except Snapshot.DoesNotExist:
raise
except Snapshot.DoesNotExist:
@ -491,7 +543,7 @@ class SnapshotView(View):
status=404,
)
except Snapshot.MultipleObjectsReturned:
snapshots = direct_snapshots_queryset(request, SnapshotView.find_snapshots_for_url(path))
snapshots = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path))
snapshot_hrefs = mark_safe("<br/>").join(
format_html(
'{} <code style="font-size: 0.8em">{}</code> <a href="/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
@ -548,15 +600,8 @@ class SnapshotPathView(View):
snapshot = None
snapshots_qs = direct_snapshots_queryset(request, Snapshot.objects.select_related("crawl", "crawl__created_by"))
if snapshot_id:
try:
snapshot = snapshots_qs.get(pk=snapshot_id)
except Snapshot.DoesNotExist:
try:
snapshot = snapshots_qs.get(id__startswith=snapshot_id)
except Snapshot.DoesNotExist:
snapshot = None
except Snapshot.MultipleObjectsReturned:
snapshot = snapshots_qs.filter(id__startswith=snapshot_id).first()
matches = list(filter_queryset_by_uuid_substring(snapshots_qs, snapshot_id)[:2])
snapshot = matches[0] if matches else None
else:
# fuzzy lookup by date + domain/url (most recent)
username_lookup = "system" if username == "web" else username
@ -830,8 +875,35 @@ def _serve_responses_path(request, responses_root: Path, rel_path: str, show_ind
def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
request_config = get_config(snapshot=snapshot, resolve_plugins=False)
request.archivebox_config = request_config
request.archivebox_snapshot_url = snapshot.url
snapshot._runtime_config = request_config
rel_path = path or ""
# POLICY EXCEPTION: archivebox normally does not depend on any specific
# plugin. The WACZ/WARC embedded-replay viewer needs ``/replay/sw.js`` and
# ``/replay/ui.js`` served same-origin on each snapshot host so its
# service worker can register. Until plugins can register their own URL
# routes generically, we conditionally import the archivewebpage plugin's
# ``replay_preview`` module here and let it handle these paths. If the
# plugin is not installed the import fails and the path falls through to
# the regular snapshot file lookup (which 404s, the expected behavior).
if rel_path.startswith("replay/") or rel_path == "replay":
try:
from abx_plugins.plugins.archivewebpage import replay_preview as _awp_preview
except ImportError:
_awp_preview = None
if _awp_preview is not None:
response = _awp_preview.serve_replay_asset(rel_path, request_config)
if response is not None:
return response
if rel_path == "progress.json":
# Host routing forwards every snap-* path to SnapshotHostView, so we forward
# /progress.json on through to the same view used everywhere else. The caller
# passes snapshot_id explicitly in the query string — we don't read it from the
# subdomain (this keeps the endpoint identical across all security modes).
return live_progress_view(request)
is_directory_request = bool(path) and path.endswith("/")
show_indexes = bool(request.GET.get("files")) or (request_config.USES_SUBDOMAIN_ROUTING and is_directory_request)
if not show_indexes and (not rel_path or rel_path == "index.html"):
@ -1150,6 +1222,7 @@ class AddView(UserPassesTestMixin, FormView):
"title": "Create Crawl",
# We can't just call request.build_absolute_uri in the template, because it would include query parameters
"absolute_add_path": self.request.build_absolute_uri(self.request.path),
"web_base_url": build_web_url("", request=self.request),
"VERSION": VERSION,
"FOOTER_INFO": request_config.FOOTER_INFO,
"required_search_plugin": required_search_plugin,
@ -1411,18 +1484,47 @@ def live_progress_view(request):
from archivebox.core.models import Snapshot, ArchiveResult
from archivebox.machine.models import Process, Machine
if not request.user.is_authenticated or not request.user.is_active or not request.user.is_staff:
return JsonResponse({"error": "Permission denied"}, status=403)
snapshot_id_filter = (request.GET.get("snapshot_id") or "").strip()
crawl_id_filter = (request.GET.get("crawl_id") or "").strip()
is_admin = is_admin_user(request)
scoped_snapshot = None
if snapshot_id_filter:
import uuid as _uuid
try:
_uuid.UUID(snapshot_id_filter)
except (TypeError, ValueError):
return JsonResponse({"error": "Invalid snapshot_id"}, status=400)
scoped_snapshot = Snapshot.objects.filter(id=snapshot_id_filter).select_related("crawl").first()
if scoped_snapshot is None or not can_view_snapshot(request, scoped_snapshot):
return JsonResponse({"error": "Permission denied"}, status=403)
elif crawl_id_filter:
# Crawl-only scope still requires staff: there's no per-crawl ACL helper,
# and a crawl can mix snapshot permissions levels.
if not is_admin:
return JsonResponse({"error": "Permission denied"}, status=403)
else:
if not is_admin:
return JsonResponse({"error": "Permission denied"}, status=403)
request_config = request.archivebox_config
now = timezone.now()
crawl_scope = Crawl.objects.all()
snapshot_scope = Snapshot.objects.all()
archiveresult_scope = ArchiveResult.objects.all()
if not request.user.is_superuser:
if is_admin and not request.user.is_superuser:
crawl_scope = crawl_scope.filter(created_by=request.user)
snapshot_scope = snapshot_scope.filter(crawl__created_by=request.user)
archiveresult_scope = archiveresult_scope.filter(snapshot__crawl__created_by=request.user)
if scoped_snapshot is not None:
snapshot_scope = Snapshot.objects.filter(id=scoped_snapshot.id)
crawl_scope = Crawl.objects.filter(id=scoped_snapshot.crawl_id)
archiveresult_scope = ArchiveResult.objects.filter(snapshot_id=scoped_snapshot.id)
elif crawl_id_filter:
snapshot_scope = snapshot_scope.filter(crawl_id=crawl_id_filter)
crawl_scope = crawl_scope.filter(id=crawl_id_filter)
archiveresult_scope = archiveresult_scope.filter(snapshot__crawl_id=crawl_id_filter)
def is_current_run_timestamp(event_ts, run_started_at) -> bool:
if run_started_at is None:
@ -1550,6 +1652,8 @@ def live_progress_view(request):
url = str(url or "")
return url if len(url) <= 96 else f"{url[:93]}..."
api_base = get_api_base_url(request=request, config=request_config) if scoped_snapshot is not None else ""
def screencast_frame_url(crawl_id: str, crawl_dir: Path) -> str:
frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
try:
@ -1560,7 +1664,8 @@ def live_progress_view(request):
return ""
if now.timestamp() - frame_stat.st_mtime > 15:
return ""
return f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
return f"{api_base}{rel}" if api_base else rel
machine_id = Machine.current().id
orchestrator_proc = (
@ -2263,6 +2368,11 @@ def live_progress_view(request):
)
payload = {
"is_admin": is_admin,
"scope": {
"snapshot_id": str(scoped_snapshot.id) if scoped_snapshot is not None else "",
"crawl_id": crawl_id_filter,
},
"orchestrator_running": orchestrator_running,
"orchestrator_pid": orchestrator_pid,
"total_workers": total_workers,

8
archivebox/crawls/admin.py
View File

@ -63,8 +63,10 @@ def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50,
filtered_qs = snapshots_qs
if query:
id_query = query.replace("-", "")
filtered_qs = filtered_qs.filter(Q(id__icontains=id_query) | Q(url__icontains=query) | Q(title__icontains=query))
from archivebox.misc.util import filter_queryset_by_uuid_substring
id_match_pks = list(filter_queryset_by_uuid_substring(Snapshot.objects.all(), query).values_list("pk", flat=True)[:100])
filtered_qs = filtered_qs.filter(Q(pk__in=id_match_pks) | Q(url__icontains=query) | Q(title__icontains=query))
if status_filter in valid_statuses:
filtered_qs = filtered_qs.filter(status=status_filter)
@ -1182,7 +1184,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
return super().change_view(request, object_id, form_url, extra_context)
def add_view(self, request, form_url="", extra_context=None):
return redirect("/add/?focus=schedule")
return redirect("/add/#schedule")
def get_fieldsets(self, request, obj=None):
if obj is None:

4
archivebox/crawls/models.py
View File

@ -865,7 +865,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
crawl_tag_names = self.current_tag_names()
tags_by_name: dict[str, Tag] = {}
config = get_config(crawl=self)
only_new_urls = bool(config.ONLY_NEW) and not bool(config.OVERWRITE)
only_new_urls = bool(config.ONLY_NEW)
for line in self.urls.splitlines():
if not line.strip():
@ -1039,7 +1039,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
if not deduped_records:
return []
existing_scope = Snapshot.objects if bool(config.ONLY_NEW) and not bool(config.OVERWRITE) else self.snapshot_set
existing_scope = Snapshot.objects if bool(config.ONLY_NEW) else self.snapshot_set
existing_urls = set(existing_scope.filter(url__in=deduped_records.keys()).values_list("url", flat=True))
urls = [url for url in deduped_records.keys() if url not in existing_urls]
remaining = self.remaining_snapshot_capacity()

7
archivebox/hooks.py
View File

@ -53,6 +53,7 @@ from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, TypedDict
from abx_plugins import get_plugins_dir
from django.utils.safestring import mark_safe
from archivebox.config.constants import CONSTANTS
from archivebox.config.version import VERSION
from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
if TYPE_CHECKING:
@ -400,6 +401,7 @@ def run_hook(
env["DATA_DIR"] = str(resolved_config.DATA_DIR)
env["ARCHIVE_DIR"] = str(resolved_config.ARCHIVE_DIR)
env["ABX_RUNTIME"] = "archivebox"
env["LIBRARY_VERSION"] = VERSION
env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
resolved_output_dir = output_dir.resolve()
@ -679,13 +681,10 @@ def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any
return [str(plugin).strip() for plugin in value if str(plugin).strip()]
return [str(value).strip()] if str(value).strip() else []
# Support explicit ENABLED_PLUGINS override (legacy)
# Support explicit ENABLED_PLUGINS override
enabled_plugins = config.get("ENABLED_PLUGINS")
if enabled_plugins:
return normalize_enabled_plugins(enabled_plugins)
enabled_extractors = config.get("ENABLED_EXTRACTORS")
if enabled_extractors:
return normalize_enabled_plugins(enabled_extractors)
# Filter all plugins by enabled status
all_plugins = get_plugins()

6
archivebox/misc/checks.py
View File

@ -61,7 +61,7 @@ def check_migrations(*, blocking: bool = True, auto_apply: bool = False, cancel_
from archivebox.misc.db import apply_migrations, migration_state, pending_migrations
pending, missing_from_code, rollback_targets = migration_state()
is_migrating = any(arg in sys.argv for arg in ["makemigrations", "migrate", "init"])
is_migrating = any(arg in sys.argv for arg in ["makemigrations", "migrate", "init"]) or os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1"
if missing_from_code:
print(
@ -240,7 +240,9 @@ def check_data_dir_permissions(config=None, **config_kwargs):
# Check /lib dir permissions
check_lib_dir(lib_dir, throw=False, must_exist=True, config=config)
os.umask(0o777 - int(config.DIR_OUTPUT_PERMISSIONS, base=8))
# Derive directory mode from file mode by OR-ing the execute bits (matches
# the old DIR_OUTPUT_PERMISSIONS=755 vs OUTPUT_PERMISSIONS=644 convention).
os.umask(0o777 - (int(config.OUTPUT_PERMISSIONS, base=8) | 0o111))
def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs):

85
archivebox/misc/db.py
View File

@ -19,6 +19,91 @@ from archivebox.config import DATA_DIR
from archivebox.misc.util import enforce_types
def run_db_analyze_batch(
remaining: list[str] | None,
*,
max_seconds_per_table: float = 120.0,
) -> list[str]:
"""Advance one step of a batched SQLite ``ANALYZE`` sweep.
Without periodic ANALYZE the optimizer's table stats go stale as
snapshot/archiveresult tables grow, causing it to start large joins from
``auth_user`` instead of using the indexed url column and blowing snapshot
detail page render time from ~50ms to ~500ms+.
The whole sweep is spread across many calls instead of running as one
blocking ``ANALYZE``: pass ``None`` to start a fresh sweep (this call
enumerates user tables and runs ``ANALYZE`` on the first one); pass the
returned list to advance one more table on each subsequent call. An
empty return value means the sweep is complete (or has been aborted) and
the next caller should pass ``None`` again. Caller is responsible for
throttling new sweeps (orchestrator starts at most one per 24hr while
idle) and enforcing a hard upper bound on total sweep wall time.
Safety guarantees:
- **Never raises**: every database call is wrapped; on any failure the
function returns ``[]`` (abandoning the rest of the sweep) so the
orchestrator never crashes on maintenance errors.
- **Bounded per-call wall time**: a SQLite progress handler aborts the
current ``ANALYZE`` statement once ``max_seconds_per_table`` is
exceeded, so a single pathological table cannot wedge the call.
- **Never leaves the db locked**: each ``ANALYZE`` runs as a single
statement transaction that auto-commits (or rolls back on
abort/error). The cursor and progress handler are always cleaned up
in ``finally`` blocks even if Python raises mid-call.
- Silent no-op on non-SQLite backends.
WAL journal mode (set in Django settings) keeps readers fully unblocked
throughout; the writer lock is only held for the brief ``sqlite_stat*``
flush after each table completes.
"""
from django.db import connection
if connection.vendor != "sqlite":
return []
if remaining is None:
try:
with connection.cursor() as cursor:
cursor.execute(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
)
remaining = [row[0] for row in cursor.fetchall()]
except Exception:
return []
if not remaining:
return []
next_table, *rest = remaining
raw_conn = getattr(connection, "connection", None)
progress_handler_set = False
if raw_conn is not None and max_seconds_per_table > 0:
deadline = time.monotonic() + max_seconds_per_table
try:
raw_conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 10000)
progress_handler_set = True
except Exception:
progress_handler_set = False
try:
with connection.cursor() as cursor:
cursor.execute(f'ANALYZE "{next_table}"')
except Exception:
# Aborted by progress handler, locked db, or any other failure — skip
# this table and continue the sweep. ANALYZE is idempotent so we can
# retry on the next 24hr sweep.
pass
finally:
if progress_handler_set and raw_conn is not None:
try:
raw_conn.set_progress_handler(None, 0)
except Exception:
pass
return rest
def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:
parts = [str(part) for part in (cmdline or []) if str(part)]
if not parts:

2
archivebox/misc/monkey_patches.py
View File

@ -59,7 +59,7 @@ class ModifiedAccessLogGenerator(access.AccessLogGenerator):
return
if "GET /health/" in request:
return
if "GET /admin/live-progress/" in request and (time_taken is None or time_taken < 1.0):
if "GET /progress.json" in request and (time_taken is None or time_taken < 1.0):
return
if "GET /api/v1/crawls/crawl/" in request and "/files/chrome_screencast/latest.jpg" in request:
return

47
archivebox/misc/serve_static.py
View File

@ -792,6 +792,9 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
bool(request.GET.get("preview")) and content_type.startswith("image/") and not content_type.startswith("image/svg+xml")
)
preview_as_mhtml_html = bool(request.GET.get("preview")) and fullpath.suffix.lower() in {".mhtml", ".mht"}
preview_as_archivewebpage_html = bool(request.GET.get("preview")) and (
fullpath.suffix.lower() in {".wacz", ".warc"} or fullpath.name.lower().endswith(".warc.gz")
)
# Respect the If-Modified-Since header for non-markdown responses.
if not (content_type.startswith("text/plain") or content_type.startswith("text/html")):
@ -859,6 +862,50 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
except Exception:
pass
if preview_as_archivewebpage_html:
# POLICY EXCEPTION: archivebox normally does not depend on any specific
# plugin. The WACZ/WARC embedded-replay viewer (ui.js + sw.js + the
# rendered preview HTML) is plugin-owned, but it has to be served at
# plugin-defined paths on the snapshot host so the same-origin service
# worker registration works. There is no clean generic "plugin
# contributes a preview handler" extension hook in archivebox yet, so
# we conditionally import the archivewebpage plugin's
# ``replay_preview`` module here. If the plugin is not installed, the
# import fails and we fall through to default static-file serving.
try:
from abx_plugins.plugins.archivewebpage import replay_preview as _awp_preview
except ImportError:
_awp_preview = None
if _awp_preview is not None:
try:
raw_query = request.GET.copy()
raw_query.pop("preview", None)
raw_output_path = request.path
if raw_query:
raw_output_path = f"{raw_output_path}?{raw_query.urlencode()}"
snapshot_url_fallback = getattr(request, "archivebox_snapshot_url", "") or ""
rendered = _awp_preview.render_preview_html(
fullpath.name,
raw_output_path,
wacz_path=fullpath,
fallback_url=snapshot_url_fallback,
)
response = HttpResponse(rendered, content_type="text/html; charset=utf-8")
response.headers["Last-Modified"] = http_date(statobj.st_mtime)
if etag:
response.headers["ETag"] = etag
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable"
else:
response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300"
response.headers["Content-Disposition"] = f'inline; filename="{fullpath.stem}.html"'
for key, value in _awp_preview.preview_response_headers().items():
response.headers[key] = value
if encoding:
response.headers["Content-Encoding"] = encoding
return response
except Exception:
pass
if preview_as_mhtml_html:
try:
raw_query = request.GET.copy()

21
archivebox/misc/util.py
View File

@ -20,6 +20,27 @@ from base32_crockford import encode as base32_encode
from .logging import COLOR_DICT
def filter_queryset_by_uuid_substring(queryset, slug: str, field: str = "id"):
"""Filter a queryset to UUID-column matches by prefix or suffix (case-insensitive).
Avoids ``id__icontains`` (an unindexed full-table scan over the UUID column) by
stripping non-hex chars from ``slug`` and matching with ``istartswith`` /
``iendswith``. Returns an empty queryset for inputs with fewer than 8 hex chars
to avoid overly broad matches. A full 32-char hex string falls back to an
exact-equality lookup.
"""
from django.db.models import Q
normalized = re.sub(r"[^0-9a-fA-F]", "", slug or "").lower()
if len(normalized) < 8:
return queryset.none()
if len(normalized) == 32:
return queryset.filter(**{field: normalized})
prefix = f"{field}__istartswith"
suffix = f"{field}__iendswith"
return queryset.filter(Q(**{prefix: normalized}) | Q(**{suffix: normalized}))
def detect_encoding(rawdata):
try:
import chardet # type:ignore

8
archivebox/search/__init__.py
View File

@ -241,9 +241,6 @@ def query_search_index(
from archivebox.core.models import Snapshot
config = config or get_config(**config_kwargs)
if not config.USE_SEARCHING_BACKEND:
return Snapshot.objects.none()
search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config)
search_mode_base = get_search_mode_base(search_mode, config=config)
if search_mode_base == "meta":
@ -262,9 +259,6 @@ def iter_query_search_ids(
):
"""Yield snapshot IDs from configured search backends as soon as each backend produces them."""
config = config or get_config(**config_kwargs)
if not config.USE_SEARCHING_BACKEND:
return
search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config)
search_mode_base = get_search_mode_base(search_mode, config=config)
forced_backend = get_search_mode_backend(search_mode, config=config)
@ -342,7 +336,7 @@ def flush_search_index(snapshots: QuerySet, config: dict[str, Any] | None = None
Remove snapshots from the search index.
"""
config = config or get_config(**config_kwargs)
if not config.USE_INDEXING_BACKEND or not snapshots:
if not snapshots:
return
backend = get_backend(config=config)

75
archivebox/services/runner.py
View File

@ -60,6 +60,7 @@ from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelled
from archivebox.config.configset import BaseConfigSet
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.misc.db import run_db_analyze_batch
from archivebox.core.shutdown_util import foreground_shutdown_signals
from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS
@ -290,6 +291,14 @@ class CrawlRunner:
if self._signal_abort_requested:
return True
if self.allow_maintenance_on_inactive_crawl:
# SEALED is the normal terminal state of a finished crawl, not a
# cancellation signal for maintenance work on its already-sealed
# snapshots (search backend backfill, fs migration, etc.). When the
# runner is invoked with explicit snapshot_ids + selected_plugins,
# treat sealed as completed rather than cancelled so the requested
# maintenance hooks can actually run.
return False
return await Crawl.objects.filter(id=self.crawl.id, status=Crawl.StatusChoices.SEALED).aexists()
async def crawl_is_paused(self) -> bool:
@ -311,7 +320,16 @@ class CrawlRunner:
return filter_plugins(self.plugins, self.selected_plugins, include_providers=True) if self.selected_plugins else self.plugins
@property
def allow_paused_snapshot_maintenance(self) -> bool:
def allow_maintenance_on_inactive_crawl(self) -> bool:
"""Run the requested hooks on a snapshot whose parent crawl is paused or sealed.
Maintenance entry paths — direct ``snapshot_ids + selected_plugins`` invocations
for search backend backfill, fs migration, plugin-targeted updates — are
legitimately allowed to operate on finished/paused crawls. Without this gate,
``crawl_is_cancelled`` would treat a SEALED parent as a cancellation signal
and short-circuit every guard before any hook ran, leaving the queued
ArchiveResult rows stuck and the orchestrator looping on them.
"""
return bool(self.initial_snapshot_ids and self.selected_plugins)
async def run(self) -> None:
@ -384,7 +402,7 @@ class CrawlRunner:
async def enqueue_snapshot(self, snapshot_id: str, crawl_start_event: CrawlStartEvent | None = None) -> None:
if await self.crawl_is_cancelled():
return
if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance:
if await self.crawl_is_paused() and not self.allow_maintenance_on_inactive_crawl:
return
task = self.snapshot_tasks.get(snapshot_id)
if task is not None and not task.done():
@ -462,7 +480,7 @@ class CrawlRunner:
task_errors.append(err)
stop_scheduling = True
if self.snapshot_tasks and (
await self.crawl_is_cancelled() or (await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance)
await self.crawl_is_cancelled() or (await self.crawl_is_paused() and not self.allow_maintenance_on_inactive_crawl)
):
stop_scheduling = True
if not stop_scheduling:
@ -520,7 +538,7 @@ class CrawlRunner:
return
if await self.crawl_is_cancelled():
return
if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance:
if await self.crawl_is_paused() and not self.allow_maintenance_on_inactive_crawl:
return
await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)()
@ -725,7 +743,7 @@ class CrawlRunner:
from archivebox.hooks import collect_urls_from_plugins
await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)()
if self.crawl.is_paused and not self.allow_paused_snapshot_maintenance:
if self.crawl.is_paused and not self.allow_maintenance_on_inactive_crawl:
return
if int(snapshot_payload["depth"]) >= self.crawl.max_depth:
return
@ -844,7 +862,7 @@ class CrawlRunner:
break
if await self.crawl_is_cancelled():
break
if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance:
if await self.crawl_is_paused() and not self.allow_maintenance_on_inactive_crawl:
break
await self.enqueue_snapshot(snapshot_id)
await self.wait_for_snapshot_tasks()
@ -855,7 +873,9 @@ class CrawlRunner:
cancel_watcher = asyncio.create_task(self.watch_for_cancelled_crawl(event))
try:
try:
if not await self.crawl_is_cancelled() and (not await self.crawl_is_paused() or self.allow_paused_snapshot_maintenance):
if not await self.crawl_is_cancelled() and (
not await self.crawl_is_paused() or self.allow_maintenance_on_inactive_crawl
):
await _run_event_now(
event.emit(
CrawlSetupEvent(
@ -868,7 +888,9 @@ class CrawlRunner:
),
crawl_setup_phase_timeout,
)
if not await self.crawl_is_cancelled() and (not await self.crawl_is_paused() or self.allow_paused_snapshot_maintenance):
if not await self.crawl_is_cancelled() and (
not await self.crawl_is_paused() or self.allow_maintenance_on_inactive_crawl
):
crawl_start_event = CrawlStartEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
@ -1720,6 +1742,10 @@ def run_pending_crawls(
)
last_recovery_at = 0.0
last_retention_at = 0.0
last_analyze_at = 0.0
analyze_queue: list[str] | None = None
analyze_sweep_started_at = 0.0
orchestrator_started_at = time.monotonic()
while True:
now_monotonic = time.monotonic()
if now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
@ -1873,6 +1899,39 @@ def run_pending_crawls(
if now_monotonic - last_recovery_at >= 30.0:
recover_orchestrator_state()
last_recovery_at = now_monotonic
# SQLite query plans degrade as the snapshot/archiveresult tables grow
# past their last ANALYZE — stale stats make the optimizer start large
# joins from auth_user/crawl instead of using the url index, blowing the
# snapshot detail page out to ~500ms. Refresh stats at most once per
# 24hr while the queue is idle, and only after the orchestrator has
# been alive for at least an hour so short server boots / one-off work
# never pay the cost. The sweep is batched one table per idle tick;
# individual table ANALYZE statements abort after 2min (progress
# handler) and the whole sweep is hard-capped at 5min so a
# pathological table cannot wedge maintenance forever. Any failure
# inside the maintenance hook is swallowed — orchestrator must never
# be taken down by stats refresh.
try:
if (
analyze_queue is None
and now_monotonic - orchestrator_started_at >= 3600.0
and now_monotonic - last_analyze_at >= 86400.0
):
analyze_sweep_started_at = now_monotonic
analyze_queue = run_db_analyze_batch(None)
elif analyze_queue and now_monotonic - analyze_sweep_started_at >= 300.0:
# Sweep blew past the 5min hard cap — abandon what's left
# and don't retry until the next 24hr window.
analyze_queue = None
last_analyze_at = now_monotonic
elif analyze_queue:
analyze_queue = run_db_analyze_batch(analyze_queue)
if analyze_queue is not None and not analyze_queue:
analyze_queue = None
last_analyze_at = now_monotonic
except Exception:
analyze_queue = None
last_analyze_at = now_monotonic
time.sleep(2.0)
continue
return 0

1
archivebox/templates/admin/base.html
View File

@ -1660,6 +1660,7 @@
<body class="{% if is_popup %}popup {% endif %}{% block bodyclass %}{% endblock %}" data-admin-utc-offset="{% now "Z" %}">
{% security_mode_banner %}
{% include 'progressbar.html' %}
<div id="container">

77
archivebox/templates/admin/core/tag/change_list.html
View File

@ -247,6 +247,13 @@
color: #075985;
font-size: 11px;
font-weight: 700;
text-decoration: none;
}
.tag-card__count:hover {
background: #bae6fd;
color: #0c4a6e;
text-decoration: none;
}
.tag-card__actions {
@ -308,58 +315,6 @@
font-size: 12px;
}
.tag-card__snapshots {
display: grid;
gap: 8px;
grid-template-columns: repeat(auto-fit, minmax(130px, 1fr));
}
.tag-snapshot-badge {
display: flex;
align-items: center;
gap: 8px;
min-width: 0;
padding: 6px 8px;
border-radius: 12px;
border: 1px solid #dbe4ee;
background: rgba(255, 255, 255, 0.86);
text-decoration: none;
color: #0f172a;
}
.tag-snapshot-badge img {
width: 16px;
height: 16px;
border-radius: 4px;
flex: 0 0 auto;
background: #f8fafc;
}
.tag-snapshot-badge span {
min-width: 0;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
font-size: 11px;
font-weight: 600;
}
.tag-card__empty {
padding: 14px;
border-radius: 14px;
border: 1px dashed #cbd5e1;
background: #f8fafc;
color: #64748b;
font-size: 13px;
}
.tag-card__empty[href] {
display: block;
color: #075985;
font-weight: 600;
text-decoration: none;
}
.tag-toast {
position: sticky;
top: 12px;
@ -626,17 +581,6 @@ document.addEventListener('DOMContentLoaded', function () {
}
grid.innerHTML = cards.map(function (card) {
const snapshotCount = Number(card.num_snapshots || 0);
const snapshotHtml = (card.snapshots || []).length
? card.snapshots.map(function (snapshot) {
return '' +
'<a class="tag-snapshot-badge" href="' + escapeHtml(snapshot.admin_url) + '" title="' + escapeHtml(snapshot.url) + '">' +
'<img src="' + escapeHtml(snapshot.favicon_url) + '" alt="" onerror="this.hidden=true">' +
'<span>' + escapeHtml(snapshot.title) + '</span>' +
'</a>';
}).join('')
: '<a class="tag-card__empty tag-snapshot-badge" href="' + escapeHtml(card.filter_url) + '">' + escapeHtml(snapshotCount) + ' snapshot' + (snapshotCount === 1 ? '' : 's') + ' tagged</a>';
return '' +
'<article class="tag-card" data-id="' + escapeHtml(card.id) + '" data-slug="' + escapeHtml(card.slug) + '" data-filter-url="' + escapeHtml(card.filter_url) + '" data-rename-url="' + escapeHtml(card.rename_url) + '" data-delete-url="' + escapeHtml(card.delete_url) + '" data-export-urls-url="' + escapeHtml(card.export_urls_url) + '" data-export-jsonl-url="' + escapeHtml(card.export_jsonl_url) + '">' +
'<div class="tag-card__header">' +
@ -655,10 +599,9 @@ document.addEventListener('DOMContentLoaded', function () {
'<button type="button" class="tag-chip-button" data-action="copy-urls">Copy URLs</button>' +
'<button type="button" class="tag-chip-button" data-action="download-jsonl">JSONL</button>' +
'<button type="button" class="tag-chip-button is-danger" data-action="delete">Delete</button>' +
'<span class="tag-card__count">' + escapeHtml(card.num_snapshots) + '</span>' +
'<a class="tag-card__count" href="' + escapeHtml(card.filter_url) + '" title="View tagged snapshots">' + escapeHtml(card.num_snapshots) + '</a>' +
'</div>' +
'</div>' +
'<div class="tag-card__snapshots">' + snapshotHtml + '</div>' +
'</article>';
}).join('');
}
@ -849,8 +792,8 @@ document.addEventListener('DOMContentLoaded', function () {
grid.addEventListener('click', async function (event) {
const actionButton = event.target.closest('[data-action]');
const snapshotLink = event.target.closest('.tag-snapshot-badge');
if (snapshotLink) return;
const filterLink = event.target.closest('.tag-card__count');
if (filterLink) return;
const cardEl = event.target.closest('.tag-card');
if (!cardEl) return;

42
archivebox/templates/admin/login.html
View File

@ -1,5 +1,5 @@
{% extends "admin/base_site.html" %}
{% load i18n static %}
{% load i18n static core_tags %}
{% block extrastyle %}{{ block.super }}<link rel="stylesheet" type="text/css" href="{% static "admin/css/login.css" %}">
{{ form.media }}
@ -7,11 +7,11 @@
{% block bodyclass %}{{ block.super }} login{% endblock %}
<a href="{% url 'Home' %}">
{% block branding %}<h1>ArchiveBox Admin</h1>{% endblock %}
{% block usertools %}
</a>
<br/>
<a href="{% url 'Home' %}">Back to Main Index</a>
{% block usertools %}
{% endblock %}
{% block nav-global %}{% endblock %}
@ -53,22 +53,16 @@
<br/>
<form action="{{ app_path }}" method="post" id="login-form">{% csrf_token %}
<div class="form-row">
<div class="form-row" style="gap: 0;">
{{ form.username.errors }}
{{ form.username.label_tag }} {{ form.username }}
</div>
<div class="form-row">
<div class="form-row" style="gap: 0;">
{{ form.password.errors }}
{{ form.password.label_tag }} {{ form.password }}
<input type="hidden" name="next" value="{{ next }}">
</div>
{% url 'admin_password_reset' as password_reset_url %}
{% if password_reset_url %}
<div class="password-reset-link">
<a href="{{ password_reset_url }}">{% trans 'Forgotten your password or username?' %}</a>
</div>
{% endif %}
<div class="submit-row">
<div class="submit-row" style="border: none;">
<label>&nbsp;</label><input type="submit" value="{% trans 'Log in' %}">
</div>
</form>
@ -77,22 +71,20 @@
<br/><br/>
<hr/>
<br/>
If you forgot your password, <a href="/accounts/password_reset/">reset it here</a> or run:<br/>
To create a new admin user or reset a password, run:<br/>
<pre>
archivebox manage changepassword USERNAME
cd data/ # run commands inside your data folder
archivebox manage createsuperuser &lt;username&gt;
archivebox manage changepassword &lt;username&gt;
</pre>
<br/><br/>
<hr/>
<br/>
To create a new admin user, run the following:
<pre>
archivebox manage createsuperuser
</pre>
<br/>
<hr/>
<small><i>(cd into your archive folder before running commands)</i></small>
</pre>
{% has_real_admin_users as real_admins_exist %}
{% if not real_admins_exist %}
(or set env vars <code>ADMIN_USERNAME</code> + <code>ADMIN_PASSWORD</code>)
{% endif %}
<br/>
</center>

20
archivebox/templates/admin/progress_monitor.html
View File

@ -187,6 +187,17 @@
#progress-monitor.collapsed .progress-content {
display: none;
}
/* Hide admin-only controls when viewer is not staff, or when the monitor is
embedded on a non-admin host (snap-* subdomain) where cross-origin POSTs
to /api would violate the subdomain isolation model. */
#progress-monitor.is-guest .crawl-action-btn,
#progress-monitor.is-guest .cancel-item-btn,
#progress-monitor.is-guest .pause-item-btn,
#progress-monitor[data-progress-scope="snapshot"] .crawl-action-btn,
#progress-monitor[data-progress-scope="snapshot"] .cancel-item-btn,
#progress-monitor[data-progress-scope="snapshot"] .pause-item-btn {
display: none !important;
}
/* Chrome Screencast */
#progress-monitor .screencast-panel {
@ -1037,7 +1048,9 @@
</style>
<div id="progress-monitor" class="collapsed">
<div id="progress-monitor" class="collapsed"
data-progress-endpoint="{{ progress_endpoint|default:'/progress.json' }}"
data-progress-scope="{{ progress_scope|default:'global' }}">
<div class="header-bar">
<div class="header-left">
<div class="orchestrator-status">
@ -1824,8 +1837,10 @@
updateDurationBadges();
}
const progressEndpoint = monitor.dataset.progressEndpoint || '/progress.json';
function fetchProgress() {
fetch('/admin/live-progress/')
fetch(progressEndpoint, { credentials: 'same-origin' })
.then(response => response.json())
.then(data => {
if (data.error) {
@ -1833,6 +1848,7 @@
idleMessage.textContent = 'API Error: ' + data.error;
idleMessage.style.color = '#f85149';
}
monitor.classList.toggle('is-guest', data.is_admin === false);
updateProgress(data);
})
.catch(error => {

88
archivebox/templates/core/add.html
View File

@ -16,7 +16,7 @@
{% block body %}
<div style="max-width: 1440px; margin: auto; float: none">
<br/><br/>
<br/>
{% if stdout %}
<h1>Add new URLs to your archive: results</h1>
<pre id="stdout">
@ -40,11 +40,16 @@
<form id="add-form" method="POST" class="p-form">{% csrf_token %}
<center>
<h1>Create a new Crawl</h1>
<p class="crawl-subtitle">
A <strong>Crawl</strong> is a job that processes URLs and creates <strong>Snapshots</strong> (archived copies) for each URL discovered.
<br/>The settings below apply to the entire crawl and all snapshots it creates.
</p>
</center>
<div class="crawl-explanation">
<p>
A <strong>Crawl</strong> is a job that processes URLs and creates <strong>Snapshots</strong> (archived copies) for each URL discovered.
The settings below apply to the entire crawl and all snapshots it creates.
<p class="crawl-tip" style="user-select: none;">
<a href="https://github.com/ArchiveBox/archivebox-browser-extension" style="float: right"><img src="{% static 'chrome_extension_icon.png' %}" alt="Chrome Extension" style="height: 33px; margin-top: -5px;">&nbsp; Get the extension</a>
💡 <strong>Tip:</strong> Instantly save a single URL by visiting:
<code class="crawl-tip-url" style="user-select: text;"><span class="crawl-tip-url-prefix">{{ web_base_url }}/web/</span><span class="crawl-tip-url-target">https://example.com/url_to_save</span></code>
</p>
</div>
@ -246,7 +251,7 @@
</p>
<div class="plugin-presets">
<span class="preset-label">Quick Select:</span>
<span class="preset-label">Presets:</span>
<a href="/admin/personas/persona/add/" class="preset-btn persona-create-btn" target="_blank" rel="noopener" title="Create new profile / import from Chrome" aria-label="Create new profile / import from Chrome">➕</a>
{% for persona in recent_personas %}
<span class="persona-preset-wrap">
@ -268,7 +273,7 @@
<summary><h3>Advanced Crawl Options</h3></summary>
<p class="section-description">Additional settings that control how this crawl processes URLs and creates snapshots.</p>
<div class="form-field">
<div class="form-field" id="schedule">
{{ form.schedule.label_tag }}
{{ form.schedule }}
{% if form.schedule.errors %}
@ -321,6 +326,20 @@
</center> -->
{% endif %}
<script>
// Triple-click the example URL in the tip to select just the URL (not the surrounding text)
document.querySelectorAll('.crawl-tip-url').forEach((el) => {
el.addEventListener('mousedown', (e) => {
if (e.detail >= 3) {
e.preventDefault();
const range = document.createRange();
range.selectNodeContents(el);
const sel = window.getSelection();
sel.removeAllRanges();
sel.addRange(range);
}
});
});
// URL preview / counter
const urlTextarea = document.querySelector('textarea[name="url"]');
const urlCounter = document.getElementById('url-counter');
@ -471,38 +490,65 @@
updater();
}
function isTruthyConfigValue(value) {
return value === true || value === 'true' || value === '1' || value === 'on' || value === 1;
}
function syncSectionTogglesFromEffectiveConfig(effectiveConfig) {
document.querySelectorAll('.plugin-card[data-plugin-enabled-key]').forEach(card => {
const enabledKey = card.dataset.pluginEnabledKey;
if (!enabledKey || !Object.prototype.hasOwnProperty.call(effectiveConfig, enabledKey)) return;
const sectionToggle = card.querySelector('.plugin-section-toggle');
if (!sectionToggle) return;
sectionToggle.checked = isTruthyConfigValue(effectiveConfig[enabledKey]);
});
}
function updatePersonaButtonHighlights(personaName) {
document.querySelectorAll('.persona-preset-btn').forEach(btn => {
const isActive = (btn.dataset.persona || '') === (personaName || '');
btn.classList.toggle('persona-preset-btn-active', isActive);
btn.setAttribute('aria-pressed', isActive ? 'true' : 'false');
});
}
function applyPersonaConfig(personaName) {
updatePersonaButtonHighlights(personaName);
const personaData = personaConfigMap[personaName];
if (!personaData) return;
const effectiveConfig = personaData.effective_config || {};
const concurrencyInput = document.querySelector('input[name="crawl_max_concurrent_snapshots"]');
const concurrencyValue = personaData.effective_config?.CRAWL_MAX_CONCURRENT_SNAPSHOTS;
const concurrencyValue = effectiveConfig.CRAWL_MAX_CONCURRENT_SNAPSHOTS;
if (concurrencyInput && concurrencyValue) {
concurrencyInput.value = concurrencyValue;
}
const deleteAfterInput = document.querySelector('input[name="delete_after"]');
const deleteAfterValue = personaData.effective_config?.DELETE_AFTER;
const deleteAfterValue = effectiveConfig.DELETE_AFTER;
if (deleteAfterInput && deleteAfterValue !== undefined && deleteAfterValue !== null) {
deleteAfterInput.value = deleteAfterValue || '0';
}
const timeoutInput = document.querySelector('input[name="timeout"]');
const timeoutValue = personaData.effective_config?.TIMEOUT;
const timeoutValue = effectiveConfig.TIMEOUT;
if (timeoutInput && timeoutValue !== undefined && timeoutValue !== null) {
timeoutInput.value = timeoutValue || '0';
}
const permissionsSelect = document.querySelector('select[name="permissions"]');
const permissionsValue = personaData.effective_config?.PERMISSIONS;
const permissionsValue = effectiveConfig.PERMISSIONS;
if (permissionsSelect && permissionsValue) {
permissionsSelect.value = permissionsValue;
}
const onlyNewInput = document.querySelector('input[name="url_filters_only_new"]');
const onlyNewValue = personaData.effective_config?.ONLY_NEW;
const onlyNewValue = effectiveConfig.ONLY_NEW;
if (onlyNewInput && onlyNewValue !== undefined && onlyNewValue !== null) {
onlyNewInput.checked = Boolean(onlyNewValue);
}
if (typeof window.archiveboxSetPluginConfigValues === 'function') {
window.archiveboxSetPluginConfigValues(personaData.effective_config || {}, personaData.binary_urls || {});
window.archiveboxSetPluginConfigValues(effectiveConfig, personaData.binary_urls || {});
}
syncSectionTogglesFromEffectiveConfig(effectiveConfig);
replaceConfigRows(personaData.config || {});
syncEnabledPluginsConfig();
updateChromeToggleButton();
updateURLPreview();
}
@ -1155,7 +1201,12 @@
// Plugin Presets
const presetConfigs = {
'text-only': ['wget', 'readability', 'mercury', 'htmltotext', 'title', 'favicon']
'text-only': [
'readability', 'mercury', 'htmltotext', 'title', 'favicon',
'dom', 'infiniscroll', 'forumdl', 'defuddle', 'trafilatura', 'liteparse',
'opendataloader', 'staticfile', 'papersdl', 'parse_dom_outlinks',
'parse_txt_urls', 'headers', 'redirects', 'seo', 'hashes',
],
};
document.querySelectorAll('.persona-preset-btn').forEach(btn => {
@ -1316,17 +1367,6 @@
loadFormState();
const focusTarget = new URLSearchParams(window.location.search).get('focus');
if (focusTarget === 'schedule') {
const scheduleInput = document.querySelector('[name="schedule"]');
const advancedSection = scheduleInput?.closest('details');
if (advancedSection) advancedSection.open = true;
if (scheduleInput) {
scheduleInput.scrollIntoView({ block: 'center' });
scheduleInput.focus();
}
}
// Form submission handler
document.getElementById('add-form').addEventListener('submit', function(event) {
document.getElementById('in-progress').style.display = 'block'

1
archivebox/templates/core/base.html
View File

@ -19,6 +19,7 @@
{% endblock %}
</head>
<body>
{% security_mode_banner %}
<div id="container">
<div id="header">
<div id="branding">

14
archivebox/templates/core/navigation.html
View File

@ -5,8 +5,8 @@
<a href="{% url 'Home' %}">Snapshots</a> |
<a href="/admin/core/tag/">Tags</a> |
<a href="/admin/core/archiveresult/?o=-1">Log</a> &nbsp; &nbsp;
<a href="{% url 'Docs' %}" target="_blank" rel="noopener noreferrer">Docs</a> |
<a href="/api/v1/docs">API</a> |
<a href="{% url 'Docs' %}" target="_blank" rel="noopener noreferrer">Docs</a> |
<a href="/api/v1/docs">API</a> |
<a href="/admin/">Admin</a>
&nbsp; &nbsp;
{% if user.is_authenticated %}
@ -20,6 +20,16 @@
{% endif %}
<a href="{% url 'admin:logout' %}">{% trans 'Log out' %}</a>
{% endblock %}
{% elif request.COOKIES.archivebox_admin_logged_in == "1" %}
{% comment %}
Authenticated on the admin host but the session cookie is admin-host-
scoped (security boundary — public.* must NEVER see the session).
The hint cookie is the only signal that crosses, so we render the
logged-out state's `Account` / `Log out` links pointing at admin host
so the user can still reach those pages from public.*/web.*.
{% endcomment %}
<a href="/admin/password_change/" title="Change your account password">Account</a> /
<a href="/admin/logout/">{% trans 'Log out' %}</a>
{% else %}
<a href="{% url 'admin:login' %}">{% trans 'Log in' %}</a>
{% endif %}

228
archivebox/templates/core/plugin_config_grid.html
View File

@ -23,14 +23,38 @@
border-radius: 6px;
}
.plugin-group-header {
.plugin-group > .plugin-group-header {
display: flex;
justify-content: space-between;
justify-content: flex-start;
align-items: center;
gap: 12px;
margin-bottom: 12px;
padding-bottom: 8px;
border-bottom: 2px solid #004882;
cursor: pointer;
user-select: none;
list-style: none;
}
.plugin-group[open] > .plugin-group-header {
margin-bottom: 12px;
}
.plugin-group > .plugin-group-header::-webkit-details-marker {
display: none;
}
.plugin-group-caret {
display: inline-block;
color: #004882;
font-size: 11px;
line-height: 1;
transition: transform 0.15s ease;
width: 12px;
flex: 0 0 12px;
}
.plugin-group[open] > .plugin-group-header > .plugin-group-caret {
transform: rotate(90deg);
}
.plugin-group-header label {
@ -38,6 +62,11 @@
color: #004882;
font-size: 15px;
font-weight: 700;
cursor: pointer;
}
.plugin-group-header .select-all-btn {
margin-left: auto;
}
.plugin-group-note {
@ -83,7 +112,8 @@
.plugin-card-main {
display: grid;
grid-template-columns: 18px minmax(0, 1fr) auto auto auto;
gap: 8px;
column-gap: 8px;
row-gap: 2px;
align-items: start;
min-width: 0;
padding: 8px 10px;
@ -150,43 +180,43 @@
}
.plugin-config-form .plugin-choice-description {
grid-column: 2;
grid-column: 1 / -1;
grid-row: 2;
display: inline-block;
display: block !important;
min-width: 0;
margin-left: 0;
color: #7a7a7a !important;
font-size: 12px;
width: 100%;
margin: 0;
color: #7a7a7a;
font-size: 12px !important;
font-weight: 400;
line-height: 1.35;
text-align: left;
text-decoration: none !important;
cursor: pointer;
user-select: none;
}
.plugin-config-form .plugin-checkboxes label a.plugin-choice-description:link,
.plugin-config-form .plugin-checkboxes label a.plugin-choice-description:visited,
.plugin-config-form .plugin-checkboxes label a.plugin-choice-description:active {
color: #7a7a7a !important;
text-decoration: none !important;
.plugin-config-form .plugin-choice-description:hover {
color: #4b5563;
text-decoration: underline;
}
.plugin-config-form .plugin-checkboxes label a.plugin-choice-description:hover,
.plugin-config-form .plugin-checkboxes label a.plugin-choice-description:focus {
color: #4b5563 !important;
text-decoration: underline !important;
.plugin-config-toggle-hack {
display: none;
}
.plugin-config-details {
.plugin-config-marker {
grid-column: 5;
grid-row: 1;
min-width: 0;
}
.plugin-card-main--static .plugin-config-details {
.plugin-card-main--static .plugin-config-marker {
grid-column: 4;
}
.plugin-card-badge,
.plugin-config-details summary {
display: inline-flex;
.plugin-config-form .plugin-config-marker {
display: inline-flex !important;
align-items: center;
gap: 6px;
min-height: 26px;
@ -195,7 +225,7 @@
border-radius: 999px;
background: #f8fafc;
color: #475569;
font-size: 12px;
font-size: 12px !important;
font-weight: 700;
cursor: pointer;
list-style: none;
@ -203,6 +233,11 @@
white-space: nowrap;
}
.plugin-config-marker:hover {
background: #eef4fa;
color: #1f2937;
}
.plugin-card-badge {
text-decoration: none !important;
}
@ -237,31 +272,30 @@
grid-column: 3;
}
.plugin-config-details summary::-webkit-details-marker {
display: none;
}
.plugin-config-details summary:before {
.plugin-config-marker::before {
content: "▶";
display: inline-block;
font-size: 10px;
transition: transform 0.2s;
}
.plugin-config-details[open] {
grid-column: 1 / -1;
grid-row: 2;
.plugin-config-toggle-hack:checked ~ .plugin-card-main .plugin-config-marker::before {
transform: rotate(90deg);
}
.plugin-config-details {
width: 100%;
min-width: 0;
margin-top: 0;
padding-top: 0;
display: none;
}
.plugin-config-toggle-hack:checked ~ .plugin-config-details {
display: block;
margin-top: 8px;
padding-top: 8px;
}
.plugin-config-details[open] summary {
margin-bottom: 8px;
}
.plugin-config-details[open] summary:before {
transform: rotate(90deg);
border-top: 1px solid #eef0f3;
}
.plugin-config-count {
@ -277,8 +311,8 @@
font-size: 11px;
}
.plugin-config-meta,
.plugin-config-empty {
grid-column: 1 / -1;
margin-bottom: 8px;
padding: 7px 9px;
border: 1px solid #e5edf5;
@ -289,6 +323,34 @@
line-height: 1.4;
}
.plugin-config-uses {
grid-column: 1 / -1;
margin-top: 4px;
padding-top: 8px;
border-top: 1px solid #e5edf5;
color: #64748b;
font-size: 11px;
line-height: 1.5;
}
.plugin-config-uses a {
color: #475569;
text-decoration: none;
}
.plugin-config-uses a:hover {
color: #1f2937;
text-decoration: underline;
}
.plugin-config-uses code {
padding: 1px 5px;
background: #f6f8fa;
border: 1px solid #d0d7de;
border-radius: 999px;
font-size: 11px;
}
.plugin-config-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
@ -350,7 +412,9 @@
}
.plugin-config-field textarea {
min-height: 70px;
min-height: 0 !important;
height: auto !important;
field-sizing: content;
resize: vertical;
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
}
@ -404,13 +468,18 @@
grid-template-columns: minmax(0, 1fr) auto auto auto;
}
.plugin-config-details,
.plugin-card-main--static .plugin-config-details {
.plugin-config-marker,
.plugin-card-main--static .plugin-config-marker {
grid-column: 1 / -1;
grid-row: 3;
justify-content: flex-start;
}
.plugin-choice-description {
grid-row: 2;
}
.plugin-group-header {
.plugin-group > .plugin-group-header {
flex-direction: column;
align-items: flex-start;
gap: 10px;
@ -421,18 +490,19 @@
<div class="plugin-groups-grid">
{% for group in plugin_groups %}
{% if group.plugins %}
<div class="plugin-group">
<div class="plugin-group-header">
<details class="plugin-group"{% if group.field_name != "metadata_plugins" and group.field_name != "postprocessing_plugins" %} open{% endif %}>
<summary class="plugin-group-header">
<span class="plugin-group-caret" aria-hidden="true">▶</span>
<label>{{ group.title }}</label>
{% if group.note %}
<span class="plugin-group-note">{{ group.note }}</span>
{% endif %}
{% if group.show_selectors and group.select_all_group %}
<button type="button" class="select-all-btn" data-group="{{ group.select_all_group }}">
<button type="button" class="select-all-btn" data-group="{{ group.select_all_group }}" onclick="event.stopPropagation();">
Select All Chrome
</button>
{% endif %}
</div>
</summary>
<div class="plugin-checkboxes"{% if group.dom_id %} id="{{ group.dom_id }}"{% endif %}>
{% for plugin in group.plugins %}
<div
@ -440,35 +510,29 @@
data-plugin-name="{{ plugin.name }}"
{% if plugin.enabled_config_key %}data-plugin-enabled-key="{{ plugin.enabled_config_key }}"{% endif %}
>
<input type="checkbox" id="plugin-config-toggle-{{ group.field_name }}-{{ plugin.name }}" class="plugin-config-toggle-hack">
<div class="plugin-card-main{% if not group.show_selectors %} plugin-card-main--static{% endif %}">
{% if group.show_selectors %}
<input type="checkbox" name="{{ group.field_name }}" value="{{ plugin.name }}" id="{{ plugin.checkbox_id }}" class="plugin-section-toggle" {% if plugin.checked %}checked{% endif %}>
<label for="{{ plugin.checkbox_id }}" class="plugin-card-label">
{{ plugin.label }}
</label>
{% else %}
<div class="plugin-card-label plugin-card-label-static">
{{ plugin.label }}
</div>
{% endif %}
<a class="plugin-card-badge plugin-source-badge" href="{{ plugin.source_url }}" target="_blank" rel="noopener noreferrer">Source</a>
<a class="plugin-card-badge plugin-docs-badge" href="{{ plugin.docs_url }}" target="_blank" rel="noopener noreferrer">Docs</a>
<details class="plugin-config-details">
<summary>
<span>Config</span>
<span class="plugin-config-count">{{ plugin.config_count }}</span>
</summary>
{% if plugin.required_plugins or plugin.required_binaries_count %}
<div class="plugin-config-meta">
{% if plugin.required_plugins %}
Requires {{ plugin.required_plugins|join:", " }}
{% endif %}
{% if plugin.required_binaries_count %}
{% if plugin.required_plugins %} · {% endif %}
{{ plugin.required_binaries_count }} binary requirement{{ plugin.required_binaries_count|pluralize }}
{% endif %}
</div>
{% endif %}
{% else %}
<div class="plugin-card-label plugin-card-label-static">
{{ plugin.label }}
</div>
{% endif %}
{% if plugin.description %}
<label for="plugin-config-toggle-{{ group.field_name }}-{{ plugin.name }}" class="plugin-choice-description">{{ plugin.description }}</label>
{% endif %}
<a class="plugin-card-badge plugin-source-badge" href="{{ plugin.source_url }}" target="_blank" rel="noopener noreferrer">Source</a>
<a class="plugin-card-badge plugin-docs-badge" href="{{ plugin.docs_url }}" target="_blank" rel="noopener noreferrer">Docs</a>
<label for="plugin-config-toggle-{{ group.field_name }}-{{ plugin.name }}" class="plugin-config-marker">
<span class="plugin-config-marker-label">Config</span>
<span class="plugin-config-count">{{ plugin.config_count }}</span>
</label>
</div>
<div class="plugin-config-details">
{% if plugin.config_fields %}
<div class="plugin-config-grid">
{% for field in plugin.config_fields %}
@ -535,16 +599,29 @@
</div>
</div>
{% endfor %}
{% if plugin.required_binary_links %}
<div class="plugin-config-uses">
Uses:
{% for binary in plugin.required_binary_links %}<a href="{{ binary.url }}" target="_blank" rel="noopener"><code>{{ binary.name }}</code></a>{% if not forloop.last %}, {% endif %}{% endfor %}
</div>
{% endif %}
</div>
{% else %}
<div class="plugin-config-empty">This plugin has no crawl-configurable options.</div>
<div class="plugin-config-empty">
This plugin has no crawl-configurable options.
{% if plugin.required_binary_links %}
<div class="plugin-config-uses">
Uses:
{% for binary in plugin.required_binary_links %}<a href="{{ binary.url }}" target="_blank" rel="noopener"><code>{{ binary.name }}</code></a>{% if not forloop.last %}, {% endif %}{% endfor %}
</div>
{% endif %}
</div>
{% endif %}
</details>
</div>
</div>
{% endfor %}
</div>
</div>
</details>
{% endif %}
{% endfor %}
</div>
@ -678,5 +755,6 @@
syncSectionToggleFromEnabledInput(event.target);
}
});
})();
</script>

56
archivebox/templates/core/security_mode_banner.html Normal file
View File

@ -0,0 +1,56 @@
{% comment %}
Fixed red badge that hangs off the top center of the page. Two trigger
conditions (see ``security_mode_banner`` in core_tags.py):
* mode="unsafe" — server is in a non-subdomain SERVER_SECURITY_MODE.
Archived content shares an origin with the admin UI.
* mode="unconfigured" — BASE_URL is not set. Always shown until the
operator pins it explicitly, even when CSRF
auto-derive or request-host fallback are keeping
the server functional.
{% endcomment %}
{% if mode == "unsafe" %}
<div id="archivebox-security-banner" role="alert" aria-live="polite"
style="position:fixed;top:-10px;left:50%;transform:translateX(-50%);
z-index:2147483647;background:#aa1e55;color:#fff;
font:600 11px/1.3 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Arial,sans-serif;
padding:14px 14px 4px 14px;border-radius:0 0 6px 6px;
box-shadow:0 2px 10px rgba(0,0,0,0.35);pointer-events:auto;
white-space:nowrap;letter-spacing:0.3px;text-transform:uppercase;">
<span style="display:inline-block;background:#fff;color:#aa1e55;
border-radius:3px;padding:1px 5px;margin-right:6px;font-weight:800;">
⚠ unsafe
</span>
ArchiveBox single-domain mode &mdash; archived pages share an origin with this site
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes"
target="_blank" rel="noopener noreferrer"
style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
</div>
{% elif mode == "unconfigured" %}
<div id="archivebox-security-banner" role="alert" aria-live="polite"
style="position:fixed;top:-10px;left:50%;transform:translateX(-50%);
z-index:2147483647;background:#dc2626;color:#fff;
font:600 11px/1.3 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Arial,sans-serif;
padding:14px 14px 4px 14px;border-radius:0 0 6px 6px;
box-shadow:0 2px 10px rgba(0,0,0,0.35);pointer-events:auto;
max-width:90vw;letter-spacing:0.3px;text-transform:uppercase;">
<span style="display:inline-block;background:#fff;color:#dc2626;
border-radius:3px;padding:1px 5px;margin-right:6px;font-weight:800;">
⚠ base_url not set
</span>
To prevent unauthorized requests, you must set your intended server URL in env or config:
<code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">
BASE_URL={% if suggested_base_url %}{{ suggested_base_url }}{% else %}http://*.archivebox.localhost:8000{% endif %}
</code>
{% if machine_admin_url %}
<a href="{{ machine_admin_url }}#BASE_URL"
style="display:inline-block;background:#fff;color:#dc2626;padding:1px 6px;border-radius:3px;
font-weight:800;text-decoration:none;margin-left:6px;text-transform:uppercase;">
pin via admin →
</a>
{% endif %}
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes#base_url"
target="_blank" rel="noopener noreferrer"
style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
</div>
{% endif %}

129
archivebox/templates/core/snapshot.html
View File

@ -245,7 +245,7 @@
}
.header-tags .permission-pill {
border-style: dashed;
text-transform: lowercase;
text-transform: uppercase;
}
.header-badges {
display: flex;
@ -311,7 +311,7 @@
.header-toggle {
line-height: 12px;
font-size: 70px;
vertical-align: -12px;
margin-top: -13px;
margin-left: 4px;
}
@media(max-width: 900px) {
@ -448,16 +448,86 @@
overflow: hidden;
order: 2;
flex: 0 0 auto;
display: flex;
flex-direction: column;
}
#main-frame-wrapper iframe {
width: 100%;
height: 100%;
border: none;
}
#snapshot-progress-wrapper {
flex: 0 0 auto;
width: 100%;
}
#main-frame-wrapper > #main-frame,
#main-frame-wrapper > #snapshot-empty-state {
flex: 1 1 auto;
min-height: 0;
}
.full-page-wrapper {
width: 100%;
height: calc(100vh - 210px);
}
.snapshot-empty-state {
display: flex;
align-items: center;
justify-content: center;
width: 100%;
height: 100%;
background: #f6f8fa;
color: #475569;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
}
.snapshot-empty-state-inner {
max-width: 520px;
padding: 32px 36px;
text-align: center;
}
.snapshot-empty-icon {
font-size: 56px;
line-height: 1;
margin-bottom: 12px;
}
.snapshot-empty-state h2 {
margin: 0 0 12px;
color: #1f2937;
font-size: 22px;
font-weight: 600;
}
.snapshot-empty-state p {
margin: 4px 0;
color: #64748b;
font-size: 14px;
line-height: 1.5;
}
.snapshot-empty-actions {
display: flex;
justify-content: center;
flex-wrap: wrap;
gap: 10px;
margin-top: 22px;
}
.snapshot-empty-btn {
display: inline-flex;
align-items: center;
gap: 6px;
padding: 8px 16px;
background: #ffffff;
border: 1px solid #d0d7de;
border-radius: 6px;
color: #1f2937;
font-size: 13px;
font-weight: 500;
text-decoration: none;
transition: background-color 0.15s, border-color 0.15s;
}
.snapshot-empty-btn:hover {
background: #eef4fa;
border-color: #b7d3ea;
color: #004882;
text-decoration: none;
}
.thumbnail-wrapper {
height: 100px;
overflow: hidden;
@ -975,7 +1045,13 @@
</style>
</head>
<body>
<div id="main-frame-wrapper" class="full-page-wrapper">
<div id="main-frame-wrapper" class="full-page-wrapper" data-has-outputs="{{ has_outputs|yesno:'1,0' }}" data-snapshot-state="{{ snapshot_state }}">
{% if snapshot_state == 'queued' or snapshot_state == 'started' or snapshot_state == 'paused' %}
<div id="snapshot-progress-wrapper">
{% include "admin/progress_monitor.html" with progress_endpoint=progress_endpoint progress_scope="snapshot" %}
</div>
{% endif %}
{% if has_outputs %}
<iframe id="main-frame"
sandbox="allow-same-origin allow-top-navigation-by-user-activation allow-scripts allow-forms"
class="full-page-iframe"
@ -984,7 +1060,47 @@
name="preview"
loading="eager"
fetchpriority="high"></iframe>
{% else %}
<div id="snapshot-empty-state" class="snapshot-empty-state" data-snapshot-state="{{ snapshot_state }}">
<div class="snapshot-empty-state-inner">
{% if snapshot_state == 'paused' %}
<div class="snapshot-empty-icon">⏸️</div>
<h2>Crawl paused</h2>
<p>This snapshot's crawl is paused — no extractors have run yet.</p>
<p>Resume the crawl to start archiving content.</p>
{% elif snapshot_state == 'started' %}
<div class="snapshot-empty-icon">⏳</div>
<h2>Archiving in progress…</h2>
<p>Extractors are running. Results will appear automatically.</p>
{% elif snapshot_state == 'queued' %}
<div class="snapshot-empty-icon">🕓</div>
<h2>Queued for archiving</h2>
<p>This snapshot is waiting in the queue. Results will appear automatically once a worker picks it up.</p>
{% else %}
<div class="snapshot-empty-icon">📭</div>
<h2>No archive outputs yet</h2>
<p>This snapshot has no archived content to display.</p>
{% endif %}
<div class="snapshot-empty-actions">
<button type="button" id="snapshot-empty-refresh" class="snapshot-empty-btn">🔄 Refresh page</button>
<a href="{{ url }}" target="_blank" rel="noopener noreferrer" class="snapshot-empty-btn">🌐 Open original URL</a>
<a href="{% admin_base_url %}/admin/core/snapshot/{{ snapshot_id }}/change/" class="snapshot-empty-btn">✏️ Edit in admin</a>
</div>
</div>
</div>
{% endif %}
</div>
<script>
(() => {
const refreshBtn = document.getElementById('snapshot-empty-refresh');
if (refreshBtn) refreshBtn.addEventListener('click', () => location.reload());
const monitor = document.getElementById('progress-monitor');
if (!monitor) return;
const wrapper = document.getElementById('main-frame-wrapper');
const hasOutputs = wrapper && wrapper.dataset.hasOutputs === '1';
if (!hasOutputs) monitor.classList.remove('collapsed');
})();
</script>
<script>
(() => {
const frame = document.getElementById('main-frame')
@ -1038,14 +1154,14 @@
<span class="header-title-text">{{title|truncatechars:120|safe}}</span>
{% if title_tags %}
<span class="header-tags">
<span class="tag-pill permission-pill">{{ snapshot_permissions_icon }} [{{ snapshot_permissions }}]</span>
<span class="tag-pill permission-pill">{{ snapshot_permissions_icon }} {{ snapshot_permissions }}</span>
{% for tag in title_tags %}
<span class="tag-pill" style="{{ tag.style }}">{{ tag.name }}</span>
{% endfor %}
</span>
{% else %}
<span class="header-tags">
<span class="tag-pill permission-pill">{{ snapshot_permissions_icon }} [{{ snapshot_permissions }}]</span>
<span class="tag-pill permission-pill">{{ snapshot_permissions_icon }} {{ snapshot_permissions }}</span>
</span>
{% endif %}
<a href="#" class="header-toggle header-toggle-trigger">▾</a>
@ -1407,6 +1523,9 @@
function ensureMainFrame(forceReplace=false) {
let frame = document.getElementById('main-frame')
const wrapper = document.getElementById('main-frame-wrapper')
if (wrapper && wrapper.querySelector('#snapshot-empty-state')) {
return null
}
if (!frame || forceReplace) {
const previousFrame = frame
frame = createMainFrame(previousFrame)

48
archivebox/templates/static/add.css
View File

@ -98,6 +98,20 @@ select {
min-height: 40px;
}
/* Crawl subtitle (grey explanation under the title) */
.crawl-subtitle {
max-width: 760px;
margin: 4px auto 16px;
color: #6b7280;
font-size: 13px;
line-height: 1.5;
}
.crawl-subtitle strong {
color: #475569;
font-weight: 600;
}
/* Crawl explanation box */
.crawl-explanation {
background-color: #e8f4f8;
@ -113,6 +127,28 @@ select {
color: #333;
}
#add-form .crawl-tip a {
text-decoration: none !important;
}
.crawl-tip-url {
padding: 2px 6px;
background: #f6f8fa;
border: 1px solid #d0d7de;
border-radius: 4px;
font-size: 13px;
white-space: nowrap;
}
.crawl-tip-url-prefix {
color: #6b7280;
}
.crawl-tip-url-target {
color: #116329;
font-weight: 600;
}
/* Form sections */
.form-section {
margin-bottom: 30px;
@ -794,6 +830,18 @@ a.preset-btn:active {
background-color: #f5fbff;
}
.persona-preset-btn.persona-preset-btn-active {
color: #ffffff;
border-color: #004882;
background-color: #004882;
box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.25);
}
.persona-preset-btn.persona-preset-btn-active:hover {
background-color: #003060;
border-color: #003060;
}
.persona-preset-wrap {
display: inline-flex;
align-items: stretch;

BIN
archivebox/templates/static/chrome_extension_icon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.3 KiB

2
archivebox/tests/test_cli_run.py
View File

@ -350,7 +350,6 @@ class TestRunDaemonMode:
"DATA_DIR": str(initialized_archive),
"USE_COLOR": "False",
"SHOW_PROGRESS": "False",
"USE_INDEXING_BACKEND": "False",
},
)
proc = subprocess.Popen(
@ -417,7 +416,6 @@ class TestRunDaemonMode:
"DATA_DIR": str(initialized_archive),
"USE_COLOR": "False",
"SHOW_PROGRESS": "False",
"USE_INDEXING_BACKEND": "False",
},
)
procs = [

14
archivebox/tests/test_cli_server.py
View File

@ -98,7 +98,6 @@ def _free_port():
def test_server_daemon_starts_real_plugin_owned_sonic_worker(archivebox_daemon_server):
server = archivebox_daemon_server(
USE_INDEXING_BACKEND="True",
SEARCH_BACKEND_ENGINE="sqlite",
)
state = server.wait_for_workers(("worker_daphne", "worker_sonic", "worker_runner"))
@ -109,14 +108,14 @@ def test_server_daemon_starts_real_plugin_owned_sonic_worker(archivebox_daemon_s
assert "sonic" in state["worker_sonic"]["name"]
def test_sonic_worker_is_disabled_by_real_indexing_config(tmp_path):
def test_sonic_worker_is_disabled_when_sonic_disabled_and_engine_not_sonic(tmp_path):
from archivebox.workers.supervisord_util import get_sonic_supervisord_worker_from_plugin
worker = get_sonic_supervisord_worker_from_plugin(
SimpleNamespace(
DATA_DIR=str(tmp_path),
SEARCH_BACKEND_ENGINE="sqlite",
USE_INDEXING_BACKEND=False,
SEARCH_BACKEND_ENGINE="ripgrep",
SEARCH_BACKEND_SONIC_ENABLED=False,
SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1",
SEARCH_BACKEND_SONIC_PORT=_free_port(),
SEARCH_BACKEND_SONIC_PASSWORD="SecretPassword",
@ -135,8 +134,7 @@ def test_sonic_daemon_event_handler_accepts_real_running_worker(archivebox_daemo
sonic_port = _free_port()
server = archivebox_daemon_server(
USE_INDEXING_BACKEND="True",
SEARCH_BACKEND_ENGINE="sqlite",
SEARCH_BACKEND_ENGINE="sonic",
SEARCH_BACKEND_SONIC_PORT=str(sonic_port),
)
state = server.wait_for_workers(("worker_sonic",))
@ -145,8 +143,7 @@ def test_sonic_daemon_event_handler_accepts_real_running_worker(archivebox_daemo
daemon_event = prepare_sonic_daemon(
SimpleNamespace(
DATA_DIR=str(server.data_dir),
SEARCH_BACKEND_ENGINE="sqlite",
USE_INDEXING_BACKEND=True,
SEARCH_BACKEND_ENGINE="sonic",
SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1",
SEARCH_BACKEND_SONIC_PORT=sonic_port,
SEARCH_BACKEND_SONIC_PASSWORD="SecretPassword",
@ -192,7 +189,6 @@ def test_supervisord_sync_does_not_start_duplicate_sonic_listener(tmp_path, proc
SimpleNamespace(
DATA_DIR=str(tmp_path),
SEARCH_BACKEND_ENGINE="sonic",
USE_INDEXING_BACKEND=True,
SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1",
SEARCH_BACKEND_SONIC_PORT=sonic_port,
SEARCH_BACKEND_SONIC_PASSWORD="SecretPassword",

6
archivebox/tests/test_cli_update_reindex_snapshots.py
View File

@ -88,9 +88,7 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
(output_dir / "dom" / "output.html").write_text("<html><body>Example searchable text</body></html>")
original_engine = os.environ.get("SEARCH_BACKEND_ENGINE")
original_indexing = os.environ.get("USE_INDEXING_BACKEND")
os.environ["SEARCH_BACKEND_ENGINE"] = "sqlite"
os.environ["USE_INDEXING_BACKEND"] = "true"
try:
stats = reindex_snapshots(
Snapshot.objects.filter(id=snapshot.id),
@ -102,10 +100,6 @@ def test_reindex_snapshots_resets_existing_search_results_and_reruns_requested_p
os.environ.pop("SEARCH_BACKEND_ENGINE", None)
else:
os.environ["SEARCH_BACKEND_ENGINE"] = original_engine
if original_indexing is None:
os.environ.pop("USE_INDEXING_BACKEND", None)
else:
os.environ["USE_INDEXING_BACKEND"] = original_indexing
result.refresh_from_db()

2
archivebox/tests/test_crawl_admin.py
View File

@ -71,7 +71,7 @@ def test_crawl_schedule_admin_add_redirects_to_add_page_schedule_field(client, a
response = client.get(reverse("admin:crawls_crawlschedule_add"), HTTP_HOST=ADMIN_HOST)
assert response.status_code == 302
assert response["Location"] == "/add/?focus=schedule"
assert response["Location"] == "/add/#schedule"
def test_crawl_admin_form_saves_tags_editor_to_tags_str(crawl, admin_user):

2
archivebox/tests/test_crawl_pause.py
View File

@ -160,8 +160,6 @@ def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(t
port,
PLUGINS="search_backend_sqlite",
SEARCH_BACKEND_ENGINE="sqlite",
USE_INDEXING_BACKEND="True",
USE_SEARCHING_BACKEND="True",
)
update_process = subprocess.run(
[

2
archivebox/tests/test_search.py
View File

@ -17,7 +17,6 @@ def test_search_backend_env_exposes_resolved_runtime_config(tmp_path):
"SEARCH_BACKEND_SONIC_HOST_NAME": "sonic",
"SEARCH_BACKEND_SONIC_PORT": 1491,
"SEARCH_BACKEND_SONIC_PASSWORD": "SecretPassword",
"USE_INDEXING_BACKEND": True,
"IGNORED_NONE_VALUE": None,
},
)
@ -30,7 +29,6 @@ def test_search_backend_env_exposes_resolved_runtime_config(tmp_path):
assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "sonic"
assert os.environ["SEARCH_BACKEND_SONIC_PORT"] == "1491"
assert os.environ["SEARCH_BACKEND_SONIC_PASSWORD"] == "SecretPassword"
assert os.environ["USE_INDEXING_BACKEND"] == "True"
assert "IGNORED_NONE_VALUE" not in os.environ
assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "old-host"

4
archivebox/tests/test_search_backends_e2e.py
View File

@ -26,8 +26,6 @@ def test_real_fulltext_search_backends_survive_reindex_transition(tmp_path):
"SAVE_WARC": "False",
"WGET_WARC_ENABLED": "False",
"WGET_TIMEOUT": "20",
"USE_SEARCHING_BACKEND": "true",
"USE_INDEXING_BACKEND": "true",
},
)
if env:
@ -66,8 +64,6 @@ def test_real_fulltext_search_backends_survive_reindex_transition(tmp_path):
"USE_COLOR": "False",
"SHOW_PROGRESS": "False",
"SEARCH_BACKEND_ENGINE": "sonic",
"USE_SEARCHING_BACKEND": "true",
"USE_INDEXING_BACKEND": "true",
"SEARCH_BACKEND_SONIC_PORT": str(sonic_port),
},
)

4
archivebox/tests/test_tag_admin.py
View File

@ -106,10 +106,10 @@ def test_tag_search_api_returns_card_payload(client, api_token, tagged_data):
assert payload["tags"][0]["id"] == tag.id
assert payload["tags"][0]["name"] == "Alpha Research"
assert payload["tags"][0]["num_snapshots"] == 2
assert payload["tags"][0]["snapshots"][0]["title"] in {"Example One", "Example Two"}
assert payload["tags"][0]["snapshots"] == []
assert payload["tags"][0]["export_jsonl_url"].endswith(f"/api/v1/core/tag/{tag.id}/snapshots.jsonl")
assert payload["tags"][0]["filter_url"].endswith(f"/admin/core/snapshot/?tags__id__exact={tag.id}")
assert {snapshot["url"] for snapshot in payload["tags"][0]["snapshots"]} == {snap.url for snap in snapshots}
assert {snap.url for snap in snapshots} == {"https://example.com/one", "https://example.com/two"}
def test_tag_search_api_respects_sort_and_filters(client, api_token, admin_user, crawl, tagged_data):

1
archivebox/tests/test_ui_add_view.py
View File

@ -175,7 +175,6 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 5
assert crawl.config["URL_ALLOWLIST"] == "example.com\n*.example.com"
assert crawl.config["URL_DENYLIST"] == "cdn.example.com"
assert "OVERWRITE" not in crawl.config
assert "ONLY_NEW" not in crawl.config

2
archivebox/tests/test_ui_add_view_runtime.py
View File

@ -199,7 +199,7 @@ def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursiv
assert list((tmp_path / "archive/users").rglob("snapshots/**/wget/**/*.html"))
progress = session.get(
f"http://127.0.0.1:{port}/admin/live-progress/",
f"http://127.0.0.1:{port}/progress.json",
headers={"Host": f"admin.archivebox.localhost:{port}"},
timeout=10,
)

7
archivebox/tests/test_ui_admin_config_widget.py
View File

@ -99,12 +99,6 @@ def test_key_value_widget_shows_array_and_object_examples_and_binary_rules(monke
"default": [],
"description": "Extra arguments to append to wget command",
},
"SAVE_ALLOWLIST": {
"plugin": "base",
"type": "object",
"default": {},
"description": "Regex allowlist mapped to enabled methods",
},
"WGET_BINARY": {
"plugin": "wget",
"type": "string",
@ -117,7 +111,6 @@ def test_key_value_widget_shows_array_and_object_examples_and_binary_rules(monke
html = str(KeyValueWidget().render("config", {}, attrs={"id": "id_config"}))
assert 'Example: ["--extra-arg"]' in html
assert 'Example: {"^https://example\\\\.com": ["wget"]}' in html
assert "Example: wget or /usr/bin/wget" in html
assert "validateBinaryValue_id_config" in html
assert "meta.key.endsWith('_BINARY')" in html

5
archivebox/tests/test_ui_admin_views.py
View File

@ -649,7 +649,7 @@ class TestAdminSnapshotListView:
status=ArchiveResult.StatusChoices.QUEUED,
)
response = client.get("/admin/live-progress/", HTTP_HOST=ADMIN_HOST)
response = client.get("/progress.json", HTTP_HOST=ADMIN_HOST)
assert response.status_code == 200
payload = response.json()
@ -688,7 +688,7 @@ class TestAdminSnapshotListView:
start_ts=now - timedelta(minutes=5),
)
response = client.get("/admin/live-progress/", HTTP_HOST=ADMIN_HOST)
response = client.get("/progress.json", HTTP_HOST=ADMIN_HOST)
assert response.status_code == 200
payload = response.json()
@ -1833,7 +1833,6 @@ class TestPublicIndexSearch:
from archivebox.core.views import PublicIndexView
monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep")
monkeypatch.setenv("USE_INDEXING_BACKEND", "false")
metadata_snapshot = Snapshot.objects.create(
url="https://public-example.com/google-meta",
title="Google Metadata Match",

15
archivebox/workers/supervisord_util.py
View File

@ -875,13 +875,20 @@ def run_runner_worker(args: list[str], *, name: str = "worker_runner_once", inte
except KeyboardInterrupt:
if not interactive_interrupts or forwarded_interrupt:
raise
forwarded_interrupt = True
# Route the signal through supervisord by worker name rather than
# raw os.kill on a cached PID. The cached proc["pid"] can be
# stale: if the worker exited between supervisord's last status
# poll and the user's Ctrl+C, the OS may have already reused
# that pid for an unrelated process (e.g. another shell the
# user has open) and raw os.kill would target it instead of the
# crawl hook. signalProcess goes through supervisord, which
# only signals workers it still owns.
proc = get_worker(supervisor, name)
pid = int(proc.get("pid") or 0) if proc else 0
if pid <= 0:
if proc is None or proc.get("statename") != "RUNNING":
raise
supervisor.signalProcess(name, "SIGINT")
forwarded_interrupt = True
print("[yellow][*] Forwarding Ctrl+C to the active crawl hook...[/yellow]")
os.kill(pid, signal.SIGINT)
finally:
log_handle.close()

57
docs/Configuration.md
View File

@ -47,11 +47,6 @@ By default, ArchiveBox will only archive new links on each import. If you want i
*Note: Regardless of how this is set, ArchiveBox will never re-download sites that have already succeeded previously. When this is `False` it only attempts to fix previous pages have *missing* archive extractor outputs, it does not re-archive pages that have already been successfully archived.*
---
#### `OVERWRITE`
**Possible Values:** [`False`]/`True`
When set to `True`, ArchiveBox will re-archive URLs even if they have already been successfully archived before, overwriting any existing output.
---
#### `TIMEOUT`
**Possible Values:** [`60`]/`120`/...
@ -59,11 +54,6 @@ Maximum allowed download time per archive method for each link in seconds. If y
*Note: Do not set this to anything less than `5` seconds as it will cause Chrome to hang indefinitely and many sites to fail completely.*
---
#### `MAX_URL_ATTEMPTS`
**Possible Values:** [`50`]/`100`/...
Maximum number of times ArchiveBox will attempt to archive a URL before giving up. Useful for handling transient failures.
---
#### `RESOLUTION`
**Possible Values:** [`1440,2000`]/`1024,768`/...
@ -111,7 +101,7 @@ The persona profile to use by default when archiving. Personas allow you to have
A regex expression used to exclude certain URLs from archiving.
*Related options:*
[`URL_ALLOWLIST`](#url_allowlist), [`SAVE_ALLOWLIST`](#save_allowlist), [`SAVE_DENYLIST`](#save_denylist)
[`URL_ALLOWLIST`](#url_allowlist)
---
#### `URL_ALLOWLIST`
@ -119,16 +109,6 @@ A regex expression used to exclude certain URLs from archiving.
A regex expression used to exclude all URLs that don't match the given pattern from archiving. Useful for recursive crawling within a single domain.
---
#### `SAVE_ALLOWLIST`
**Possible Values:** [`{}`]/`{".*example\\.com.*": ["screenshot", "pdf"]}`/...
A JSON dictionary mapping URL regex patterns to lists of archive methods. Only the specified methods will be used for URLs matching each pattern.
---
#### `SAVE_DENYLIST`
**Possible Values:** [`{}`]/`{".*\\.pdf$": ["screenshot", "dom"]}`/...
A JSON dictionary mapping URL regex patterns to lists of archive methods to *skip*.
---
#### `TAG_SEPARATOR_PATTERN`
**Possible Values:** [`[,]`]/`[,;]`/...
@ -185,11 +165,6 @@ Comma-separated list of allowed HTTP Host header values. Set this to your domain
**Possible Values:** [`40`]/`100`/...
Maximum number of Snapshots to show per page on Snapshot list pages.
---
#### `PREVIEW_ORIGINALS`
**Possible Values:** [`True`]/`False`
Whether to show inline previews of the original URL on snapshot detail pages.
---
#### `FOOTER_INFO`
**Possible Values:** [`Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.`]/...
@ -326,11 +301,6 @@ User and Group ID that the data directory should be owned by.
- https://docs.linuxserver.io/general/understanding-puid-and-pgid/
- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues
---
#### `RESTRICT_FILE_NAMES`
**Possible Values:** [`windows`]/`unix`/`ascii`/...
Restrict output filenames to be compatible with the given filesystem type.
---
#### `ENFORCE_ATOMIC_WRITES`
**Possible Values:** [`True`]/`False`
@ -357,26 +327,11 @@ Path where installed binaries are symlinked for convenient manual access.
*Options for full-text search backend configuration.*
---
#### `USE_INDEXING_BACKEND`
**Possible Values:** [`True`]/`False`
Enable the search indexing backend.
---
#### `USE_SEARCHING_BACKEND`
**Possible Values:** [`True`]/`False`
Enable the search querying backend.
---
#### `SEARCH_BACKEND_ENGINE`
**Possible Values:** [`ripgrep`]/`sqlite`/`sonic`
Which search backend engine to use. `ripgrep` (default) requires no setup. `sqlite` uses FTS5. `sonic` requires a running Sonic instance.
---
#### `SEARCH_PROCESS_HTML`
**Possible Values:** [`True`]/`False`
Whether to strip HTML tags before indexing content for search.
---
## Shell Options
@ -449,11 +404,6 @@ Enable favicon downloading
**Default:** [`30`] *(falls back to [`TIMEOUT`](#timeout))*
Timeout for favicon fetch in seconds
---
#### `FAVICON_USER_AGENT`
**Default:** [`""`] *(falls back to [`USER_AGENT`](#user_agent))*
User agent string
### Wget Settings
@ -918,11 +868,6 @@ Submit URLs to archive.org Wayback Machine
**Default:** [`60`] *(falls back to [`TIMEOUT`](#timeout))*
Timeout for archive.org submission in seconds
---
#### `ARCHIVEDOTORG_USER_AGENT`
**Default:** [`""`] *(falls back to [`USER_AGENT`](#user_agent))*
User agent string
### Chrome Settings