The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

release: archivebox 0.9.34rc71

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-07 20:51:28 -07:00
parent 667f1658f0
commit 4fa90e484a
No known key found for this signature in database
47 changed files with 737 additions and 444 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Dockerfile
View File

@ -47,9 +47,10 @@ ENV ARCHIVEBOX_USER=archivebox \
ENV CODE_DIR=/app \
DATA_DIR=/data \
LIB_DIR=/opt/archivebox/lib \
ABXPKG_LIB_DIR=/opt/archivebox/lib \
PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
CONFIG_DIR=/home/archivebox/.config/abx \
LIB_DIR=/home/archivebox/.config/abx/lib \
ABXPKG_LIB_DIR=/home/archivebox/.config/abx/lib \
PLAYWRIGHT_BROWSERS_PATH=/home/archivebox/.config/abx/lib/playwright/cache \
PERSONAS_DIR=/data/personas \
CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
CHROME_HEADLESS=true \
@ -77,7 +78,7 @@ ENV UV_COMPILE_BYTECODE=false \
UV_LINK_MODE=copy \
UV_PROJECT_ENVIRONMENT=/venv \
VIRTUAL_ENV=/venv \
PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
PATH="/venv/bin:/opt/node/bin:$PATH"
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
WORKDIR "$CODE_DIR"
@ -168,7 +169,7 @@ COPY --from=archivebox-builder /app /app
COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
&& printf 'export PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
&& printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
&& ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
&& ln -sf /venv/bin/daphne /usr/local/bin/daphne \
&& ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \

11
Dockerfile.multistage
View File

@ -47,9 +47,10 @@ ENV ARCHIVEBOX_USER=archivebox \
ENV CODE_DIR=/app \
DATA_DIR=/data \
LIB_DIR=/opt/archivebox/lib \
ABXPKG_LIB_DIR=/opt/archivebox/lib \
PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
CONFIG_DIR=/home/archivebox/.config/abx \
LIB_DIR=/home/archivebox/.config/abx/lib \
ABXPKG_LIB_DIR=/home/archivebox/.config/abx/lib \
PLAYWRIGHT_BROWSERS_PATH=/home/archivebox/.config/abx/lib/playwright/cache \
PERSONAS_DIR=/data/personas \
CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
CHROME_HEADLESS=true \
@ -78,7 +79,7 @@ ENV UV_COMPILE_BYTECODE=false \
UV_LINK_MODE=copy \
UV_PROJECT_ENVIRONMENT=/venv \
VIRTUAL_ENV=/venv \
PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
PATH="/venv/bin:/opt/node/bin:$PATH"
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
WORKDIR "$CODE_DIR"
@ -197,7 +198,7 @@ RUN echo "[+] Initializing image collection..." \
RUN chmod +x "$CODE_DIR"/bin/*.sh \
&& chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
RUN "$LIB_DIR/bin/chromium" --version | tee -a /VERSION.txt \
RUN "$LIB_DIR/playwright/bin/chromium" --version | tee -a /VERSION.txt \
&& "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
&& /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
&& /usr/local/bin/sonic --version | tee -a /VERSION.txt \

2
archivebox/cli/archivebox_add.py
View File

@ -128,7 +128,7 @@ def add(
admitted_urls: list[str] = []
admitted_snapshot_ids: list[str] | None = [] if snapshot_ids else None
for index, url in enumerate(url_list):
if Snapshot.is_archivebox_internal_url(url):
if Snapshot.is_archivebox_internal_url(url, config=runtime_config):
print(f"[yellow][!] Skipping internal ArchiveBox URL: {url}[/yellow]")
continue
admitted_urls.append(url)

1
archivebox/cli/archivebox_init.py
View File

@ -163,7 +163,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
if working_lib_dir:
working_lib_dir.mkdir(parents=True, exist_ok=True)
(working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
if install:
from archivebox.cli.archivebox_install import install as install_method

22
archivebox/config/common.py
View File

@ -176,6 +176,10 @@ def build_crawl_config_snapshot(
plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields)
effective = get_config(persona=persona, base_config=base_config)
frozen = effective.for_crawl_frozen(persona=persona)
for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
value = getattr(effective, key, None)
if value is not None:
frozen[key] = value
if persona is not None:
persona_config = persona.get_derived_config()
for key in plugin_owned_keys - explicit_overrides:
@ -185,6 +189,10 @@ def build_crawl_config_snapshot(
resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False)
resolved_payload = normalize_runtime_config(resolved)
frozen = resolved.for_crawl_frozen(persona=persona)
for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
value = getattr(resolved, key, None)
if value is not None:
frozen[key] = value
for key in plugin_owned_keys & explicit_overrides:
if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload:
frozen[key] = resolved_payload[key]
@ -238,7 +246,7 @@ class StorageConfig(BaseConfigSet):
_scope: str = PrivateAttr(default=_SCOPE_SERVER)
# TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
# must be a short path due to unix path length restrictions for socket files (<100 chars)
# must be a short path due to unix path length restrictions for socket files (<90 chars)
# must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
@ -247,10 +255,6 @@ class StorageConfig(BaseConfigSet):
# should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
# LIB_BIN_DIR is an optional human-facing symlink convenience directory.
# Runtime lookup must use provider-specific paths under LIB_DIR instead.
LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
OUTPUT_PERMISSIONS: str = Field(default="644")
ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
ALLOW_NO_UNIX_SOCKETS: bool = Field(default=False, alias="ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS")
@ -736,13 +740,6 @@ class ArchiveBoxBaseConfig(
lib_dir = CONSTANTS.DATA_DIR / lib_dir
self.LIB_DIR = lib_dir.resolve()
lib_bin_dir = self.LIB_BIN_DIR.expanduser()
if lib_bin_dir == CONSTANTS.DEFAULT_LIB_BIN_DIR and self.LIB_DIR != CONSTANTS.DEFAULT_LIB_DIR:
lib_bin_dir = self.LIB_DIR / "bin"
elif not lib_bin_dir.is_absolute():
lib_bin_dir = CONSTANTS.DATA_DIR / lib_bin_dir
self.LIB_BIN_DIR = lib_bin_dir.resolve()
return self
@model_validator(mode="after")
@ -1137,7 +1134,6 @@ def get_config(
if is_sensitive_config_key(key) and value not in (None, ""):
setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
os.environ["LIB_DIR"] = str(config.LIB_DIR)
os.environ["LIB_BIN_DIR"] = str(config.LIB_BIN_DIR)
os.environ["ABXPKG_LIB_DIR"] = str(config.LIB_DIR)
archiving_warning_key = (config.TIMEOUT, config.USE_COLOR)
if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:

1
archivebox/config/constants.py
View File

@ -118,7 +118,6 @@ class ConstantsDict:
"LIB_DIR",
user_config_path("abx") / LIB_DIR_NAME,
)
DEFAULT_LIB_BIN_DIR: Path = _env_path("LIB_BIN_DIR", DEFAULT_LIB_DIR / "bin")
RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
(

68
archivebox/config/paths.py
View File

@ -20,6 +20,8 @@ if TYPE_CHECKING:
PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent # archivebox source code dir
DATA_DIR: Path = Path(os.getcwd()).resolve() # archivebox user data dir
MAX_TMP_SOCKET_URL_LENGTH = 90
SUPERVISORD_SOCKET_FILENAME = "supervisord.sock"
def _env_path(key: str, default: Path) -> Path:
@ -176,33 +178,45 @@ def create_and_chown_dir(dir_path: Path) -> None:
def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
socket_file = dir_path.absolute().resolve() / "supervisord.sock"
return len(f"file://{socket_file}") <= 96
socket_file = dir_path.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
return len(f"file://{socket_file}") < MAX_TMP_SOCKET_URL_LENGTH
def tmp_dir_candidates(config: "ArchiveBoxConfig") -> list[Path]:
from archivebox.config.constants import CONSTANTS
collection_id = get_collection_id()
collection_id_short = collection_id[:4]
system_tmp_dir = Path(tempfile.gettempdir())
candidates = [
config.TMP_DIR, # <user-specified>
CONSTANTS.DEFAULT_TMP_DIR, # ./data/tmp/<machine_id>
Path("/var/run/archivebox") / collection_id,
Path("/tmp") / "archivebox" / collection_id,
Path("~/.tmp/archivebox").expanduser() / collection_id,
system_tmp_dir / "archivebox" / collection_id,
system_tmp_dir / "archivebox" / collection_id_short,
system_tmp_dir / "abx" / collection_id_short,
]
seen = set()
unique_candidates = []
for path in candidates:
path_key = str(path.expanduser().absolute())
if path_key in seen:
continue
seen.add(path_key)
unique_candidates.append(path)
return unique_candidates
def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
from archivebox.config.constants import CONSTANTS
from archivebox.config.common import get_config
from archivebox.misc.checks import check_tmp_dir
config = config or get_config(**config_kwargs)
# try a few potential directories in order of preference
CANDIDATES = [
config.TMP_DIR, # <user-specified>
CONSTANTS.DEFAULT_TMP_DIR, # ./data/tmp/<machine_id>
Path("/var/run/archivebox") / get_collection_id(), # /var/run/archivebox/abc5d8512
Path("/tmp") / "archivebox" / get_collection_id(), # /tmp/archivebox/abc5d8512
Path("~/.tmp/archivebox").expanduser() / get_collection_id(), # ~/.tmp/archivebox/abc5d8512
Path(tempfile.gettempdir())
/ "archivebox"
/ get_collection_id(), # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d8512
Path(tempfile.gettempdir())
/ "archivebox"
/ get_collection_id()[:4], # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d
Path(tempfile.gettempdir()) / "abx" / get_collection_id()[:4], # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/abx/abc5
]
candidates = tmp_dir_candidates(config)
fallback_candidate = None
for candidate in CANDIDATES:
for candidate in candidates:
try:
create_and_chown_dir(candidate)
except Exception:
@ -231,7 +245,7 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxC
return fallback_candidate
if not quiet:
raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!")
raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {candidates}!")
def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@ -329,7 +343,10 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
"TMP_DIR": {
"path": tmp_dir.resolve(),
"enabled": True,
"is_valid": os.path.isdir(tmp_dir) and os.access(tmp_dir, os.R_OK) and os.access(tmp_dir, os.W_OK), # read + write
"is_valid": os.path.isdir(tmp_dir)
and os.access(tmp_dir, os.R_OK)
and os.access(tmp_dir, os.W_OK)
and tmp_dir_socket_path_is_short_enough(tmp_dir),
},
# "CACHE_DIR": {
# "path": CACHE_DIR.resolve(),
@ -351,8 +368,6 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
except Exception:
lib_dir = config.LIB_DIR
lib_bin_dir = lib_dir / "bin"
return AttrDict(
{
"PACKAGE_DIR": {
@ -380,12 +395,5 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
"enabled": True,
"is_valid": os.path.isdir(lib_dir) and os.access(lib_dir, os.R_OK) and os.access(lib_dir, os.W_OK), # read + write
},
"LIB_BIN_DIR": {
"path": lib_bin_dir.resolve(),
"enabled": True,
"is_valid": os.path.isdir(lib_bin_dir)
and os.access(lib_bin_dir, os.R_OK)
and os.access(lib_bin_dir, os.W_OK), # read + write
},
},
)

5
archivebox/config/views.py
View File

@ -18,7 +18,6 @@ from archivebox.misc.util import parse_date
from archivebox.machine.models import Binary
LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
@ -32,10 +31,6 @@ def format_parsed_datetime(value: object) -> str:
return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""
def get_live_config_url(key: str) -> str:
return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
def get_environment_binary_url(name: str) -> str:
return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/"

35
archivebox/core/admin_snapshots.py
View File

@ -30,6 +30,7 @@ from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_p
from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
from archivebox.core.models import Tag, Snapshot, ArchiveResult
from archivebox.crawls.models import Crawl
from archivebox.core.admin_archiveresults import render_archiveresults_list
from archivebox.core.preview_util import EXTENSION_SCREENSHOT_PLUGIN
from archivebox.progressmonitor.views import progress_endpoint
@ -344,7 +345,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
list_select_related = ()
list_display = (
"permissions_badge",
"created_at_display",
"created_at",
"preview_icon",
"title_str",
"tags_inline",
@ -352,7 +353,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
"files",
"size_with_stats",
)
list_display_links = ("created_at_display",)
list_display_links = ("created_at",)
sort_fields = ("title_str", "created_at", "status", "crawl")
readonly_fields = (
"admin_actions",
@ -453,7 +454,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
),
)
ordering = ["-id"]
ordering = ["-created_at"]
actions = [
"add_tags",
"remove_tags",
@ -470,7 +471,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
paginator = AcceleratedPaginator
save_on_top = True
show_full_result_count = False
show_full_result_count = True
def get_changelist(self, request, **kwargs):
return SnapshotChangeList
@ -480,10 +481,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
return []
return super().get_ordering(request)
@admin.display(description="Created", ordering="id")
def created_at_display(self, obj):
return obj.created_at
def change_view(self, request, object_id, form_url="", extra_context=None):
self.request = request
extra_context = extra_context or {}
@ -650,10 +647,22 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
),
),
)
else:
prefetches.append(
Prefetch(
"crawl",
queryset=Crawl.objects.select_related("created_by").only(
"id",
"created_by_id",
"created_by__id",
"created_by__username",
),
),
)
qs = super().get_queryset(request).select_related("crawl__created_by")
qs = super().get_queryset(request)
if is_change_view:
qs = qs.defer("notes")
qs = qs.select_related("crawl__created_by").defer("notes")
else:
qs = qs.only(
"id",
@ -667,12 +676,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
"fs_version",
"output_size",
"permissions",
"crawl__id",
"crawl__persona_id",
"crawl__status",
"crawl__created_by_id",
"crawl__created_by__id",
"crawl__created_by__username",
)
qs = qs.prefetch_related(*prefetches)
if needs_files_sort:

39
archivebox/core/middleware.py
View File

@ -3,6 +3,7 @@ __package__ = "archivebox.core"
import ipaddress
import re
from pathlib import Path
from django.conf import settings
from django.utils import timezone
from django.contrib.auth.middleware import RemoteUserMiddleware
from django.contrib.auth.models import AnonymousUser
@ -24,7 +25,6 @@ from archivebox.core.routes_util import (
get_base_host,
get_listen_host,
get_listen_subdomain,
get_public_host,
get_web_host,
host_matches,
is_snapshot_subdomain,
@ -42,7 +42,7 @@ def _admin_login_hint_cookie_domain(config) -> str | None:
NOTE: this cookie carries only the single bit "user is logged in on
admin somewhere"; it MUST NOT be confused with the session cookie,
which stays admin-host-scoped (see core/settings.py
SESSION_COOKIE_DOMAIN comment — admin/public is a security boundary).
SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).
Returns the hostname portion of ``get_base_host`` (which respects
``BASE_URL`` and falls back to the local-bind mapping). Strips the
@ -85,6 +85,34 @@ def TimezoneMiddleware(get_response):
return middleware
def AdminCookieIsolationMiddleware(get_response):
def middleware(request):
response = get_response(request)
config = request.__dict__.get("archivebox_config")
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config
if not config.USES_SUBDOMAIN_ROUTING:
return response
request_host = (request.get_host() or "").lower()
if host_matches(request_host, get_admin_host(config=config)):
return response
if host_matches(request_host, get_web_host(config=config)):
for cookie_name in tuple(response.cookies.keys()):
if cookie_name != ADMIN_LOGIN_HINT_COOKIE:
response.cookies.pop(cookie_name, None)
return response
response.cookies.pop(settings.SESSION_COOKIE_NAME, None)
response.cookies.pop(settings.CSRF_COOKIE_NAME, None)
return response
return middleware
def CacheControlMiddleware(get_response):
snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
static_cache_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
@ -176,7 +204,6 @@ def HostRoutingMiddleware(get_response):
admin_host = get_admin_host(config=config)
web_host = get_web_host(config=config)
api_host = get_api_host(config=config)
public_host = get_public_host(config=config)
listen_host = get_listen_host(config=config)
subdomain = get_listen_subdomain(request_host, config=config)
@ -278,12 +305,6 @@ def HostRoutingMiddleware(get_response):
request._cached_user = request.user
return get_response(request)
if host_matches(request_host, public_host):
if request.COOKIES.get(ADMIN_LOGIN_HINT_COOKIE) == "1" and (request.path == "/public" or request.path.startswith("/public/")):
target = build_admin_url("/admin/core/snapshot/", request=request)
return redirect(target)
return get_response(request)
if subdomain:
view = OriginalDomainHostView.as_view()
return view(request, domain=subdomain, path=request.path.lstrip("/"))

54
archivebox/core/models.py
View File

@ -1,7 +1,7 @@
__package__ = "archivebox.core"
from typing import TYPE_CHECKING, Optional, Any
from collections.abc import Iterable, Sequence
from collections.abc import Iterable, Mapping, Sequence
import uuid
from archivebox.uuid_compat import CompactUUIDField, uuid7
from datetime import datetime, timedelta
@ -782,7 +782,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
)
@classmethod
def is_archivebox_internal_url(cls, url: str) -> bool:
def is_archivebox_internal_url(cls, url: str, *, config: Mapping[str, Any] | Any | None = None) -> bool:
parsed = urlparse((url or "").strip())
if parsed.scheme not in ("http", "https") or not parsed.hostname:
return False
@ -792,15 +792,29 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
get_api_host,
get_base_host,
get_listen_host,
get_public_host,
get_web_host,
split_host_port,
)
config = get_config()
if config is None:
config = get_config()
elif isinstance(config, Mapping):
route_config = config
class RouteConfig:
BIND_ADDR = str(route_config.get("BIND_ADDR") or "")
BASE_URL = str(route_config.get("BASE_URL") or "")
CSRF_TRUSTED_ORIGINS = str(route_config.get("CSRF_TRUSTED_ORIGINS") or "")
SERVER_SECURITY_MODE = str(route_config.get("SERVER_SECURITY_MODE") or "")
@property
def USES_SUBDOMAIN_ROUTING(self) -> bool:
return self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay"
config = RouteConfig()
host = parsed.hostname.lower().strip(".")
port = str(parsed.port) if parsed.port else None
protected_subdomains = {"admin", "web", "api", "public"}
protected_subdomains = {"admin", "web", "api"}
protected_hosts: set[tuple[str, str | None]] = set()
protected_roots: set[tuple[str, str | None]] = set()
for host_value in (
@ -809,7 +823,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
get_admin_host(config=config),
get_web_host(config=config),
get_api_host(config=config),
get_public_host(config=config),
):
if not host_value:
continue
@ -865,17 +878,15 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
def ensure_permissions_config(self, crawl_permissions: str | None = None) -> bool:
config = dict(self.config or {})
permission = str(config.get("PERMISSIONS") or "").strip().lower()
from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
from archivebox.core.permissions import PERMISSIONS_PUBLIC, PERMISSIONS_VALUES, normalize_permissions
if permission not in PERMISSIONS_VALUES:
if self.crawl_id:
crawl = getattr(self, "crawl", None)
crawl_permissions = crawl_permissions or getattr(crawl, "permissions", None)
if not crawl_permissions:
crawl_permissions = Crawl.objects.filter(pk=self.crawl_id).values_list("permissions", flat=True).first()
config["PERMISSIONS"] = normalize_permissions(
crawl_permissions,
default=normalize_permissions(get_config(include_machine=True).PERMISSIONS),
default=PERMISSIONS_PUBLIC,
)
self.config = config
return True
@ -888,7 +899,15 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
def save(self, *args, **kwargs):
update_fields = kwargs.get("update_fields")
validate_url_field = self._state.adding or update_fields is None or "url" in update_fields
if self.ensure_permissions_config():
crawl_config_for_save = None
crawl_permissions_for_save = None
if self.crawl_id and validate_url_field:
crawl_row = Crawl.objects.filter(pk=self.crawl_id).values("config", "permissions").first()
if crawl_row:
crawl_config_for_save = crawl_row.get("config") or {}
crawl_permissions_for_save = crawl_row.get("permissions")
if self.ensure_permissions_config(crawl_permissions=crawl_permissions_for_save):
if update_fields is not None:
kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
@ -898,7 +917,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
except ValueError as err:
raise ValidationError({"url": str(err)}) from err
if self.is_archivebox_internal_url(self.url):
if self.is_archivebox_internal_url(self.url, config=crawl_config_for_save if self.crawl_id else None):
raise ValidationError({"url": "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs."})
if not self.bookmarked_at:
@ -910,12 +929,13 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
self.title = self._normalize_title_candidate(self.title, snapshot_url=self.url or "") or None
# Migrate filesystem if needed (happens automatically on save)
if self.pk and self.fs_migration_needed:
existing_snapshot = self.pk and not self._state.adding
if existing_snapshot and self.fs_migration_needed:
self.migrate_filesystem_to_current_version()
update_fields = kwargs.get("update_fields")
if update_fields is not None:
kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "fs_version", "modified_at"]))
elif self.pk:
elif existing_snapshot:
current_dir = self.get_storage_path_for_version(self._fs_current_version())
source_dir = Path(self.output_dir)
if source_dir.exists() and source_dir != current_dir and not source_dir.is_symlink():
@ -928,8 +948,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
def finish_snapshot_save():
self.ensure_legacy_archive_symlink()
self.ensure_crawl_symlink()
crawl = self.crawl
if not crawl.url_passes_filters(self.url, snapshot=self):
crawl = Crawl.objects.filter(pk=self.crawl_id).first()
if crawl is None:
return
if not crawl.url_passes_filters(self.url, snapshot=self, use_effective_config=False):
return
# Best-effort skip if our URL is already recorded on the crawl;
# the atomic UPDATE below is what actually prevents clobbering.

31
archivebox/core/routes_util.py
View File

@ -9,7 +9,7 @@ from archivebox.config.common import get_config
_SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
_SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api", "public")
_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api")
def split_host_port(host: str) -> tuple[str, str | None]:
@ -102,7 +102,7 @@ def _with_port(host: str, port: str | None) -> str:
def strip_role_subdomain(host: str) -> str:
"""Strip leading ``admin.`` / ``web.`` / ``api.`` / ``public.`` / ``snap-*.``
"""Strip leading ``admin.`` / ``web.`` / ``api.`` / ``snap-*.``
labels from a host (preserving the port). Strips repeatedly so an
already-compounded host like ``snap-X.snap-X.<base>`` reduces all the
way down to ``<base>``.
@ -172,13 +172,12 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw
return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
# C) Per-request fallback: when ``BASE_URL`` is unset and CSRF didn't
# give us a single origin, trust the request's Host header — but first
# peel off any ``admin.`` / ``web.`` / ``api.`` / ``public.`` /
# ``snap-*.`` label. Otherwise the URL builders below prepend their own
# role label onto a host that already carries one, producing the
# ``snap-X.snap-X.snap-X.<base>`` compounding bug. Django has already
# admitted the host via ALLOWED_HOSTS; the misconfig banner surfaces
# the case where the resulting URL doesn't match what the operator
# probably intended.
# peel off any ``admin.`` / ``web.`` / ``api.`` / ``snap-*.`` label.
# Otherwise the URL builders below prepend their own role label onto a
# host that already carries one, producing the ``snap-X.snap-X.snap-X``
# compounding bug. Django has already admitted the host via
# ALLOWED_HOSTS; the misconfig banner surfaces the case where the
# resulting URL doesn't match what the operator probably intended.
canonical_host = strip_role_subdomain(request.get_host())
return f"{scheme}://{canonical_host}"
@ -220,13 +219,6 @@ def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) ->
return _build_base_host("api", config=config)
def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return get_base_host(config=config)
return _build_base_host("public", config=config)
def get_snapshot_subdomain(snapshot_id: str) -> str:
normalized = re.sub(r"[^0-9a-fA-F]", "", snapshot_id or "")
suffix = (normalized[-12:] if len(normalized) >= 12 else normalized).lower()
@ -330,13 +322,6 @@ def get_api_base_url(request=None, config: dict[str, Any] | None = None, **confi
return _build_base_url_for_host(_build_base_host("api", request=request, config=config), request=request, config=config)
def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return get_base_url(request=request, config=config)
return _build_base_url_for_host(_build_base_host("public", request=request, config=config), request=request, config=config)
def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:

3
archivebox/core/settings.py
View File

@ -84,6 +84,7 @@ DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST"
MIDDLEWARE = [
"archivebox.core.middleware.TimezoneMiddleware",
"django.middleware.security.SecurityMiddleware",
"archivebox.core.middleware.AdminCookieIsolationMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"archivebox.api.middleware.ApiCorsMiddleware",
@ -440,7 +441,7 @@ SESSION_COOKIE_HTTPONLY = True
SESSION_COOKIE_NAME = f"archivebox_sessionid_{CONSTANTS.COLLECTION_ID}"
CSRF_COOKIE_NAME = f"archivebox_csrftoken_{CONSTANTS.COLLECTION_ID}"
# Auth cookies are intentionally scoped to the exact host that set them so
# the admin session is NOT readable from public.* / web.* / api.* — that
# the admin session is NOT readable from web.* / api.* — that
# split is a security boundary, not a UX choice. Subdomains that need to
# render an "is the user logged in?" indicator must use the single-bit
# `archivebox_admin_logged_in` hint cookie set by core/middleware.py

21
archivebox/core/templatetags/core_tags.py
View File

@ -20,7 +20,6 @@ from archivebox.plugins.discovery import (
from archivebox.core.routes_util import (
canonical_base_host_for_request,
get_admin_base_url,
get_public_base_url,
get_web_base_url,
get_snapshot_base_url,
build_snapshot_url,
@ -498,11 +497,6 @@ def web_base_url(context) -> str:
return get_web_base_url(request=context.get("request"), config=context.get("CONFIG"))
@register.simple_tag(takes_context=True)
def public_base_url(context) -> str:
return get_public_base_url(request=context.get("request"), config=context.get("CONFIG"))
@register.simple_tag(takes_context=True)
def snapshot_base_url(context, snapshot) -> str:
snapshot_id = _snapshot_id(snapshot)
@ -521,16 +515,11 @@ def snapshot_index_row(context, link) -> str:
request = context.get("request")
config = context.get("CONFIG")
snapshot_base = get_snapshot_base_url(snapshot_id, request=request, config=config)
screenshot_plugin_url = build_snapshot_url(snapshot_id, "screenshot/screenshot.png", request=request, config=config)
extension_screenshot_1_url = build_snapshot_url(
snapshot_id,
"chrome_extension_screenshot/screenshot-1.png",
request=request,
config=config,
)
extension_screenshot_url = build_snapshot_url(snapshot_id, "chrome_extension_screenshot/screenshot.png", request=request, config=config)
favicon_plugin_url = build_snapshot_url(snapshot_id, "favicon/favicon.ico", request=request, config=config)
favicon_root_url = build_snapshot_url(snapshot_id, "favicon.ico", request=request, config=config)
screenshot_plugin_url = f"{snapshot_base}/screenshot/screenshot.png"
extension_screenshot_1_url = f"{snapshot_base}/chrome_extension_screenshot/screenshot-1.png"
extension_screenshot_url = f"{snapshot_base}/chrome_extension_screenshot/screenshot.png"
favicon_plugin_url = f"{snapshot_base}/favicon/favicon.ico"
favicon_root_url = f"{snapshot_base}/favicon.ico"
status = getattr(link, "status", None) or "unknown"
bookmarked_at = getattr(link, "bookmarked_at", None)

10
archivebox/core/views.py
View File

@ -68,7 +68,9 @@ from archivebox.search.views import get_cached_public_search_state
from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
from archivebox.core.permissions import (
PERMISSIONS_PRIVATE,
PERMISSIONS_PUBLIC,
PERMISSIONS_UNLISTED,
can_view_snapshot,
direct_snapshots_queryset,
filter_personas_by_permissions,
@ -1128,7 +1130,7 @@ class PublicIndexView(ListView):
model = Snapshot
ordering = ["-bookmarked_at", "-created_at"]
paginator_class = AcceleratedPaginator
public_page_scan_chunk_size = 500
public_page_scan_chunk_size = 50
def get_paginate_by(self, queryset):
runtime_config = self.__dict__.get("runtime_config")
@ -1155,7 +1157,7 @@ class PublicIndexView(ListView):
target_count = page_number * page_size
public_snapshots: list[Snapshot] = []
scanned = 0
chunk_size = max(self.public_page_scan_chunk_size, page_size * 4)
chunk_size = max(self.public_page_scan_chunk_size, page_size)
ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
while len(public_snapshots) < target_count:
@ -1271,7 +1273,9 @@ class PublicIndexView(ListView):
return context
def get_exact_public_snapshot_count(self) -> int:
return Snapshot.objects.filter(permissions=PERMISSIONS_PUBLIC).count()
hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
return Snapshot.objects.count() - hidden_count
def get_queryset(self, **kwargs):
qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())

70
archivebox/crawls/models.py
View File

@ -588,22 +588,29 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
except re.error:
return False
def get_current_config(self, *, refresh: bool = False) -> dict[str, Any]:
if refresh and self.pk:
config = type(self).objects.filter(pk=self.pk).values_list("config", flat=True).first()
if config is not None:
self.config = config
return dict(self.config or {})
def get_url_allowlist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
if use_effective_config:
from archivebox.config.common import get_config
config = get_config(crawl=self, snapshot=snapshot)
config = self.get_current_config(refresh=True)
else:
config = self.config or {}
config = self.get_current_config()
if snapshot is not None and snapshot.config:
config.update(snapshot.config)
return self.split_filter_patterns(config.get("URL_ALLOWLIST", ""))
def get_url_denylist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
if use_effective_config:
from archivebox.config.common import get_config
config = get_config(crawl=self, snapshot=snapshot)
config = self.get_current_config(refresh=True)
else:
config = self.config or {}
config = self.get_current_config()
if snapshot is not None and snapshot.config:
config.update(snapshot.config)
return self.split_filter_patterns(config.get("URL_DENYLIST", ""))
def url_passes_filters(self, url: str, *, snapshot=None, use_effective_config: bool = True) -> bool:
@ -704,9 +711,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
return len(urls)
def remaining_url_capacity(self) -> int | None:
from archivebox.config.common import get_config
max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
max_urls = int(self._config_value(self.get_current_config(refresh=True), "CRAWL_MAX_URLS", 0) or 0)
if max_urls <= 0:
return None
return max(max_urls - self.count_urls_for_limit(), 0)
@ -716,9 +721,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
return remaining is None or remaining > 0
def remaining_snapshot_capacity(self) -> int | None:
from archivebox.config.common import get_config
max_urls = int(get_config(crawl=self).CRAWL_MAX_URLS or 0)
max_urls = int(self._config_value(self.get_current_config(refresh=True), "CRAWL_MAX_URLS", 0) or 0)
if max_urls <= 0:
return None
return max(max_urls - self.snapshot_set.count(), 0)
@ -948,7 +951,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
List of newly created Snapshot objects
"""
from archivebox.core.models import Snapshot, Tag
from archivebox.config.common import get_config
from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
if self.status == self.StatusChoices.SEALED:
@ -957,12 +959,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
created_snapshots = []
crawl_tag_names = self.current_tag_names()
tags_by_name: dict[str, Tag] = {}
config = get_config(crawl=self)
only_new_urls = bool(config.ONLY_NEW)
for line in self.urls.splitlines():
if not line.strip():
continue
config = self.get_current_config(refresh=True)
only_new_urls = bool(self._config_value(config, "ONLY_NEW", True))
# Parse JSONL or plain URL
try:
@ -988,10 +990,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
except ValueError as err:
print(f"[yellow][!] Skipping invalid snapshot URL: {url[:120]}... ({err})[/yellow]")
continue
if Snapshot.is_archivebox_internal_url(url):
if Snapshot.is_archivebox_internal_url(url, config=config):
print(f"[yellow][!] Skipping internal ArchiveBox snapshot URL: {url}[/yellow]")
continue
if not self.url_passes_filters(url):
if not self.url_passes_filters(url, use_effective_config=False):
continue
if only_new_urls and Snapshot.objects.filter(url=url).exists():
continue
@ -1099,7 +1101,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
) -> list["Snapshot"]:
"""Create child snapshots from discovered URL records after filtering and deduping once."""
from archivebox.core.models import Snapshot, SnapshotTag, Tag
from archivebox.config.common import get_config
from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
if self.status == self.StatusChoices.SEALED:
@ -1109,7 +1110,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
return []
crawl_tag_names = self.current_tag_names()
config = get_config(crawl=self, snapshot=parent_snapshot)
config = self.get_current_config(refresh=True)
if parent_snapshot is not None and parent_snapshot.config:
config.update(parent_snapshot.config)
allowlist = self.split_filter_patterns(config.get("URL_ALLOWLIST", ""))
denylist = self.split_filter_patterns(config.get("URL_DENYLIST", ""))
@ -1123,7 +1126,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
except ValueError as err:
print(f"[yellow][!] Skipping invalid discovered snapshot URL: {url[:120]}... ({err})[/yellow]")
continue
if Snapshot.is_archivebox_internal_url(url):
if Snapshot.is_archivebox_internal_url(url, config=config):
print(f"[yellow][!] Skipping internal ArchiveBox discovered snapshot URL: {url}[/yellow]")
continue
if self.url_passes_compiled_filters(url, allowlist=allowlist, denylist=denylist):
@ -1132,7 +1135,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
if not deduped_records:
return []
existing_scope = Snapshot.objects if bool(config.ONLY_NEW) else self.snapshot_set
existing_scope = Snapshot.objects if bool(self._config_value(config, "ONLY_NEW", True)) else self.snapshot_set
existing_urls = set(existing_scope.filter(url__in=deduped_records.keys()).values_list("url", flat=True))
urls = [url for url in deduped_records.keys() if url not in existing_urls]
remaining = self.remaining_snapshot_capacity()
@ -1162,7 +1165,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
for index, url in enumerate(urls)
]
for snapshot in snapshots:
snapshot.set_delete_at_from_config(config.DELETE_AFTER)
snapshot.set_delete_at_from_config(self._config_value(config, "DELETE_AFTER", "0"))
created_snapshots = []
for snapshot in snapshots:
@ -1189,7 +1192,10 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
for snapshot in created_snapshots:
tag_names = {
*crawl_tag_names,
*self.parse_tag_names(str(deduped_records[snapshot.url].get("tags") or ""), pattern=config.TAG_SEPARATOR_PATTERN),
*self.parse_tag_names(
str(deduped_records[snapshot.url].get("tags") or ""),
pattern=self._config_value(config, "TAG_SEPARATOR_PATTERN", r"[,]"),
),
}
if tag_names:
tag_names_by_url[snapshot.url] = tag_names
@ -1312,7 +1318,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
The root Snapshot for this crawl, or None for system crawls that don't create snapshots
"""
import time
from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records, is_finite_background_hook
from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records
from archivebox.config.common import get_config
from archivebox.machine.models import Binary, Machine
@ -1370,19 +1376,15 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
print(f"[yellow]⏱️ Hook {hook.name} took {hook_elapsed:.2f}s[/yellow]")
if process.status == process.StatusChoices.RUNNING:
if not is_finite_background_hook(hook.name):
return set()
try:
process.wait(timeout=process.timeout)
except Exception:
if process.poll() is None:
return set()
from archivebox.plugins.hooks import extract_records_from_process
records = []
# Finite background hooks can exit before their completed Process
# metadata is visible. Give successful hooks a brief chance to
# flush JSONL stdout into the Process row before downstream hooks.
# A hook can exit before its completed Process metadata is visible.
# Give successful hooks a brief chance to flush JSONL stdout into
# the Process row before downstream hooks.
for delay in (0.0, 0.05, 0.1, 0.25, 0.5):
if delay:
time.sleep(delay)

14
archivebox/machine/models.py
View File

@ -681,7 +681,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
)
from archivebox.config.common import get_config
binary.symlink_to_lib_bin_after_commit(get_config().LIB_BIN_DIR)
binary.symlink_to_lib_bin_after_commit(get_config().LIB_DIR / "bin")
return binary
# Case 2: From binaries.json - create queued binary (needs installation)
@ -714,7 +714,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
)
from archivebox.config.common import get_config
binary.symlink_to_lib_bin_after_commit(get_config().LIB_BIN_DIR)
binary.symlink_to_lib_bin_after_commit(get_config().LIB_DIR / "bin")
return binary
return None
@ -750,7 +750,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
def symlink_to_lib_bin(self, lib_bin_dir: str | Path) -> Path | None:
"""
Symlink this binary into LIB_BIN_DIR for human-facing convenience.
Symlink this binary into a derived lib/bin directory for human-facing convenience.
After a binary is installed by any binprovider (pip, npm, brew, apt, etc),
we can optionally expose a flat convenience directory for shell users.
@ -758,7 +758,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
paths, not this indirection.
Args:
lib_bin_dir: Path to LIB_BIN_DIR (e.g., /data/lib/arm64-darwin/bin)
lib_bin_dir: Path to the derived convenience bin dir (e.g., /data/lib/bin)
Returns:
Path to the created symlink, or None if symlinking failed
@ -782,14 +782,14 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
except StopIteration:
app_index = -1
# Create LIB_BIN_DIR if it doesn't exist
# Create the derived convenience bin dir if it doesn't exist.
try:
lib_bin_dir.mkdir(parents=True, exist_ok=True)
except (OSError, PermissionError) as e:
print(f"Failed to create LIB_BIN_DIR {lib_bin_dir}: {e}", file=sys.stderr)
print(f"Failed to create lib/bin convenience dir {lib_bin_dir}: {e}", file=sys.stderr)
return None
# Expose the canonical Binary.name in LIB_BIN_DIR. Some providers point
# Expose the canonical Binary.name in the convenience bin dir. Some providers point
# abspath at implementation files like cli.js or manifest.json; those
# are valid targets, but they are not user-facing binary names.
binary_name = _canonical_binary_name(self.name) or binary_abspath.name

19
archivebox/misc/checks.py
View File

@ -289,7 +289,14 @@ def check_data_dir_permissions(config=None, **config_kwargs):
def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs):
from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir
from archivebox.config.paths import (
MAX_TMP_SOCKET_URL_LENGTH,
SUPERVISORD_SOCKET_FILENAME,
assert_dir_can_contain_unix_sockets,
dir_is_writable,
get_or_create_working_tmp_dir,
tmp_dir_socket_path_is_short_enough,
)
from archivebox.misc.logging import STDERR
from archivebox.misc.logging_util import pretty_path
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
@ -297,11 +304,11 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, confi
config = config or get_config(**config_kwargs)
tmp_dir = tmp_dir or config.TMP_DIR
socket_file = tmp_dir.absolute().resolve() / "supervisord.sock"
socket_file = tmp_dir.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
if not must_exist and not os.path.isdir(tmp_dir):
# just check that its viable based on its length (because dir may not exist yet, we cant check if its writable)
return len(f"file://{socket_file}") <= 96
return tmp_dir_socket_path_is_short_enough(tmp_dir)
tmp_is_valid = False
try:
@ -309,8 +316,10 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, confi
if not config.ALLOW_NO_UNIX_SOCKETS:
tmp_is_valid = tmp_is_valid and assert_dir_can_contain_unix_sockets(tmp_dir)
assert tmp_is_valid, f"ArchiveBox user PUID={ARCHIVEBOX_USER} PGID={ARCHIVEBOX_GROUP} is unable to write to TMP_DIR={tmp_dir}"
assert len(f"file://{socket_file}") <= 96, (
f"ArchiveBox TMP_DIR={tmp_dir} is too long, dir containing unix socket files must be <90 chars."
socket_url_len = len(f"file://{socket_file}")
assert tmp_dir_socket_path_is_short_enough(tmp_dir), (
f"ArchiveBox TMP_DIR={tmp_dir} is too long, file://{socket_file} is {socket_url_len} chars "
f"and must be <{MAX_TMP_SOCKET_URL_LENGTH} chars."
)
return True
except Exception as e:

1
archivebox/personas/importers.py
View File

@ -378,7 +378,6 @@ def discover_persona_template_profiles(personas_dir: Path | None = None) -> list
candidate_roots.extend(
[
CONSTANTS.PERSONAS_DIR.expanduser(),
Path.home() / ".config" / "abx" / "personas",
],
)

12
archivebox/plugins/hooks.py
View File

@ -32,7 +32,7 @@ Execution order:
- After all foreground hooks complete, background hooks receive SIGTERM and must finalize
Hook naming convention:
on_{EventFamily}__{run_order}_{description}[.finite.bg|.daemon.bg].{ext}
on_{EventFamily}__{run_order}_{description}[.bg].{ext}
API:
discover_hooks(event) -> List[Path] Find hook scripts for a hook-backed event family
@ -104,11 +104,6 @@ def is_background_hook(hook_name: str) -> bool:
return ".bg." in hook_name or "__background" in hook_name
def is_finite_background_hook(hook_name: str) -> bool:
"""Check if a background hook is finite-lived and should be awaited."""
return ".finite.bg." in hook_name
def normalize_hook_event_name(event_name: str) -> str | None:
"""
Normalize a hook event family or event class name to its on_* prefix.
@ -410,7 +405,6 @@ def run_hook(
"PATH",
"LIB_DIR",
"ABXPKG_LIB_DIR",
"LIB_BIN_DIR",
"NODE_PATH",
"NODE_MODULES_DIR",
"NODE_MODULE_DIR",
@ -434,8 +428,8 @@ def run_hook(
# Create output directory if needed
output_dir.mkdir(parents=True, exist_ok=True)
# Detect if this is a background hook (long-running daemon)
# Background hooks use the .daemon.bg. or .finite.bg. filename convention.
# Detect if this is a background hook.
# Background hooks use the .bg. filename marker.
# Old convention: __background in stem (for backwards compatibility)
is_background = ".bg." in script.name or "__background" in script.stem

6
archivebox/plugins/views.py
View File

@ -14,13 +14,13 @@ from django.utils.safestring import mark_safe
from admin_data_views.typing import ItemContext, SectionData, TableContext
from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
from archivebox.config.common import get_live_config_url
from archivebox.config.views import get_environment_binary_url, is_superuser
from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, discover_plugin_configs, iter_plugin_dirs
ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/"
ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/"
LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
LIVE_PLUGIN_BASE_URL = "/admin/environment/plugins/"
@ -73,10 +73,6 @@ def get_plugin_hook_source_url(plugin_name: str, hook_name: str) -> str:
return f"{ABX_PLUGINS_GITHUB_BASE_URL}{quote(plugin_name)}/{quote(hook_name)}"
def get_live_config_url(key: str) -> str:
return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
def get_machine_admin_url() -> str | None:
from archivebox.machine.models import Machine

71
archivebox/search/views.py
View File

@ -149,7 +149,7 @@ def iter_url_prefix_search_ids(prefix: str, queryset):
yield snapshot_id
def iter_admin_meta_search_ids(query, queryset):
def iter_meta_search_ids(query, queryset):
"""Yield metadata search matches from a filtered Snapshot queryset."""
seen = set()
try:
@ -195,30 +195,63 @@ def iter_admin_meta_search_ids(query, queryset):
yield pk
def iter_admin_backend_search_ids(iterator, queryset):
"""Yield backend search IDs that still match the filtered queryset."""
def normalize_search_result_id(snapshot_id) -> str | None:
"""Return a compact Snapshot ID string from a search provider result."""
snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
if len(snapshot_id) != 32:
return None
return snapshot_id
def iter_filtered_search_result_ids(iterator, queryset, *, flush_max_delay=0.05):
"""Yield provider IDs that still match the filtered queryset.
This is the single intersection/dedupe path used for metadata and every
search backend. It flushes by elapsed time so sparse providers stream rows
as soon as IDs are found instead of waiting for a fixed batch size.
"""
batch = []
seen = set()
queued = set()
last_flush_at = 0.0
def flush_batch():
valid = {str(pk) for pk in queryset.filter(pk__in=batch).values_list("pk", flat=True)}
for snapshot_id in batch:
nonlocal batch, queued, last_flush_at
if not batch:
return
batch_ids = batch
batch = []
queued = set()
last_flush_at = time.monotonic()
valid = {str(pk).replace("-", "") for pk in queryset.filter(pk__in=batch_ids).values_list("pk", flat=True)}
for snapshot_id in batch_ids:
if snapshot_id in valid and snapshot_id not in seen:
seen.add(snapshot_id)
yield snapshot_id
for snapshot_id in iterator:
snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
if len(snapshot_id) != 32:
snapshot_id = normalize_search_result_id(snapshot_id)
if not snapshot_id or snapshot_id in seen or snapshot_id in queued:
continue
batch.append(snapshot_id)
if len(batch) >= (1 if not seen else 200):
queued.add(snapshot_id)
if not seen or time.monotonic() - last_flush_at >= flush_max_delay:
yield from flush_batch()
batch = []
if batch:
yield from flush_batch()
def iter_search_result_ids(query, base_queryset, *, search_mode, config):
"""Yield filtered Snapshot IDs from the selected search provider."""
search_mode_base = get_search_mode_base(search_mode, config=config)
provider = (
iter_meta_search_ids(query, base_queryset)
if search_mode_base == "meta"
else iter_query_search_ids(query, search_mode=search_mode, config=config)
)
yield from iter_filtered_search_result_ids(provider, base_queryset)
def snapshot_search_stream_response(query, base_queryset, *, search_mode, config, cache_key, thread_name):
"""Stream Snapshot search progress and cache matching IDs for a list view."""
if not query:
@ -229,7 +262,6 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
ids = []
last_sent = 0
last_sent_at = time.monotonic()
stream_batch_size = 3
stream_max_delay = 0.05
stream_padding = " " * 4096
cache.set(cache_key, {"ids": [], "done": False}, SEARCH_RESULT_CACHE_TTL)
@ -246,8 +278,7 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
def publish_count(done=False):
nonlocal last_sent, last_sent_at
if done:
cache.set(cache_key, {"ids": list(ids), "done": True}, SEARCH_RESULT_CACHE_TTL)
cache.set(cache_key, {"ids": list(ids), "done": done}, SEARCH_RESULT_CACHE_TTL)
last_sent = len(ids)
last_sent_at = time.monotonic()
emit(f"{last_sent}{stream_padding}\n")
@ -255,24 +286,16 @@ def snapshot_search_stream_response(query, base_queryset, *, search_mode, config
def run_search():
iterator = None
try:
search_mode_base = get_search_mode_base(search_mode, config=config)
iterator = (
iter_admin_meta_search_ids(query, base_queryset)
if search_mode_base == "meta"
else iter_admin_backend_search_ids(
iter_query_search_ids(query, search_mode=search_mode, config=config),
base_queryset,
)
)
iterator = iter_search_result_ids(query, base_queryset, search_mode=search_mode, config=config)
for snapshot_id in iterator:
if stop_event.is_set():
break
snapshot_id = str(snapshot_id).strip().lower().replace("-", "")
if len(snapshot_id) != 32 or snapshot_id in seen:
snapshot_id = normalize_search_result_id(snapshot_id)
if not snapshot_id or snapshot_id in seen:
continue
seen.add(snapshot_id)
ids.append(snapshot_id)
if len(ids) == 1 or len(ids) - last_sent >= stream_batch_size or time.monotonic() - last_sent_at >= stream_max_delay:
if len(ids) == 1 or time.monotonic() - last_sent_at >= stream_max_delay:
publish_count()
if not stop_event.is_set() and len(ids) != last_sent:
publish_count(done=True)

3
archivebox/services/binary_service.py
View File

@ -124,7 +124,6 @@ class ArchiveBoxDBBinaryCacheBackend:
)
async def set(self, request: BinaryRequestEvent | None, binary: AbxBinary) -> None:
from archivebox.config.common import get_config
from archivebox.machine.models import Binary, Machine, _canonical_binary_name
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
@ -160,8 +159,6 @@ class ArchiveBoxDBBinaryCacheBackend:
await existing.asave(
update_fields=["abspath", "version", "sha256", "binproviders", "binprovider", "overrides", "status", "retry_at", "modified_at"],
)
lib_bin_dir = await sync_to_async(lambda: get_config().LIB_BIN_DIR, thread_sensitive=True)()
await sync_to_async(existing.symlink_to_lib_bin_after_commit, thread_sensitive=True)(lib_bin_dir)
async def invalidate(self, request: BinaryRequestEvent, binary: AbxBinary, reason: str) -> None:
from archivebox.machine.models import Binary, Machine, _canonical_binary_name

28
archivebox/templates/admin/snapshot_search_stream.html
View File

@ -40,6 +40,33 @@
}
}
function setChangelistSearchingState(controller) {
if (activeSearch !== controller) return;
var changelist = document.getElementById("changelist");
if (!changelist) return;
var results = changelist.querySelector(".results");
if (!results) {
results = document.createElement("div");
var toolbar = changelist.querySelector("#toolbar");
if (toolbar && toolbar.nextSibling) {
toolbar.parentNode.insertBefore(results, toolbar.nextSibling);
} else {
changelist.appendChild(results);
}
}
results.className = "results search-empty-state";
while (results.firstChild) results.removeChild(results.firstChild);
var message = document.createElement("p");
message.textContent = "Searching matching snapshots...";
results.appendChild(message);
var paginator = changelist.querySelector(".paginator");
if (paginator) paginator.hidden = true;
var actions = changelist.querySelector(".actions");
if (actions) actions.hidden = true;
}
function streamSnapshotSearch(eventOrForm) {
var form = eventOrForm && eventOrForm.target ? eventOrForm.target : eventOrForm;
if (!form || form.id !== "changelist-search") return;
@ -55,6 +82,7 @@
var controller = new AbortController();
activeSearch = controller;
setSearchLoading(controller, "Searching...");
setChangelistSearchingState(controller);
var resultVersion = 0;
var pendingRender = null;
var renderInFlight = false;

4
archivebox/templates/core/navigation.html
View File

@ -27,10 +27,10 @@
{% elif request.COOKIES.archivebox_admin_logged_in == "1" %}
{% comment %}
Authenticated on the admin host but the session cookie is admin-host-
scoped (security boundary — public.* must NEVER see the session).
scoped (security boundary — web.* must NEVER see the session).
The hint cookie is the only signal that crosses, so we render the
logged-out state's `Account` / `Log out` links pointing at admin host
so the user can still reach those pages from public.*/web.*.
so the user can still reach those pages from web.*.
{% endcomment %}
<a href="/admin/password_change/" title="Change your account password">Account</a> /
<a href="/admin/logout/">{% trans 'Log out' %}</a>

6
archivebox/templates/core/snapshot.html
View File

@ -1137,9 +1137,9 @@
<div class="header-top">
<div class="header-nav">
<div class="header-col header-left" style="line-height: 58px; vertical-align: middle">
{% public_base_url as public_base %}
<a href="{% if public_base %}{{ public_base }}/public/{% else %}/{% endif %}" class="header-archivebox" title="Go to Public Index...">
<img src="{% if public_base %}{{ public_base }}/static/archive.png{% else %}{% static 'archive.png' %}{% endif %}" alt="Archive Icon">
{% web_base_url as web_base %}
<a href="{% if web_base %}{{ web_base }}/public/{% else %}/{% endif %}" class="header-archivebox" title="Go to Public Index...">
<img src="{% if web_base %}{{ web_base }}/static/archive.png{% else %}{% static 'archive.png' %}{% endif %}" alt="Archive Icon">
ArchiveBox
</a>
</div>

1
archivebox/tests/conftest.py
View File

@ -947,7 +947,6 @@ def run_python_cwd(
API_TEST_HOST = "api.archivebox.localhost:8000"
ADMIN_TEST_HOST = "admin.archivebox.localhost:8000"
PUBLIC_TEST_HOST = "public.archivebox.localhost:8000"
WEB_TEST_HOST = "web.archivebox.localhost:8000"

3
archivebox/tests/test_binary_service.py
View File

@ -29,7 +29,6 @@ def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]:
assert archivebox_bin, "archivebox console script must be available for CLI tests"
return {
"LIB_DIR": str(data_dir / "lib"),
"LIB_BIN_DIR": str(data_dir / "lib" / "bin"),
"ABXPKG_LIB_DIR": str(data_dir / "lib"),
"PATH": os.pathsep.join([str(bin_dir), str(Path(archivebox_bin).parent), "/usr/bin", "/bin", "/usr/sbin", "/sbin"]),
}
@ -129,7 +128,6 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
assert first_abspath.resolve() == Path(shutil.which("rg") or "").resolve()
assert first_abspath.is_relative_to(initialized_archive / "lib")
assert (initialized_archive / "lib" / "env" / "bin" / name).exists()
assert (initialized_archive / "lib" / "bin" / name).exists()
assert (initialized_archive / "machines" / machine_id / "binaries" / name / "index.jsonl").exists()
assert binary_processes
assert binary_processes[-1].status == Process.StatusChoices.EXITED
@ -152,7 +150,6 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
assert binary.version in version_stdout
first_abspath.unlink()
(initialized_archive / "lib" / "bin" / name).unlink(missing_ok=True)
_link_real_binary(bootstrap_bin_dir, name, source="rg")
_cmd_result = run_archivebox_cmd(

3
archivebox/tests/test_cli_version.py
View File

@ -8,6 +8,7 @@ import os
import re
import tempfile
from pathlib import Path
from archivebox.config.paths import tmp_dir_socket_path_is_short_enough
from archivebox.tests.conftest import run_archivebox_cmd
@ -123,7 +124,7 @@ def test_version_auto_selects_short_tmp_dir_for_deep_collection_path(tmp_path):
assert reported_tmp_dir.exists()
assert not reported_tmp_dir.is_relative_to(default_tmp_dir)
assert len(f"file://{reported_tmp_dir / 'supervisord.sock'}") <= 96
assert tmp_dir_socket_path_is_short_enough(reported_tmp_dir)
def test_version_help_lists_quiet_flag(tmp_path):

2
archivebox/tests/test_machine_models.py
View File

@ -451,7 +451,7 @@ class TestBinaryModel:
@pytest.mark.django_db(transaction=True)
def test_binary_lib_bin_symlink_waits_for_outer_transaction_commit(self, tmp_path):
"""Binary DB projection writes can be direct, but LIB_BIN_DIR writes must run after commit."""
"""Binary DB projection writes can be direct, but convenience symlinks must run after commit."""
source = tmp_path / "provider" / "bin" / "abx-test-binary"
source.parent.mkdir(parents=True)
source.write_text("#!/bin/sh\nexit 0\n")

4
archivebox/tests/test_machine_service.py
View File

@ -38,7 +38,6 @@ def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]:
]
return {
"LIB_DIR": str(data_dir / "lib"),
"LIB_BIN_DIR": str(data_dir / "lib" / "bin"),
"ABXPKG_LIB_DIR": str(data_dir / "lib"),
"LITEPARSE_ENABLED": "True",
"TIMEOUT": "180",
@ -75,10 +74,8 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
process = Process.objects.filter(process_type=Process.TypeChoices.BINARY).latest("created_at")
installed_liteparse_path = Path(liteparse_binary.abspath)
lib_bin_path = initialized_archive / "lib" / "bin" / installed_liteparse_path.name
assert installed_liteparse_path.exists()
assert installed_liteparse_path.is_relative_to(initialized_archive / "lib")
assert lib_bin_path.exists()
assert binaries
assert process.status == Process.StatusChoices.EXITED
assert process.exit_code == 0
@ -152,7 +149,6 @@ def test_install_persists_machine_binary_config_and_recovers_stale_path(initiali
assert "lit" in version_stdout
installed_liteparse_path.unlink()
(initialized_archive / "lib" / "bin" / installed_liteparse_path.name).unlink(missing_ok=True)
_cmd_result = run_archivebox_cmd(
["version"],

262
archivebox/tests/test_search.py
View File

@ -5,7 +5,6 @@ import shutil
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from threading import Thread
from types import SimpleNamespace
from datetime import timedelta
from urllib.parse import urlencode
@ -33,7 +32,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
User = get_user_model()
ADMIN_HOST = "admin.archivebox.localhost:8000"
PUBLIC_HOST = "public.archivebox.localhost:8000"
WEB_HOST = "web.archivebox.localhost:8000"
@pytest.fixture
@ -88,16 +87,6 @@ def consume_streaming_response(response):
return b"".join(response.streaming_content)
def collect_streaming_response_chunks(response):
if response.is_async:
async def consume():
return [chunk async for chunk in response.streaming_content]
return async_to_sync(consume)()
return list(response.streaming_content)
def populate_admin_search_cache(client, path, params):
search_url = f"{path}?{urlencode(params)}"
response = client.get(
@ -285,11 +274,8 @@ class TestAdminSnapshotSearch:
response = populate_admin_search_cache(client, path, params)
assert response.status_code == 200
from archivebox.search.views import get_admin_search_cache_key
cached = cache.get(get_admin_search_cache_key(SimpleNamespace(user=admin_user), f"{path}?{urlencode(params)}"))
assert cached["ids"][:3] == [str(prefix_snapshot.pk), str(title_snapshot.pk), str(contains_snapshot.pk)]
result_ids = list(response.context["cl"].queryset.values_list("pk", flat=True))
assert result_ids[:3] == [prefix_snapshot.pk, title_snapshot.pk, contains_snapshot.pk]
assert {title_snapshot.pk, contains_snapshot.pk, prefix_snapshot.pk}.issubset(result_ids)
def test_admin_contents_search_stream_uses_real_backend_results(self, client, admin_user, crawl, monkeypatch):
@ -398,7 +384,7 @@ class TestPublicIndexSearch:
@override_settings(PUBLIC_INDEX=True)
def test_public_search_by_url(self, client, public_snapshot):
cache.clear()
response = client.get("/public/", {"q": "public-example.com"}, HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", {"q": "public-example.com"}, HTTP_HOST=WEB_HOST)
assert response.status_code == 200
assert b"matching snapshots..." in response.content
@ -408,7 +394,7 @@ class TestPublicIndexSearch:
def test_public_search_mode_selector_defaults_to_configured_deep_backend_for_ripgrep(self, client, monkeypatch):
monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep")
response = client.get("/public/", HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", HTTP_HOST=WEB_HOST)
assert response.status_code == 200
assert response.context["search_mode"] == "deep:ripgrep"
@ -444,20 +430,19 @@ class TestPublicIndexSearch:
stream_response = client.get(
"/public/search-stream/",
{**search_params, "search_url": search_url},
HTTP_HOST=PUBLIC_HOST,
HTTP_HOST=WEB_HOST,
)
assert stream_response.status_code == 200
assert consume_streaming_response(stream_response)
response = client.get("/public/", search_params, HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", search_params, HTTP_HOST=WEB_HOST)
content = response.content.decode()
assert content.index(str(metadata_snapshot.url)) < content.index(str(fulltext_snapshot.url))
@override_settings(PUBLIC_INDEX=True)
def test_public_metadata_search_prioritizes_common_url_prefixes(self, crawl):
def test_public_metadata_search_prioritizes_common_url_prefixes(self, client, crawl):
from archivebox.core.models import Snapshot
from archivebox.search.views import iter_admin_meta_search_ids
broad_match = Snapshot.objects.create(
url="https://late.example.com/path/to/iana",
@ -472,44 +457,24 @@ class TestPublicIndexSearch:
status=Snapshot.StatusChoices.SEALED,
)
ids = list(iter_admin_meta_search_ids("iana", Snapshot.objects.order_by("created_at")))
assert str(ids[0]) == str(prefix_match.pk)
assert str(broad_match.pk) in {str(snapshot_id) for snapshot_id in ids}
@override_settings(PUBLIC_INDEX=True)
def test_public_search_stream_flushes_first_result_and_small_batches(self, client, crawl):
from archivebox.core.models import Snapshot
Snapshot.objects.bulk_create(
[
Snapshot(
url=f"https://www.fast-stream-{index}.example.com",
title=f"Fast Stream {index}",
crawl=crawl,
status=Snapshot.StatusChoices.SEALED,
timestamp=str(2_000_000_000 + index),
)
for index in range(6)
],
)
search_params = {"q": "fast-stream", "search_mode": "meta"}
search_params = {"q": "iana", "search_mode": "meta"}
search_url = f"/public/?{urlencode(search_params)}"
response = client.get(
stream_response = client.get(
"/public/search-stream/",
{**search_params, "search_url": search_url},
HTTP_HOST=PUBLIC_HOST,
HTTP_HOST=WEB_HOST,
)
assert stream_response.status_code == 200
assert consume_streaming_response(stream_response)
assert response.status_code == 200
chunks = collect_streaming_response_chunks(response)
counts = [int(chunk.strip() or b"0") for chunk in chunks if chunk.strip()]
assert counts[:4] == [0, 1, 4, 6]
response = client.get("/public/", search_params, HTTP_HOST=WEB_HOST)
content = response.content.decode()
assert content.index(str(prefix_match.url)) < content.index(str(broad_match.url))
@override_settings(PUBLIC_INDEX=True)
def test_public_search_by_title(self, client, public_snapshot):
response = client.get("/public/", {"q": "Public Example"}, HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", {"q": "Public Example"}, HTTP_HOST=WEB_HOST)
assert response.status_code == 200
assert b"archivebox-search-stream-status" in response.content
@ -522,14 +487,14 @@ class TestPublicIndexSearch:
response = client.get(
"/public/search-stream/",
{**search_params, "search_url": search_url},
HTTP_HOST=PUBLIC_HOST,
HTTP_HOST=WEB_HOST,
)
assert response.status_code == 200
assert response["X-Accel-Buffering"] == "no"
assert consume_streaming_response(response)
response = client.get("/public/", search_params, HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", search_params, HTTP_HOST=WEB_HOST)
assert response.status_code == 200
assert b"Public Example Website" in response.content
@ -570,7 +535,7 @@ class TestPublicIndexSearch:
],
)
response = client.get("/public/", HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", HTTP_HOST=WEB_HOST)
assert response.status_code == 200
assert response.context["paginator"].count == 125
@ -584,7 +549,7 @@ class TestPublicIndexSearch:
assert "last &raquo;" in content
assert "private-page-test" not in content
last_response = client.get("/public/", {"page": 3}, HTTP_HOST=PUBLIC_HOST)
last_response = client.get("/public/", {"page": 3}, HTTP_HOST=WEB_HOST)
assert last_response.status_code == 200
assert last_response.context["paginator"].count == 125
@ -612,7 +577,7 @@ class TestPublicIndexSearch:
},
)
response = client.get("/public/", HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", HTTP_HOST=WEB_HOST)
assert response.status_code == 200
content = response.content.decode()
@ -633,7 +598,7 @@ class TestPublicIndexSearch:
status=Snapshot.StatusChoices.STARTED,
)
response = client.get("/public/", HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", HTTP_HOST=WEB_HOST)
assert response.status_code == 200
content = response.content.decode()
@ -645,7 +610,7 @@ class TestPublicIndexSearch:
public_snapshot.title = ""
public_snapshot.save(update_fields=["title"])
response = client.get("/public/", HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", HTTP_HOST=WEB_HOST)
assert response.status_code == 200
content = response.content.decode()
@ -654,19 +619,19 @@ class TestPublicIndexSearch:
@override_settings(PUBLIC_INDEX=True)
def test_public_search_query_type_meta(self, client, public_snapshot):
response = client.get("/public/", {"q": "example", "query_type": "meta"}, HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", {"q": "example", "query_type": "meta"}, HTTP_HOST=WEB_HOST)
assert response.status_code == 200
@override_settings(PUBLIC_INDEX=True)
def test_public_search_query_type_url(self, client, public_snapshot):
response = client.get("/public/", {"q": "public-example.com", "query_type": "url"}, HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", {"q": "public-example.com", "query_type": "url"}, HTTP_HOST=WEB_HOST)
assert response.status_code == 200
@override_settings(PUBLIC_INDEX=True)
def test_public_search_query_type_title(self, client, public_snapshot):
response = client.get("/public/", {"q": "Website", "query_type": "title"}, HTTP_HOST=PUBLIC_HOST)
response = client.get("/public/", {"q": "Website", "query_type": "title"}, HTTP_HOST=WEB_HOST)
assert response.status_code == 200
@ -690,6 +655,10 @@ class TestSearchBackendsE2E:
title_only_needle = "livematrixtitleprecisionunique"
tag_only_needle = "livematrixtagprecisionunique"
order_needle = "orderneedle"
first_batch_content_needle = "firstbatchcontentonlyneedle"
second_batch_content_needle = "secondbatchcontentonlyneedle"
shared_content_needle = "sharedcontentneedle"
overlapping_content_needle = "overlappingcontentneedle"
title_prefix_order_title = "Orderneedle Title Prefix Page"
title_contains_order_title = "Contains Orderneedle Later Page"
pages = {
@ -698,7 +667,10 @@ class TestSearchBackendsE2E:
f"<title>Search Matrix Page {index:03d}</title>"
"</head><body>"
f"archivebox-ui-stream-needle page-{index:03d} "
"real wget output for public admin search matrix"
"real mercury output for public admin search matrix "
f"{shared_content_needle} "
f"{first_batch_content_needle if index < 50 else second_batch_content_needle} "
f"{overlapping_content_needle if index in (0, 1, 2, 50, 51) else ''}"
"</body></html>"
).encode()
for index in range(page_count)
@ -782,13 +754,9 @@ class TestSearchBackendsE2E:
sonic_port = get_free_port()
env = cli_env(
live=True,
PLUGINS="wget,search_backend_ripgrep,search_backend_sqlite,search_backend_sonic",
PLUGINS="mercury,search_backend_ripgrep,search_backend_sqlite,search_backend_sonic",
SAVE_TITLE="True",
SAVE_WGET="True",
SAVE_WARC="False",
WGET_WARC_ENABLED="False",
SAVE_WGET_REQUISITES="False",
WGET_TIMEOUT="20",
MERCURY_ENABLED="True",
TIMEOUT="20",
PUBLIC_INDEX="True",
PUBLIC_ADD_VIEW="True",
@ -805,23 +773,46 @@ class TestSearchBackendsE2E:
)
create_admin_and_token(initialized_archive)
bulk_wget_urls = [*matrix_urls]
add_result = run_archivebox_cmd(
mercury_capture_urls = [*matrix_urls]
first_mercury_urls = mercury_capture_urls[:50]
second_mercury_urls = mercury_capture_urls[50:]
first_add_result = run_archivebox_cmd(
[
"add",
"--depth=0",
f"--max-urls={len(bulk_wget_urls)}",
f"--max-urls={len(first_mercury_urls)}",
"--crawl-max-concurrent-snapshots=4",
"--parser=url_list",
"--plugins=wget",
"--plugins=mercury",
"--tag=search-matrix",
*bulk_wget_urls,
*first_mercury_urls,
],
cwd=initialized_archive,
env={
**env,
"SEARCH_BACKEND_SONIC_ENABLED": "False",
"SEARCH_BACKEND_SQLITE_ENABLED": "False",
},
timeout=120,
)
assert first_add_result.returncode == 0, first_add_result.stderr or first_add_result.stdout
second_add_result = run_archivebox_cmd(
[
"add",
"--depth=0",
f"--max-urls={len(second_mercury_urls)}",
"--crawl-max-concurrent-snapshots=4",
"--parser=url_list",
"--plugins=mercury",
"--tag=search-matrix",
*second_mercury_urls,
],
cwd=initialized_archive,
env=env,
timeout=180,
timeout=120,
)
assert add_result.returncode == 0, add_result.stderr or add_result.stdout
assert second_add_result.returncode == 0, second_add_result.stderr or second_add_result.stdout
metadata_snapshot_records = [
{
@ -918,7 +909,7 @@ class TestSearchBackendsE2E:
)
wait_for_http(
archivebox_port,
host=f"public.archivebox.localhost:{archivebox_port}",
host=f"web.archivebox.localhost:{archivebox_port}",
path="/public/",
process=archivebox_server,
)
@ -929,16 +920,23 @@ class TestSearchBackendsE2E:
process=archivebox_server,
)
backend_expectations = (
(shared_content_needle, matrix_urls),
(first_batch_content_needle, first_mercury_urls),
(second_batch_content_needle, second_mercury_urls),
(overlapping_content_needle, [*first_mercury_urls[:3], *second_mercury_urls[:2]]),
)
for backend_name in ("ripgrep", "sqlite", "sonic"):
backend_result = run_archivebox_cmd(
["list", "--search=contents", "--csv=url", "public admin search matrix"],
cwd=initialized_archive,
env={**env, "SEARCH_BACKEND_ENGINE": backend_name},
timeout=60,
)
assert backend_result.returncode == 0, backend_result.stderr or backend_result.stdout
backend_urls = [line.strip().strip('"') for line in backend_result.stdout.splitlines() if line.strip()]
assert set(backend_urls) == set(matrix_urls), (backend_name, backend_result.stdout)
for query, expected_urls in backend_expectations:
backend_result = run_archivebox_cmd(
["list", "--search=contents", "--csv=url", query],
cwd=initialized_archive,
env={**env, "SEARCH_BACKEND_ENGINE": backend_name},
timeout=60,
)
assert backend_result.returncode == 0, backend_result.stderr or backend_result.stdout
backend_urls = [line.strip().strip('"') for line in backend_result.stdout.splitlines() if line.strip()]
assert set(backend_urls) == set(expected_urls), (backend_name, query, backend_result.stdout)
session = requests.Session()
login_page = session.get(
@ -968,7 +966,7 @@ class TestSearchBackendsE2E:
public_default = requests.get(
f"http://127.0.0.1:{archivebox_port}/public/",
headers={"Host": f"public.archivebox.localhost:{archivebox_port}"},
headers={"Host": f"web.archivebox.localhost:{archivebox_port}"},
timeout=10,
)
assert public_default.status_code == 200
@ -990,7 +988,7 @@ class TestSearchBackendsE2E:
for surface_name, host, stream_path, list_path, requester in (
(
"public",
f"public.archivebox.localhost:{archivebox_port}",
f"web.archivebox.localhost:{archivebox_port}",
"/public/search-stream/",
"/public/",
requests,
@ -1003,14 +1001,40 @@ class TestSearchBackendsE2E:
session,
),
):
for search_mode, query in (
("meta", "search-matrix"),
("deep:ripgrep", "public admin search matrix"),
("deep:sqlite", "public admin search matrix"),
("deep:sonic", "public admin search matrix"),
for search_mode, query, expected_urls in (
("meta", "search-matrix", urls),
("deep:ripgrep", first_batch_content_needle, first_mercury_urls),
("deep:ripgrep", second_batch_content_needle, second_mercury_urls),
("deep:ripgrep", shared_content_needle, matrix_urls),
("deep:ripgrep", overlapping_content_needle, [*first_mercury_urls[:3], *second_mercury_urls[:2]]),
("deep:sqlite", first_batch_content_needle, first_mercury_urls),
("deep:sqlite", second_batch_content_needle, second_mercury_urls),
("deep:sqlite", shared_content_needle, matrix_urls),
("deep:sqlite", overlapping_content_needle, [*first_mercury_urls[:3], *second_mercury_urls[:2]]),
("deep:sonic", first_batch_content_needle, first_mercury_urls),
("deep:sonic", second_batch_content_needle, second_mercury_urls),
("deep:sonic", shared_content_needle, matrix_urls),
("deep:sonic", overlapping_content_needle, [*first_mercury_urls[:3], *second_mercury_urls[:2]]),
):
params = {"q": query, "search_mode": search_mode}
search_url = f"{list_path}?{urlencode(params)}"
expected_count = len(expected_urls)
partial_count = min(10, expected_count)
initial_page_started = time.monotonic()
initial_page = requester.get(
f"http://127.0.0.1:{archivebox_port}{list_path}",
headers={"Host": host},
params=params,
timeout=10,
)
initial_page_elapsed = time.monotonic() - initial_page_started
assert initial_page.status_code == 200
assert "archivebox-search-stream-status" in initial_page.text
assert query in initial_page.text
assert f'value="{search_mode}" selected' in initial_page.text
assert initial_page_elapsed < 1.0, (surface_name, search_mode, initial_page_elapsed)
stream_started = time.monotonic()
stream_response = requester.get(
f"http://127.0.0.1:{archivebox_port}{stream_path}",
@ -1023,30 +1047,68 @@ class TestSearchBackendsE2E:
assert stream_response.headers.get("X-Accel-Buffering") == "no"
counts = []
count_events = []
first_positive_elapsed = None
first_partial_page_checked = False
later_partial_page_checked = False
for line in stream_response.iter_lines(decode_unicode=True):
if not line:
continue
now = time.monotonic()
count = int(line.strip())
counts.append(count)
count_events.append((count, now - stream_started))
if count > 0 and first_positive_elapsed is None:
first_positive_elapsed = time.monotonic() - stream_started
first_positive_elapsed = now - stream_started
if count > 0 and not first_partial_page_checked:
first_partial_page_checked = True
partial_page_started = time.monotonic()
partial_page = requester.get(
f"http://127.0.0.1:{archivebox_port}{list_path}",
headers={"Host": host},
params=params,
timeout=10,
)
partial_page_elapsed = time.monotonic() - partial_page_started
assert partial_page.status_code == 200
assert "No snapshots found." not in partial_page.text
assert "127.0.0.1" in partial_page.text
assert partial_page_elapsed < 1.0, (surface_name, search_mode, count, partial_page_elapsed)
if count >= partial_count and not later_partial_page_checked:
later_partial_page_checked = True
partial_page = requester.get(
f"http://127.0.0.1:{archivebox_port}{list_path}",
headers={"Host": host},
params=params,
timeout=10,
)
assert partial_page.status_code == 200
assert "No snapshots found." not in partial_page.text
assert "127.0.0.1" in partial_page.text
total_elapsed = time.monotonic() - stream_started
assert counts[0] == 0, (surface_name, search_mode, counts[:10])
assert counts[1] == 1, (surface_name, search_mode, counts[:10])
expected_count = total_snapshot_count if search_mode == "meta" else page_count
expected_elapsed = 2.0 if expected_count == total_snapshot_count else 1.0
assert counts[-1] == expected_count, (surface_name, search_mode, counts[-10:])
assert counts == sorted(counts), (surface_name, search_mode, counts[:20])
assert any(1 < count < page_count for count in counts), (surface_name, search_mode, counts)
assert first_positive_elapsed is not None and first_positive_elapsed < 1.0, (
assert first_partial_page_checked
assert later_partial_page_checked
positive_events = [(count, elapsed) for count, elapsed in count_events if count > 0]
assert len(positive_events) >= 2, (surface_name, search_mode, positive_events)
max_progress_gap = max(
later_elapsed - earlier_elapsed
for (_, earlier_elapsed), (_, later_elapsed) in zip(positive_events, positive_events[1:])
)
assert max_progress_gap < 1.0, (surface_name, search_mode, max_progress_gap, positive_events[:10])
assert first_positive_elapsed is not None and first_positive_elapsed < 0.75, (
surface_name,
search_mode,
first_positive_elapsed,
counts[:10],
)
assert total_elapsed < expected_elapsed, (surface_name, search_mode, total_elapsed, counts[-10:])
assert total_elapsed < 2.0, (surface_name, search_mode, total_elapsed, counts[-10:])
rendered_page = requester.get(
f"http://127.0.0.1:{archivebox_port}{list_path}",
@ -1056,7 +1118,7 @@ class TestSearchBackendsE2E:
)
assert rendered_page.status_code == 200
assert "No snapshots found." not in rendered_page.text
assert "127.0.0.1" in rendered_page.text
assert any(expected_url in rendered_page.text for expected_url in expected_urls)
assert search_mode in rendered_page.text
cleared_page = requester.get(

1
archivebox/tests/test_ui_admin_snapshot.py
View File

@ -29,6 +29,7 @@ def test_snapshot_changelist_uses_stable_ordering_without_unordered_paginator_wa
assert response.context["cl"].queryset.ordered is True
assert response.context["cl"].queryset.query.order_by[0] == "-created_at"
assert b"archivebox-search-stream-status" in response.content
assert b"Searching matching snapshots..." in response.content
def test_snapshot_changelist_bulk_permissions_action_updates_selected_snapshots(client, admin_user, crawl, snapshot):

223
archivebox/tests/test_ui_public_snapshot.py
View File

@ -1,5 +1,6 @@
"""Public snapshot UI tests."""
import json
import re
import time
@ -7,12 +8,14 @@ import pytest
import requests
from django.test import override_settings
from archivebox.tests.conftest import PUBLIC_TEST_HOST, WEB_TEST_HOST
from archivebox.core.middleware import ADMIN_LOGIN_HINT_COOKIE
from archivebox.tests.conftest import WEB_TEST_HOST
from archivebox.tests.conftest import (
cli_env,
create_admin_and_token,
get_free_port,
init_archive,
run_archivebox_cmd,
start_archivebox_server,
stop_server,
wait_for_http,
@ -119,6 +122,99 @@ def _replay_cookies(session: requests.Session):
return [cookie for cookie in session.cookies if cookie.name.startswith("archivebox_replay_")]
def _create_admin_user_with_cli(data_dir) -> None:
result = run_archivebox_cmd(
[
"manage",
"createsuperuser",
"--noinput",
"--username",
"apitestadmin",
"--email",
"apitestadmin@example.com",
],
cwd=data_dir,
env=cli_env(DJANGO_SUPERUSER_PASSWORD="testpass123"),
timeout=60,
)
assert result.returncode == 0, result.stderr or result.stdout
def _create_public_snapshot_with_cli(data_dir, url: str) -> str:
result = run_archivebox_cmd(
["snapshot", "create", "--status", "sealed", "--tag", "public-mode-matrix", url],
cwd=data_dir,
env=cli_env(PERMISSIONS="public"),
timeout=60,
)
assert result.returncode == 0, result.stderr or result.stdout
records = [json.loads(line) for line in result.stdout.splitlines() if line.strip().startswith("{")]
assert records, result.stdout
snapshot_id = str(records[-1]["id"])
updated = run_archivebox_cmd(
["snapshot", "update", "--status", "sealed"],
cwd=data_dir,
env=cli_env(PERMISSIONS="public"),
input=result.stdout,
timeout=60,
)
assert updated.returncode == 0, updated.stderr or updated.stdout
listed = run_archivebox_cmd(
["snapshot", "list", "--url__icontains", url, "--csv=id,url,status"],
cwd=data_dir,
env=cli_env(),
timeout=60,
)
assert listed.returncode == 0, listed.stderr or listed.stdout
assert snapshot_id in listed.stdout
assert url in listed.stdout
assert "sealed" in listed.stdout
return snapshot_id
def _login_admin_session_over_http(port: int, host: str) -> requests.Session:
session = requests.Session()
login_page = session.get(
f"http://127.0.0.1:{port}/admin/login/",
headers={"Host": host},
timeout=10,
)
assert login_page.status_code == 200, login_page.text[:500]
csrf_match = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', login_page.text)
assert csrf_match, login_page.text[:500]
login_response = session.post(
f"http://127.0.0.1:{port}/admin/login/",
headers={"Host": host, "Referer": f"http://{host}/admin/login/"},
data={
"username": "apitestadmin",
"password": "testpass123",
"csrfmiddlewaretoken": csrf_match.group(1),
"next": "/admin/core/snapshot/",
},
timeout=10,
allow_redirects=False,
)
assert login_response.status_code in (302, 303), login_response.text[:500]
return session
def _response_cookie_names(response: requests.Response) -> set[str]:
return {cookie.name for cookie in response.cookies}
def _assert_no_admin_cookies_set(response: requests.Response) -> None:
cookie_names = _response_cookie_names(response)
assert not any(name.startswith("archivebox_sessionid_") for name in cookie_names), response.headers.get("Set-Cookie", "")
assert not any(name.startswith("archivebox_csrftoken_") for name in cookie_names), response.headers.get("Set-Cookie", "")
def _assert_only_hint_cookie_set(response: requests.Response) -> None:
_assert_no_admin_cookies_set(response)
assert _response_cookie_names(response) <= {ADMIN_LOGIN_HINT_COOKIE}, response.headers.get("Set-Cookie", "")
class TestPublicIndex:
"""Tests for public index visibility and redirects."""
@ -177,7 +273,7 @@ class TestPublicIndex:
status=Snapshot.StatusChoices.SEALED,
)
response = client.get("/public/", HTTP_HOST=PUBLIC_TEST_HOST)
response = client.get("/public/", HTTP_HOST=WEB_TEST_HOST)
assert response.status_code == 200
assert b"Public Snapshot" in response.content
@ -287,8 +383,129 @@ class TestPublicIndex:
@override_settings(PUBLIC_INDEX=True)
def test_public_index_redirects_logged_in_users_to_admin_snapshot_list(self, client, admin_user):
client.force_login(admin_user)
client.cookies[ADMIN_LOGIN_HINT_COOKIE] = "1"
response = client.get("/public/", HTTP_HOST=PUBLIC_TEST_HOST)
response = client.get("/public/", HTTP_HOST=WEB_TEST_HOST)
assert response.status_code == 302
assert response["Location"] == "/admin/core/snapshot/"
@pytest.mark.timeout(240)
@pytest.mark.parametrize(
"mode",
[
"safe-subdomains-fullreplay",
"safe-onedomain-nojsreplay",
"unsafe-onedomain-noadmin",
"danger-onedomain-fullreplay",
],
)
def test_public_web_routing_and_auth_cookie_behavior_over_real_server_in_all_security_modes(tmp_path, mode):
init_archive(tmp_path)
_create_admin_user_with_cli(tmp_path)
public_url = f"https://public-mode-{mode}.example"
_create_public_snapshot_with_cli(tmp_path, public_url)
port = get_free_port()
base_host = f"archivebox.localhost:{port}"
admin_host = f"admin.archivebox.localhost:{port}" if mode == "safe-subdomains-fullreplay" else base_host
web_host = f"web.archivebox.localhost:{port}" if mode == "safe-subdomains-fullreplay" else base_host
api_host = f"api.archivebox.localhost:{port}" if mode == "safe-subdomains-fullreplay" else base_host
env = cli_env(
port=port,
server=True,
BASE_URL=f"http://archivebox.localhost:{port}",
SERVER_SECURITY_MODE=mode,
PUBLIC_INDEX="True",
PUBLIC_ADD_VIEW="False",
PERMISSIONS="public",
)
try:
start_archivebox_server(tmp_path, env=env, port=port)
wait_for_http(port, host=web_host, path="/public/")
public_page = requests.get(
f"http://127.0.0.1:{port}/public/",
headers={"Host": web_host},
timeout=10,
allow_redirects=False,
)
assert public_page.status_code == 200, public_page.text[:500]
assert public_url in public_page.text
if mode == "safe-subdomains-fullreplay":
_assert_only_hint_cookie_set(public_page)
add_page = requests.get(
f"http://127.0.0.1:{port}/add/",
headers={"Host": web_host},
timeout=10,
allow_redirects=False,
)
if mode == "unsafe-onedomain-noadmin":
assert add_page.status_code == 403
else:
assert add_page.status_code in (301, 302), add_page.text[:500]
assert "/admin/login/" in add_page.headers["Location"] or "/add/" in add_page.headers["Location"]
if mode == "safe-subdomains-fullreplay":
_assert_only_hint_cookie_set(add_page)
admin_login = requests.get(
f"http://127.0.0.1:{port}/admin/login/",
headers={"Host": admin_host},
timeout=10,
allow_redirects=False,
)
if mode == "unsafe-onedomain-noadmin":
assert admin_login.status_code == 403
unsafe_post = requests.post(
f"http://127.0.0.1:{port}/public/",
headers={"Host": web_host},
data={"x": "1"},
timeout=10,
allow_redirects=False,
)
assert unsafe_post.status_code == 403
api_docs = requests.get(
f"http://127.0.0.1:{port}/api/v1/docs",
headers={"Host": api_host},
timeout=10,
allow_redirects=False,
)
assert api_docs.status_code == 403
return
assert admin_login.status_code == 200, admin_login.text[:500]
session = _login_admin_session_over_http(port, admin_host)
assert any(cookie.name.startswith("archivebox_sessionid_") for cookie in session.cookies)
for cookie in list(session.cookies):
if cookie.name == ADMIN_LOGIN_HINT_COOKIE:
session.cookies.clear(domain=cookie.domain, path=cookie.path, name=cookie.name)
logged_in_public_page = session.get(
f"http://127.0.0.1:{port}/public/",
headers={"Host": web_host},
timeout=10,
allow_redirects=False,
)
if mode == "safe-subdomains-fullreplay":
assert logged_in_public_page.status_code == 200, logged_in_public_page.headers.get("Location")
assert public_url in logged_in_public_page.text
_assert_only_hint_cookie_set(logged_in_public_page)
session.cookies.set(ADMIN_LOGIN_HINT_COOKIE, "1", path="/")
hinted_public_page = session.get(
f"http://127.0.0.1:{port}/public/",
headers={"Host": web_host},
timeout=10,
allow_redirects=False,
)
assert hinted_public_page.status_code in (301, 302)
assert hinted_public_page.headers["Location"] == f"http://{admin_host}/admin/core/snapshot/"
_assert_only_hint_cookie_set(hinted_public_page)
else:
assert logged_in_public_page.status_code in (301, 302)
assert logged_in_public_page.headers["Location"] == "/admin/core/snapshot/"
finally:
stop_server(tmp_path)

22
archivebox/tests/test_urls.py
View File

@ -60,7 +60,6 @@ def _build_script(body: str) -> str:
get_api_host,
get_web_host,
get_web_base_url,
get_public_host,
get_snapshot_subdomain,
get_snapshot_host,
get_original_host,
@ -197,7 +196,7 @@ class TestUrlRouting:
result = run_archivebox_cmd(["config", "--set", *settings], cwd=self.data_dir)
assert result.returncode == 0, result.stderr
def test_routes_util_and_public_redirect(self) -> None:
def test_routes_util_and_web_public_redirect(self) -> None:
self._run(
"""
snapshot = get_snapshot()
@ -207,7 +206,6 @@ class TestUrlRouting:
web_host = get_web_host()
admin_host = get_admin_host()
api_host = get_api_host()
public_host = get_public_host()
snapshot_subdomain = get_snapshot_subdomain(snapshot_id)
snapshot_host = get_snapshot_host(snapshot_id)
original_host = get_original_host(domain)
@ -223,7 +221,6 @@ class TestUrlRouting:
assert web_host == "web.archivebox.localhost:8000"
assert admin_host == "admin.archivebox.localhost:8000"
assert api_host == "api.archivebox.localhost:8000"
assert public_host == "public.archivebox.localhost:8000"
assert snapshot_subdomain == f"snap-{snapshot_id[-12:].lower()}"
assert snapshot_host == f"{snapshot_subdomain}.archivebox.localhost:8000"
assert original_host == f"{domain}.archivebox.localhost:8000"
@ -258,7 +255,7 @@ class TestUrlRouting:
""",
)
def test_api_archive_redirect_uses_public_web_base_url(self) -> None:
def test_api_archive_redirect_uses_web_base_url(self) -> None:
try:
config_result = run_archivebox_cmd(
["config", "--set", "BASE_URL=https://archivebox.io"],
@ -296,7 +293,6 @@ class TestUrlRouting:
snapshot = get_snapshot()
client = Client()
web_host = get_web_host()
public_host = get_public_host()
admin_host = get_admin_host()
snapshot_host = get_snapshot_host(str(snapshot.id))
original_host = get_original_host(snapshot.domain)
@ -305,10 +301,6 @@ class TestUrlRouting:
assert resp.status_code in (301, 302)
assert admin_host in resp["Location"]
resp = client.get("/admin/login/?next=/admin/", HTTP_HOST=public_host)
assert resp.status_code in (301, 302)
assert resp["Location"] == f"http://{admin_host}/admin/login/?next=/admin/"
resp = client.get("/admin/login/?next=/admin/", HTTP_HOST=snapshot_host)
assert resp.status_code in (301, 302)
assert resp["Location"] == f"http://{admin_host}/admin/login/?next=/admin/"
@ -869,7 +861,6 @@ class TestUrlRouting:
snapshot_host = get_snapshot_host(snapshot_id)
admin_host = get_admin_host()
web_host = get_web_host()
public_host = get_public_host()
client = Client()
@ -880,6 +871,11 @@ class TestUrlRouting:
ensure_admin_user()
assert client.login(username="testadmin", password="testpassword")
resp = client.get("/public/", HTTP_HOST=web_host)
assert resp.status_code == 200
assert not getattr(resp.wsgi_request.user, "is_authenticated", False)
resp = client.get("/admin/", HTTP_HOST=admin_host)
assert resp.status_code == 200
assert client.cookies[ADMIN_LOGIN_HINT_COOKIE].value == "1"
@ -892,7 +888,7 @@ class TestUrlRouting:
assert resp.status_code == 200
live_html = response_body(resp).decode("utf-8", "ignore")
assert f"http://{snapshot_host}/" in live_html
assert f"http://{public_host}/static/archive.png" in live_html
assert f"http://{web_host}/static/archive.png" in live_html
assert "?preview=1" in live_html
assert "function createMainFrame(previousFrame)" in live_html
assert "function activateCardPreview(card, link, updateHash=true)" in live_html
@ -918,7 +914,7 @@ class TestUrlRouting:
static_html = Path(snapshot.output_dir, "index.html").read_text(encoding="utf-8", errors="ignore")
assert f"http://{snapshot_host}/" in static_html
assert f"http://{public_host}/static/archive.png" in static_html
assert f"http://{web_host}/static/archive.png" in static_html
assert "?preview=1" in static_html
assert "function createMainFrame(previousFrame)" in static_html
assert "function activateCardPreview(card, link, updateHash=true)" in static_html

22
archivebox/workers/supervisord_util.py
View File

@ -21,7 +21,7 @@ from xmlrpc.client import Fault, ServerProxy
from archivebox.config import CONSTANTS
from archivebox.config.common import rprint as print
from archivebox.config.paths import get_or_create_working_tmp_dir
from archivebox.config.paths import SUPERVISORD_SOCKET_FILENAME, get_or_create_working_tmp_dir
from archivebox.config.permissions import ARCHIVEBOX_USER
from archivebox.core.shutdown_util import (
configured_stopwaitsecs,
@ -299,9 +299,8 @@ def get_sock_file():
"""Get the path to the supervisord socket file.
Supervisord-managed workers inherit SUPERVISOR_SERVER_URL from their parent
supervisord. They must keep using that socket even when crawl hooks override
TMP_DIR, otherwise a hook can accidentally start a nested supervisord for a
crawl-scoped temp dir.
supervisord. They must keep using that socket so worker code cannot
accidentally start a nested supervisord.
"""
server_url = os.environ.get("SUPERVISOR_SERVER_URL", "")
if server_url.startswith("unix://"):
@ -309,9 +308,7 @@ def get_sock_file():
TMP_DIR = get_or_create_working_tmp_dir(autofix=True, quiet=False)
assert TMP_DIR, "Failed to find or create a writable TMP_DIR!"
socket_file = TMP_DIR / "supervisord.sock"
return socket_file
return TMP_DIR / SUPERVISORD_SOCKET_FILENAME
def create_supervisord_config():
@ -320,12 +317,21 @@ def create_supervisord_config():
CONFIG_FILE = SOCK_FILE.parent / CONFIG_FILE_NAME
PID_FILE = SOCK_FILE.parent / PID_FILE_NAME
LOG_FILE = CONSTANTS.LOGS_DIR / LOG_FILE_NAME
environment = ",".join(
f"{key}={json.dumps(str(value))}"
for key, value in {
"IS_SUPERVISORD_PARENT": "true",
"COLUMNS": "200",
"DATA_DIR": CONSTANTS.DATA_DIR,
"TMP_DIR": SOCK_FILE.parent,
}.items()
)
CONSTANTS.LOGS_DIR.mkdir(parents=True, exist_ok=True)
config_content = f"""
[supervisord]
nodaemon = true
environment = IS_SUPERVISORD_PARENT="true",COLUMNS="200"
environment = {environment}
pidfile = {PID_FILE}
logfile = {LOG_FILE}
childlogdir = {CONSTANTS.LOGS_DIR}

5
bin/docker_entrypoint.sh
View File

@ -29,12 +29,13 @@ ulimit -c 0 >/dev/null 2>&1 || true
# Load global invariants (set by Dockerfile during image build time, not intended to be customized by users at runtime)
export DATA_DIR="${DATA_DIR:-/data}"
export CONFIG_DIR="${CONFIG_DIR:-/home/archivebox/.config/abx}"
export TMP_DIR="${TMP_DIR:-/tmp/archivebox}"
export LIB_DIR="${LIB_DIR:-/opt/archivebox/lib}"
export LIB_DIR="${LIB_DIR:-/home/archivebox/.config/abx/lib}"
export ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-$LIB_DIR}"
export ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
export PERSONAS_DIR="${PERSONAS_DIR:-$DATA_DIR/personas}"
export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-/browsers}"
export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-$LIB_DIR/playwright/cache}"
# Global default PUID and PGID if data dir is empty and no intended PUID+PGID is set manually by user
export DEFAULT_PUID=911

6
bin/validate_docker_uid_gid.sh
View File

@ -103,9 +103,9 @@ if grep -E "(/[[:alnum:]_.-]+/)?pip install|npm install|uv pip install" /tmp/arc
exit 1
fi
archivebox version 2>&1 | tee /tmp/archivebox-version.log
grep -Eq "/opt/archivebox/lib/(pip/venv/bin|env/bin)/trafilatura" /tmp/archivebox-version.log
grep -Eq "/opt/archivebox/lib/(npm/node_modules/.bin|env/bin)/defuddle" /tmp/archivebox-version.log
grep -Eq "/opt/archivebox/lib/env/bin/sonic" /tmp/archivebox-version.log
grep -Eq "/home/archivebox/.config/abx/lib/(pip/venv/bin|env/bin)/trafilatura" /tmp/archivebox-version.log
grep -Eq "/home/archivebox/.config/abx/lib/(npm/node_modules/.bin|env/bin)/defuddle" /tmp/archivebox-version.log
grep -Eq "/home/archivebox/.config/abx/lib/env/bin/sonic" /tmp/archivebox-version.log
archivebox add --depth=0 https://example.com/ 2>&1 | tee /tmp/archivebox-add.log
archivebox update --index-only 2>&1 | tee /tmp/archivebox-update.log
snapshot_dir="$(find /data/archive/users/system/snapshots -mindepth 3 -maxdepth 3 -type d | head -n 1)"

2
docker-compose.yml
View File

@ -74,7 +74,7 @@ services:
### TLS / HTTPS ingress (opt-in, everything below is driven by env vars only).
#
# ArchiveBox serves the admin/web/api/public control plane AND every archived
# ArchiveBox serves the admin/web/api control plane AND every archived
# snapshot on its own subdomain for security isolation, so a public deployment
# needs wildcard DNS + TLS for *.your.domain. Pick ONE of the two ingress options
# below by activating its profile (e.g. put COMPOSE_PROFILES=https or =tunnel in a

10
docs/Configuration.md
View File

@ -376,15 +376,15 @@ IPv6 literal addresses must be bracketed: `[::1]:8000`, not `::1:8000`.
#### `BASE_URL`
**Possible Values:** [`""`]/`https://archive.example.com`/`http://archivebox.localhost:8000`/...
The canonical public URL of your ArchiveBox instance. Used to build absolute links in templates, redirects (`/admin/login/?next=...`), admin notification emails, OG/meta tags, and — in subdomain security mode — to derive the `admin.`, `web.`, `api.`, `public.`, and per-snapshot `snap-<id>.` subdomains.
The canonical public URL of your ArchiveBox instance. Used to build absolute links in templates, redirects (`/admin/login/?next=...`), admin notification emails, OG/meta tags, and — in subdomain security mode — to derive the `admin.`, `web.`, `api.`, and per-snapshot `snap-<id>.` subdomains.
**When `BASE_URL` is set explicitly**, ArchiveBox treats it as the source of truth and ignores the incoming `Host` header for URL building. In `safe-subdomains-fullreplay` mode setting it is **required for redirects to work correctly**.
**When `BASE_URL` is empty**, the value is resolved at request time from the incoming request's `Host` header (with any leading `admin.` / `web.` / `api.` / `public.` / `snap-*.` label stripped to recover the canonical base). Loopback hostnames (`localhost`, `127.0.0.1`, `0.0.0.0`, `::`) are rewritten to `archivebox.localhost` so subdomain routing works without `/etc/hosts` edits. If there's no live request, [`BIND_ADDR`](#bind_addr) is used as a last resort.
**When `BASE_URL` is empty**, the value is resolved at request time from the incoming request's `Host` header (with any leading `admin.` / `web.` / `api.` / `snap-*.` label stripped to recover the canonical base). Loopback hostnames (`localhost`, `127.0.0.1`, `0.0.0.0`, `::`) are rewritten to `archivebox.localhost` so subdomain routing works without `/etc/hosts` edits. If there's no live request, [`BIND_ADDR`](#bind_addr) is used as a last resort.
The scheme is taken from the explicit `BASE_URL` if set, otherwise from the request (so put a reverse proxy in front for HTTPS and trust `X-Forwarded-Proto`).
ArchiveBox automatically derives the underlying Django `ALLOWED_HOSTS` and `CSRF_TRUSTED_ORIGINS` settings from `BASE_URL` + [`SERVER_SECURITY_MODE`](#server_security_mode), so you do **not** set those directly — the system widens them as needed to admit the admin/web/api/public subdomains.
ArchiveBox automatically derives the underlying Django `ALLOWED_HOSTS` and `CSRF_TRUSTED_ORIGINS` settings from `BASE_URL` + [`SERVER_SECURITY_MODE`](#server_security_mode), so you do **not** set those directly — the system widens them as needed to admit the admin/web/api subdomains.
> [!NOTE]
> **Pin `BASE_URL` explicitly on any deployment using `safe-subdomains-fullreplay` mode.** A misconfig banner will surface in the rendered UI until you do.
@ -401,11 +401,11 @@ ArchiveBox automatically derives the underlying Django `ALLOWED_HOSTS` and `CSRF
The top-level security posture of the server. Controls how archived content is served, whether the admin/API control plane is reachable, and which host(s) the UI is split across. **This is the most important security knob** — pick the most restrictive mode that still works for your use case.
ArchiveBox splits its surfaces across four logical hosts: `admin.*` (Django admin + session cookies, the entire control plane), `web.*` (logged-in browsing UI), `api.*` (REST/JSON endpoints), and `public.*` (unauthenticated browsing of `PERMISSIONS=public` snapshots). In subdomain mode each gets its own host derived from [`BASE_URL`](#base_url); session/CSRF cookies are scoped to `admin.*` only, so a compromised replay page on `snap-<id>.*` can't read admin auth.
ArchiveBox splits its surfaces across three logical hosts: `admin.*` (Django admin + session cookies, the entire control plane), `web.*` (anonymous public browsing UI, optional public add view, and public snapshot index), and `api.*` (REST/JSON endpoints). In subdomain mode each gets its own host derived from [`BASE_URL`](#base_url); session/CSRF cookies are scoped to `admin.*` only, so `web.*`, `api.*`, and compromised replay pages on `snap-<id>.*` can't read admin auth.
| Mode | Host layout | JS replay | Control plane | Use when |
|---|---|---|---|---|
| **`safe-subdomains-fullreplay`** *(default, recommended)* | admin/web/api/public/snap-* on separate subdomains | Full JS replay enabled | Enabled on `admin.*` only | You have wildcard DNS (`*.archive.example.com`) and a TLS cert that covers it. Archived JS runs sandboxed away from the admin origin. |
| **`safe-subdomains-fullreplay`** *(default, recommended)* | admin/web/api/snap-* on separate subdomains | Full JS replay enabled | Enabled on `admin.*` only | You have wildcard DNS (`*.archive.example.com`) and a TLS cert that covers it. Archived JS runs sandboxed away from the admin origin. |
| **`safe-onedomain-nojsreplay`** | Everything on one host | JS in replays is neutered (served as `text/plain` or stripped) | Enabled | You can't get wildcard DNS. Trades replay fidelity for same-origin safety — archived pages won't execute scripts. |
| **`unsafe-onedomain-noadmin`** | Everything on one host | Full JS replay enabled | **Disabled** — `/admin`, `/accounts`, `/api`, `/add`, `/web` return 403; only GET/HEAD/OPTIONS allowed | Read-only public archive on a single host. Operate the instance via CLI only; the web admin is unreachable. |
| **`danger-onedomain-fullreplay`** | Everything on one host | Full JS replay enabled | Enabled | Local dev / trusted-network only. Archived JS runs on the **same origin as the admin UI** — a malicious archived page can call admin endpoints with your session. **Do not expose this mode to the internet.** |

2
docs/Setting-up-Authentication.md
View File

@ -207,7 +207,7 @@ curl -X 'GET' \
> Browsers enforce that requests made to the ArchiveBox API from *other origins* will not include any session cookies by default. This is is a [foundational security principle of the web](https://docs.djangoproject.com/en/5.0/ref/csrf/) that protects you from API requests being initiated by JS on websites you don't control (aka CSRF/CORS attacks).
>
> To allow incoming POST/PUT/DELETE requests from other domains **that you trust**, set [`BASE_URL`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#base_url) to the public URL of your instance — ArchiveBox derives Django's `ALLOWED_HOSTS` and `CSRF_TRUSTED_ORIGINS` from `BASE_URL` + [`SERVER_SECURITY_MODE`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode) automatically, including widening them to admit the admin/web/api/public subdomains. If your setup needs something the auto-derivation doesn't cover, [open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new/choose).
> To allow incoming POST/PUT/DELETE requests from other domains **that you trust**, set [`BASE_URL`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#base_url) to the public URL of your instance — ArchiveBox derives Django's `ALLOWED_HOSTS` and `CSRF_TRUSTED_ORIGINS` from `BASE_URL` + [`SERVER_SECURITY_MODE`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode) automatically, including widening them to admit the admin/web/api subdomains. If your setup needs something the auto-derivation doesn't cover, [open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new/choose).
Log in via the Admin Web UI: `/admin/login/`, you can then re-use your login session id (stored in the `sessionid` cookie) for REST API requests. By default, this only allows you to make requests from the same domain ArchiveBox is being served on (e.g. from browser devtools open on an ArchiveBox page or CLI tools).

11
docs/apidocs/archivebox/archivebox.config.common.md
View File

@ -462,17 +462,6 @@ Bases: {py:obj}`archivebox.config.configset.BaseConfigSet`
````
````{py:attribute} LIB_BIN_DIR
:canonical: archivebox.config.common.StorageConfig.LIB_BIN_DIR
:type: pathlib.Path
:value: >
'Field(...)'
```{autodoc2-docstring} archivebox.config.common.StorageConfig.LIB_BIN_DIR
```
````
````{py:attribute} CUSTOM_TEMPLATES_DIR
:canonical: archivebox.config.common.StorageConfig.CUSTOM_TEMPLATES_DIR
:type: pathlib.Path

11
docs/apidocs/archivebox/archivebox.config.constants.md
View File

@ -606,17 +606,6 @@
````
````{py:attribute} DEFAULT_LIB_BIN_DIR
:canonical: archivebox.config.constants.ConstantsDict.DEFAULT_LIB_BIN_DIR
:type: pathlib.Path
:value: >
'_env_path(...)'
```{autodoc2-docstring} archivebox.config.constants.ConstantsDict.DEFAULT_LIB_BIN_DIR
```
````
````{py:attribute} RESERVED_ARCHIVE_DIR_NAMES
:canonical: archivebox.config.constants.ConstantsDict.RESERVED_ARCHIVE_DIR_NAMES
:type: frozenset[str]

22
docs/apidocs/archivebox/archivebox.core.host_util.md
View File

@ -91,10 +91,6 @@
- ```{autodoc2-docstring} archivebox.core.host_util.get_api_host
:summary:
```
* - {py:obj}`get_public_host <archivebox.core.host_util.get_public_host>`
- ```{autodoc2-docstring} archivebox.core.host_util.get_public_host
:summary:
```
* - {py:obj}`get_snapshot_subdomain <archivebox.core.host_util.get_snapshot_subdomain>`
- ```{autodoc2-docstring} archivebox.core.host_util.get_snapshot_subdomain
:summary:
@ -143,10 +139,6 @@
- ```{autodoc2-docstring} archivebox.core.host_util.get_api_base_url
:summary:
```
* - {py:obj}`get_public_base_url <archivebox.core.host_util.get_public_base_url>`
- ```{autodoc2-docstring} archivebox.core.host_util.get_public_base_url
:summary:
```
* - {py:obj}`get_snapshot_base_url <archivebox.core.host_util.get_snapshot_base_url>`
- ```{autodoc2-docstring} archivebox.core.host_util.get_snapshot_base_url
:summary:
@ -362,13 +354,6 @@
```
````
````{py:function} get_public_host(config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str
:canonical: archivebox.core.host_util.get_public_host
```{autodoc2-docstring} archivebox.core.host_util.get_public_host
```
````
````{py:function} get_snapshot_subdomain(snapshot_id: str) -> str
:canonical: archivebox.core.host_util.get_snapshot_subdomain
@ -453,13 +438,6 @@
```
````
````{py:function} get_public_base_url(request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str
:canonical: archivebox.core.host_util.get_public_base_url
```{autodoc2-docstring} archivebox.core.host_util.get_public_base_url
```
````
````{py:function} get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, typing.Any] | None = None, **config_kwargs: typing.Any) -> str
:canonical: archivebox.core.host_util.get_snapshot_base_url

11
docs/apidocs/archivebox/archivebox.core.templatetags.core_tags.md
View File

@ -119,10 +119,6 @@
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.web_base_url
:summary:
```
* - {py:obj}`public_base_url <archivebox.core.templatetags.core_tags.public_base_url>`
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.public_base_url
:summary:
```
* - {py:obj}`snapshot_base_url <archivebox.core.templatetags.core_tags.snapshot_base_url>`
- ```{autodoc2-docstring} archivebox.core.templatetags.core_tags.snapshot_base_url
:summary:
@ -508,13 +504,6 @@
```
````
````{py:function} public_base_url(context) -> str
:canonical: archivebox.core.templatetags.core_tags.public_base_url
```{autodoc2-docstring} archivebox.core.templatetags.core_tags.public_base_url
```
````
````{py:function} snapshot_base_url(context, snapshot) -> str
:canonical: archivebox.core.templatetags.core_tags.snapshot_base_url

2
etc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.34rc70",
"version": "0.9.34rc71",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {

8
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.34rc70"
version = "0.9.34rc71"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -79,9 +79,9 @@ dependencies = [
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus==2.5.10", # EventBus API
"abxpkg>=1.11.185", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.189", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.189", # shared ArchiveBox downloader package with blocking install preflight
"abxpkg>=1.11.186", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.190", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.190", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]