The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Consolidate runtime config handling

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-01 15:03:40 -07:00
parent 453d998e7d
commit c075d654d8
No known key found for this signature in database
96 changed files with 1809 additions and 1188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
archivebox/__init__.py
View File

@ -79,16 +79,10 @@ def __getattr__(name: str):
return VERSION
if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
from abx_plugins import get_plugins_dir
from .config.constants import CONSTANTS
builtin_plugins_dir = Path(get_plugins_dir()).resolve()
user_plugins_dir = (
Path(
os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR")
or os.environ.get("USER_PLUGINS_DIR")
or os.environ.get("DATA_DIR", os.getcwd()),
)
/ "custom_plugins"
)
user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
plugins = {
"builtin": builtin_plugins_dir,
"user": user_plugins_dir,

6
archivebox/api/v1_api.py
View File

@ -68,14 +68,14 @@ class NinjaAPIWithIOCapture(NinjaAPI):
# response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
# Add Auth Headers to response
api_token_attr = getattr(request, "_api_token", None)
api_token_attr = request.__dict__.get("_api_token")
api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
response["X-ArchiveBox-Auth-Expires"] = token_expiry
response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
# import ipdb; ipdb.set_trace()

20
archivebox/api/v1_cli.py
View File

@ -150,8 +150,8 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
"snapshot_ids": snapshot_ids,
"queued_urls": args.urls,
}
stdout = getattr(request, "stdout", None)
stderr = getattr(request, "stderr", None)
stdout = request.__dict__.get("stdout")
stderr = request.__dict__.get("stderr")
return {
"success": True,
@ -177,8 +177,8 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
continuous=args.continuous,
stop_daemon_stack=False,
)
stdout = getattr(request, "stdout", None)
stderr = getattr(request, "stderr", None)
stdout = request.__dict__.get("stdout")
stderr = request.__dict__.get("stderr")
return {
"success": True,
"errors": [],
@ -211,8 +211,8 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
config=config_overrides or None,
)
stdout = getattr(request, "stdout", None)
stderr = getattr(request, "stderr", None)
stdout = request.__dict__.get("stdout")
stderr = request.__dict__.get("stderr")
return {
"success": True,
"errors": [],
@ -249,8 +249,8 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
elif args.as_csv:
result_format = "csv"
stdout = getattr(request, "stdout", None)
stderr = getattr(request, "stderr", None)
stdout = request.__dict__.get("stdout")
stderr = request.__dict__.get("stderr")
return {
"success": True,
"errors": [],
@ -290,8 +290,8 @@ def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
"removed_snapshot_ids": removed_snapshot_ids,
"remaining_snapshots": Snapshot.objects.count(),
}
stdout = getattr(request, "stdout", None)
stderr = getattr(request, "stderr", None)
stdout = request.__dict__.get("stdout")
stderr = request.__dict__.get("stderr")
return {
"success": True,
"errors": [],

27
archivebox/api/v1_core.py
View File

@ -273,7 +273,7 @@ def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any]
def _get_archiveresult_upload_data(request: HttpRequest):
cached = getattr(request, "_archiveresult_upload_data", None)
cached = request.__dict__.get("_archiveresult_upload_data")
if cached is not None:
return cached
@ -468,12 +468,7 @@ def _write_archiveresult_files(
)
guessed_mime = mimetypes.guess_type(relative_output_path)[0]
output_mime_type = (
(mime_types[0] if mime_types else "")
or getattr(uploaded_file, "content_type", None)
or guessed_mime
or "application/octet-stream"
)
output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
output_files[relative_output_path] = {
"extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
"mimetype": output_mime_type,
@ -501,7 +496,7 @@ def _write_archiveresult_files(
guessed_mime = mimetypes.guess_type(saved_output_path)[0]
output_mime_type = (
(mime_types[index] if index < len(mime_types) else "")
or getattr(uploaded_file, "content_type", None)
or uploaded_file.content_type
or guessed_mime
or "application/octet-stream"
)
@ -702,7 +697,7 @@ class SnapshotSchema(Schema):
@staticmethod
def resolve_archiveresults(obj, context):
if bool(getattr(context["request"], "with_archiveresults", False)):
if bool(context["request"].__dict__.get("with_archiveresults", False)):
return obj.archiveresult_set.all().distinct()
return ArchiveResult.objects.none()
@ -883,7 +878,7 @@ def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], wi
if not query:
return queryset
runtime_config = getattr(request, "archivebox_config", None)
runtime_config = request.archivebox_config
search_mode = get_search_mode(filters.search_mode, config=runtime_config)
try:
return apply_snapshot_search(
@ -1081,7 +1076,7 @@ class TagSchema(Schema):
def resolve_created_by_username(obj):
user_model = get_user_model()
user = user_model.objects.get(id=obj.created_by_id)
username = getattr(user, "username", None)
username = user.username
return username if isinstance(username, str) else str(user)
@staticmethod
@ -1090,7 +1085,7 @@ class TagSchema(Schema):
@staticmethod
def resolve_snapshots(obj, context):
if bool(getattr(context["request"], "with_snapshots", False)):
if bool(context["request"].__dict__.get("with_snapshots", False)):
return obj.snapshot_set.all().distinct()
return Snapshot.objects.none()
@ -1294,8 +1289,8 @@ def _public_tag_listing_enabled() -> bool:
def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
user = getattr(request, "user", None)
if getattr(user, "is_authenticated", False):
user = request.user
if user.is_authenticated:
return True
token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
@ -1315,7 +1310,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
if not _request_has_tag_autocomplete_access(request):
raise HttpError(401, "Authentication required")
public_only = not getattr(request.user, "is_authenticated", False) and not getattr(request, "_api_token", None)
public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
queryset = get_matching_tags(q)
public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
if public_only:
@ -1324,7 +1319,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
return {
"tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags],
"tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
}

13
archivebox/api/v1_crawls.py
View File

@ -58,7 +58,7 @@ class CrawlSchema(Schema):
def resolve_created_by_username(obj):
user_model = get_user_model()
user = user_model.objects.get(id=obj.created_by_id)
username = getattr(user, "username", None)
username = user.username
return username if isinstance(username, str) else str(user)
@staticmethod
@ -71,7 +71,7 @@ class CrawlSchema(Schema):
@staticmethod
def resolve_snapshots(obj, context):
if bool(getattr(context["request"], "with_snapshots", False)):
if bool(context["request"].__dict__.get("with_snapshots", False)):
return obj.snapshot_set.all().distinct()
return Snapshot.objects.none()
@ -122,8 +122,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
tags = normalize_tag_list(data.tags, data.tags_str)
config = dict(data.config or {})
request_user = request.user if request.user.is_authenticated else None
config.setdefault("PERMISSIONS", str(get_config(user=request_user).PERMISSIONS))
config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
crawl = Crawl.objects.create(
urls="\n".join(urls),
max_depth=data.max_depth,
@ -164,8 +163,8 @@ def crawl_file(request: HttpRequest, crawl_id: str, path: str):
# Determine the effective viewer: session user takes precedence, otherwise
# fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or
# Authorization: Bearer ... (so that programmatic clients still work).
user = getattr(request, "user", None)
is_authenticated = bool(getattr(user, "is_authenticated", False) and getattr(user, "is_active", False))
user = request.user
is_authenticated = bool(user.is_authenticated and user.is_active)
if not is_authenticated:
token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
auth_header = request.headers.get("Authorization", "")
@ -183,7 +182,7 @@ def crawl_file(request: HttpRequest, crawl_id: str, path: str):
# must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
if not is_admin_user(request):
permissions = normalize_permissions(crawl.permissions)
is_owner = bool(is_authenticated and getattr(crawl, "created_by_id", None) == getattr(user, "id", None))
is_owner = bool(is_authenticated and crawl.created_by_id == user.id)
if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
raise HttpError(404, "Crawl not found")

2
archivebox/api/v1_personas.py
View File

@ -136,7 +136,7 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
created = persona is None
if persona is None:
persona = Persona(name=name)
if getattr(request.user, "is_authenticated", False):
if request.user.is_authenticated:
persona.created_by = request.user
persona.config = {

46
archivebox/base_models/admin.py
View File

@ -5,7 +5,7 @@ __package__ = "archivebox.base_models"
import json
import uuid
from collections.abc import Mapping
from typing import NotRequired, TypedDict
from typing import NotRequired, TypedDict, cast
from django import forms
from django.contrib import admin
@ -76,39 +76,23 @@ class KeyValueWidget(forms.Widget):
def _get_config_options(self) -> dict[str, ConfigOption]:
"""Get available config options from plugins."""
try:
from archivebox.config.common import ArchiveBoxConfig
from archivebox.plugins.discovery import discover_plugin_configs
from archivebox.config.common import config_field_metadata
options: dict[str, ConfigOption] = {}
skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "SNAP_DIR"}
for key, field in ArchiveBoxConfig.model_fields.items():
if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
continue
default = field.default
try:
json.dumps(default)
except TypeError:
default = str(default)
options[key] = {
"plugin": "archivebox",
"type": str(field.annotation),
"default": default,
"description": field.description or "",
for key, metadata in config_field_metadata().items():
option_type = metadata.get("type", "string")
option: ConfigOption = {
"plugin": str(metadata.get("plugin", "archivebox")),
"type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
"default": metadata.get("default", ""),
"description": str(metadata.get("description", "")),
}
plugin_configs = discover_plugin_configs()
for plugin_name, schema in plugin_configs.items():
for key, prop in schema.get("properties", {}).items():
option: ConfigOption = {
"plugin": plugin_name,
"type": prop.get("type", "string"),
"default": prop.get("default", ""),
"description": prop.get("description", ""),
}
schema = metadata.get("schema")
if isinstance(schema, Mapping):
for schema_key in ("enum", "pattern", "minimum", "maximum"):
if schema_key in prop:
option[schema_key] = prop[schema_key]
options[key] = option
if schema_key in schema:
option[schema_key] = schema[schema_key]
options[key] = option
return options
except Exception:
return {}
@ -839,7 +823,7 @@ class ConfigEditorMixin(admin.ModelAdmin):
"""
from archivebox.config.common import is_sensitive_config_key
if change and obj.pk and getattr(obj, "config", None) is not None:
if change and obj.pk and obj.config is not None:
try:
stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
except Exception:

8
archivebox/base_models/models.py
View File

@ -20,6 +20,8 @@ from django.conf import settings
from django_stubs_ext.db.models import TypedModelMeta
from archivebox.config import CONSTANTS
def normalize_config_json_values(config: Any) -> Any:
if not isinstance(config, dict):
@ -165,7 +167,7 @@ class ModelWithDeleteAfter(models.Model):
def get_delete_after_config_value(self):
from archivebox.config.common import get_config
return get_config(include_machine=False).DELETE_AFTER
return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
def set_delete_at_from_config(self, config_value=None) -> bool:
if self.delete_at is not None:
@ -253,9 +255,7 @@ class ModelWithOutputDir(ModelWithUUID):
@classmethod
def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
from archivebox.config.common import get_config
data_dir = get_config().DATA_DIR.resolve()
data_dir = CONSTANTS.DATA_DIR.resolve()
safe_paths = []
for raw_path in paths:
path = Path(raw_path)

4
archivebox/cli/__init__.py
View File

@ -139,9 +139,9 @@ class ArchiveBoxGroup(click.Group):
# print(f'LAZY LOADING {import_path}')
mod = import_module(modname)
func = getattr(mod, funcname)
func = vars(mod)[funcname]
if not hasattr(func, "__doc__"):
if func.__doc__ is None:
raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
# if not isinstance(cmd, click.BaseCommand):

9
archivebox/cli/archivebox_add.py
View File

@ -113,11 +113,9 @@ def add(
from archivebox.misc.logging_util import printable_filesize
from archivebox.misc.system import get_dir_size
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
from django.contrib.auth import get_user_model
from django.utils import timezone
created_by_id = created_by_id or get_or_create_system_user_pk()
created_by = get_user_model().objects.filter(pk=created_by_id).first()
started_at = timezone.now()
if isinstance(urls, str):
@ -151,7 +149,7 @@ def add(
plugins = plugins or ""
persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
persona_obj.ensure_dirs()
effective_persona_config = get_config(persona=persona_obj, user=created_by)
effective_persona_config = get_config(persona=persona_obj)
crawl_config = {
"PERMISSIONS": str(effective_persona_config.PERMISSIONS),
@ -172,9 +170,8 @@ def add(
**({"URL_DENYLIST": url_denylist} if url_denylist else {}),
}
# Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
# priority — they win over persona/plugin/env defaults and get stamped
# directly onto crawl.config so the runtime resolution and admin UI both
# reflect them faithfully.
# priority for crawl-frozen keys. Runtime-derived execution keys are
# stripped by Crawl.save() and rederived when hooks run.
crawl_config.update(config_overrides)
crawl = Crawl.objects.create(

4
archivebox/cli/archivebox_config.py
View File

@ -58,7 +58,7 @@ def config(
]
matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
for config_section in CONFIGS.values():
aliases = getattr(config_section, "aliases", {})
aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}
for search_key in config_options:
# search all aliases in the section
@ -90,7 +90,7 @@ def config(
# Display core config sections
for config_section in CONFIGS.values():
section_header = getattr(config_section, "toml_section_header", "")
section_header = config_section.toml_section_header
if isinstance(section_header, str) and section_header:
print(f"[grey53]\\[{section_header}][/grey53]")
else:

4
archivebox/cli/archivebox_help.py
View File

@ -85,8 +85,8 @@ def help() -> None:
[link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
""")
config = get_config()
if os.access(config.ARCHIVE_DIR, os.R_OK) and config.ARCHIVE_DIR.is_dir():
get_config()
if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
EXAMPLE_USAGE = f"""
[light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]

24
archivebox/cli/archivebox_init.py
View File

@ -25,7 +25,7 @@ def _display_data_path(path: Path, data_dir: Path) -> str:
def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
"""Initialize a new ArchiveBox collection in the current directory"""
from archivebox.config import CONSTANTS, VERSION, DATA_DIR
from archivebox.config import CONSTANTS, VERSION
from archivebox.config.common import get_config
from archivebox.config.collection import write_config_file
from archivebox.misc.db import apply_migrations
@ -37,7 +37,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
# print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
# print("[red] You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
if is_empty and not existing_index:
print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
@ -65,23 +65,23 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
else:
print("\n[green][+] Building archive folder structure...[/green]")
archive_path = _display_data_path(config.ARCHIVE_DIR, DATA_DIR)
sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, DATA_DIR)
logs_path = _display_data_path(CONSTANTS.LOGS_DIR, DATA_DIR)
archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
print(f" + {archive_path}, {sources_path}, {logs_path}...")
Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
config.USERS_DIR.mkdir(parents=True, exist_ok=True)
CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
for path in (Path(CONSTANTS.SOURCES_DIR), config.ARCHIVE_DIR, config.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
print(f" + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
print(f" + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")
# create the .archivebox_id file with a unique ID for this collection
from archivebox.config.paths import _get_collection_id
_get_collection_id(DATA_DIR, force_create=True)
_get_collection_id(CONSTANTS.DATA_DIR, force_create=True)
# create the ArchiveBox.conf file
write_config_file({"SECRET_KEY": config.SECRET_KEY})
@ -96,12 +96,12 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
setup_django()
check_migrations(blocking=True, auto_apply=False)
for migration_line in apply_migrations(DATA_DIR):
for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
sys.stdout.write(f" {migration_line}\n")
assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
print()
print(f" √ {_display_data_path(CONSTANTS.DATABASE_FILE, DATA_DIR)}")
print(f" √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
# from django.contrib.auth.models import User
# call_command("createsuperuser", interactive=True)

5
archivebox/cli/archivebox_install.py
View File

@ -22,12 +22,11 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
"""
from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.config.common import get_config
from archivebox.config import CONSTANTS
from archivebox.misc.logging import stderr
from archivebox.cli.archivebox_init import init
config = get_config()
archive_dir = config.ARCHIVE_DIR
archive_dir = CONSTANTS.ARCHIVE_DIR
if dry_run:
print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")

6
archivebox/cli/archivebox_persona.py
View File

@ -372,7 +372,7 @@ def create_personas(
if not is_tty:
write_record(
{
"id": str(persona.id) if hasattr(persona, "id") else None,
"id": str(persona.id),
"name": persona.name,
"path": str(persona.path),
"CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@ -424,7 +424,7 @@ def list_personas(
else:
write_record(
{
"id": str(persona.id) if hasattr(persona, "id") else None,
"id": str(persona.id),
"name": persona.name,
"path": str(persona.path),
"CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@ -500,7 +500,7 @@ def update_personas(name: str | None = None) -> int:
if not is_tty:
write_record(
{
"id": str(persona.id) if hasattr(persona, "id") else None,
"id": str(persona.id),
"name": persona.name,
"path": str(persona.path),
},

4
archivebox/cli/archivebox_remove.py
View File

@ -12,7 +12,7 @@ import rich_click as click
from django.db import OperationalError
from django.db.models import QuerySet
from archivebox.config import DATA_DIR
from archivebox.config import CONSTANTS
from archivebox.config.django import setup_django
from archivebox.misc.util import enforce_types, docstring
from archivebox.misc.checks import check_data_folder
@ -33,7 +33,7 @@ def remove(
after: float | None = None,
before: float | None = None,
yes: bool = False,
out_dir: Path = DATA_DIR,
out_dir: Path = CONSTANTS.DATA_DIR,
) -> QuerySet:
"""Remove the specified URLs from the archive"""

30
archivebox/cli/archivebox_run.py
View File

@ -40,14 +40,23 @@ Examples:
__package__ = "archivebox.cli"
__command__ = "archivebox run"
import sys
import asyncio
import os
import signal
import sys
from collections import defaultdict
import rich_click as click
from rich import print as rprint
RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
os._exit(128 + int(sig))
def process_stdin_records() -> int:
"""
Process JSONL records from stdin.
@ -281,8 +290,16 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
# hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
# orchestrators should shut down immediately and cleanly on the first signal.
interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
if daemon:
os.environ[RUNNER_DAEMON_ENV] = "1"
try:
with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
with (
foreground_shutdown_signals(
on_signal=_exit_daemon_runner_on_signal if daemon else None,
raise_on_first_signal=not daemon,
),
foreground_parent_watchdog(enabled=not daemon),
):
run_pending_crawls(
daemon=daemon,
crawl_id=crawl_id,
@ -322,7 +339,14 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
if daemon and not snapshot_id and not binary_id and not crawl_id:
try:
with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=False):
os.environ[RUNNER_DAEMON_ENV] = "1"
with (
foreground_shutdown_signals(
on_signal=_exit_daemon_runner_on_signal,
raise_on_first_signal=False,
),
foreground_parent_watchdog(enabled=False),
):
sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
except KeyboardInterrupt:
sys.exit(0)

4
archivebox/cli/archivebox_search.py
View File

@ -12,7 +12,7 @@ import rich_click as click
from django.db.models import Q, QuerySet
from archivebox.config import DATA_DIR
from archivebox.config import CONSTANTS
from archivebox.config.common import get_config
from archivebox.misc.logging import stderr
from archivebox.misc.util import enforce_types, docstring
@ -146,7 +146,7 @@ def get_snapshots(
filter_type: str = "substring",
after: float | None = None,
before: float | None = None,
out_dir: Path = DATA_DIR,
out_dir: Path = CONSTANTS.DATA_DIR,
) -> QuerySet["Snapshot", "Snapshot"]:
"""Filter and return Snapshots matching the given criteria."""
from archivebox.core.models import Snapshot

20
archivebox/cli/archivebox_server.py
View File

@ -12,6 +12,7 @@ from collections.abc import Iterable
import rich_click as click
from rich import print
from archivebox.config import CONSTANTS
from archivebox.misc.util import docstring, enforce_types
@ -259,8 +260,6 @@ def server(
host, port = _parse_and_validate_bind_spec(bind_spec)
if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
from archivebox.config import CONSTANTS
log_path = CONSTANTS.LOGS_DIR / "server.log"
log_path.parent.mkdir(parents=True, exist_ok=True)
daemon_env = os.environ.copy()
@ -337,21 +336,25 @@ def server(
runtime_config = get_config()
_print_server_startup_warnings(runtime_config, host, port)
bind_url = f"http://{host}:{port}"
command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url)
command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)
def still_owns_runtime_stack() -> bool:
from django.db import connections
try:
return command_owns_runtime_stack(command, data_dir=config.DATA_DIR)
return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
finally:
connections.close_all()
shutdown_state = None
try:
with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"):
with (
foreground_shutdown_signals() as shutdown_state,
foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"),
):
while True:
standby_result = standby_until_runtime_stack_needed(command, data_dir=config.DATA_DIR)
older_owner = runtime_stack_owner(data_dir=config.DATA_DIR, exclude_id=command.id)
standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id)
takeover_components = active_supervisord_runtime_components(config=config)
if older_owner and takeover_components:
print(
@ -386,7 +389,8 @@ def server(
except KeyboardInterrupt:
pass
finally:
command.mark_exited()
if not shutdown_state or not shutdown_state.signal_name:
command.mark_exited()
print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")

8
archivebox/cli/archivebox_snapshot.py
View File

@ -313,13 +313,7 @@ def list_snapshots(
for snapshot in queryset.iterator(chunk_size=500):
if set(cols).issubset(simple_cols):
rows.append(
",".join(
to_json(
value.isoformat() if hasattr((value := getattr(snapshot, col, "")), "isoformat") else value,
indent=None,
)
for col in cols
),
",".join(to_json(snapshot.serializable_value(col), indent=None) for col in cols),
)
else:
rows.append(snapshot.to_csv(cols=cols, separator=","))

8
archivebox/cli/archivebox_status.py
View File

@ -8,7 +8,7 @@ import rich_click as click
from rich import print
from archivebox.misc.util import enforce_types, docstring
from archivebox.config import DATA_DIR, CONSTANTS
from archivebox.config import CONSTANTS
from archivebox.config.common import get_config
from archivebox.misc.system import get_dir_size
from archivebox.misc.logging_util import printable_filesize
@ -18,7 +18,7 @@ MAX_STATUS_FS_DIR_SCAN = 5000
@enforce_types
def status(out_dir: Path = DATA_DIR) -> None:
def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
"""Print out some info and statistics about the archive collection"""
from django.contrib.auth import get_user_model
@ -38,7 +38,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
snapshots_qs = Snapshot.objects.all()
num_sql_links = snapshots_qs.count()
archive_dir = config.ARCHIVE_DIR
archive_dir = CONSTANTS.ARCHIVE_DIR
legacy_snapshot_dirs = []
if archive_dir.exists():
legacy_snapshot_dirs = [
@ -48,7 +48,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
print(f" > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
print()
print("[green]\\[*] Scanning archive data directories...[/green]")
users_dir = config.USERS_DIR
users_dir = CONSTANTS.USERS_DIR
scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
print(f"[yellow] {scan_roots_display}[/yellow]")

7
archivebox/cli/archivebox_update.py
View File

@ -401,7 +401,7 @@ def update(
resume = None
except (KeyboardInterrupt, asyncio.CancelledError) as err:
exit_code = 130
exact_resume = getattr(err, "archivebox_resume", None)
exact_resume = err.__dict__.get("archivebox_resume")
resume_cmd = ["archivebox", "update"]
if migrate_only:
resume_cmd.append("--migrate-only")
@ -464,7 +464,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
1:1 mapping between DB and filesystem.
"""
from archivebox.core.models import Snapshot
from archivebox.config.common import get_config
from archivebox.config import CONSTANTS
from archivebox.crawls.models import Crawl
from django.utils import timezone
@ -473,8 +473,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
crawl_url_sets: dict[str, set[str]] = {}
dirty_crawl_ids: set[str] = set()
runtime_config = get_config()
archive_dir = runtime_config.ARCHIVE_DIR
archive_dir = CONSTANTS.ARCHIVE_DIR
if not archive_dir.exists():
return stats

7
archivebox/cli/archivebox_version.py
View File

@ -32,7 +32,6 @@ def _format_binary_abspath(
(lib_dir, "LIB_DIR/"),
(Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
(personas_dir, "PERSONAS_DIR/"),
(Path(os.environ.get("PERSONAS_DIR", "")), "PERSONAS_DIR/") if os.environ.get("PERSONAS_DIR") else (Path(), ""),
(home, "~/"),
)
@ -155,7 +154,7 @@ def version(
)
prnt()
if not (os.access(config.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
PANEL_TEXT = "\n".join(
(
"",
@ -219,7 +218,7 @@ def version(
installed.abspath,
pwd=Path.cwd(),
lib_dir=config.LIB_DIR,
personas_dir=config.PERSONAS_DIR,
personas_dir=CONSTANTS.PERSONAS_DIR,
home=Path.home(),
)
if compact_paths
@ -275,7 +274,7 @@ def version(
prnt(f" [red]Error getting code locations: {e}[/red]")
prnt()
if os.access(config.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
try:
for name, path in get_data_locations().items():

3
archivebox/config/collection.py
View File

@ -3,6 +3,7 @@ __package__ = "archivebox.config"
import io
import json
import os
from collections.abc import Mapping
from typing import Any
from archivebox.config.constants import CONSTANTS
@ -54,7 +55,7 @@ def _coerce_to_str_dict(config: Any) -> dict[str, str]:
"""
if not config:
return {}
if not hasattr(config, "items"):
if not isinstance(config, Mapping):
return {}
flat: dict[str, str] = {}
for key, value in config.items():

538
archivebox/config/common.py
View File

@ -8,11 +8,13 @@ import re
import secrets
import sys
import shutil
import inspect
from functools import lru_cache
from collections.abc import Mapping
from datetime import timedelta
from typing import Any, ClassVar, cast
from pathlib import Path
from urllib.parse import quote
from rich.console import Console
from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
@ -30,6 +32,7 @@ from .permissions import IN_DOCKER
ConfigOverrides = Mapping[str, object]
ConfigPayload = dict[str, object]
PluginSchemaDocuments = dict[str, dict[str, Any]]
LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
###################### Config ##########################
@ -64,6 +67,16 @@ def permissions_from_legacy_public_flags(raw_config: Mapping[str, object]) -> st
return None
def resolve_delete_after_config_value(*configs: Mapping[str, Any] | None) -> str:
for config in configs:
if config is None:
continue
value = config.get("DELETE_AFTER")
if value:
return str(value)
return "0"
_SENSITIVE_CONFIG_KEY_NEEDLES = ("TOKEN", "SECRET", "API_KEY", "APIKEY", "PASSWORD")
SENSITIVE_CONFIG_VALUE_REDACTED = "********"
_SCOPE_CRAWL_FROZEN = "crawl_frozen"
@ -118,8 +131,15 @@ def redact_sensitive_config(config: Mapping[str, Any] | None) -> dict[str, Any]:
return redacted
def normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | None) -> dict[str, Any]:
"""Return a JSON-safe config dict suitable for storage or event payloads."""
def normalize_runtime_config(
config: BaseConfigSet | Mapping[str, Any] | str | None,
*,
only_crawl_execution: bool = False,
exclude_runtime_derived: bool = False,
exclude_crawl_execution: bool = False,
json_safe: bool = True,
) -> dict[str, Any]:
"""Return config filtered for runtime/frozen usage, optionally JSON-safe."""
if config is None:
return {}
if isinstance(config, BaseConfigSet):
@ -128,21 +148,41 @@ def normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | N
config = json.loads(config)
else:
config = dict(config)
return {key: value for key, value in json.loads(json.dumps(config, default=str)).items() if value is not None}
runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys() if exclude_runtime_derived else frozenset()
filtered = {
key: value
for key, value in config.items()
if (
value is not None
and (not only_crawl_execution or ArchiveBoxConfig.scope_for_key(str(key)) == _SCOPE_CRAWL_EXECUTION)
and (not exclude_runtime_derived or str(key) not in runtime_derived_keys)
and (not exclude_crawl_execution or ArchiveBoxConfig.scope_for_key(str(key)) != _SCOPE_CRAWL_EXECUTION)
)
}
if not json_safe:
return filtered
return {key: value for key, value in json.loads(json.dumps(filtered, default=str)).items() if value is not None}
def build_crawl_config_snapshot(
*,
user: Any = None,
persona: Any = None,
overrides: Mapping[str, Any] | None = None,
base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
) -> dict[str, Any]:
"""Build the frozen runtime config stored on Crawl.config at creation time."""
effective = get_config(user=user, persona=persona, base_config=base_config)
frozen = effective.for_crawl_frozen()
"""Build the frozen crawl config stored on Crawl.config at creation time."""
explicit_overrides = set(overrides or {})
plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields)
effective = get_config(persona=persona, base_config=base_config)
frozen = effective.for_crawl_frozen(persona=persona)
if overrides:
frozen = get_config(base_config=frozen, overrides=overrides, include_machine=False).for_crawl_frozen()
resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False)
resolved_payload = normalize_runtime_config(resolved)
frozen = resolved.for_crawl_frozen(persona=persona)
for key in plugin_owned_keys & explicit_overrides:
if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload:
frozen[key] = resolved_payload[key]
return frozen
@ -187,11 +227,6 @@ class StorageConfig(BaseConfigSet):
toml_section_header: str = "STORAGE_CONFIG"
_scope: str = PrivateAttr(default=_SCOPE_SERVER)
# ARCHIVE_DIR / USERS_DIR are resolved dynamically via get_config().
ARCHIVE_DIR: Path = Field(default=CONSTANTS.ARCHIVE_DIR)
USERS_DIR: Path = Field(default=CONSTANTS.USERS_DIR)
PERSONAS_DIR: Path = Field(default=CONSTANTS.PERSONAS_DIR)
# TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
# must be a short path due to unix path length restrictions for socket files (<100 chars)
# must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
@ -206,10 +241,6 @@ class StorageConfig(BaseConfigSet):
# Runtime lookup must use provider-specific paths under LIB_DIR instead.
LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
# CUSTOM_TEMPLATES_DIR allows users to override default templates
# defaults to DATA_DIR / 'user_templates' but can be configured
CUSTOM_TEMPLATES_DIR: Path = Field(default=CONSTANTS.CUSTOM_TEMPLATES_DIR)
OUTPUT_PERMISSIONS: str = Field(default="644")
ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
ALLOW_NO_UNIX_SOCKETS: bool = Field(default=False, alias="ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS")
@ -244,8 +275,6 @@ class ServerConfig(BaseConfigSet):
FOOTER_INFO: str = Field(
default="Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.",
)
# CUSTOM_TEMPLATES_DIR: Path = Field(default=None) # this is now a constant
PUBLIC_INDEX: bool = Field(default=True)
PUBLIC_ADD_VIEW: bool = Field(default=False)
@ -355,7 +384,7 @@ class ArchivingConfig(BaseConfigSet):
URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
DEFAULT_PERSONA: str = Field(default="Default")
DEFAULT_PERSONA: str = Field(default="Default", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
PERMISSIONS: str = Field(
default="public",
description="Snapshot visibility: public lists and serves content, unlisted serves direct links only, private requires admin login.",
@ -469,9 +498,7 @@ def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
schemas: PluginSchemaDocuments = {}
if BASE_CONFIG_PATH.exists():
schemas["base"] = {
"properties": json.loads(BASE_CONFIG_PATH.read_text()).get("properties", {}),
}
schemas["base"] = json.loads(BASE_CONFIG_PATH.read_text())
schemas.update(discover_plugin_configs())
return schemas
@ -524,10 +551,6 @@ class ArchiveBoxBaseConfig(
populate_by_name=True,
)
DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
ABX_RUNTIME: str = Field(default="archivebox", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
CRAWL_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
SNAP_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
computed_config_keys: ClassVar[tuple[str, ...]] = COMPUTED_CONFIG_KEYS
@classmethod
@ -566,64 +589,146 @@ class ArchiveBoxBaseConfig(
@classmethod
def _plugin_field_scope(cls, key: str) -> str | None:
scope = None
for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
properties = schema.get("properties") if isinstance(schema, dict) else None
if not isinstance(properties, dict) or key not in properties:
continue
prop_schema = properties.get(key) or {}
if isinstance(prop_schema, Mapping) and prop_schema.get("x-scope"):
return str(prop_schema["x-scope"])
if str(plugin_name).startswith("search_backend_"):
return _SCOPE_SERVER
return _SCOPE_CRAWL_FROZEN
return None
scope = str(prop_schema["x-scope"])
elif scope is None:
if str(plugin_name).startswith("search_backend_"):
scope = _SCOPE_SERVER
else:
scope = _SCOPE_CRAWL_FROZEN
return scope
@classmethod
@lru_cache(maxsize=None)
def scope_for_key(cls, key: str) -> str:
for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
if str(plugin_name).startswith("search_backend_"):
continue
properties = schema.get("properties") if isinstance(schema, dict) else None
if isinstance(properties, dict) and key == f"{str(plugin_name).upper()}_ENABLED" and key in properties:
return _SCOPE_CRAWL_EXECUTION
if key.endswith("_BINARY"):
return _SCOPE_CRAWL_EXECUTION
return cls._core_field_scope(key) or cls._plugin_field_scope(key) or _SCOPE_SERVER
def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
allowed_scopes = {_SCOPE_CRAWL_FROZEN}
if include_execution:
allowed_scopes.add(_SCOPE_CRAWL_EXECUTION)
return {key: value for key, value in normalize_runtime_config(self).items() if type(self).scope_for_key(key) in allowed_scopes}
@classmethod
@lru_cache(maxsize=1)
def _scope_by_key(cls) -> dict[str, str]:
return {key: cls.scope_for_key(key) for key in cls.model_fields}
def for_crawl_execution(self) -> dict[str, Any]:
"""Config safe to pass to crawl/snapshot hook execution."""
@classmethod
@lru_cache(maxsize=1)
def _crawl_frozen_keys(cls) -> frozenset[str]:
return frozenset(key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_FROZEN)
@classmethod
@lru_cache(maxsize=1)
def _crawl_runtime_keys(cls) -> frozenset[str]:
return frozenset(key for key, scope in cls._scope_by_key().items() if scope in {_SCOPE_CRAWL_FROZEN, _SCOPE_CRAWL_EXECUTION})
@classmethod
@lru_cache(maxsize=1)
def runtime_derived_config_keys(cls) -> frozenset[str]:
runtime_derived_keys = {
"ABX_INSTALL_CACHE",
"ACTIVE_PERSONA",
"CHROME_DOWNLOADS_DIR",
"CHROME_USER_DATA_DIR",
"DEFAULT_PERSONA",
"EXTRA_CONTEXT",
}
return frozenset(
key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_EXECUTION and key in runtime_derived_keys
)
def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
keys = type(self)._crawl_runtime_keys() if include_execution else type(self)._crawl_frozen_keys()
payload = self.model_dump(mode="json")
return {key: payload[key] for key in keys if payload.get(key) is not None}
def for_crawl(self) -> dict[str, Any]:
"""Config scoped to crawl execution, without runtime object overlays."""
return self._scoped_config(include_execution=True)
def for_crawl_frozen(self) -> dict[str, Any]:
def for_crawl_frozen(self, *, persona: Any = None) -> dict[str, Any]:
"""Config safe to persist permanently on Crawl.config."""
return self._scoped_config(include_execution=False)
frozen = self._scoped_config(include_execution=False)
if persona is not None:
persona_config = dict(persona.config or {})
scope_by_key = type(self)._scope_by_key()
for key in persona.get_derived_config():
if key not in persona_config and scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
frozen.pop(key, None)
return frozen
def for_crawl_runtime(
self,
*,
crawl: Any = None,
snapshot: Any = None,
persona: Any = None,
runtime_overrides: Mapping[str, Any] | None = None,
extra_context: Mapping[str, Any] | None = None,
crawl_output_dir: Any = None,
snapshot_output_dir: Any = None,
) -> dict[str, Any]:
"""Config payload safe to pass to crawl/snapshot hook execution."""
config = self.for_crawl()
scope_by_key = type(self)._scope_by_key()
model_fields = type(self).model_fields
for key in type(self).runtime_derived_config_keys():
config.pop(key, None)
if persona is not None:
for key, value in persona.get_derived_config().items():
if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
config[key] = value
if crawl is not None:
for key, value in dict(crawl.config or {}).items():
if key in model_fields and scope_by_key.get(key) != _SCOPE_CRAWL_EXECUTION:
config[key] = value
config["CRAWL_DIR"] = str(crawl_output_dir if crawl_output_dir is not None else crawl.output_dir)
if snapshot is not None:
for key, value in dict(snapshot.config or {}).items():
if key in model_fields and scope_by_key.get(key) != _SCOPE_CRAWL_EXECUTION:
config[key] = value
config["SNAP_DIR"] = str(snapshot_output_dir if snapshot_output_dir is not None else snapshot.output_dir)
if runtime_overrides:
config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
if extra_context:
context: dict[str, Any] = {}
if config.get("EXTRA_CONTEXT"):
parsed_extra_context = json.loads(str(config["EXTRA_CONTEXT"]))
if not isinstance(parsed_extra_context, dict):
raise TypeError("EXTRA_CONTEXT must decode to an object")
context = parsed_extra_context
context.update(dict(extra_context))
config["EXTRA_CONTEXT"] = json.dumps(context, separators=(",", ":"), sort_keys=True)
_derive_plugin_enabled_config(config)
return config
@model_validator(mode="after")
def resolve_runtime_paths(self):
self.DATA_DIR = self.DATA_DIR.expanduser().resolve()
archive_dir = self.ARCHIVE_DIR.expanduser()
if archive_dir == (CONSTANTS.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME) and self.DATA_DIR != CONSTANTS.DATA_DIR:
archive_dir = self.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME
if not archive_dir.is_absolute():
archive_dir = self.DATA_DIR / archive_dir
self.ARCHIVE_DIR = archive_dir.resolve()
users_dir = self.USERS_DIR.expanduser()
if users_dir == (CONSTANTS.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME):
users_dir = self.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME
if not users_dir.is_absolute():
users_dir = self.ARCHIVE_DIR / users_dir
self.USERS_DIR = users_dir.resolve()
lib_dir = self.LIB_DIR.expanduser()
if not lib_dir.is_absolute():
lib_dir = self.DATA_DIR / lib_dir
lib_dir = CONSTANTS.DATA_DIR / lib_dir
self.LIB_DIR = lib_dir.resolve()
lib_bin_dir = self.LIB_BIN_DIR.expanduser()
if lib_bin_dir == CONSTANTS.DEFAULT_LIB_BIN_DIR and self.LIB_DIR != CONSTANTS.DEFAULT_LIB_DIR:
lib_bin_dir = self.LIB_DIR / "bin"
elif not lib_bin_dir.is_absolute():
lib_bin_dir = self.DATA_DIR / lib_bin_dir
lib_bin_dir = CONSTANTS.DATA_DIR / lib_bin_dir
self.LIB_BIN_DIR = lib_bin_dir.resolve()
return self
@ -649,15 +754,204 @@ PLUGIN_CONFIG_SCHEMAS = _discover_plugin_config_schemas()
ArchiveBoxConfig = _build_archivebox_config_model(PLUGIN_CONFIG_SCHEMAS)
def _normalize_plugins_config_value(value: Any) -> set[str]:
if value is None:
return set()
if isinstance(value, str):
raw = value.strip()
if not raw:
return set()
if raw.startswith("["):
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
parsed = None
if isinstance(parsed, list):
return {str(plugin).strip().lower() for plugin in parsed if str(plugin).strip()}
return {plugin.strip().lower() for plugin in raw.split(",") if plugin.strip()}
if isinstance(value, (list, tuple, set)):
return {str(plugin).strip().lower() for plugin in value if str(plugin).strip()}
normalized = str(value).strip().lower()
return {normalized} if normalized else set()
@lru_cache(maxsize=1)
def _plugin_enabled_config_keys() -> dict[str, str]:
enabled_keys: dict[str, str] = {}
for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
properties = schema.get("properties") if isinstance(schema, dict) else None
if not isinstance(properties, dict):
continue
enabled_key = f"{str(plugin_name).upper()}_ENABLED"
if enabled_key in properties and ArchiveBoxConfig.scope_for_key(enabled_key) == _SCOPE_CRAWL_EXECUTION:
enabled_keys[str(plugin_name).lower()] = enabled_key
return enabled_keys
def _plugins_with_required_plugins(plugin_names: set[str]) -> set[str]:
selected = set(plugin_names)
pending = list(selected)
while pending:
plugin_name = pending.pop()
schema = PLUGIN_CONFIG_SCHEMAS.get(plugin_name, {})
required_plugins = schema.get("required_plugins") if isinstance(schema, dict) else None
if not isinstance(required_plugins, list):
continue
for required_plugin in required_plugins:
required_plugin_name = str(required_plugin).strip().lower()
if required_plugin_name and required_plugin_name not in selected:
selected.add(required_plugin_name)
pending.append(required_plugin_name)
return selected
def _derive_plugin_enabled_config(config: dict[str, Any]) -> None:
plugin_names = _normalize_plugins_config_value(config.get("PLUGINS"))
if not plugin_names:
return
selected_plugins = _plugins_with_required_plugins(plugin_names)
for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
config[enabled_key] = plugin_name in selected_plugins
def get_live_config_url(key: str) -> str:
return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
@lru_cache(maxsize=1)
def config_field_metadata() -> dict[str, dict[str, Any]]:
"""Return one centralized metadata map for core and plugin config fields."""
metadata: dict[str, dict[str, Any]] = {}
for key, field in ArchiveBoxConfig.model_fields.items():
if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_EXECUTION or key in ArchiveBoxConfig.computed_config_keys:
continue
default = field.default
try:
json.dumps(default)
except TypeError:
default = str(default)
metadata[key] = {
"plugin": "archivebox",
"section": find_config_section(key),
"type": config_field_type(key),
"default": default,
"description": field.description or "",
"scope": ArchiveBoxConfig.scope_for_key(key),
"sensitive": is_sensitive_config_key(key),
}
for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
properties = schema.get("properties") if isinstance(schema, dict) else None
if not isinstance(properties, dict):
continue
for key, prop in properties.items():
if not isinstance(prop, Mapping):
continue
if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_EXECUTION:
continue
metadata[key] = {
**metadata.get(key, {}),
"plugin": plugin_name,
"section": "PLUGINS",
"type": prop.get("type", metadata.get(key, {}).get("type", "string")),
"default": prop.get("default", metadata.get(key, {}).get("default", "")),
"description": prop.get("description", metadata.get(key, {}).get("description", "")),
"scope": ArchiveBoxConfig.scope_for_key(key),
"sensitive": bool(prop.get("x-sensitive")) or is_sensitive_config_key(key),
"schema": dict(prop),
}
return metadata
def find_config_section(key: str) -> str:
from archivebox.config import CONSTANTS_CONFIG
if key in CONSTANTS_CONFIG:
return "CONSTANT"
for section_id, section in get_all_configs().items():
if key in type(section).model_fields:
return section_id
if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
return "PLUGINS"
return "DYNAMIC"
def find_config_default(key: str) -> str:
from archivebox.config import CONSTANTS_CONFIG
if key in CONSTANTS_CONFIG:
return str(CONSTANTS_CONFIG[key])
field = ArchiveBoxConfig.model_fields.get(key)
if field is None:
return ""
default_val = field.default
if callable(default_val):
default_val = inspect.getsource(default_val).split("lambda", 1)[-1].split(":", 1)[-1].replace("\n", " ").strip()
if default_val.count(")") > default_val.count("("):
default_val = default_val[:-1]
else:
default_val = str(default_val)
return default_val
def config_field_type(key: str) -> str:
field = ArchiveBoxConfig.model_fields.get(key)
if field is None:
return "str"
annotation = field.annotation
try:
return annotation.__name__
except AttributeError:
return str(annotation)
def find_config_type(key: str) -> str:
return config_field_type(key)
def find_config_source(key: str, merged_config: Mapping[str, Any]) -> str:
"""Determine where a config value comes from."""
try:
from archivebox.machine.models import Machine
machine = Machine.current()
if machine.config and key in machine.config:
return "Machine"
except Exception:
pass
if key in os.environ:
return "Environment"
file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
if key in file_config:
return "File"
if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
return "Plugin Default"
return "Default"
def get_request_config(request: Any, *, resolve_plugins: bool = False) -> ArchiveBoxBaseConfig:
"""Return the per-request ArchiveBox config, upgrading to plugin resolution if needed."""
request_state = request.__dict__
request_config = request_state.get("archivebox_config")
request_config_resolves_plugins = bool(request_state.get("_archivebox_config_resolves_plugins", False))
if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
request_config = get_config(resolve_plugins=resolve_plugins)
request.archivebox_config = request_config
request._archivebox_config_resolves_plugins = resolve_plugins
return request_config
def get_config(
defaults: ConfigOverrides | None = None,
overrides: ConfigOverrides | None = None,
base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
persona: Any = None,
user: Any = None,
crawl: Any = None,
snapshot: Any = None,
archiveresult: Any = None,
machine: Any = None,
include_machine: bool = True,
resolve_plugins: bool = True,
@ -666,28 +960,20 @@ def get_config(
"""
Get merged config from all sources.
Priority (highest to lowest):
1. Explicit overrides
2. Per-ArchiveResult config
3. Per-snapshot config and output path
4. Frozen per-crawl config and output path
5. Per-user config (only when resolving outside a crawl)
6. Per-persona derived config (only when resolving outside a crawl)
7. Current machine derived config (only when resolving outside a crawl)
8. Environment variables (only when resolving outside a crawl)
9. Config file (ArchiveBox.conf, only when resolving outside a crawl)
10. Plugin schema defaults
11. Core config defaults
"""
if snapshot is None and archiveresult is not None:
snapshot = archiveresult.snapshot
Defaults are hydrated by pydantic from core/plugin defaults,
ArchiveBox.conf, and environment variables. Persisted Machine/Persona
values then apply for live crawl-execution scope, while Crawl/Snapshot
rows apply their frozen crawl-scope values. Explicit overrides win last.
Crawl-execution config is not persisted on Crawl.config. It is rederived
from current Machine/Persona state and hydrated process defaults each time.
"""
if crawl is None and snapshot is not None:
crawl = snapshot.crawl
crawl_config_base = crawl is not None and base_config is None
if include_machine and machine is None and not crawl_config_base:
if include_machine and machine is None:
try:
from django.apps import apps
@ -698,58 +984,62 @@ def get_config(
except Exception:
machine = None
if persona is None and crawl is not None and not crawl_config_base:
if persona is None and crawl is not None:
persona = crawl.resolve_persona()
config_data: ConfigPayload = dict(defaults or {})
base_config_payload: ConfigPayload = {}
if crawl_config_base:
config_data.update(dict(crawl.config or {}))
config_data.update(
normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
)
config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
elif base_config is not None:
if isinstance(base_config, ArchiveBoxBaseConfig):
base_config_payload.update(base_config.model_dump(mode="json"))
else:
base_config_payload.update(dict(base_config))
config_data.update(base_config_payload)
config_data.update(normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False))
else:
config_data.update(ArchiveBoxConfig().model_dump(mode="json"))
config_data.update(
normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
)
legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
if legacy_permissions:
config_data["PERMISSIONS"] = legacy_permissions
scope_overrides: ConfigPayload = {}
if not crawl_config_base:
if include_machine and machine is not None and machine.config:
from archivebox.machine.models import _sanitize_machine_config
if include_machine and machine is not None and machine.config:
from archivebox.machine.models import _sanitize_machine_config
scope_overrides.update(_sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")))
scope_overrides.update(
normalize_runtime_config(
_sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")),
only_crawl_execution=crawl_config_base,
exclude_runtime_derived=True,
json_safe=False,
),
)
if persona is not None:
scope_overrides.update(persona.get_derived_config())
user_config = getattr(user, "config", None)
if user_config:
scope_overrides.update(user_config)
if persona is not None:
scope_overrides.update(
normalize_runtime_config(
persona.get_derived_config(),
only_crawl_execution=crawl_config_base,
exclude_runtime_derived=not crawl_config_base,
json_safe=False,
),
)
if crawl is not None and crawl.config and not crawl_config_base:
scope_overrides.update(crawl.config)
if crawl is not None:
if not overrides or "CRAWL_DIR" not in overrides:
scope_overrides["CRAWL_DIR"] = crawl.output_dir
scope_overrides.update(normalize_runtime_config(crawl.config, exclude_crawl_execution=True, json_safe=False))
if snapshot is not None and snapshot.config:
scope_overrides.update(snapshot.config)
if snapshot is not None:
scope_overrides["SNAP_DIR"] = snapshot.output_dir
if archiveresult is not None and archiveresult.config:
scope_overrides.update(archiveresult.config)
scope_overrides.update(normalize_runtime_config(snapshot.config, exclude_crawl_execution=True, json_safe=False))
if overrides:
scope_overrides.update(overrides)
scope_overrides.update(normalize_runtime_config(overrides, exclude_crawl_execution=True, json_safe=False))
legacy_scope_permissions = permissions_from_legacy_public_flags(scope_overrides)
if legacy_scope_permissions:
@ -759,27 +1049,47 @@ def get_config(
config_data.update(archivebox_scope_overrides)
if resolve_plugins:
plugin_schemas = {
plugin_name: schema.get("properties", {}) for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)
}
plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
plugin_user_config = _plugin_user_config(scope_overrides)
plugin_user_config = _plugin_user_config(
{
**normalize_runtime_config(config_data, only_crawl_execution=True, json_safe=False),
**scope_overrides,
},
)
if not crawl_config_base:
plugin_user_config = {**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **plugin_user_config}
plugin_user_config = {
**normalize_runtime_config(
BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE),
exclude_runtime_derived=True,
json_safe=False,
),
**plugin_user_config,
}
plugin_sections = resolve_plugin_configs(
plugin_schemas,
global_config=plugin_global_config,
user_config=plugin_user_config,
environ={},
)
for plugin_config in plugin_sections.values():
config_data.update(plugin_config)
for key, value in plugin_config.items():
if key in ArchiveBoxBaseConfig.model_fields and key not in archivebox_scope_overrides and key not in base_config_payload:
continue
config_data[key] = value
if base_config_payload:
config_data.update({key: value for key, value in base_config_payload.items() if key in _archivebox_config_input_names()})
config_data.update(
{
key: value
for key, value in normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False).items()
if key in _archivebox_config_input_names()
},
)
if crawl_config_base:
config_data.update(dict(crawl.config or {}))
config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
config_data.update(archivebox_scope_overrides)
config_data["ABX_RUNTIME"] = "archivebox"
_derive_plugin_enabled_config(config_data)
# Decode JSON-encoded complex values (dict/list fields) that came from
# string-only sources before validation. ``IniConfigSettingsSource`` does
@ -807,7 +1117,7 @@ def get_config(
config = ArchiveBoxConfig.model_validate(config_data)
if redact_sensitive:
for key in type(config).model_fields:
value = getattr(config, key, None)
value = config[key]
if is_sensitive_config_key(key) and value not in (None, ""):
setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
os.environ["LIB_DIR"] = str(config.LIB_DIR)

34
archivebox/config/paths.py
View File

@ -19,7 +19,7 @@ if TYPE_CHECKING:
#############################################################################################
PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent # archivebox source code dir
DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve() # archivebox user data dir
DATA_DIR: Path = Path(os.getcwd()).resolve() # archivebox user data dir
def _env_path(key: str, default: Path) -> Path:
@ -29,8 +29,8 @@ def _env_path(key: str, default: Path) -> Path:
return path.resolve()
ARCHIVE_DIR: Path = _env_path("ARCHIVE_DIR", DATA_DIR / "archive") # archivebox snapshot data dir
USERS_DIR: Path = _env_path("USERS_DIR", ARCHIVE_DIR / "users") # archivebox user-scoped crawl/snapshot data dir
ARCHIVE_DIR: Path = DATA_DIR / "archive" # archivebox snapshot data dir
USERS_DIR: Path = ARCHIVE_DIR / "users" # archivebox user-scoped crawl/snapshot data dir
IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
@ -290,20 +290,20 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
"is_mount": os.path.ismount(DATABASE_FILE.resolve()),
},
"ARCHIVE_DIR": {
"path": config.ARCHIVE_DIR.resolve(),
"path": CONSTANTS.ARCHIVE_DIR.resolve(),
"enabled": True,
"is_valid": os.path.isdir(config.ARCHIVE_DIR)
and os.access(config.ARCHIVE_DIR, os.R_OK)
and os.access(config.ARCHIVE_DIR, os.W_OK),
"is_mount": os.path.ismount(config.ARCHIVE_DIR.resolve()),
"is_valid": os.path.isdir(CONSTANTS.ARCHIVE_DIR)
and os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK)
and os.access(CONSTANTS.ARCHIVE_DIR, os.W_OK),
"is_mount": os.path.ismount(CONSTANTS.ARCHIVE_DIR.resolve()),
},
"USERS_DIR": {
"path": config.USERS_DIR.resolve(),
"enabled": os.path.isdir(config.USERS_DIR),
"is_valid": os.path.isdir(config.USERS_DIR)
and os.access(config.USERS_DIR, os.R_OK)
and os.access(config.USERS_DIR, os.W_OK),
"is_mount": os.path.ismount(config.USERS_DIR.resolve()),
"path": CONSTANTS.USERS_DIR.resolve(),
"enabled": os.path.isdir(CONSTANTS.USERS_DIR),
"is_valid": os.path.isdir(CONSTANTS.USERS_DIR)
and os.access(CONSTANTS.USERS_DIR, os.R_OK)
and os.access(CONSTANTS.USERS_DIR, os.W_OK),
"is_mount": os.path.ismount(CONSTANTS.USERS_DIR.resolve()),
},
"SOURCES_DIR": {
"path": CONSTANTS.SOURCES_DIR.resolve(),
@ -366,9 +366,9 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
"is_valid": os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK), # read + list
},
"CUSTOM_TEMPLATES_DIR": {
"path": config.CUSTOM_TEMPLATES_DIR.resolve(),
"enabled": os.path.isdir(config.CUSTOM_TEMPLATES_DIR),
"is_valid": os.path.isdir(config.CUSTOM_TEMPLATES_DIR) and os.access(config.CUSTOM_TEMPLATES_DIR, os.R_OK), # read
"path": CONSTANTS.CUSTOM_TEMPLATES_DIR.resolve(),
"enabled": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR),
"is_valid": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK), # read
},
"USER_PLUGINS_DIR": {
"path": CONSTANTS.USER_PLUGINS_DIR.resolve(),

2
archivebox/config/permissions.py
View File

@ -15,8 +15,6 @@ from contextlib import contextmanager
#############################################################################################
DATA_DIR = Path(os.getcwd())
if os.environ.get("DATA_DIR") and Path(os.environ["DATA_DIR"]).resolve() != DATA_DIR.resolve():
raise SystemExit(f"[X] DATA_DIR={os.environ['DATA_DIR']} must equal cwd={DATA_DIR}; cd into the data dir before running archivebox")
try:
DATA_DIR_STAT = DATA_DIR.stat()

10
archivebox/config/views.py
View File

@ -24,7 +24,7 @@ INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
def is_superuser(request: HttpRequest) -> bool:
return bool(getattr(request.user, "is_superuser", False))
return bool(request.user.is_superuser)
def format_parsed_datetime(value: object) -> str:
@ -44,7 +44,7 @@ def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | N
if binary is None or not binary.id:
return None
base_url = getattr(binary, "admin_change_url", None) or f"{INSTALLED_BINARIES_BASE_URL}{binary.id}/change/"
base_url = binary.admin_change_url
changelist_filters = urlencode({"q": name})
return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}"
@ -158,7 +158,7 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
for name in all_binary_names:
binary = db_binaries.get(name)
binary_is_valid = bool(binary and getattr(binary, "is_valid", getattr(binary, "abspath", None)))
binary_is_valid = bool(binary and binary.is_valid)
rows["Binary Name"].append(ItemLink(name, key=name))
@ -185,9 +185,9 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
"youtube-dl": "yt-dlp",
}.get(key, key)
db_binary = get_db_binaries_by_name().get(key)
binary_is_valid = bool(db_binary and getattr(db_binary, "is_valid", getattr(db_binary, "abspath", None)))
binary_is_valid = bool(db_binary and db_binary.is_valid)
if binary_is_valid:
binary_data = db_binary.to_json() if hasattr(db_binary, "to_json") else db_binary.__dict__
binary_data = db_binary.to_json()
section: SectionData = {
"name": key,
"description": mark_safe(render_binary_detail_description(key, binary_data, db_binary)),

59
archivebox/core/admin_archiveresults.py
View File

@ -19,8 +19,6 @@ from django.urls import reverse, resolve
from django.utils import timezone
from django.utils.text import smart_split
from archivebox.config import DATA_DIR
from archivebox.config.common import get_config
from archivebox.misc.paginators import AcceleratedPaginator
from archivebox.base_models.admin import BaseModelAdmin
from archivebox.plugins.discovery import get_plugin_icon
@ -35,7 +33,7 @@ from archivebox.core.models import ArchiveResult, Snapshot
def _get_replay_source_url(result: ArchiveResult) -> str:
process = result.process_record
return str(getattr(process, "url", None) or result.snapshot.url or "")
return str((process.url if process else None) or result.snapshot.url or "")
def build_abx_dl_display_command(result: ArchiveResult) -> str:
@ -51,8 +49,8 @@ def build_abx_dl_display_command(result: ArchiveResult) -> str:
def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
display_command = build_abx_dl_display_command(result)
process = getattr(result, "process", None)
env_items = env_to_shell_exports(getattr(process, "env", None) or {})
process = result.process
env_items = env_to_shell_exports(process.env if process else {})
if config is not None:
result.snapshot._runtime_config = config
snapshot_dir = shlex.quote(str(result.pwd or result.snapshot_dir))
@ -164,8 +162,8 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
cmd_attr = html.escape(replay_cmd, quote=True)
# Build output link - use embed_path() which checks output_files first
embed_path = result.embed_path() if hasattr(result, "embed_path") else None
snapshot_id = str(getattr(result, "snapshot_id", ""))
embed_path = result.embed_path()
snapshot_id = str(result.snapshot_id)
if embed_path and result.status == "succeeded":
output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
else:
@ -354,8 +352,7 @@ class ArchiveResultInline(admin.TabularInline):
def get_formset(self, request, obj=None, **kwargs):
formset = super().get_formset(request, obj, **kwargs)
snapshot = self.get_parent_object_from_request(request)
form_class = getattr(formset, "form", None)
base_fields = getattr(form_class, "base_fields", {})
base_fields = formset.form.base_fields
snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
# import ipdb; ipdb.set_trace()
@ -491,12 +488,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
def change_view(self, request, object_id, form_url="", extra_context=None):
self.request = request
request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
return super().change_view(request, object_id, form_url, extra_context)
def changelist_view(self, request, extra_context=None):
self.request = request
request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
saved_list_per_page = self.list_per_page
self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
try:
@ -573,20 +568,20 @@ class ArchiveResultAdmin(BaseModelAdmin):
return queryset.filter(reduce(and_, filters)).distinct(), True
def get_snapshot_view_url(self, result: ArchiveResult) -> str:
request = getattr(self, "request", None)
return build_snapshot_url(str(result.snapshot_id), request=request, config=getattr(request, "archivebox_config", None))
request = self.request
return build_snapshot_url(str(result.snapshot_id), request=request, config=request.archivebox_config)
def get_output_view_url(self, result: ArchiveResult) -> str:
request = getattr(self, "request", None)
config = getattr(request, "archivebox_config", None)
output_path = result.embed_path() if hasattr(result, "embed_path") else None
request = self.request
config = request.archivebox_config
output_path = result.embed_path()
if not output_path:
output_path = result.plugin or ""
return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config)
def get_output_files_url(self, result: ArchiveResult) -> str:
request = getattr(self, "request", None)
return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
request = self.request
return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=request.archivebox_config)}/?files=1"
def get_output_zip_url(self, result: ArchiveResult) -> str:
return f"{self.get_output_files_url(result)}&download=zip"
@ -612,10 +607,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
)
def snapshot_info(self, result):
snapshot_id = str(result.snapshot_id)
request = getattr(self, "request", None)
request = self.request
return format_html(
'<a href="{}"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
snapshot_id[:8],
result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"),
result.snapshot.url[:128],
@ -687,9 +682,9 @@ class ArchiveResultAdmin(BaseModelAdmin):
@admin.display(description="Command")
def cmd_str(self, result):
request = getattr(self, "request", None)
request = self.request
display_cmd = build_abx_dl_display_command(result)
replay_cmd = build_abx_dl_replay_command(result, config=getattr(request, "archivebox_config", None))
replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
return format_html(
"""
<div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
@ -710,10 +705,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
)
def output_display(self, result):
request = getattr(self, "request", None)
config = getattr(request, "archivebox_config", None)
request = self.request
config = request.archivebox_config
# Determine output link path - use embed_path() which checks output_files
embed_path = result.embed_path() if hasattr(result, "embed_path") else None
embed_path = result.embed_path()
output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
snapshot_id = str(result.snapshot_id)
return format_html(
@ -728,12 +723,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
if not output_text:
return "-"
request = getattr(self, "request", None)
live_path = result.embed_path() if hasattr(result, "embed_path") else None
request = self.request
live_path = result.embed_path()
if live_path:
return format_html(
'<a href="{}" title="{}"><code>{}</code></a>',
build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=getattr(request, "archivebox_config", None)),
build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=request.archivebox_config),
output_text,
output_text,
)
@ -785,18 +780,18 @@ class ArchiveResultAdmin(BaseModelAdmin):
)
def output_summary(self, result):
snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1]
snapshot_dir = Path(result.snapshot.output_dir)
output_html = format_html(
'<pre style="display: inline-block">{}</pre><br/>',
result.output_str_for_display(),
)
snapshot_id = str(result.snapshot_id)
request = getattr(self, "request", None)
request = self.request
output_html += format_html(
'<a href="{}#all">See result files ...</a><br/><pre><code>',
build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
)
embed_path = result.embed_path() if hasattr(result, "embed_path") else ""
embed_path = result.embed_path() or ""
path_from_embed = snapshot_dir / (embed_path or "")
output_html += format_html(
'<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>',

92
archivebox/core/admin_snapshots.py
View File

@ -200,7 +200,6 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
("failed", ">50% failed"),
("running", ">50% running"),
("pending", ">50% queued"),
("backoff", ">50% waiting to retry"),
("noresults", ">50% noresults"),
)
@ -216,7 +215,7 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
"succeeded": ArchiveResult.StatusChoices.SUCCEEDED,
"failed": ArchiveResult.StatusChoices.FAILED,
"running": ArchiveResult.StatusChoices.STARTED,
"pending": ArchiveResult.StatusChoices.QUEUED,
"pending": (ArchiveResult.StatusChoices.QUEUED, ArchiveResult.StatusChoices.BACKOFF),
"backoff": ArchiveResult.StatusChoices.BACKOFF,
"noresults": ArchiveResult.StatusChoices.NORESULTS,
}
@ -226,21 +225,28 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
# "succeeded" is overwhelmingly common in large collections.
# Scan Snapshots in admin order and do indexed per-snapshot
# probes so page 1 can stop after list_per_page matches.
return queryset.alias(
queryset = queryset.alias(
total_results=ArchiveResult.snapshot_count_expr(),
matching_results=ArchiveResult.snapshot_count_expr(status=status),
).filter(matching_results__gt=F("total_results") / 2)
queryset._archivebox_count_hint = "model_estimate"
queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
return queryset
# Rare statuses are faster status-first: use the
# (status, snapshot_id) index to find candidate snapshots.
return queryset.filter(pk__in=ArchiveResult.snapshot_ids_with_majority_status(status))
snapshot_ids = ArchiveResult.cached_snapshot_ids_with_majority_status(status)
queryset = queryset.filter(pk__in=snapshot_ids)
queryset._archivebox_count_hint = len(snapshot_ids)
queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
return queryset
return queryset
class SnapshotChangeList(SearchResultsChangeList):
def __init__(self, request, *args, **kwargs):
super().__init__(request, *args, **kwargs)
resolver_name = getattr(getattr(request, "resolver_match", None), "url_name", "")
resolver_name = request.resolver_match.url_name
self.embedded_changelist = request.GET.get("_embedded") == "crawl"
self.snapshot_is_grid_view = not self.embedded_changelist and (
resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
@ -323,9 +329,7 @@ class SnapshotAdminForm(forms.ModelForm):
# Handle tags_editor field
if commit:
instance.save()
save_m2m = getattr(self, "_save_m2m", None)
if callable(save_m2m):
save_m2m()
self._save_m2m()
# Parse and save tags from tags_editor
tags_str = self.cleaned_data.get("tags_editor", "")
@ -489,7 +493,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
return super().get_ordering(request)
def change_view(self, request, object_id, form_url="", extra_context=None):
request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
self.request = request
extra_context = extra_context or {}
extra_context["CONFIG"] = request.archivebox_config
snapshot = self.get_object(request, object_id)
@ -504,7 +508,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
def changelist_view(self, request, extra_context=None):
self.request = request
request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
saved_list_per_page = self.list_per_page
embedded_changelist = request.GET.get("_embedded") == "crawl"
if embedded_changelist:
@ -540,12 +543,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
return super().lookup_allowed(lookup, value, request=request)
def get_snapshot_view_url(self, obj: Snapshot) -> str:
request = getattr(self, "request", None)
return build_snapshot_url(str(obj.id), request=request, config=getattr(request, "archivebox_config", None))
request = self.request
return build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)
def get_snapshot_files_url(self, obj: Snapshot) -> str:
request = getattr(self, "request", None)
return f"{build_snapshot_url(str(obj.id), request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
request = self.request
return f"{build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)}/?files=1"
def get_snapshot_zip_url(self, obj: Snapshot) -> str:
return f"{self.get_snapshot_files_url(obj)}&download=zip"
@ -650,7 +653,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
ordering_fields = self._get_ordering_fields(request)
needs_files_sort = "files" in ordering_fields
needs_tags_sort = "tags_inline" in ordering_fields
is_change_view = getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_change"
is_change_view = request.resolver_match.url_name == "core_snapshot_change"
prefetches = ["tags"]
if is_change_view:
prefetches.append(
@ -702,7 +705,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
@admin.display(description="👁", ordering="permissions")
def permissions_badge(self, obj):
permissions = getattr(obj, "snapshot_permissions", None)
permissions = obj.__dict__.get("snapshot_permissions")
if permissions is None:
if obj.permissions:
permissions = obj.permissions
@ -872,8 +875,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
)
def status_info(self, obj):
request = getattr(self, "request", None)
config = getattr(request, "archivebox_config", None)
request = self.request
config = request.archivebox_config
favicon_url = build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)
return format_html(
"""
@ -890,16 +893,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
@admin.display(description="Archive Results")
def archiveresults_list(self, obj):
request = getattr(self, "request", None)
return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=getattr(request, "archivebox_config", None))
request = self.request
return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=request.archivebox_config)
@admin.display(
description="Title",
ordering="title",
)
def title_str(self, obj):
request = getattr(self, "request", None)
config = getattr(request, "archivebox_config", None)
request = self.request
config = request.archivebox_config
title_raw = (obj.title or "").strip()
url_raw = (obj.url or "").strip()
title_normalized = title_raw.lower()
@ -952,8 +955,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
return mark_safe(f'<span class="tags-inline-editor">{tags_html}</span>')
def _get_preview_data(self, obj):
request = getattr(self, "request", None)
config = getattr(request, "archivebox_config", None)
request = self.request
config = request.archivebox_config
results = self._get_prefetched_results(obj)
if results is not None:
has_screenshot = any(r.plugin == "screenshot" for r in results)
@ -1005,8 +1008,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
if not preview:
return ""
request = getattr(self, "request", None)
config = getattr(request, "archivebox_config", None)
request = self.request
config = request.archivebox_config
favicon_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
fallback_list = ",".join([build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)])
onerror_js = (
@ -1044,8 +1047,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
@admin.display(description=" ", empty_value="")
def snapshot_summary(self, obj):
request = getattr(self, "request", None)
config = getattr(request, "archivebox_config", None)
request = self.request
config = request.archivebox_config
preview = self._get_preview_data(obj)
stats = self._get_progress_stats(obj)
archive_size = stats["output_size"] or 0
@ -1109,8 +1112,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
)
visible_results = sorted_results[:14]
output = []
request = getattr(self, "request", None)
config = getattr(request, "archivebox_config", None)
request = self.request
config = request.archivebox_config
for result in visible_results:
icon = mark_safe(get_plugin_icon(result.plugin))
if not icon.strip():
@ -1141,8 +1144,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
# ordering='archiveresult_count'
)
def size(self, obj):
request = getattr(self, "request", None)
config = getattr(request, "archivebox_config", None)
request = self.request
config = request.archivebox_config
archive_size = self._get_progress_stats(obj)["output_size"] or 0
if archive_size:
size_txt = printable_filesize(archive_size)
@ -1256,7 +1259,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
)
def _get_progress_stats(self, obj):
cached_stats = getattr(obj, "_admin_progress_stats", None)
cached_stats = obj.__dict__.get("_admin_progress_stats")
if cached_stats is not None:
return cached_stats
@ -1302,31 +1305,31 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
def _get_prefetched_results(self, obj):
if "_admin_archiveresults" in obj.__dict__:
return obj.__dict__["_admin_archiveresults"]
if hasattr(obj, "_prefetched_objects_cache") and "archiveresult_set" in obj._prefetched_objects_cache:
if "archiveresult_set" in obj.__dict__.get("_prefetched_objects_cache", {}):
return obj.archiveresult_set.all()
return None
def _get_expected_hook_total(self, obj) -> int:
try:
request = getattr(self, "request", None)
if getattr(getattr(request, "resolver_match", None), "url_name", "") in {"core_snapshot_changelist", "core_snapshot_change"}:
request = self.request
if request.resolver_match.url_name in {"core_snapshot_changelist", "core_snapshot_change"}:
return 0
crawl = getattr(obj, "crawl", None)
snapshot_config = getattr(obj, "config", None) or {}
crawl_config = getattr(crawl, "config", None) or {}
crawl = obj.crawl
snapshot_config = obj.config or {}
crawl_config = crawl.config or {}
has_scoped_config = bool(snapshot_config or crawl_config)
if request is not None and not has_scoped_config:
cached_total = getattr(request, "archivebox_expected_snapshot_hook_total", None)
cached_total = request.__dict__.get("archivebox_expected_snapshot_hook_total")
if cached_total is None:
config = getattr(request, "archivebox_config", None) or get_config()
config = request.archivebox_config
cached_total = len(discover_hooks("Snapshot", config=config))
request.archivebox_expected_snapshot_hook_total = cached_total
return cached_total
if request is not None:
scoped_cache = getattr(request, "archivebox_expected_snapshot_hook_totals_by_scope", None)
scoped_cache = request.__dict__.get("archivebox_expected_snapshot_hook_totals_by_scope")
if scoped_cache is None:
scoped_cache = {}
request.archivebox_expected_snapshot_hook_totals_by_scope = scoped_cache
@ -1346,8 +1349,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
return 0
def _get_prefetched_tags(self, obj):
if hasattr(obj, "_prefetched_objects_cache") and "tags" in obj._prefetched_objects_cache:
return list(obj._prefetched_objects_cache["tags"])
prefetched_cache = obj.__dict__.get("_prefetched_objects_cache", {})
if "tags" in prefetched_cache:
return list(prefetched_cache["tags"])
return None
def _get_ordering_fields(self, request):

2
archivebox/core/admin_tags.py
View File

@ -231,7 +231,7 @@ class TagAdmin(BaseModelAdmin):
@admin.display(description="Snapshots", ordering="num_snapshots")
def num_snapshots(self, tag: Tag):
count = getattr(tag, "num_snapshots", None)
count = tag.__dict__.get("num_snapshots")
if count is None:
count = tag.snapshot_set.count()
return format_html(

2
archivebox/core/admin_users.py
View File

@ -47,7 +47,7 @@ class CustomUserAdmin(UserAdmin):
def snapshot_count_badge(self, obj):
snapshots_url = f"/admin/core/snapshot/?created_by__id__exact={obj.pk}"
snapshot_count = getattr(obj, "snapshot_count", 0)
snapshot_count = obj.__dict__.get("snapshot_count", 0)
snapshot_label = "snapshot" if snapshot_count == 1 else "snapshots"
return format_html(
(

2
archivebox/core/forms.py
View File

@ -31,6 +31,8 @@ DEPTH_CHOICES = (
class AddLinkForm(PluginConfigFormMixin, forms.Form):
allow_crawl_execution_config_fields = False
# Basic fields
url = forms.CharField(
label="URLs",

8
archivebox/core/middleware.py
View File

@ -119,7 +119,7 @@ def CacheControlMiddleware(get_response):
if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
if not response.get("Cache-Control"):
config = getattr(request, "archivebox_config", None)
config = request.__dict__.get("archivebox_config")
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config
@ -136,7 +136,7 @@ def ServerSecurityModeMiddleware(get_response):
allowed_methods = {"GET", "HEAD", "OPTIONS"}
def middleware(request):
config = getattr(request, "archivebox_config", None)
config = request.__dict__.get("archivebox_config")
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config
@ -169,7 +169,7 @@ def HostRoutingMiddleware(get_response):
return get_response(request)
request_host = (request.get_host() or "").lower()
config = getattr(request, "archivebox_config", None)
config = request.__dict__.get("archivebox_config")
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config
@ -323,7 +323,7 @@ class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
header = "HTTP_REMOTE_USER"
def process_request(self, request):
config = getattr(request, "archivebox_config", None)
config = request.__dict__.get("archivebox_config")
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config

14
archivebox/core/migrations/0048_remove_archiveresult_config.py Normal file
View File

@ -0,0 +1,14 @@
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("core", "0047_archiveresult_status_snapshot_index"),
]
operations = [
migrations.RemoveField(
model_name="archiveresult",
name="config",
),
]

170
archivebox/core/models.py
View File

@ -63,6 +63,13 @@ if TYPE_CHECKING:
from archivebox.config.common import ArchiveBoxBaseConfig
class UngroupedSubquery(models.Subquery):
"""Scalar subquery that should not be copied into the outer GROUP BY."""
def get_group_by_cols(self):
return []
class Tag(ModelWithUUID):
# Keep AutoField for compatibility with main branch migrations
# Don't use UUIDField here - requires complex FK transformation
@ -203,7 +210,7 @@ class SnapshotQuerySet(models.QuerySet):
offset += chunk_size
return
unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if getattr(field, "unique", False))}
unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
if not any(field_name in unique_field_names for field_name in ordered_field_names):
offset = 0
while True:
@ -659,7 +666,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
self.sm.seal()
def get_delete_after_config_value(self):
return get_config(snapshot=self).DELETE_AFTER
from archivebox.config.common import resolve_delete_after_config_value
return resolve_delete_after_config_value(self.config, self.crawl.config)
@classmethod
def missing_delete_at_candidates(cls):
@ -818,7 +827,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
# work and crawl maintenance so SQLite commits before touching the disk.
transaction.on_commit(finish_snapshot_save)
migration_cleanup = getattr(self, "_pending_fs_migration_cleanup", None)
migration_cleanup = self.__dict__.get("_pending_fs_migration_cleanup")
if migration_cleanup:
old_dir, new_dir = migration_cleanup
transaction.on_commit(lambda: self._cleanup_old_migration_dir(old_dir, new_dir))
@ -890,9 +899,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
runtime_config = config or get_config()
if source_dir and current == target:
current_dir = self.get_storage_path_for_version(target, config=runtime_config)
current_dir = self.get_storage_path_for_version(target)
cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, target_dir=current_dir)
crawl_dir = self.crawl.output_dir_for_config(runtime_config)
crawl_dir = self.crawl.output_dir
old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex)))
if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink():
crawl_dir.parent.mkdir(parents=True, exist_ok=True)
@ -932,9 +941,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
def _fs_migrate_from_0_9_0_to_0_9_4(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None):
runtime_config = config or get_config()
target_dir = self.get_storage_path_for_version("0.9.4", config=runtime_config)
target_dir = self.get_storage_path_for_version("0.9.4")
cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir or self.output_dir, target_dir=target_dir, config=runtime_config)
crawl_dir = self.crawl.output_dir_for_config(runtime_config)
crawl_dir = self.crawl.output_dir
old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex)))
if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink():
crawl_dir.parent.mkdir(parents=True, exist_ok=True)
@ -956,8 +965,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
import filecmp
import shutil
old_dir = Path(source_dir) if source_dir else self.get_storage_path_for_version("0.8.0", config=config)
new_dir = Path(target_dir) if target_dir else self.get_storage_path_for_version("0.9.0", config=config)
old_dir = Path(source_dir) if source_dir else self.get_storage_path_for_version("0.8.0")
new_dir = Path(target_dir) if target_dir else self.get_storage_path_for_version("0.9.0")
if old_dir == new_dir:
return None
@ -1087,7 +1096,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
except Exception:
return "unknown"
def get_storage_path_for_version(self, version: str, config: "ArchiveBoxBaseConfig | None" = None) -> Path:
def get_storage_path_for_version(self, version: str) -> Path:
"""
Calculate storage path for specific filesystem version.
Centralizes path logic so it's reusable.
@ -1095,10 +1104,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
0.7.x/0.8.x: archive/{timestamp}
0.9.x: archive/users/{username}/snapshots/YYYYMMDD/{domain}/{uuid}/
"""
runtime_config = config or get_config()
if version in ("0.7.0", "0.8.0"):
return runtime_config.ARCHIVE_DIR / self.timestamp
return CONSTANTS.ARCHIVE_DIR / self.timestamp
elif version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
username = self.created_by.username
@ -1108,10 +1115,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
domain = self.extract_domain_from_url(self.url)
return runtime_config.USERS_DIR / username / CONSTANTS.SNAPSHOTS_DIR_NAME / date_str / domain / str(self.id)
return CONSTANTS.USERS_DIR / username / CONSTANTS.SNAPSHOTS_DIR_NAME / date_str / domain / str(self.id)
else:
# Unknown version - use current
return self.get_storage_path_for_version(self._fs_current_version(), config=runtime_config)
return self.get_storage_path_for_version(self._fs_current_version())
# =========================================================================
# Loading and Creation from Filesystem (Used by archivebox update ONLY)
@ -1702,7 +1709,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
f.write(json.dumps(process.to_json()) + "\n")
# Write ArchiveResult record
f.write(json.dumps(ar.to_json()) + "\n")
f.write(json.dumps(ar.to_json(snapshot_output_dir=output_dir)) + "\n")
os.replace(tmp_index_path, index_path)
def read_index_jsonl(self, output_dir: Path | None = None) -> dict:
@ -1978,7 +1985,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
@admin.display(description="Tags")
def tags_str(self, nocache=True) -> str | None:
calc_tags_str = lambda: ",".join(sorted(tag.name for tag in self.tags.all()))
prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
if "tags" in prefetched_cache:
return calc_tags_str()
cache_key = f"{self.pk}-tags"
@ -1988,12 +1995,12 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
"""Generate HTML icons showing which extractor plugins have succeeded for this snapshot"""
from django.utils.html import format_html
compact_icons = getattr(self, "_icons_compact", False)
compact_icons = self.__dict__.get("_icons_compact", False)
cache_key = f"result_icons:{self.pk}:{'compact' if compact_icons else 'full'}:{(self.downloaded_at or self.modified_at or self.created_at or self.bookmarked_at).timestamp()}"
def calc_icons():
if compact_icons and self.status == self.StatusChoices.STARTED:
progress_stats = getattr(self, "_icons_progress_stats", None) or self.get_progress_stats()
progress_stats = self.__dict__.get("_icons_progress_stats") or self.get_progress_stats()
total = int(progress_stats.get("total") or 0)
succeeded = int(progress_stats.get("succeeded") or 0)
failed = int(progress_stats.get("failed") or 0)
@ -2025,8 +2032,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
running,
)
precomputed_archive_results = getattr(self, "_icons_archive_results", None)
prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
precomputed_archive_results = self.__dict__.get("_icons_archive_results")
prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
if precomputed_archive_results is not None and compact_icons:
archive_results = {plugin: True for plugin in precomputed_archive_results}
elif "archiveresult_set" in prefetched_cache:
@ -2049,7 +2056,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a>'
# Get all plugins from hooks system (sorted by numeric prefix)
all_plugins = getattr(self, "_icons_plugin_names", None)
all_plugins = self.__dict__.get("_icons_plugin_names")
if all_plugins is None and not compact_icons:
all_plugins = [get_plugin_name(e) for e in get_plugins()]
elif all_plugins is None:
@ -2200,8 +2207,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
"""The filesystem path to the snapshot's output directory."""
import os
runtime_config = getattr(self, "_runtime_config", None) or get_config()
current_path = self.get_storage_path_for_version(self.fs_version, config=runtime_config)
current_path = self.get_storage_path_for_version(self.fs_version)
if current_path.exists():
return current_path
@ -2212,7 +2218,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
return hyphen_path
# Check for backwards-compat symlink
old_path = runtime_config.ARCHIVE_DIR / self.timestamp
old_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
if old_path.is_symlink():
link_target = Path(os.readlink(old_path))
return (old_path.parent / link_target).resolve() if not link_target.is_absolute() else link_target.resolve()
@ -2225,7 +2231,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
"""Ensure the legacy archive/<timestamp> path resolves to this snapshot."""
import os
legacy_path = get_config().ARCHIVE_DIR / self.timestamp
legacy_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
target = Path(self.get_storage_path_for_version(self._fs_current_version()))
if target == legacy_path:
@ -2301,8 +2307,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
username = "web"
crawl = getattr(self, "crawl", None)
if crawl and getattr(crawl, "created_by_id", None):
crawl = self.crawl if self.crawl_id else None
if crawl and crawl.created_by_id:
username = crawl.created_by.username
if username == "system":
username = "web"
@ -2326,7 +2332,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
output_dir = Path(self.output_dir).resolve()
try:
rel_users_path = output_dir.relative_to(get_config().USERS_DIR)
rel_users_path = output_dir.relative_to(CONSTANTS.USERS_DIR)
except Exception:
rel_users_path = None
@ -2836,7 +2842,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
@cached_property
def is_archived(self) -> bool:
cached_is_archived = getattr(self, "_is_archived_cached", None)
cached_is_archived = self.__dict__.get("_is_archived_cached")
if cached_is_archived is not None:
return bool(cached_is_archived)
@ -2890,10 +2896,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
@cached_property
def num_outputs(self) -> int:
if hasattr(self, "num_outputs_cached"):
return int(self.num_outputs_cached or 0)
if "num_outputs_cached" in self.__dict__:
return int(self.__dict__["num_outputs_cached"] or 0)
prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
if "archiveresult_set" in prefetched_cache:
return sum(1 for result in self.archiveresult_set.all() if result.status == "succeeded")
@ -2901,10 +2907,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
@cached_property
def num_failures(self) -> int:
if hasattr(self, "num_failures_cached"):
return int(self.num_failures_cached or 0)
if "num_failures_cached" in self.__dict__:
return int(self.__dict__["num_failures_cached"] or 0)
prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
if "archiveresult_set" in prefetched_cache:
return sum(1 for result in self.archiveresult_set.all() if result.status == "failed")
@ -3458,7 +3464,7 @@ class SnapshotMachine(BaseStateMachine):
# and the runner still needs to enqueue those child snapshots.
class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
class StatusChoices(models.TextChoices):
QUEUED = "queued", "Queued"
STARTED = "started", "Started"
@ -3523,7 +3529,17 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
qs = cls.objects.filter(snapshot_id=models.OuterRef(outer_ref))
if status is not None:
qs = qs.filter(status=status)
return qs.order_by().values("snapshot_id").annotate(count=models.Count("pk")).values("count")
return qs.order_by().values("snapshot_id").annotate(count=models.Count("*")).values("count")
@classmethod
def snapshot_half_count_subquery(cls, *, outer_ref: str = "snapshot_id") -> QuerySet:
return (
cls.objects.filter(snapshot_id=models.OuterRef(outer_ref))
.order_by()
.values("snapshot_id")
.annotate(half=models.Count("*") / models.Value(2))
.values("half")
)
@classmethod
def snapshot_count_expr(cls, *, status: str | None = None, outer_ref: str = "pk"):
@ -3539,28 +3555,50 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
return {status: qs.filter(status=status).count() for status in (statuses or cls.StatusChoices.values)}
@classmethod
def snapshot_ids_with_majority_status(cls, status: str) -> QuerySet:
def snapshot_ids_with_majority_status(cls, status: str | Iterable[str]) -> QuerySet:
"""Return Snapshot IDs where more than half of ArchiveResults have ``status``.
Start from ArchiveResult.status for every majority-status filter. The
``(status, snapshot_id)`` index keeps the plan predictable even when a
user's collection has an unusual status distribution.
"""
statuses = tuple(status) if not isinstance(status, str) else (status,)
total_half = UngroupedSubquery(cls.snapshot_half_count_subquery(outer_ref="snapshot_id"), output_field=models.IntegerField())
return (
cls.objects.filter(status=status)
cls.objects.filter(status__in=statuses)
.order_by()
.values("snapshot_id")
.annotate(
matching_results=models.Count("pk"),
total_results=models.Subquery(
cls.snapshot_count_subquery(outer_ref="snapshot_id"),
output_field=models.IntegerField(),
),
matching_results=models.Count("*"),
total_half=total_half,
)
.filter(matching_results__gt=models.F("total_results") / 2)
.filter(matching_results__gt=models.F("total_half"))
.values("snapshot_id")
)
@classmethod
def cached_snapshot_ids_with_majority_status(cls, status: str | Iterable[str], *, timeout: int = 60) -> tuple[str, ...]:
statuses = tuple(status) if not isinstance(status, str) else (status,)
cache_key = f"archivebox:archiveresult:majority_status:{':'.join(sorted(statuses))}"
cached_ids = cache.get(cache_key)
if cached_ids is not None:
return tuple(cached_ids)
snapshot_ids = tuple(
str(snapshot_id) for snapshot_id in cls.snapshot_ids_with_majority_status(statuses).values_list("snapshot_id", flat=True)
)
cache.set(cache_key, snapshot_ids, timeout=timeout)
return snapshot_ids
@classmethod
def clear_majority_status_cache(cls) -> None:
cache.delete_many(
[
*(f"archivebox:archiveresult:majority_status:{status}" for status in cls.StatusChoices.values),
f"archivebox:archiveresult:majority_status:{':'.join(sorted((cls.StatusChoices.BACKOFF, cls.StatusChoices.QUEUED)))}",
],
)
# UUID primary key (migrated from integer in 0029)
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
created_at = models.DateTimeField(default=timezone.now, db_index=True)
@ -3609,7 +3647,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
class Meta(
ModelWithDeleteAfter.Meta,
ModelWithOutputDir.Meta,
ModelWithConfig.Meta,
ModelWithNotes.Meta,
):
app_label = "core"
@ -3644,14 +3681,15 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
return "\n".join(self._format_output_line_for_display(line) for line in str(self.output_str or "").splitlines())
def get_delete_after_config_value(self):
return get_config(archiveresult=self).DELETE_AFTER
snapshot = self.snapshot
from archivebox.config.common import resolve_delete_after_config_value
return resolve_delete_after_config_value(snapshot.config, snapshot.crawl.config)
@classmethod
def missing_delete_at_candidates(cls):
return cls.objects.filter(delete_at__isnull=True).filter(
Q(config__has_key="DELETE_AFTER")
| Q(snapshot__config__has_key="DELETE_AFTER")
| Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
)
@property
@ -3659,12 +3697,21 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
"""Convenience property to access the user who created this archive result via its snapshot's crawl."""
return self.snapshot.crawl.created_by
def to_json(self) -> dict:
def to_json(self, *, snapshot_output_dir: Path | None = None) -> dict:
"""
Convert ArchiveResult model instance to a JSON-serializable dict.
"""
from archivebox.config import VERSION
process = self.process_record
pwd = (
process.pwd
if process and process.pwd
else str((snapshot_output_dir / self.plugin) if snapshot_output_dir is not None else self.output_dir)
)
cmd = process.cmd if process else []
cmd_version = process.cmd_version if process else ""
record = {
"type": "ArchiveResult",
"schema_version": VERSION,
@ -3686,13 +3733,12 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
record["output_size"] = self.output_size
if self.output_mimetypes:
record["output_mimetypes"] = self.output_mimetypes
if self.pwd:
record["pwd"] = self.pwd
if self.cmd:
record["cmd"] = self.cmd
if self.cmd_version:
record["cmd_version"] = self.cmd_version
process = self.process_record
if pwd:
record["pwd"] = pwd
if cmd:
record["cmd"] = cmd
if cmd_version:
record["cmd_version"] = cmd_version
if process:
record["process_id"] = str(process.id)
return record
@ -3767,6 +3813,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
if refresh_snapshot_size:
snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, self.snapshot_id) if snapshot_id}
transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids))
if is_new or update_fields is None or "status" in update_fields or "snapshot" in update_fields or "snapshot_id" in update_fields:
transaction.on_commit(type(self).clear_majority_status_cache)
# if is_new:
# from archivebox.misc.logging_util import log_worker_event
@ -3788,6 +3836,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
deleted = super().delete(*args, **kwargs)
if snapshot_id:
transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes({snapshot_id}))
transaction.on_commit(type(self).clear_majority_status_cache)
return deleted
@staticmethod
@ -4495,7 +4544,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
def _url_passes_filters(self, url: str) -> bool:
"""Check if URL passes URL_ALLOWLIST and URL_DENYLIST config filters.
Uses proper config hierarchy: defaults -> file -> env -> machine -> user -> crawl -> snapshot
Uses the centralized config resolver so frozen crawl/snapshot values
and live Machine/Persona execution values apply in their scoped order.
"""
return self.snapshot.crawl.url_passes_filters(url, snapshot=self.snapshot)

8
archivebox/core/settings.py
View File

@ -174,9 +174,9 @@ except ImportError:
STATIC_URL = "/static/"
TEMPLATES_DIR_NAME = "templates"
CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONFIG.CUSTOM_TEMPLATES_DIR) and os.access(CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK)
CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK)
STATICFILES_DIRS = [
*([str(CONFIG.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
*([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
# *[
# str(plugin_dir / 'static')
# for plugin_dir in PLUGIN_DIRS.values()
@ -187,7 +187,7 @@ STATICFILES_DIRS = [
]
TEMPLATE_DIRS = [
*([str(CONFIG.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
*([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
# *[
# str(plugin_dir / 'templates')
# for plugin_dir in PLUGIN_DIRS.values()
@ -334,7 +334,7 @@ STORAGES = {
"BACKEND": "django.core.files.storage.FileSystemStorage",
"OPTIONS": {
"base_url": "/archive/",
"location": CONFIG.ARCHIVE_DIR,
"location": CONSTANTS.ARCHIVE_DIR,
},
},
# "snapshots": {

7
archivebox/core/shutdown_util.py
View File

@ -134,7 +134,10 @@ def foreground_shutdown_signals(
# signal for hard foreground-command shutdown. Server/update/run and
# other non-interactive commands raise immediately so their finally
# blocks can stop owned children without prompting.
if raise_on_first_signal or already_requested:
if already_requested:
os.write(sys.stdout.fileno(), f"\n[🛑] Got {sig.name} again, exiting immediately.\n".encode())
os._exit(130)
if raise_on_first_signal:
raise KeyboardInterrupt
try:
@ -143,6 +146,8 @@ def foreground_shutdown_signals(
signal.signal(sig, raise_keyboard_interrupt)
yield state
finally:
if state.signal_name and previous_active_state is not None and not previous_active_state.signal_name:
previous_active_state.signal_name = state.signal_name
_active_shutdown_state = previous_active_state
for sig, previous_handler in previous_handlers.items():
signal.signal(sig, previous_handler)

2
archivebox/core/tag_util.py
View File

@ -246,7 +246,7 @@ def _build_snapshot_preview_map(
def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = None) -> dict[str, Any]:
count = getattr(tag, "num_snapshots", None)
count = tag.__dict__.get("num_snapshots")
if count is None:
count = tag.snapshot_set.count()
return {

47
archivebox/core/templatetags/core_tags.py
View File

@ -76,6 +76,12 @@ def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
return {}
def _snapshot_id(value: Any) -> Any:
from archivebox.core.models import Snapshot
return value.id if isinstance(value, Snapshot) else value
def _coerce_output_file_size(value: Any) -> int | None:
try:
return max(int(value or 0), 0)
@ -85,7 +91,7 @@ def _coerce_output_file_size(value: Any) -> int | None:
def _count_media_files(result) -> int:
try:
output_files = _normalize_output_files(getattr(result, "output_files", None) or {})
output_files = _normalize_output_files(result.output_files or {})
except Exception:
output_files = {}
@ -121,7 +127,7 @@ def _list_media_files(result) -> list[dict]:
except Exception:
return media_files
output_files = _normalize_output_files(getattr(result, "output_files", None) or {})
output_files = _normalize_output_files(result.output_files or {})
candidates: list[tuple[Path, int | None]] = []
if output_files:
for path, metadata in output_files.items():
@ -300,7 +306,7 @@ def result_list(context, cl):
"""
num_sorted_fields = 0
request = context.get("request")
config = getattr(request, "archivebox_config", None) or context.get("CONFIG")
config = request.__dict__.get("archivebox_config") if request is not None else context.get("CONFIG")
results = cl.result_list
if config is not None:
for obj in results:
@ -443,7 +449,7 @@ def _unconfigured_banner_context(request) -> dict:
# operator's clipboard already aligned with subdomain routing. The config
# parser strips the leading ``*.`` so users can paste it verbatim.
suggested_base_url = f"{scheme}://*.{canonical_host}" if canonical_host else ""
user = getattr(request, "user", None)
user = request.user
is_superuser = bool(user and user.is_authenticated and user.is_superuser)
machine_admin_url = ""
if is_superuser:
@ -496,19 +502,19 @@ def public_base_url(context) -> str:
@register.simple_tag(takes_context=True)
def snapshot_base_url(context, snapshot) -> str:
snapshot_id = getattr(snapshot, "id", snapshot)
snapshot_id = _snapshot_id(snapshot)
return get_snapshot_base_url(str(snapshot_id), request=context.get("request"), config=context.get("CONFIG"))
@register.simple_tag(takes_context=True)
def snapshot_url(context, snapshot, path: str = "") -> str:
snapshot_id = getattr(snapshot, "id", snapshot)
snapshot_id = _snapshot_id(snapshot)
return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
@register.simple_tag(takes_context=True)
def snapshot_preview_url(context, snapshot, path: str = "") -> str:
snapshot_id = getattr(snapshot, "id", snapshot)
snapshot_id = _snapshot_id(snapshot)
return _build_snapshot_preview_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
@ -538,16 +544,18 @@ def plugin_card(context, result) -> str:
- output_path: Path to output relative to snapshot dir (from embed_path())
- plugin: Plugin base name
"""
if result is None or not hasattr(result, "plugin"):
from archivebox.core.models import ArchiveResult
if result is None or not isinstance(result, ArchiveResult):
return ""
plugin = get_plugin_name(result.plugin)
template_str = get_plugin_template(plugin, "card")
# Use embed_path() for the display path
raw_output_path = result.embed_path() if hasattr(result, "embed_path") else ""
raw_output_path = result.embed_path() or ""
output_url = build_snapshot_url(
str(getattr(result, "snapshot_id", "")),
str(result.snapshot_id),
raw_output_path or "",
request=context.get("request"),
config=context.get("CONFIG"),
@ -558,7 +566,7 @@ def plugin_card(context, result) -> str:
media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
if media_files:
snapshot_id = str(getattr(result, "snapshot_id", ""))
snapshot_id = str(result.snapshot_id)
request = context.get("request")
config = context.get("CONFIG")
for item in media_files:
@ -592,7 +600,7 @@ def plugin_card(context, result) -> str:
pass
if force_text_preview:
preview = _render_text_file_preview(getattr(result, "snapshot_dir", None), raw_output_path, plugin, icon_html)
preview = _render_text_file_preview(result.snapshot_dir, raw_output_path, plugin, icon_html)
if preview:
return mark_safe(preview)
@ -608,7 +616,7 @@ def plugin_card(context, result) -> str:
def output_card(snapshot, output_path: str, plugin: str) -> str:
plugin_name = get_plugin_name(plugin)
icon_html = get_plugin_icon(plugin_name)
preview = _render_text_file_preview(getattr(snapshot, "output_dir", None), output_path, plugin_name, icon_html)
preview = _render_text_file_preview(snapshot.output_dir, output_path, plugin_name, icon_html)
if preview:
return mark_safe(preview)
@ -624,7 +632,9 @@ def plugin_full(context, result) -> str:
Usage: {% plugin_full result %}
"""
if result is None or not hasattr(result, "plugin"):
from archivebox.core.models import ArchiveResult
if result is None or not isinstance(result, ArchiveResult):
return ""
plugin = get_plugin_name(result.plugin)
@ -634,14 +644,13 @@ def plugin_full(context, result) -> str:
return ""
raw_output_path = ""
if hasattr(result, "embed_path_db"):
raw_output_path = result.embed_path_db() or ""
if not raw_output_path and hasattr(result, "embed_path"):
raw_output_path = result.embed_path_db() or ""
if not raw_output_path:
raw_output_path = result.embed_path() or ""
if _is_root_snapshot_output_path(raw_output_path):
return ""
output_url = build_snapshot_url(
str(getattr(result, "snapshot_id", "")),
str(result.snapshot_id),
raw_output_path,
request=context.get("request"),
config=context.get("CONFIG"),
@ -685,7 +694,7 @@ def api_token(context) -> str:
from archivebox.api.auth import get_or_create_api_token
request = context.get("request")
user = getattr(request, "user", None)
user = request.user
if not user or not user.is_authenticated:
return ""

171
archivebox/core/views.py
View File

@ -5,9 +5,7 @@ import os
import posixpath
from glob import glob, escape
from django.utils import timezone
import inspect
from typing import cast
from collections.abc import Callable
from pathlib import Path
from urllib.parse import quote, urlparse
@ -30,7 +28,17 @@ from admin_data_views.utils import render_with_table_view, render_with_item_view
from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, VERSION
from archivebox.config.common import SENSITIVE_CONFIG_VALUE_REDACTED, get_config, get_all_configs, redact_sensitive_config
from archivebox.config.common import (
SENSITIVE_CONFIG_VALUE_REDACTED,
find_config_default,
find_config_section,
find_config_source,
find_config_type,
get_config,
get_all_configs,
get_request_config,
redact_sensitive_config,
)
from archivebox.config.configset import BaseConfigSet
from archivebox.misc.paginators import CountlessPaginator
from archivebox.misc.util import (
@ -80,16 +88,6 @@ from archivebox.plugins.views import get_config_definition_link
from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
def _get_request_config(request: HttpRequest, *, resolve_plugins: bool = False):
request_config = getattr(request, "archivebox_config", None)
request_config_resolves_plugins = bool(getattr(request, "_archivebox_config_resolves_plugins", False))
if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
request_config = get_config(resolve_plugins=resolve_plugins)
request.archivebox_config = request_config
request._archivebox_config_resolves_plugins = resolve_plugins
return request_config
def _files_index_target(snapshot: Snapshot, archivefile: str | None) -> str:
target = archivefile or ""
if target == "index.html":
@ -122,14 +120,14 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
def _admin_login_redirect_or_forbidden(request: HttpRequest):
if _get_request_config(request).CONTROL_PLANE_ENABLED:
if get_request_config(request).CONTROL_PLANE_ENABLED:
return redirect(f"/admin/login/?next={request.path}")
return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.")
class HomepageView(View):
def get(self, request):
request_config = _get_request_config(request)
request_config = get_request_config(request)
if request.user.is_authenticated and request_config.CONTROL_PLANE_ENABLED:
return redirect("/admin/core/snapshot/")
@ -200,7 +198,7 @@ class SnapshotView(View):
# Reuse the middleware-attached config; never re-bootstrap from env + plugin
# schemas just to render a snapshot page (that pays ~30ms for no reason).
runtime_config = _get_request_config(request)
runtime_config = get_request_config(request)
snapshot._runtime_config = runtime_config
snapshot_permissions = get_snapshot_permissions(snapshot)
hidden_card_plugins = {"archivedotorg", "favicon", "title"}
@ -734,7 +732,7 @@ def _replay_path_visible(request: HttpRequest, path: Path) -> bool:
snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first()
if not snapshot or not can_view_snapshot(request, snapshot):
return False
request.archivebox_config = _get_request_config(request, resolve_plugins=False)
request.archivebox_config = get_request_config(request, resolve_plugins=False)
return True
@ -845,11 +843,14 @@ def _serve_responses_path(request, responses_root: Path, rel_path: str, show_ind
def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
request_config = _get_request_config(request, resolve_plugins=False)
rel_path = path or ""
request_config = get_request_config(
request,
resolve_plugins=rel_path.startswith("replay/") or rel_path == "replay",
)
request.archivebox_config = request_config
request.archivebox_snapshot_url = snapshot.url
snapshot._runtime_config = request_config
rel_path = path or ""
if rel_path.startswith("replay/") or rel_path == "replay":
response = archivewebpage_replay.serve_replay_asset_response(rel_path, request_config, HttpResponse)
@ -899,7 +900,8 @@ def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str =
def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = ""):
request_config = _get_request_config(request)
request_config = get_request_config(request, resolve_plugins=False)
request.archivebox_config = request_config
requested_root_index = path in ("", "index.html") or path.endswith("/")
rel_path = path or ""
if not rel_path or rel_path.endswith("/"):
@ -909,13 +911,13 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
raise Http404
domain = domain.lower()
match = _latest_response_match(request, domain, rel_path, data_root=request_config.USERS_DIR)
match = _latest_response_match(request, domain, rel_path, data_root=CONSTANTS.USERS_DIR)
if not match and "." not in Path(rel_path).name:
index_path = f"{rel_path.rstrip('/')}/index.html"
match = _latest_response_match(request, domain, index_path, data_root=request_config.USERS_DIR)
match = _latest_response_match(request, domain, index_path, data_root=CONSTANTS.USERS_DIR)
if not match and "." not in Path(rel_path).name:
html_path = f"{rel_path}.html"
match = _latest_response_match(request, domain, html_path, data_root=request_config.USERS_DIR)
match = _latest_response_match(request, domain, html_path, data_root=CONSTANTS.USERS_DIR)
show_indexes = bool(request.GET.get("files"))
if match:
@ -924,7 +926,7 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
if response is not None:
return response
responses_root = _latest_responses_root(request, domain, data_root=request_config.USERS_DIR)
responses_root = _latest_responses_root(request, domain, data_root=CONSTANTS.USERS_DIR)
if responses_root:
response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
if response is not None:
@ -946,7 +948,7 @@ class SnapshotHostView(View):
"""Serve snapshot directory contents on <snapshot-subdomain>.<listen_host>/<path>."""
def get(self, request, snapshot_id: str, path: str = ""):
request_config = _get_request_config(request)
request_config = get_request_config(request)
snapshot = _find_snapshot_by_ref(snapshot_id)
if not snapshot:
@ -998,15 +1000,15 @@ class PublicIndexView(ListView):
paginator_class = CountlessPaginator
def get_paginate_by(self, queryset):
runtime_config = getattr(self, "runtime_config", None)
runtime_config = self.__dict__.get("runtime_config")
if runtime_config is None:
self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=False)
self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
return runtime_config.SNAPSHOTS_PER_PAGE
def get_context_data(self, **kwargs):
runtime_config = getattr(self, "runtime_config", None)
runtime_config = self.__dict__.get("runtime_config")
if runtime_config is None:
self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=False)
self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config)
context = {
@ -1023,7 +1025,7 @@ class PublicIndexView(ListView):
self.request.GET.get("q")
and get_search_mode_base(search_mode, config=runtime_config) == "deep"
and search_mode_backend
and getattr(context.get("paginator"), "count", 0) == 0,
and context["paginator"].count == 0,
)
snapshots = list(context.get("object_list") or ())
icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
@ -1086,7 +1088,7 @@ class PublicIndexView(ListView):
if not query:
return qs
runtime_config = getattr(self, "runtime_config", None)
runtime_config = self.__dict__.get("runtime_config")
search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
try:
return apply_snapshot_search(
@ -1105,7 +1107,7 @@ class PublicIndexView(ListView):
def get(self, *args, **kwargs):
if self.request.user.is_authenticated:
return redirect("/admin/core/snapshot/")
if _get_request_config(self.request).PUBLIC_INDEX:
if get_request_config(self.request).PUBLIC_INDEX:
response = super().get(*args, **kwargs)
return response
else:
@ -1132,7 +1134,7 @@ class AddView(UserPassesTestMixin, FormView):
return kwargs
def test_func(self):
return _get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
return get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
def _can_override_crawl_config(self) -> bool:
return is_admin_user(self.request)
@ -1146,11 +1148,11 @@ class AddView(UserPassesTestMixin, FormView):
if not self._can_override_crawl_config():
return {}
return custom_config
return {str(key): value for key, value in custom_config.items() if not str(key).endswith("_BINARY")}
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
request_config = _get_request_config(self.request, resolve_plugins=True)
request_config = get_request_config(self.request, resolve_plugins=True)
required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
can_override_crawl_config = self._can_override_crawl_config()
plugin_configs = discover_plugin_configs() if can_override_crawl_config else {}
@ -1252,7 +1254,7 @@ class AddView(UserPassesTestMixin, FormView):
created_by_id = get_or_create_system_user_pk()
created_by_name = getattr(self.request.user, "username", "web") if self.request.user.is_authenticated else "web"
created_by_name = self.request.user.username if self.request.user.is_authenticated else "web"
# 1. save the provided urls to sources/2024-11-05__23-59-59__web_ui_add_by_user_<user_pk>.txt
sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__web_ui_add_by_user_{created_by_id}.txt"
@ -1267,8 +1269,7 @@ class AddView(UserPassesTestMixin, FormView):
config = {}
if plugins:
config["PLUGINS"] = plugins
request_user = self.request.user if self.request.user.is_authenticated else None
effective_config = get_config(persona=persona, user=request_user) if persona else get_config(user=request_user)
effective_config = get_config(persona=persona) if persona else get_config()
if crawl_max_concurrent_snapshots != int(effective_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS):
config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
if delete_after != str(effective_config.DELETE_AFTER):
@ -1378,11 +1379,7 @@ class WebAddView(AddView):
return redirect(f"/{snapshot.url_path}")
request_host = (request.get_host() or "").lower()
if (
request.user.is_authenticated
and not _get_request_config(request).PUBLIC_ADD_VIEW
and host_matches(request_host, get_web_host())
):
if request.user.is_authenticated and not get_request_config(request).PUBLIC_ADD_VIEW and host_matches(request_host, get_web_host()):
return redirect(build_admin_url(request.get_full_path(), request=request))
if not self.test_func():
@ -1461,79 +1458,11 @@ class HealthCheckView(View):
return HttpResponse("OK", content_type="text/plain", status=200)
def find_config_section(key: str) -> str:
CONFIGS = get_all_configs()
if key in CONSTANTS_CONFIG:
return "CONSTANT"
matching_sections = [section_id for section_id, section in CONFIGS.items() if key in dict(section)]
section = matching_sections[0] if matching_sections else "DYNAMIC"
return section
def find_config_default(key: str) -> str:
CONFIGS = get_all_configs()
if key in CONSTANTS_CONFIG:
return str(CONSTANTS_CONFIG[key])
default_val = None
for config in CONFIGS.values():
if key in dict(config):
default_val = type(config).model_fields[key].default
break
if isinstance(default_val, Callable):
default_val = inspect.getsource(default_val).split("lambda", 1)[-1].split(":", 1)[-1].replace("\n", " ").strip()
if default_val.count(")") > default_val.count("("):
default_val = default_val[:-1]
else:
default_val = str(default_val)
return default_val
def find_config_type(key: str) -> str:
CONFIGS = get_all_configs()
for config in CONFIGS.values():
if key in type(config).model_fields:
annotation = type(config).model_fields[key].annotation
return getattr(annotation, "__name__", str(annotation))
return "str"
def find_config_source(key: str, merged_config: dict) -> str:
"""Determine where a config value comes from."""
from archivebox.machine.models import Machine
# Environment variables override all persistent config sources.
if key in os.environ:
return "Environment"
# Machine.config overrides ArchiveBox.conf.
try:
machine = Machine.current()
if machine.config and key in machine.config:
return "Machine"
except Exception:
pass
# Check if it's from archivebox.config.file
file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
if key in file_config:
return "Config File"
# Otherwise it's using the default
return "Default"
@render_with_table_view
def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
CONFIGS = get_all_configs()
assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings."
assert request.user.is_superuser, "Must be a superuser to view configuration settings."
merged_config = get_config(redact_sensitive=True)
@ -1555,12 +1484,12 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
rows["Type"].append(format_html("<code>{}</code>", find_config_type(key)))
# Use merged config value (includes machine overrides)
actual_value = merged_config.get(key, getattr(section, key, None))
actual_value = merged_config.get(key, dict(section)[key])
rows["Value"].append(mark_safe(f"<code>{actual_value}</code>"))
# Show where the value comes from
source = find_config_source(key, merged_config)
source_colors = {"Machine": "purple", "Environment": "blue", "Config File": "green", "Default": "gray"}
source_colors = {"Machine": "purple", "Environment": "blue", "File": "green", "Plugin Default": "teal", "Default": "gray"}
rows["Source"].append(format_html('<code style="color: {}">{}</code>', source_colors.get(source, "gray"), source))
rows["Default"].append(
@ -1575,7 +1504,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
for key in CONSTANTS_CONFIG.keys():
rows["Section"].append(section) # section.replace('_', ' ').title().replace(' Config', '')
rows["Key"].append(ItemLink(key, key=key))
rows["Type"].append(format_html("<code>{}</code>", getattr(type(CONSTANTS_CONFIG[key]), "__name__", str(CONSTANTS_CONFIG[key]))))
rows["Type"].append(format_html("<code>{}</code>", type(CONSTANTS_CONFIG[key]).__name__))
rows["Value"].append(format_html("<code>{}</code>", redact_sensitive_config(CONSTANTS_CONFIG).get(key)))
rows["Source"].append(mark_safe('<code style="color: gray">Constant</code>'))
rows["Default"].append(
@ -1598,17 +1527,13 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
CONFIGS = get_all_configs()
assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings."
assert request.user.is_superuser, "Must be a superuser to view configuration settings."
merged_config = get_config(redact_sensitive=True)
# Determine all sources for this config value
sources_info = []
# Environment variable
if key in os.environ:
sources_info.append(("Environment", redact_sensitive_config(os.environ).get(key), "blue"))
# Machine config
machine = None
machine_admin_url = None
@ -1620,11 +1545,15 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
except Exception:
pass
# Environment variable
if key in os.environ:
sources_info.append(("Environment", redact_sensitive_config(os.environ).get(key), "blue"))
# Config file value
if CONSTANTS.CONFIG_FILE.exists():
file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
if key in file_config:
sources_info.append(("Config File", redact_sensitive_config(file_config).get(key), "green"))
sources_info.append(("File", redact_sensitive_config(file_config).get(key), "green"))
# Default value
default_val = find_config_default(key)

14
archivebox/core/widgets.py
View File

@ -4,6 +4,8 @@ import json
import re
import hashlib
from django import forms
from django.db.models.manager import BaseManager
from django.db.models.query import QuerySet
from django.utils.html import escape
from django.utils.safestring import mark_safe
@ -65,10 +67,12 @@ class TagEditorWidget(forms.Widget):
# Parse value to get list of tag names
tags = []
if value:
if hasattr(value, "all"): # QuerySet
if isinstance(value, (BaseManager, QuerySet)):
tags = sorted([tag.name for tag in value.all()])
elif isinstance(value, (list, tuple)):
if value and hasattr(value[0], "name"): # List of Tag objects
from archivebox.core.models import Tag
if value and isinstance(value[0], Tag): # List of Tag objects
tags = sorted([tag.name for tag in value])
else: # List of strings or IDs
# Could be tag IDs from form submission
@ -675,12 +679,14 @@ class InlineTagEditorWidget(TagEditorWidget):
# Parse value to get list of tag dicts with id and name
tag_data = []
if value:
if hasattr(value, "all"): # QuerySet
if isinstance(value, (BaseManager, QuerySet)):
for tag in value.all():
tag_data.append({"id": tag.pk, "name": tag.name})
tag_data.sort(key=lambda x: x["name"].lower())
elif isinstance(value, (list, tuple)):
if value and hasattr(value[0], "name"):
from archivebox.core.models import Tag
if value and isinstance(value[0], Tag):
for tag in value:
tag_data.append({"id": tag.pk, "name": tag.name})
tag_data.sort(key=lambda x: x["name"].lower())

46
archivebox/crawls/admin.py
View File

@ -605,9 +605,7 @@ class CrawlAdminForm(forms.ModelForm):
if commit:
instance.save()
instance.apply_crawl_config_filters()
save_m2m = getattr(self, "_save_m2m", None)
if callable(save_m2m):
save_m2m()
self._save_m2m()
return instance
@ -729,7 +727,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
super().__init__(model, admin_site)
self.crawl_admin_base_config = None
self.stop_reason_cache = {}
self.persona_limit_config_cache = {}
class Media:
css = {"all": ("admin/crawls/crawl_change.css",)}
@ -739,9 +736,8 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
self.request = request
self.crawl_admin_base_config = request.archivebox_config
self.stop_reason_cache = {}
self.persona_limit_config_cache = {}
response = super().changelist_view(request, extra_context)
cl = getattr(response, "context_data", {}).get("cl") if hasattr(response, "context_data") else None
cl = response.context_data.get("cl")
if cl is not None and not self.should_annotate_snapshot_counts(request):
self.hydrate_visible_snapshot_counts(cl.result_list)
return response
@ -790,7 +786,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
self.request = request
self.crawl_admin_base_config = request.archivebox_config
self.stop_reason_cache = {}
self.persona_limit_config_cache = {}
crawl = self.get_object(request, object_id)
if crawl:
self.hydrate_visible_snapshot_counts([crawl])
@ -982,7 +977,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
if obj.pk in self.stop_reason_cache:
return self.stop_reason_cache[obj.pk]
output_dir = obj.output_dir_for_config(self.crawl_admin_base_config)
output_dir = obj.output_dir
config = self.limit_config_for_crawl(obj, output_dir)
reason = obj.stop_reason(
config=config,
@ -994,22 +989,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
return reason
def limit_config_for_crawl(self, obj, output_dir):
config = {
"CRAWL_DIR": str(output_dir),
"CRAWL_MAX_URLS": self.crawl_admin_base_config.CRAWL_MAX_URLS,
"CRAWL_MAX_SIZE": self.crawl_admin_base_config.CRAWL_MAX_SIZE,
"CRAWL_TIMEOUT": self.crawl_admin_base_config.CRAWL_TIMEOUT,
"SNAPSHOT_MAX_SIZE": self.crawl_admin_base_config.SNAPSHOT_MAX_SIZE,
}
if obj.persona_id:
if obj.persona_id not in self.persona_limit_config_cache:
self.persona_limit_config_cache[obj.persona_id] = {
key: value for key, value in obj.persona.get_derived_config().items() if key in config
}
config.update(self.persona_limit_config_cache[obj.persona_id])
if obj.config:
config.update({key: value for key, value in obj.config.items() if key in config})
return config
from archivebox.config.common import get_config
return get_config(crawl=obj).for_crawl_runtime(
crawl=obj,
persona=obj.resolve_persona(),
crawl_output_dir=output_dir,
)
@admin.display(description="Status", ordering="status")
def status_with_stop_reason(self, obj):
@ -1106,12 +1092,10 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
@admin.display(description="Snapshots")
def snapshots_changelist(self, obj):
request = getattr(self, "request", None)
request = self.request
snapshot_changelist = reverse("admin:core_snapshot_changelist")
scoped_params = {"crawl_id": str(obj.pk)}
full_url = f"{snapshot_changelist}?{urlencode(scoped_params)}"
if request is None:
return format_html('<a class="button" href="{}">Open snapshots changelist</a>', full_url)
snapshot_admin = self.admin_site._registry[Snapshot]
changelist_request = copy(request)
@ -1305,7 +1289,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
return self.fieldsets
def save_model(self, request, obj, form, change):
if not obj.created_by_id and getattr(request, "user", None) and request.user.is_authenticated:
if not obj.created_by_id and request.user.is_authenticated:
obj.created_by = request.user
super().save_model(request, obj, form, change)
@ -1315,14 +1299,14 @@ class CrawlScheduleAdmin(BaseModelAdmin):
@admin.display(description="# Crawls", ordering="crawl_count")
def num_crawls(self, obj):
count = getattr(obj, "crawl_count", None)
count = obj.__dict__.get("crawl_count")
if count is None:
count = obj.crawl_set.count()
return count
@admin.display(description="# Snapshots", ordering="snapshot_count")
def num_snapshots(self, obj):
count = getattr(obj, "snapshot_count", None)
count = obj.__dict__.get("snapshot_count")
if count is None:
count = Snapshot.objects.filter(crawl__schedule=obj).count()
return count
@ -1338,7 +1322,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
crawl_ids = obj.crawl_set.values_list("pk", flat=True)
return render_snapshots_list(
Snapshot.objects.filter(crawl_id__in=crawl_ids),
request=getattr(self, "request", None),
request=self.request,
prefix="schedule_snapshots",
)

49
archivebox/crawls/migrations/0016_hydrate_crawl_permissions.py
View File

@ -1,5 +1,6 @@
import os
import json
import uuid
from django.db import migrations
from django.db.models import Q
@ -9,31 +10,6 @@ VALID_PERMISSIONS = {"public", "unlisted", "private"}
BATCH_SIZE = 1000
def legacy_bool(value):
if value is None:
return None
normalized = str(value).strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
return None
def permissions_from_legacy_public_flags(config):
if str(config.get("PERMISSIONS") or "").strip():
return None
public_snapshots = legacy_bool(config.get("PUBLIC_SNAPSHOTS"))
public_index = legacy_bool(config.get("PUBLIC_INDEX"))
if public_snapshots is False:
return "private"
if public_index is False:
return "unlisted"
if public_snapshots is True or public_index is True:
return "public"
return None
def normalize_permissions(value, default):
value = str(value or "").strip().lower()
return value if value in VALID_PERMISSIONS else default
@ -59,22 +35,19 @@ def raw_base_config(apps):
def resolve_permissions(config, default):
from archivebox.config.common import permissions_from_legacy_public_flags
explicit = str(config.get("PERMISSIONS") or "").strip().lower()
if explicit in VALID_PERMISSIONS:
return explicit
return permissions_from_legacy_public_flags(config) or default
def model_has_config(model):
try:
model._meta.get_field("config")
except Exception:
return False
return True
def id_values(pk):
return str(pk), getattr(pk, "hex", str(pk).replace("-", ""))
if isinstance(pk, uuid.UUID):
return str(pk), pk.hex
pk_str = str(pk)
return pk_str, pk_str.replace("-", "")
def flush_batch(cursor, table_name, batch):
@ -125,8 +98,6 @@ def _ensure_permissions_column(cursor):
def hydrate_crawl_permissions(apps, schema_editor):
Crawl = apps.get_model("crawls", "Crawl")
User = apps.get_model("auth", "User")
user_has_config = model_has_config(User)
base_config = raw_base_config(apps)
default_permissions = resolve_permissions(base_config, "public")
table_name = schema_editor.quote_name(Crawl._meta.db_table)
@ -135,17 +106,13 @@ def hydrate_crawl_permissions(apps, schema_editor):
batch = []
missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
for crawl in Crawl.objects.filter(missing_permissions).select_related("persona", "created_by").iterator(chunk_size=BATCH_SIZE):
for crawl in Crawl.objects.filter(missing_permissions).select_related("persona").iterator(chunk_size=BATCH_SIZE):
config = dict(crawl.config or {})
resolved = dict(base_config)
if crawl.persona_id:
persona_config = crawl.persona.config or {}
if isinstance(persona_config, dict):
resolved.update(persona_config)
if user_has_config:
user_config = crawl.created_by.config or {}
if isinstance(user_config, dict):
resolved.update(user_config)
resolved.update(config)
config["PERMISSIONS"] = resolve_permissions(resolved, default_permissions)
batch.append((crawl.id, config))

28
archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py
View File

@ -1,6 +1,5 @@
import json
from django.conf import settings
from django.db import migrations
@ -19,26 +18,31 @@ def _flush_updates(Crawl, db_alias, pending):
def freeze_existing_crawl_configs(apps, schema_editor):
from archivebox.config.common import build_crawl_config_snapshot
from archivebox.personas.models import Persona
from archivebox.config.constants import CONSTANTS
from archivebox.personas.models import derive_persona_config
class PersonaConfigSnapshot:
def __init__(self, persona):
self.name = persona.name
self.config = dict(persona.config or {})
def get_derived_config(self):
return derive_persona_config(name=self.name, config=self.config, persona_dir=CONSTANTS.PERSONAS_DIR / self.name)
Crawl = apps.get_model("crawls", "Crawl")
auth_app_label, auth_model_name = settings.AUTH_USER_MODEL.split(".", 1)
User = apps.get_model(auth_app_label, auth_model_name)
Persona = apps.get_model("personas", "Persona")
db_alias = schema_editor.connection.alias
rows = Crawl.objects.using(db_alias).values_list("id", "persona_id", "created_by_id", "config")
persona_ids = {persona_id for _, persona_id, _, _ in rows if persona_id}
user_ids = {user_id for _, _, user_id, _ in rows if user_id}
personas = {persona.pk: persona for persona in Persona.objects.using(db_alias).filter(pk__in=persona_ids)}
users = {user.pk: user for user in User.objects.using(db_alias).filter(pk__in=user_ids)}
rows = Crawl.objects.using(db_alias).values_list("id", "persona_id", "config")
persona_ids = {persona_id for _, persona_id, _ in rows if persona_id}
personas = {persona.pk: PersonaConfigSnapshot(persona) for persona in Persona.objects.using(db_alias).filter(pk__in=persona_ids)}
frozen_cache = {}
pending = []
for crawl_id, persona_id, user_id, current_config in rows.iterator(chunk_size=BATCH_SIZE):
for crawl_id, persona_id, current_config in rows.iterator(chunk_size=BATCH_SIZE):
current_config = dict(current_config or {})
cache_key = (persona_id, user_id, _config_cache_key(current_config))
cache_key = (persona_id, _config_cache_key(current_config))
if cache_key not in frozen_cache:
frozen_cache[cache_key] = build_crawl_config_snapshot(
user=users.get(user_id),
persona=personas.get(persona_id),
overrides=current_config,
)

60
archivebox/crawls/models.py
View File

@ -109,11 +109,10 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
template = self.template
label = template.label or self.label
persona = template.persona if template.persona_id else None
user = template.created_by if template.created_by_id else None
return Crawl.objects.create(
urls=template.urls,
config=build_crawl_config_snapshot(user=user, persona=persona, overrides=self.config or {}),
config=build_crawl_config_snapshot(persona=persona, overrides=self.config or {}),
max_depth=template.max_depth,
tags_str=template.tags_str,
persona_id=template.persona_id,
@ -197,9 +196,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
return f"[...{short_id}] {first_url[:120]}"
def get_delete_after_config_value(self):
from archivebox.config.common import get_config
from archivebox.config.common import resolve_delete_after_config_value
return get_config(crawl=self).DELETE_AFTER
return resolve_delete_after_config_value(self.config)
def pause(self, *, save: bool = True) -> bool:
return super().pause(save=save)
@ -286,15 +285,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
config = dict(self.config or {})
is_new = self._state.adding or old_crawl is None
persona = self.persona if self.persona_id else None
user = self.created_by if self.created_by_id else None
if is_new:
from archivebox.config.common import build_crawl_config_snapshot
config = build_crawl_config_snapshot(user=user, persona=persona, overrides=config)
config = build_crawl_config_snapshot(persona=persona, overrides=config)
if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
from archivebox.config.common import get_config
config["PERMISSIONS"] = normalize_permissions(get_config(persona=persona, user=user, include_machine=True).PERMISSIONS)
config["PERMISSIONS"] = normalize_permissions(get_config(persona=persona, include_machine=True).PERMISSIONS)
if "CRAWL_MAX_CONCURRENT_SNAPSHOTS" in config:
raw_concurrency = config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]
if raw_concurrency in (None, ""):
@ -469,11 +467,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
)
return crawl
def output_dir_for_config(self, runtime_config: Mapping[str, Any] | Any) -> Path:
"""
Construct output directory: archive/users/{username}/crawls/{YYYYMMDD}/{domain}/{crawl-id}
Domain is extracted from the first URL in the crawl.
"""
@property
def output_dir(self) -> Path:
from archivebox.config import CONSTANTS
from archivebox.core.models import Snapshot
@ -486,14 +481,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
break
domain = Snapshot.extract_domain_from_url(first_url) if first_url else "unknown"
users_dir = Path(runtime_config["USERS_DIR"]) if isinstance(runtime_config, Mapping) else runtime_config.USERS_DIR
return users_dir / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
@property
def output_dir(self) -> Path:
from archivebox.config.common import get_config
output_dir = self.output_dir_for_config(get_config(resolve_plugins=False))
output_dir = CONSTANTS.USERS_DIR / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
hyphen_dir = output_dir.with_name(str(uuid.UUID(hex=self.id.hex)))
return output_dir if output_dir.exists() or not hyphen_dir.exists() else hyphen_dir
@ -783,7 +771,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
def _config_value(config: Mapping[str, Any] | Any, key: str, default: Any = None) -> Any:
if isinstance(config, Mapping):
return config.get(key, default)
return getattr(config, key, default)
return config[key] if key in config else default
@classmethod
def create_scheduler_row(cls, **kwargs) -> "Crawl":
@ -794,17 +782,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
kwargs.setdefault("created_at", now)
kwargs.setdefault("modified_at", now)
config = normalize_config_json_values(kwargs.get("config") or {})
user = kwargs.get("created_by")
persona = kwargs.get("persona")
if user is None and kwargs.get("created_by_id"):
from django.contrib.auth import get_user_model
user = get_user_model().objects.filter(pk=kwargs["created_by_id"]).first()
if persona is None and kwargs.get("persona_id"):
from archivebox.personas.models import Persona
persona = Persona.objects.filter(pk=kwargs["persona_id"]).first()
kwargs["config"] = build_crawl_config_snapshot(user=user, persona=persona, overrides=config)
kwargs["config"] = build_crawl_config_snapshot(persona=persona, overrides=config)
crawl = cls(**kwargs)
if crawl.delete_at is None:
crawl.set_delete_at_from_config()
@ -820,18 +803,20 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
) -> str:
from abx_dl.limits import CrawlLimitState
if output_dir is None:
output_dir = self.output_dir
if config is None:
from archivebox.config.common import get_config
config = get_config(crawl=self, include_machine=False)
if output_dir is None:
output_dir = self.output_dir
config = get_config(crawl=self, include_machine=False).for_crawl_runtime(
crawl=self,
persona=self.resolve_persona(),
crawl_output_dir=output_dir,
)
limits_path = output_dir / ".abx-dl" / "limits.json"
if limits_path.exists():
config_with_crawl_dir = {**dict(config.items())} if isinstance(config, Mapping) else config
config_with_crawl_dir["CRAWL_DIR"] = str(output_dir)
stop_reason = CrawlLimitState.from_config(config_with_crawl_dir).get_stop_reason()
stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
if stop_reason:
return stop_reason
@ -1302,10 +1287,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
from archivebox.machine.models import Binary, Machine
def get_runtime_config():
config = get_config(crawl=self)
if persona_runtime_overrides:
config.update(persona_runtime_overrides)
return config
return get_config(crawl=self).for_crawl_runtime(
crawl=self,
persona=persona,
runtime_overrides=persona_runtime_overrides,
)
system_task = self.get_system_task()
if system_task == "archivebox://update":

23
archivebox/machine/admin.py
View File

@ -590,9 +590,10 @@ class ProcessAdmin(BaseModelAdmin):
@admin.display(description="ArchiveResult", ordering="archiveresult__plugin")
def archiveresult_link(self, process):
if not hasattr(process, "archiveresult"):
try:
ar = process.archiveresult
except Process.archiveresult.RelatedObjectDoesNotExist:
return "-"
ar = process.archiveresult
return format_html(
'<a href="/admin/core/archiveresult/{}/change">{} ← <code>{}</code></a>',
ar.id,
@ -602,9 +603,9 @@ class ProcessAdmin(BaseModelAdmin):
@admin.display(description="Snapshot", ordering="archiveresult__snapshot__id")
def snapshot_link(self, process):
ar = getattr(process, "archiveresult", None)
snapshot = getattr(ar, "snapshot", None)
if not snapshot:
try:
snapshot = process.archiveresult.snapshot
except Process.archiveresult.RelatedObjectDoesNotExist:
return "-"
return format_html(
'<a href="/admin/core/snapshot/{}/change"><code>{}</code></a>',
@ -614,10 +615,9 @@ class ProcessAdmin(BaseModelAdmin):
@admin.display(description="Crawl", ordering="archiveresult__snapshot__crawl__id")
def crawl_link(self, process):
ar = getattr(process, "archiveresult", None)
snapshot = getattr(ar, "snapshot", None)
crawl = getattr(snapshot, "crawl", None)
if not crawl:
try:
crawl = process.archiveresult.snapshot.crawl
except Process.archiveresult.RelatedObjectDoesNotExist:
return "-"
return format_html(
'<a href="/admin/crawls/crawl/{}/change"><code>{}</code></a>',
@ -682,7 +682,10 @@ class ProcessAdmin(BaseModelAdmin):
@admin.display(description="Output", ordering="archiveresult__output_size")
def output_summary(self, process):
output_files = getattr(getattr(process, "archiveresult", None), "output_files", {}) or {}
try:
output_files = process.archiveresult.output_files or {}
except Process.archiveresult.RelatedObjectDoesNotExist:
output_files = {}
if isinstance(output_files, str):
try:

27
archivebox/machine/models.py
View File

@ -19,6 +19,7 @@ from django.db.models import Q, QuerySet
from django.utils import timezone
from django.utils.functional import cached_property
from archivebox.config import CONSTANTS
from archivebox.config.common import rprint
from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
from archivebox.workers.models import BaseStateMachine, ModelWithStateMachine
@ -442,7 +443,7 @@ class NetworkInterface(ModelWithHealthStats):
if refresh or timezone.now() >= _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
updates = ["modified_at"]
for key, value in net_info.items():
if getattr(_CURRENT_INTERFACE, key) != value:
if _CURRENT_INTERFACE.__dict__.get(key) != value:
setattr(_CURRENT_INTERFACE, key, value)
updates.append(key)
if len(updates) > 1:
@ -581,10 +582,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
Get output directory for this binary's hook logs.
Path: data/machines/{machine_uuid}/binaries/{binary_name}/{binary_uuid}
"""
from archivebox.config.common import get_config
data_dir = get_config().DATA_DIR
return data_dir / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
return CONSTANTS.DATA_DIR / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
def to_json(self) -> dict:
"""
@ -1144,15 +1142,13 @@ class Process(ModelWithDeleteAfter, models.Model):
return f"Process[{self.id}] {cmd_str} ({self.status})"
def get_delete_after_config_value(self):
from archivebox.config.common import get_config
value = self.env.get("DELETE_AFTER")
if value not in (None, ""):
return value
value = (self.machine.config or {}).get("DELETE_AFTER")
if value not in (None, ""):
return value
return get_config(include_machine=False).DELETE_AFTER
return "0"
@classmethod
def missing_delete_at_candidates(cls):
@ -1174,17 +1170,18 @@ class Process(ModelWithDeleteAfter, models.Model):
@property
def plugin(self) -> str:
"""Get plugin name from associated ArchiveResult (if any)."""
if hasattr(self, "archiveresult"):
# Inline import to avoid circular dependency
try:
return self.archiveresult.plugin
return ""
except Process.archiveresult.RelatedObjectDoesNotExist:
return ""
@property
def hook_name(self) -> str:
"""Get hook name from associated ArchiveResult (if any)."""
if hasattr(self, "archiveresult"):
try:
return self.archiveresult.hook_name
return ""
except Process.archiveresult.RelatedObjectDoesNotExist:
return ""
def to_json(self) -> dict:
"""
@ -2449,16 +2446,14 @@ class Process(ModelWithDeleteAfter, models.Model):
"""
import subprocess
from importlib.resources import files
from archivebox.config.common import get_config
chrome_utils = files("abx_plugins.plugins.chrome").joinpath("chrome_utils.js")
if not chrome_utils.exists():
return 0
config = get_config()
crawl_roots = [
crawls_dir
for user_dir in config.USERS_DIR.iterdir()
for user_dir in CONSTANTS.USERS_DIR.iterdir()
if user_dir.is_dir()
for crawls_dir in [user_dir / "crawls"]
if crawls_dir.is_dir()

9
archivebox/misc/checks.py
View File

@ -66,7 +66,7 @@ def check_data_folder(config=None, **config_kwargs) -> None:
from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir
config = config or get_config(**config_kwargs)
archive_dir = config.ARCHIVE_DIR
archive_dir = CONSTANTS.ARCHIVE_DIR
archive_dir_exists = os.path.isdir(archive_dir)
if not archive_dir_exists:
print("[red][X] No archivebox index found in the current directory.[/red]", file=sys.stderr)
@ -82,7 +82,7 @@ def check_data_folder(config=None, **config_kwargs) -> None:
# Create data dir subdirs
create_and_chown_dir(CONSTANTS.SOURCES_DIR)
create_and_chown_dir(config.USERS_DIR)
create_and_chown_dir(CONSTANTS.USERS_DIR)
create_and_chown_dir(CONSTANTS.PERSONAS_DIR / "Default")
create_and_chown_dir(CONSTANTS.LOGS_DIR)
# create_and_chown_dir(CONSTANTS.CACHE_DIR)
@ -211,11 +211,10 @@ def check_not_inside_source_dir():
"""Prevent running ArchiveBox from inside its source directory (would pollute repo with data files)."""
cwd = Path(os.getcwd()).resolve()
is_source_dir = (cwd / "archivebox" / "__init__.py").exists() and (cwd / "pyproject.toml").exists()
data_dir_set_elsewhere = os.environ.get("DATA_DIR", "").strip() and Path(os.environ["DATA_DIR"]).resolve() != cwd
is_testing = "pytest" in sys.modules or "unittest" in sys.modules
if is_source_dir and not data_dir_set_elsewhere and not is_testing:
raise SystemExit("[!] Cannot run from source dir, set DATA_DIR or cd to a data folder first")
if is_source_dir and not is_testing:
raise SystemExit("[!] Cannot run from source dir, cd to a data folder first")
def check_data_dir_permissions(config=None, **config_kwargs):

23
archivebox/misc/db.py
View File

@ -1,7 +1,7 @@
"""
Database utility functions for ArchiveBox.
Post-bootstrap: requires archivebox.config (DATA_DIR) and uses Django lazily
Post-bootstrap: requires archivebox.config constants and uses Django lazily
(``from django.db import ...`` inside functions). Not safe to import pre-bootstrap.
"""
@ -17,7 +17,7 @@ from collections.abc import Callable
from contextlib import contextmanager
from sqlite3 import OperationalError as SQLiteOperationalError
from archivebox.config import DATA_DIR
from archivebox.config import CONSTANTS
from archivebox.misc.util import enforce_types
@ -79,7 +79,7 @@ def run_db_analyze_batch(
return []
next_table, *rest = remaining
raw_conn = getattr(connection, "connection", None)
raw_conn = connection.connection
progress_handler_set = False
if raw_conn is not None and max_seconds_per_table > 0:
deadline = time.monotonic() + max_seconds_per_table
@ -117,7 +117,7 @@ def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:
return " ".join([Path(parts[0]).name, *parts[1:]])[:220]
def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]:
def sqlite_lock_holders(db_path: Path = CONSTANTS.DATABASE_FILE) -> list[str]:
import psutil
db_path = db_path.resolve()
@ -146,7 +146,7 @@ def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]
return holders
def log_sqlite_lock_holders(console: Any, *, db_path: Path = DATA_DIR / "index.sqlite3", limit: int = 8) -> None:
def log_sqlite_lock_holders(console: Any, *, db_path: Path = CONSTANTS.DATABASE_FILE, limit: int = 8) -> None:
holders = sqlite_lock_holders(db_path)
if holders:
console.print("[yellow] DB holders:[/yellow]")
@ -346,7 +346,7 @@ HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(
@enforce_types
def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dict[str, str]]:
def migration_state(out_dir: Path = CONSTANTS.DATA_DIR) -> tuple[list[str], list[str], dict[str, str]]:
"""Cheaply compare migration files to django_migrations without invoking migrate."""
from django.apps import apps
from django.db import connection
@ -373,7 +373,7 @@ def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dic
loader.load_disk()
for (app_label, migration_name), migration in loader.disk_migrations.items():
disk_migrations.add((app_label, migration_name))
for replaced_app, replaced_name in getattr(migration, "replaces", ()) or ():
for replaced_app, replaced_name in migration.replaces or ():
squashed_replaced.add((replaced_app, replaced_name))
applied = {(app, name) for app, name in applied if app in app_labels}
@ -392,14 +392,19 @@ def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dic
@enforce_types
def pending_migrations(out_dir: Path = DATA_DIR) -> list[str]:
def pending_migrations(out_dir: Path = CONSTANTS.DATA_DIR) -> list[str]:
"""Return migration files on disk that have not been applied yet."""
pending, _missing_from_code, _rollback_targets = migration_state(out_dir=out_dir)
return pending
@enforce_types
def apply_migrations(out_dir: Path = DATA_DIR, stdout: TextIO | None = None, stderr: TextIO | None = None, verbosity: int = 1) -> list[str]:
def apply_migrations(
out_dir: Path = CONSTANTS.DATA_DIR,
stdout: TextIO | None = None,
stderr: TextIO | None = None,
verbosity: int = 1,
) -> list[str]:
"""Apply pending Django migrations"""
from django.core.management import call_command

10
archivebox/misc/logging_util.py
View File

@ -1,7 +1,7 @@
__package__ = "archivebox"
# Post-bootstrap CLI logging helpers (event loggers, progress bars, formatters).
# Requires archivebox.config to be loaded — imports DATA_DIR, get_config, and
# Requires archivebox.config to be loaded — imports CONSTANTS/get_config and
# references Django ORM types. For pre-bootstrap logging primitives use
# misc/logging.py, which has no archivebox or Django dependencies.
@ -24,7 +24,7 @@ if TYPE_CHECKING:
from rich import print
from rich.panel import Panel
from archivebox.config import DATA_DIR, VERSION
from archivebox.config import CONSTANTS, VERSION
from archivebox.config.common import get_config
from archivebox.misc.util import enforce_types
from archivebox.misc.logging import ANSI
@ -171,7 +171,9 @@ def log_list_finished(snapshots):
def log_removal_started(snapshots, yes: bool):
count = snapshots.count() if hasattr(snapshots, "count") else len(snapshots)
from django.db.models import QuerySet
count = snapshots.count() if isinstance(snapshots, QuerySet) else len(snapshots)
print(f"[yellow3][i] Found {count} matching URLs to remove.[/]")
file_counts = [s.num_outputs for s in snapshots if os.access(s.output_dir, os.R_OK)]
print(
@ -203,7 +205,7 @@ def log_removal_finished(remaining_links: int, removed_links: int):
@enforce_types
def pretty_path(path: Path | str, pwd: Path | str = DATA_DIR, color: bool = True) -> str:
def pretty_path(path: Path | str, pwd: Path | str = CONSTANTS.DATA_DIR, color: bool = True) -> str:
"""convert paths like .../ArchiveBox/archivebox/../output/abc into output/abc"""
pwd = str(Path(pwd)) # .resolve()
path = str(path)

32
archivebox/misc/paginators.py
View File

@ -4,6 +4,7 @@ from django.core.paginator import Paginator
from django.core.paginator import Page
from django.core.paginator import EmptyPage
from django.db import connection
from django.db.models import QuerySet
from django.utils.functional import cached_property
@ -19,9 +20,13 @@ class CountlessPage(Page):
class CountlessPaginator(Paginator):
has_exact_count = False
def __init__(self, *args, **kwargs):
self._count_hint = 0
super().__init__(*args, **kwargs)
@cached_property
def count(self):
return getattr(self, "_count_hint", 0)
return self._count_hint
@cached_property
def num_pages(self):
@ -56,16 +61,29 @@ class AcceleratedPaginator(Paginator):
@cached_property
def count(self):
query = getattr(self.object_list, "query", None)
if query is not None and (getattr(query, "distinct", False) or getattr(getattr(query, "where", None), "children", None)):
if not isinstance(self.object_list, QuerySet):
return super().count
query = self.object_list.query
count_hint = self.object_list.__dict__.get("_archivebox_count_hint")
if count_hint is None:
count_hint = query.__dict__.get("_archivebox_count_hint")
if count_hint is not None:
model = self.object_list.model
if count_hint == "model_estimate":
return self._model_count_estimate(model)
if callable(count_hint):
return count_hint()
return count_hint
if query.distinct or query.where.children:
# fallback to normal count method on filtered queryset
return super().count
model = getattr(self.object_list, "model", None)
if model is None:
return super().count
# otherwise count total rows in a separate fast query
return self._model_count_estimate(self.object_list.model)
def _model_count_estimate(self, model):
if connection.vendor == "sqlite":
table_name = model._meta.db_table
with connection.cursor() as cursor:

17
archivebox/misc/serve_static.py
View File

@ -2,6 +2,7 @@ import html
import json
import re
import os
import sys
import stat
import asyncio
import posixpath
@ -17,6 +18,7 @@ from pathlib import Path
from urllib.parse import urlencode
from django import template
from django.core.handlers.asgi import ASGIRequest
from django.contrib.staticfiles import finders
from django.template import TemplateDoesNotExist, loader
from django.views import static
@ -117,7 +119,7 @@ def _render_mhtml_preview_document(filename: str, output_path: str) -> str:
def _format_direntry_timestamp(stat_result: os.stat_result) -> str:
timestamp = getattr(stat_result, "st_birthtime", None) or stat_result.st_mtime
timestamp = stat_result.st_birthtime if sys.platform == "darwin" else stat_result.st_mtime
return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M")
@ -333,7 +335,7 @@ mimetypes.add_type("multipart/related", ".mhtml")
mimetypes.add_type("multipart/related", ".mht")
try:
_markdown = getattr(importlib.import_module("markdown"), "markdown")
_markdown = importlib.import_module("markdown").markdown
except ImportError:
_markdown: Callable[..., str] | None = None
@ -716,10 +718,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
https://github.com/satchamo/django/commit/2ce75c5c4bee2a858c0214d136bfcd351fcde11d
"""
assert document_root
config = getattr(request, "archivebox_config", None)
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config
config = request.archivebox_config
fullpath, path = _resolve_archive_path(document_root, path)
if os.access(fullpath, os.R_OK) and fullpath.is_dir():
if request.GET.get("download") == "zip" and show_indexes:
@ -727,7 +726,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
fullpath,
path,
is_archive_replay=is_archive_replay,
use_async_stream=hasattr(request, "scope"),
use_async_stream=isinstance(request, ASGIRequest),
config=config,
)
if show_indexes:
@ -872,7 +871,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
fullpath.name,
raw_output_path,
wacz_path=fullpath,
fallback_url=getattr(request, "archivebox_snapshot_url", "") or "",
fallback_url=request.archivebox_snapshot_url or "",
last_modified=http_date(statobj.st_mtime),
etag=etag or "",
cache_control=(
@ -978,7 +977,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
# setup response object
ranged_file = RangedFileReader(open(fullpath, "rb"))
response = StreamingHttpResponse(
_stream_ranged_file_async(ranged_file) if hasattr(request, "scope") else ranged_file,
_stream_ranged_file_async(ranged_file) if isinstance(request, ASGIRequest) else ranged_file,
content_type=content_type,
)
response.headers["Last-Modified"] = http_date(statobj.st_mtime)

2
archivebox/misc/toml_util.py
View File

@ -96,7 +96,7 @@ class JSONSchemaWithLambdas(GenerateJsonSchema):
def better_toml_dump_str(val: Any) -> str:
try:
dump_str = cast(Callable[[Any], str], getattr(toml.encoder, "_dump_str"))
dump_str = cast(Callable[[Any], str], toml.encoder._dump_str)
return dump_str(val)
except Exception:
# if we hit any of toml's numerous encoding bugs,

2
archivebox/misc/util.py
View File

@ -474,7 +474,7 @@ class ExtendedEncoder(pyjson.JSONEncoder):
def default(self, o):
cls_name = o.__class__.__name__
if hasattr(o, "_asdict"):
if isinstance(o, tuple) and "_asdict" in vars(type(o)):
return o._asdict()
elif isinstance(o, bytes):

49
archivebox/personas/migrations/0004_hydrate_persona_permissions.py
View File

@ -1,5 +1,6 @@
import os
import json
import uuid
from django.db import migrations
from django.db.models import Q
@ -9,31 +10,6 @@ VALID_PERMISSIONS = {"public", "unlisted", "private"}
BATCH_SIZE = 1000
def legacy_bool(value):
if value is None:
return None
normalized = str(value).strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
return None
def permissions_from_legacy_public_flags(config):
if str(config.get("PERMISSIONS") or "").strip():
return None
public_snapshots = legacy_bool(config.get("PUBLIC_SNAPSHOTS"))
public_index = legacy_bool(config.get("PUBLIC_INDEX"))
if public_snapshots is False:
return "private"
if public_index is False:
return "unlisted"
if public_snapshots is True or public_index is True:
return "public"
return None
def normalize_permissions(value, default):
value = str(value or "").strip().lower()
return value if value in VALID_PERMISSIONS else default
@ -59,22 +35,19 @@ def raw_base_config(apps):
def resolve_permissions(config, default):
from archivebox.config.common import permissions_from_legacy_public_flags
explicit = str(config.get("PERMISSIONS") or "").strip().lower()
if explicit in VALID_PERMISSIONS:
return explicit
return permissions_from_legacy_public_flags(config) or default
def model_has_config(model):
try:
model._meta.get_field("config")
except Exception:
return False
return True
def id_values(pk):
return str(pk), getattr(pk, "hex", str(pk).replace("-", ""))
if isinstance(pk, uuid.UUID):
return str(pk), pk.hex
pk_str = str(pk)
return pk_str, pk_str.replace("-", "")
def flush_batch(cursor, table_name, batch):
@ -120,8 +93,6 @@ def _ensure_permissions_column(cursor):
def hydrate_persona_permissions(apps, schema_editor):
Persona = apps.get_model("personas", "Persona")
User = apps.get_model("auth", "User")
user_has_config = model_has_config(User)
base_config = raw_base_config(apps)
default_permissions = resolve_permissions(base_config, "public")
table_name = schema_editor.quote_name(Persona._meta.db_table)
@ -130,13 +101,9 @@ def hydrate_persona_permissions(apps, schema_editor):
batch = []
missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
for persona in Persona.objects.filter(missing_permissions).select_related("created_by").iterator(chunk_size=BATCH_SIZE):
for persona in Persona.objects.filter(missing_permissions).iterator(chunk_size=BATCH_SIZE):
config = dict(persona.config or {})
resolved = dict(base_config)
if user_has_config:
user_config = persona.created_by.config or {}
if isinstance(user_config, dict):
resolved.update(user_config)
resolved.update(config)
config["PERMISSIONS"] = resolve_permissions(resolved, default_permissions)
batch.append((persona.id, config))

41
archivebox/personas/models.py
View File

@ -16,6 +16,7 @@ import sys
from contextlib import contextmanager
from pathlib import Path
from typing import TYPE_CHECKING, Any
from collections.abc import Mapping
from django.db import models
from django.db.models.fields.json import KT
@ -59,6 +60,26 @@ VOLATILE_PROFILE_FILE_NAMES = {
}
def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, persona_dir: Path) -> dict[str, Any]:
derived = dict(config or {})
if "CHROME_USER_DATA_DIR" not in derived:
derived["CHROME_USER_DATA_DIR"] = str(persona_dir / "chrome_profile")
if "CHROME_DOWNLOADS_DIR" not in derived:
derived["CHROME_DOWNLOADS_DIR"] = str(persona_dir / "chrome_downloads")
cookies_path = persona_dir / "cookies.txt"
if "COOKIES_FILE" not in derived and cookies_path.exists():
derived["COOKIES_FILE"] = str(cookies_path)
auth_path = persona_dir / "auth.json"
if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
derived["AUTH_STORAGE_FILE"] = str(auth_path)
derived["ACTIVE_PERSONA"] = name
return derived
class Persona(ModelWithConfig):
"""
Browser persona/profile for archiving sessions.
@ -99,8 +120,7 @@ class Persona(ModelWithConfig):
if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
from archivebox.config.common import get_config
user = self.created_by if self.created_by_id else None
config["PERMISSIONS"] = normalize_permissions(get_config(user=user, include_machine=True).PERMISSIONS)
config["PERMISSIONS"] = normalize_permissions(get_config(include_machine=True).PERMISSIONS)
self.config = config
update_fields = kwargs.get("update_fields")
if update_fields is not None:
@ -151,22 +171,7 @@ class Persona(ModelWithConfig):
- AUTH_STORAGE_FILE (derived from persona path, if file exists)
- ACTIVE_PERSONA (set to this persona's name)
"""
derived = dict(self.config or {})
# Add derived paths (don't override if explicitly set in config)
if "CHROME_USER_DATA_DIR" not in derived:
derived["CHROME_USER_DATA_DIR"] = self.CHROME_USER_DATA_DIR
if "CHROME_DOWNLOADS_DIR" not in derived:
derived["CHROME_DOWNLOADS_DIR"] = self.CHROME_DOWNLOADS_DIR
if "COOKIES_FILE" not in derived and self.COOKIES_FILE:
derived["COOKIES_FILE"] = self.COOKIES_FILE
if "AUTH_STORAGE_FILE" not in derived and self.AUTH_STORAGE_FILE:
derived["AUTH_STORAGE_FILE"] = self.AUTH_STORAGE_FILE
# Always set ACTIVE_PERSONA to this persona's name
derived["ACTIVE_PERSONA"] = self.name
return derived
return derive_persona_config(name=self.name, config=self.config, persona_dir=self.path)
def ensure_dirs(self) -> None:
"""Create persona directories if they don't exist."""

7
archivebox/plugins/discovery.py
View File

@ -1,7 +1,6 @@
__package__ = "archivebox.plugins"
import json
import os
from collections.abc import Iterable
from functools import lru_cache
from pathlib import Path
@ -26,9 +25,7 @@ class PluginSpecialConfig(TypedDict):
BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
USER_PLUGINS_DIR = Path(
os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or str(CONSTANTS.USER_PLUGINS_DIR),
).expanduser()
USER_PLUGINS_DIR = CONSTANTS.USER_PLUGINS_DIR
def iter_plugin_dirs() -> list[Path]:
@ -167,7 +164,7 @@ def discover_plugins_that_provide_interface(
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
if not all(hasattr(module, attr) for attr in required_attrs):
if not all(attr in vars(module) for attr in required_attrs):
continue
if plugin_prefix:

13
archivebox/plugins/forms.py
View File

@ -9,7 +9,7 @@ from typing import Any
from django import forms
from django.utils.html import format_html
from archivebox.config.common import get_config
from archivebox.config.common import ArchiveBoxConfig, get_config
from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_icon, get_plugins
@ -250,6 +250,7 @@ def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> An
class PluginConfigFormMixin:
plugin_groups: list[dict[str, Any]]
allow_crawl_execution_config_fields = True
def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
all_plugins = get_plugins()
@ -314,7 +315,10 @@ class PluginConfigFormMixin:
config_fields = [
self._build_plugin_config_field(str(plugin_name), str(config_key), prop_schema, runtime_config)
for config_key, prop_schema in properties.items()
if isinstance(prop_schema, dict)
if (
isinstance(prop_schema, dict)
and (self.allow_crawl_execution_config_fields or ArchiveBoxConfig.scope_for_key(str(config_key)) == "crawl_frozen")
)
]
cards.append(
{
@ -427,6 +431,8 @@ class PluginConfigFormMixin:
input_name = _plugin_config_input_name(plugin_name, config_key)
if input_name not in self.data:
continue
if not self.allow_crawl_execution_config_fields and ArchiveBoxConfig.scope_for_key(str(config_key)) != "crawl_frozen":
continue
raw_value: Any = self.data.get(input_name)
if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list):
@ -595,5 +601,6 @@ def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str
)
if binary is None and name != value:
binary = Binary.objects.get_valid_binary(name, machine=machine)
urls[key] = get_installed_binary_change_url(getattr(binary, "name", name), binary) or get_environment_binary_url(name)
binary_name = binary.name if binary is not None else name
urls[key] = get_installed_binary_change_url(binary_name, binary) or get_environment_binary_url(name)
return urls

45
archivebox/plugins/hooks.py
View File

@ -46,7 +46,7 @@ import json
import os
from collections.abc import Mapping
from pathlib import Path
from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard
from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, runtime_checkable
from archivebox.config.constants import CONSTANTS
from archivebox.config.version import VERSION
@ -62,12 +62,13 @@ if TYPE_CHECKING:
from archivebox.machine.models import Process
@runtime_checkable
class ConfigDump(Protocol):
def as_dict(self) -> dict[str, Any]: ...
def _has_config_dump(config: object) -> TypeGuard[ConfigDump]:
return callable(getattr(config, "as_dict", None))
return isinstance(config, ConfigDump)
def _config_to_overrides(config: ConfigLookup | Mapping[str, Any] | None) -> dict[str, Any]:
@ -259,13 +260,13 @@ def run_hook(
This is the low-level hook executor that creates a Process record and
uses Process.launch() for subprocess management.
Config is passed to hooks via environment variables. Caller MUST use
get_config() to merge all sources (file, env, machine, crawl, snapshot).
Config is passed to hooks via environment variables. Crawl/snapshot callers
should pass the runtime config produced by for_crawl_runtime().
Args:
script: Path to the hook script (.sh, .py, or .js)
output_dir: Working directory for the script (where output files go)
config: Optional pre-merged config dict from get_config(crawl=..., snapshot=...).
config: Optional runtime config dict from for_crawl_runtime().
If omitted, pass scope/override args using kwargs prefixed with config_.
timeout: Maximum execution time in seconds
If None, auto-detects from PLUGINNAME_TIMEOUT config (fallback to TIMEOUT, default 300)
@ -277,18 +278,22 @@ def run_hook(
Example:
from archivebox.config.common import get_config
config = get_config(crawl=my_crawl, snapshot=my_snapshot)
config = get_config(crawl=my_crawl, snapshot=my_snapshot).for_crawl_runtime(crawl=my_crawl, snapshot=my_snapshot)
process = run_hook(hook_path, output_dir, config=config, url=url, snapshot_id=id)
if process.status == 'exited':
records = process.get_records() # Get parsed JSONL output
"""
from archivebox.machine.models import Process, Machine, NetworkInterface
from archivebox.config.common import get_config
from archivebox.config.common import get_config, normalize_runtime_config
import sys
config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
resolved_config = get_config(overrides=_config_to_overrides(config), **config_scope)
hook_config = resolved_config.for_crawl_execution()
config_overrides = _config_to_overrides(config)
resolved_config = get_config(overrides=config_overrides, **config_scope)
hook_config = normalize_runtime_config(
config_overrides if config is not None else resolved_config.for_crawl(),
json_safe=False,
)
# Auto-detect timeout from plugin config if not explicitly provided
if timeout is None:
@ -363,31 +368,16 @@ def run_hook(
# Set up environment with base paths
env = os.environ.copy()
env["DATA_DIR"] = str(resolved_config.DATA_DIR)
env["ARCHIVE_DIR"] = str(resolved_config.ARCHIVE_DIR)
env["ABX_RUNTIME"] = "archivebox"
env["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
env["LIBRARY_VERSION"] = VERSION
env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
resolved_output_dir = output_dir.resolve()
snap_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
crawl_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.CRAWLS_DIR_NAME)
snap_dir = hook_config.get("SNAP_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
crawl_dir = hook_config.get("CRAWL_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.CRAWLS_DIR_NAME)
if snap_dir:
env["SNAP_DIR"] = str(snap_dir)
if crawl_dir:
env["CRAWL_DIR"] = str(crawl_dir)
crawl_id = kwargs.get("_crawl_id") or kwargs.get("crawl_id")
if crawl_id:
try:
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.filter(id=crawl_id).first()
if crawl:
env["CRAWL_DIR"] = str(crawl.output_dir)
except Exception:
pass
# Export runtime library roots; abx-dl/abxpkg own executable lookup env.
lib_dir = resolved_config.LIB_DIR
if lib_dir:
@ -423,7 +413,6 @@ def run_hook(
"NODE_MODULES_DIR",
"NODE_MODULE_DIR",
"DATA_DIR",
"ARCHIVE_DIR",
"MACHINE_ID",
"SNAP_DIR",
"CRAWL_DIR",

2
archivebox/plugins/views.py
View File

@ -82,7 +82,7 @@ def get_machine_admin_url() -> str | None:
from archivebox.machine.models import Machine
machine = Machine.current()
return getattr(machine, "admin_change_url", None) or f"/admin/machine/machine/{machine.id}/change/"
return machine.admin_change_url or f"/admin/machine/machine/{machine.id}/change/"
except Exception:
return None

6
archivebox/progressmonitor/views.py
View File

@ -345,9 +345,7 @@ def live_progress_view(request):
persona_objects_by_id = {}
persona_objects_by_name = {}
persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
persona_names = {
str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default") for crawl in active_crawls_list if not crawl["persona_id"]
}
persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in active_crawls_list) else set()
if persona_ids or persona_names:
from archivebox.personas.models import Persona
@ -813,7 +811,7 @@ def live_progress_view(request):
urls_preview = crawl["urls"][:60] if crawl["urls"] else None
crawl_tags = [tag.strip() for tag in (crawl["tags_str"] or "").replace("\n", ",").split(",") if tag.strip()]
persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
persona_name = persona_details["name"] if persona_details else str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default")
persona_name = persona_details["name"] if persona_details else "Default"
persona_details = persona_details or persona_details_by_name.get(persona_name)
crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0

18
archivebox/search/admin.py
View File

@ -19,8 +19,8 @@ class SearchResultsChangeList(ChangeList):
def __init__(self, request, *args, **kwargs):
"""Capture normalized search mode before Django builds results."""
self.search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
self.search_mode_backend = get_search_mode_backend(self.search_mode, config=getattr(request, "archivebox_config", None))
self.search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
self.search_mode_backend = get_search_mode_backend(self.search_mode, config=request.archivebox_config)
super().__init__(request, *args, **kwargs)
self.embedded_changelist = request.GET.get("_embedded") == "crawl"
@ -31,7 +31,7 @@ class SearchResultsChangeList(ChangeList):
self.opts.model_name == "snapshot"
and self.query
and self.result_count == 0
and get_search_mode_base(self.search_mode, config=getattr(request, "archivebox_config", None)) == "deep"
and get_search_mode_base(self.search_mode, config=request.archivebox_config) == "deep"
and self.search_mode_backend,
)
@ -55,13 +55,11 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
def get_default_search_mode(self):
"""Return the default search mode for the current request config."""
request = getattr(self, "request", None)
return get_default_search_mode(config=getattr(request, "archivebox_config", None))
return get_default_search_mode(config=self.request.archivebox_config)
def get_search_mode_options(self):
"""Return selector options for the current request config."""
request = getattr(self, "request", None)
return get_search_mode_options(config=getattr(request, "archivebox_config", None))
return get_search_mode_options(config=self.request.archivebox_config)
def get_search_results(self, request, queryset, search_term: str):
"""Apply admin search semantics to a changelist queryset."""
@ -69,14 +67,14 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
search_term = search_term.strip()
if not search_term:
return super().get_search_results(request, queryset, search_term)
search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl":
cached_ids = get_cached_admin_search_ids(request)
if cached_ids is not None:
return queryset.filter(pk__in=cached_ids) if cached_ids else queryset.none(), False
return queryset.none(), False
if get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None)) == "meta":
if get_search_mode_base(search_mode, config=request.archivebox_config) == "meta":
qs, use_distinct = super().get_search_results(request, queryset, search_term)
return qs, use_distinct
if request.GET.get("_embedded") == "crawl":
@ -85,7 +83,7 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
pk__in=query_search_index(
search_term,
search_mode=search_mode,
config=getattr(request, "archivebox_config", None),
config=request.archivebox_config,
).values("pk"),
), False
except Exception as err:

6
archivebox/search/backends.py
View File

@ -12,16 +12,16 @@ _search_backends_cache: dict | None = None
@contextmanager
def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
"""Temporarily expose resolved config through os.environ for backend code."""
"""Temporarily expose resolved search config through os.environ for backend code."""
config = config or get_config(**config_kwargs)
updates = {}
for key, value in config.items():
if not str(key).startswith("SEARCH_BACKEND_"):
continue
if value is None:
continue
if isinstance(value, (str, int, float, bool, os.PathLike)):
updates[str(key)] = str(value)
updates["DATA_DIR"] = str(config.DATA_DIR)
updates["SNAP_DIR"] = str(config.USERS_DIR)
previous = {key: os.environ.get(key) for key in updates}
os.environ.update(updates)
try:

4
archivebox/search/query.py
View File

@ -248,10 +248,8 @@ def iter_query_search_ids(
backend = backends[backend_name]
try:
with search_backend_env(config=config):
if hasattr(backend, "iter_search"):
if backend_name == "ripgrep":
ids = backend.iter_search(query, search_mode=search_mode_base)
elif backend_name == "ripgrep":
ids = backend.search(query, search_mode=search_mode_base)
else:
ids = backend.search(query)
for snapshot_id in ids:

11
archivebox/search/views.py
View File

@ -101,7 +101,8 @@ def iter_admin_backend_search_ids(iterator, queryset):
def admin_snapshot_search_stream_view(model_admin, request):
"""Stream admin Snapshot search progress and cache matching IDs."""
query = (request.GET.get("q") or "").strip()
search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
config = request.archivebox_config
search_mode = get_search_mode(request.GET.get("search_mode"), config=config)
if not query:
return StreamingHttpResponse((), content_type="text/plain")
@ -115,12 +116,12 @@ def admin_snapshot_search_stream_view(model_admin, request):
filter_request.path = target_url.path or request.path
filter_request.path_info = target_url.path or request.path_info
filter_request.GET = target_get
filter_request.archivebox_config = getattr(request, "archivebox_config", None)
filter_request.archivebox_config = config
# Build the same filtered base queryset the changelist uses, but with the
# search params stripped. The stream intersects each wave with this queryset
# before writing IDs into the short-lived cache consumed by the changelist.
current_request = getattr(model_admin, "request", None)
current_request = model_admin.__dict__.get("request")
try:
base_queryset = model_admin.get_changelist_instance(filter_request).queryset
finally:
@ -150,12 +151,12 @@ def admin_snapshot_search_stream_view(model_admin, request):
nonlocal last_sent
iterator = None
try:
search_mode_base = get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None))
search_mode_base = get_search_mode_base(search_mode, config=config)
iterator = (
iter_admin_meta_search_ids(query, base_queryset)
if search_mode_base == "meta"
else iter_admin_backend_search_ids(
iter_query_search_ids(query, search_mode=search_mode, config=getattr(request, "archivebox_config", None)),
iter_query_search_ids(query, search_mode=search_mode, config=config),
base_queryset,
)
)

29
archivebox/services/archive_result_service.py
View File

@ -5,7 +5,7 @@ import json
from collections import defaultdict
from collections.abc import Iterable
from pathlib import Path
from typing import Any
from typing import Any, Protocol, runtime_checkable
from asgiref.sync import sync_to_async
from django.db import IntegrityError
@ -18,6 +18,11 @@ from abx_dl.services.base import BaseService
from .process_service import parse_event_datetime
@runtime_checkable
class ModelDumpable(Protocol):
def model_dump(self) -> dict[str, Any]: ...
def _collect_output_metadata(plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"}
output_files: dict[str, dict] = {}
@ -97,15 +102,8 @@ def _normalize_output_files(raw_output_files: Any) -> dict[str, dict]:
if isinstance(item, str):
normalized[item] = _enrich_metadata(item, {})
continue
if hasattr(item, "model_dump"):
if isinstance(item, ModelDumpable):
item = item.model_dump()
elif hasattr(item, "path"):
item = {
"path": getattr(item, "path", ""),
"extension": getattr(item, "extension", ""),
"mimetype": getattr(item, "mimetype", ""),
"size": getattr(item, "size", 0),
}
if not isinstance(item, dict):
continue
path = str(item.get("path") or "").strip()
@ -224,14 +222,18 @@ class ArchiveResultService(BaseService):
snapshot = await Snapshot.objects.filter(id=event.snapshot_id).select_related("crawl", "crawl__created_by").afirst()
if snapshot is None:
return
plugin_dir = Path(snapshot.output_dir) / event.plugin
output_files, output_size, output_mimetypes = await sync_to_async(_resolve_output_metadata)(event.output_files, plugin_dir)
process_started = await self.bus.find(
ProcessStartedEvent,
past=True,
future=False,
where=lambda candidate: self.bus.event_is_child_of(event, candidate),
)
plugin_dir = (
Path(process_started.output_dir)
if process_started is not None and process_started.output_dir
else Path(snapshot.output_dir) / event.plugin
)
output_files, output_size, output_mimetypes = await sync_to_async(_resolve_output_metadata)(event.output_files, plugin_dir)
process = None
if process_started is not None:
started_at = parse_event_datetime(process_started.start_ts)
@ -292,14 +294,15 @@ class ArchiveResultService(BaseService):
update_fields = []
for field, value in defaults.items():
if getattr(result, field) != value:
if result.__dict__[field] != value:
setattr(result, field, value)
update_fields.append(field)
if update_fields:
await result.asave(update_fields=[*update_fields, "modified_at"])
if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
next_title = _extract_snapshot_title(str(snapshot.output_dir), event.plugin, result.output_str, snapshot_url=snapshot.url)
title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else ""
next_title = _extract_snapshot_title(str(plugin_dir.parent), event.plugin, title_output_str, snapshot_url=snapshot.url)
if next_title and _should_update_snapshot_title(snapshot.title or "", next_title, snapshot_url=snapshot.url):
snapshot.title = next_title
await snapshot.asave(update_fields=["title", "modified_at"])

13
archivebox/services/machine_service.py
View File

@ -40,19 +40,18 @@ class MachineService(BaseService):
self.bus.on(MachineEvent, self.on_MachineEvent__save_to_db)
async def on_MachineEvent__save_to_db(self, event: MachineEvent) -> None:
from archivebox.machine.models import Machine, _sanitize_machine_config
from archivebox.config.common import get_config
from archivebox.machine.models import Machine
if event.config_type != "derived":
return
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
lib_dir = await sync_to_async(lambda: get_config(include_machine=False).LIB_DIR, thread_sensitive=True)()
config = dict(machine.config or {})
old_config = dict(machine.config or {})
config = dict(old_config)
if event.config is not None:
binary_only = _strip_to_binary_keys(event.config)
config.update(_sanitize_machine_config(binary_only, lib_dir=lib_dir))
config.update(binary_only)
elif event.method == "update":
key = event.key.replace("config/", "", 1).strip()
if key and _is_binary_event_key(key):
@ -64,5 +63,7 @@ class MachineService(BaseService):
else:
return
machine.config = _sanitize_machine_config(config, lib_dir=lib_dir)
if config == old_config:
return
machine.config = config
await machine.asave(update_fields=["config", "modified_at"])

335
archivebox/services/runner.py
View File

@ -2,15 +2,12 @@ from __future__ import annotations
import asyncio
import contextvars
import json
import os
import signal
import shutil
import subprocess
import sys
import threading
import time
from collections.abc import Mapping
from contextlib import nullcontext
from datetime import timedelta
from functools import lru_cache
@ -58,10 +55,10 @@ from abxbus import BaseEvent
from abxbus.event_bus import EventBus, get_current_event, in_handler_context
from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError
from archivebox.config.common import ArchiveBoxBaseConfig
from archivebox.config.common import ArchiveBoxBaseConfig, normalize_runtime_config
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.misc.db import run_db_analyze_batch
from archivebox.core.shutdown_util import foreground_shutdown_signals
from archivebox.core.shutdown_util import foreground_shutdown_signals, raise_if_shutdown_requested
from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS
@ -74,6 +71,9 @@ from .snapshot_service import SnapshotService, finalize_completed_snapshot
from .tag_service import TagService
QUEUED_PLUGIN_RESULT_BATCH_SIZE = 100
def _bus_name(prefix: str, identifier: str) -> str:
normalized = "".join(ch if ch.isalnum() else "_" for ch in identifier)
return f"{prefix}_{normalized}"
@ -110,9 +110,9 @@ def _runner_console_line(*, crawl=None, crawl_id=None, snapshot=None, status: st
if snapshot is not None:
label = snapshot.url
else:
label = (getattr(crawl, "label", "") or "").strip()
label = (crawl.label or "").strip()
if not label:
label = (getattr(crawl, "urls", "") or "").partition("\n")[0].strip() or str(crawl_id)
label = (crawl.urls or "").partition("\n")[0].strip() or str(crawl_id)
line.append(_runner_label(label, reserve=prefix_width))
Console(highlight=False).print(line)
@ -122,14 +122,6 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str
return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
def _normalize_runtime_config(config: ArchiveBoxBaseConfig | Mapping[str, Any] | str | None) -> dict[str, Any]:
from archivebox.config.common import normalize_runtime_config
if isinstance(config, ArchiveBoxBaseConfig):
return config.for_crawl_execution()
return normalize_runtime_config(config)
def _runner_task_context() -> contextvars.Context:
context = contextvars.copy_context()
context.run(EventBus.current_event_context.set, None)
@ -149,8 +141,9 @@ async def _emit_machine_config(
derived_config: dict[str, Any],
parent_event=None,
) -> None:
user_config = _normalize_runtime_config(config)
derived_machine_config = _normalize_runtime_config(derived_config)
user_config = normalize_runtime_config(config)
user_config["ABX_RUNTIME"] = "archivebox"
derived_machine_config = normalize_runtime_config(derived_config)
user_event = MachineEvent(
config=user_config,
config_type="user",
@ -179,7 +172,6 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
if os.environ.get("PYTEST_CURRENT_TEST") and not allow_under_pytest:
return False
from archivebox.config import CONSTANTS
from archivebox.machine.models import Machine, Process
from archivebox.workers.supervisord_util import RUNNER_WORKER, get_existing_supervisord_process, get_worker, start_worker
@ -205,22 +197,7 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
except Exception:
pass
log_path = CONSTANTS.LOGS_DIR / "errors.log"
log_path.parent.mkdir(parents=True, exist_ok=True)
env = os.environ.copy()
env.setdefault("DATA_DIR", str(CONSTANTS.DATA_DIR))
with log_path.open("a", encoding="utf-8") as log_handle:
subprocess.Popen(
[sys.executable, "-m", "archivebox", "run", "--daemon"],
cwd=str(CONSTANTS.DATA_DIR),
env=env,
stdin=subprocess.DEVNULL,
stdout=log_handle,
stderr=log_handle,
start_new_session=True,
)
return True
return False
class CrawlRunner:
@ -233,6 +210,7 @@ class CrawlRunner:
process_discovered_snapshots_inline: bool = True,
show_progress: bool = True,
interactive_interrupts: bool = False,
config_overrides: dict[str, Any] | None = None,
):
self.crawl = crawl
self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0)
@ -249,6 +227,7 @@ class CrawlRunner:
self.process_discovered_snapshots_inline = process_discovered_snapshots_inline
self.show_progress = show_progress
self.interactive_interrupts = interactive_interrupts
self.config_overrides = dict(config_overrides or {})
async def ignore_snapshot(_snapshot_id: str) -> None:
return None
@ -265,7 +244,7 @@ class CrawlRunner:
self.snapshot_semaphore = asyncio.Semaphore(1)
self.max_concurrent_snapshots = 1
self.persona = None
self.base_config: dict[str, Any] = {}
self.base_config: ArchiveBoxBaseConfig | dict[str, Any] = {}
self.derived_config: dict[str, Any] = {}
self.primary_url = ""
self.crawl_output_dir = ""
@ -282,6 +261,8 @@ class CrawlRunner:
self._last_lease_heartbeat_at = 0.0
def _request_abort_from_signal(self, _sig: signal.Signals) -> None:
if os.environ.get("ARCHIVEBOX_RUNNER_DAEMON") == "1":
os._exit(128 + int(_sig))
already_requested = self._signal_abort_requested
self._signal_abort_requested = True
self._skip_wait_until_idle = True
@ -352,16 +333,19 @@ class CrawlRunner:
if self.interactive_interrupts
else "\n[🛑] Got {signal_name}, stopping gracefully...\n"
)
# interactive_interrupts is only enabled when this runner belongs
# to a foreground `archivebox add`. Runners owned by server/update/
# run should use immediate graceful shutdown instead of the
# add-specific "abort current hook or continue" flow.
self._run_task = asyncio.current_task()
# Do not raise KeyboardInterrupt directly from an OS signal while
# the asyncio loop is active. Python can inject it into whichever
# task is currently running, which produces noisy "Task exception
# was never retrieved" logs from unrelated abxbus housekeeping
# tasks. _request_abort_from_signal() cancels the runner task
# cooperatively instead; repeated signals still hard-exit in the
# shared foreground signal handler.
with foreground_shutdown_signals(
first_signal_message=first_signal_message,
on_signal=self._request_abort_from_signal,
raise_on_first_signal=not self.interactive_interrupts,
raise_on_first_signal=False,
):
self._run_task = asyncio.current_task()
snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)()
max_concurrent_snapshots = max(1, int(self.base_config.get("CRAWL_MAX_CONCURRENT_SNAPSHOTS", 1)))
self.max_concurrent_snapshots = max_concurrent_snapshots
@ -372,10 +356,7 @@ class CrawlRunner:
await heartbeat.start()
await _emit_machine_config(
self.bus,
config={
**self.base_config,
"ABX_RUNTIME": "archivebox",
},
config=self.base_config,
derived_config=self.derived_config,
)
if snapshot_ids:
@ -386,8 +367,9 @@ class CrawlRunner:
await heartbeat.stop()
await self.stop_snapshot_tasks()
try:
if not self._skip_wait_until_idle:
await self.bus.wait_until_idle(timeout=30.0)
await self.bus.wait_until_idle(timeout=1.0 if self._skip_wait_until_idle else 30.0)
except TimeoutError:
pass
finally:
await self.bus.destroy(clear=False)
bus_destroyed = True
@ -548,7 +530,7 @@ class CrawlRunner:
return
await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)()
config = await sync_to_async(lambda: get_config(crawl=self.crawl, include_machine=False), thread_sensitive=True)()
config = await sync_to_async(lambda: get_config(crawl=self.crawl), thread_sensitive=True)()
self.max_concurrent_snapshots = max(1, int(config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]))
active_snapshot_ids = [snapshot_id for snapshot_id, task in self.snapshot_tasks.items() if not task.done()]
@ -573,20 +555,27 @@ class CrawlRunner:
from archivebox.config.common import get_config
from archivebox.core.models import Snapshot
from archivebox.plugins.hooks import discover_hooks
from archivebox.machine.models import Machine, NetworkInterface, Process, _sanitize_machine_config
from archivebox.machine.models import Machine, NetworkInterface, Process
self.primary_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else ""
current_iface = NetworkInterface.current(refresh=True)
current_iface = NetworkInterface.current(refresh=not self.allow_maintenance_on_inactive_crawl)
current_process = Process.current()
if current_process.iface_id != current_iface.id or current_process.machine_id != current_iface.machine_id:
current_process.iface = current_iface
current_process.machine = current_iface.machine
current_process.save(update_fields=["iface", "machine", "modified_at"])
self.persona = self.crawl.resolve_persona()
self.base_config = get_config(crawl=self.crawl, include_machine=False)
self.derived_config = _sanitize_machine_config(Machine.current().config, lib_dir=self.base_config["LIB_DIR"])
self.base_config = get_config(crawl=self.crawl)
self.derived_config = dict(Machine.current().config or {})
self.crawl_output_dir = str(self.crawl.output_dir)
self.base_config["ABX_RUNTIME"] = "archivebox"
if self.persona:
self.base_config.update(
self.persona.prepare_runtime_for_crawl(
self.crawl,
chrome_binary=self.base_config["CHROME_BINARY"],
),
)
self.base_config.update(self.config_overrides)
if self.selected_plugins is None:
raw_plugins = str(self.base_config.get("PLUGINS") or "").strip()
if raw_plugins:
@ -597,13 +586,6 @@ class CrawlRunner:
hook.parent.name for event_name in runtime_events for hook in discover_hooks(event_name, config=self.base_config)
}
self.selected_plugins = sorted(runtime_plugins) or None
if self.persona:
self.base_config.update(
self.persona.prepare_runtime_for_crawl(
self.crawl,
chrome_binary=self.base_config["CHROME_BINARY"],
),
)
if self.initial_snapshot_ids:
# Direct snapshot maintenance paths are allowed to name paused
# snapshots explicitly. The runner still requires selected_plugins
@ -712,31 +694,32 @@ class CrawlRunner:
return live_ui
def load_snapshot_payload(self, snapshot_id: str) -> dict[str, Any]:
from archivebox.core.models import Snapshot
from archivebox.config.common import get_config
from archivebox.core.models import Snapshot
snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").get(id=snapshot_id)
self.crawl = snapshot.crawl
self.persona = snapshot.crawl.resolve_persona()
self.base_config = get_config(crawl=snapshot.crawl)
if self.persona:
self.base_config.update(
self.persona.prepare_runtime_for_crawl(
snapshot.crawl,
chrome_binary=self.base_config["CHROME_BINARY"],
),
)
self.base_config.update(self.config_overrides)
self.crawl_output_dir = str(snapshot.crawl.output_dir)
runtime_chrome_overrides = {
key: self.base_config[key] for key in ("CHROME_USER_DATA_DIR", "CHROME_DOWNLOADS_DIR") if self.base_config.get(key)
}
config = get_config(
config = self.base_config.for_crawl_runtime(
crawl=snapshot.crawl,
snapshot=snapshot,
base_config=self.base_config or None,
overrides=runtime_chrome_overrides,
include_machine=False,
persona=self.persona,
runtime_overrides=runtime_chrome_overrides,
extra_context={"snapshot_id": str(snapshot.id), "snapshot_depth": snapshot.depth},
)
config["CRAWL_DIR"] = self.crawl_output_dir
config["SNAP_DIR"] = str(snapshot.output_dir)
extra_context: dict[str, Any] = {}
if config.get("EXTRA_CONTEXT"):
parsed_extra_context = json.loads(str(config["EXTRA_CONTEXT"]))
if not isinstance(parsed_extra_context, dict):
raise TypeError("EXTRA_CONTEXT must decode to an object")
extra_context = parsed_extra_context
extra_context["snapshot_id"] = str(snapshot.id)
extra_context["snapshot_depth"] = snapshot.depth
config["EXTRA_CONTEXT"] = json.dumps(extra_context, separators=(",", ":"), sort_keys=True)
return {
"id": str(snapshot.id),
"url": snapshot.url,
@ -748,7 +731,7 @@ class CrawlRunner:
"depth": snapshot.depth,
"status": snapshot.status,
"output_dir": str(snapshot.output_dir),
"config": _normalize_runtime_config(config),
"config": normalize_runtime_config(config),
"_snapshot": snapshot,
}
@ -774,9 +757,10 @@ class CrawlRunner:
if parent_snapshot is None:
return
config = await sync_to_async(
lambda: get_config(crawl=self.crawl, snapshot=parent_snapshot, include_machine=False),
lambda: get_config(crawl=self.crawl, snapshot=parent_snapshot),
thread_sensitive=True,
)()
config = config.for_crawl_runtime(crawl=self.crawl, snapshot=parent_snapshot, persona=self.crawl.resolve_persona())
if CrawlLimitState.from_config(config).get_stop_reason() in ("crawl_max_size", "crawl_timeout"):
return
@ -790,8 +774,8 @@ class CrawlRunner:
async def run_crawl(self, root_snapshot_id: str, snapshot_ids: list[str]) -> None:
snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(root_snapshot_id)
config = _normalize_runtime_config(snapshot["config"])
derived_config = _normalize_runtime_config(self.derived_config)
config = normalize_runtime_config(snapshot["config"])
derived_config = normalize_runtime_config(self.derived_config)
output_dir = Path(self.crawl_output_dir)
plugins = self.runtime_plugins()
abx_snapshot = AbxSnapshot(
@ -1033,8 +1017,8 @@ class CrawlRunner:
):
await sync_to_async(self.seal_snapshot_due_to_limit, thread_sensitive=True)(snapshot_id)
return
config = _normalize_runtime_config(snapshot["config"])
derived_config = _normalize_runtime_config(self.derived_config)
config = normalize_runtime_config(snapshot["config"])
derived_config = normalize_runtime_config(self.derived_config)
output_dir = Path(snapshot["output_dir"])
plugins = (
filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True)
@ -1087,9 +1071,14 @@ class CrawlRunner:
# runner is the scheduler owner. Finalize idempotently here too
# so a completed snapshot cannot remain STARTED if the event was
# observed before its DB projector advanced the state machine.
await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(snapshot_id)
crawl_limit_stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
snapshot_id,
output_dir=output_dir,
crawl_limit_stop_reason=crawl_limit_stop_reason,
)
if snapshot["status"] == "sealed":
await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id)
await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
return
await self.enqueue_discovered_snapshots_from_outputs(snapshot)
await sync_to_async(
@ -1109,7 +1098,7 @@ class CrawlRunner:
def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None:
from archivebox.core.models import Snapshot
snapshot = Snapshot.objects.filter(id=snapshot_id).first()
snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
if snapshot is None or snapshot.status == Snapshot.StatusChoices.SEALED:
return
if snapshot.status == Snapshot.StatusChoices.STARTED:
@ -1129,6 +1118,7 @@ def run_crawl(
process_discovered_snapshots_inline: bool = True,
show_progress: bool = True,
interactive_interrupts: bool = False,
config_overrides: dict[str, Any] | None = None,
) -> None:
from archivebox.crawls.models import Crawl
from django.db import close_old_connections
@ -1145,6 +1135,7 @@ def run_crawl(
process_discovered_snapshots_inline=process_discovered_snapshots_inline,
show_progress=show_progress,
interactive_interrupts=interactive_interrupts,
config_overrides=config_overrides,
).run(),
)
finally:
@ -1171,15 +1162,15 @@ def run_crawl(
async def _run_binary(binary_id: str) -> None:
from archivebox.config.common import get_config
from archivebox.machine.models import Binary, Machine, _sanitize_machine_config
from archivebox.machine.models import Binary, Machine
binary = await Binary.objects.aget(id=binary_id)
plugins = discover_plugins()
config = get_config(include_machine=False)
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
derived_config = _normalize_runtime_config(_sanitize_machine_config(machine.config, lib_dir=config["LIB_DIR"]))
config["ABX_RUNTIME"] = "archivebox"
config = _normalize_runtime_config(config)
derived_config = normalize_runtime_config(machine.config)
config = config.for_crawl()
config = normalize_runtime_config(config)
bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0)
process_service = PersistedProcessService(bus)
BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
@ -1273,10 +1264,10 @@ def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
return None
def run_snapshot_maintenance(snapshot_id: str) -> bool:
def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None) -> bool:
from archivebox.core.models import ArchiveResult, Snapshot
snapshot = Snapshot.objects.filter(id=snapshot_id).first()
snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
if snapshot is None:
return False
@ -1290,9 +1281,23 @@ def run_snapshot_maintenance(snapshot_id: str) -> bool:
# branch can process those targeted plugin rows on the next tick
# This avoids reopening final/paused snapshots while also avoiding stranded
# queued ArchiveResults that have no independent scheduler.
snapshot.retry_at = timezone.now() if has_queued_results else None
snapshot.save(update_fields=["retry_at", "modified_at"])
snapshot.write_index_jsonl()
current_retry_at = snapshot.retry_at
next_retry_at = timezone.now() if has_queued_results else None
snapshot.retry_at = next_retry_at
if snapshot.fs_migration_needed:
snapshot.save(update_fields=["retry_at", "modified_at"])
else:
updated = snapshot.safe_update(
{"retry_at": next_retry_at},
refresh=False,
extra_filter={
"status": snapshot.StatusChoices.SEALED,
"retry_at": current_retry_at,
},
)
if not updated:
return False
snapshot.write_index_jsonl(output_dir=output_dir)
return True
@ -1395,8 +1400,6 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
snapshot = Snapshot.objects.get(pk=snapshot.pk)
except Snapshot.DoesNotExist:
return False
if runtime_config is not None:
snapshot._runtime_config = runtime_config
parent_reconciled = snapshot.reconcile_parent_lifecycle(lock_seconds=lock_seconds)
if parent_reconciled is not None:
return parent_reconciled
@ -1547,14 +1550,14 @@ def run_due_binary(binary, *, lock_seconds: int) -> bool:
async def _run_install(plugin_names: list[str] | None = None) -> None:
from archivebox.config.common import get_config
from archivebox.machine.models import Machine, _sanitize_machine_config
from archivebox.machine.models import Machine
plugins = discover_plugins()
config = get_config(include_machine=False)
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
derived_config = _normalize_runtime_config(_sanitize_machine_config(machine.config, lib_dir=config["LIB_DIR"]))
config["ABX_RUNTIME"] = "archivebox"
config = _normalize_runtime_config(config)
derived_config = normalize_runtime_config(machine.config)
config = config.for_crawl()
config = normalize_runtime_config(config)
bus = create_bus(name="ArchiveBox_install", total_timeout=3600.0)
PersistedProcessService(bus)
BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
@ -1721,24 +1724,84 @@ def _run_due_queued_plugin_result(
runtime_config,
) -> bool:
from archivebox.core.models import ArchiveResult, Snapshot
from django.db.models import Exists, OuterRef
if not plugin_names:
return False
queued_results = ArchiveResult.objects.filter(
queued_snapshot_ids = ArchiveResult.objects.filter(
status=ArchiveResult.StatusChoices.QUEUED,
plugin__in=plugin_names,
snapshot__status=Snapshot.StatusChoices.SEALED,
snapshot__retry_at__lte=timezone.now(),
).values("snapshot_id")
due_snapshots = Snapshot.objects.filter(
id__in=queued_snapshot_ids,
retry_at__lte=timezone.now(),
status=Snapshot.StatusChoices.SEALED,
)
if crawl_id:
queued_results = queued_results.filter(snapshot__crawl_id=crawl_id)
due_snapshot_id = queued_results.order_by("snapshot__retry_at", "snapshot__created_at").values_list("snapshot_id", flat=True).first()
return _run_due_snapshot_id(
due_snapshot_id,
lock_seconds=lock_seconds,
interactive_interrupts=interactive_interrupts,
runtime_config=runtime_config,
due_snapshots = due_snapshots.filter(crawl_id=crawl_id)
due_snapshots = due_snapshots.only("id", "crawl_id", "retry_at", "status").order_by("retry_at", "created_at")
first_due_snapshot = due_snapshots.first()
if first_due_snapshot is None:
return False
root_crawl_id = str(first_due_snapshot.crawl_id)
batch_candidates = list(
due_snapshots.filter(crawl_id=root_crawl_id)[:QUEUED_PLUGIN_RESULT_BATCH_SIZE],
)
if not batch_candidates:
return False
selected_plugins: list[str] | None = None
claimed_snapshot_ids: list[str] = []
for snapshot in batch_candidates:
snapshot_selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
if not snapshot_selected_plugins:
continue
if selected_plugins is None:
selected_plugins = snapshot_selected_plugins
if snapshot_selected_plugins != selected_plugins:
continue
if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
continue
snapshot.refresh_from_db()
snapshot.finalize_completed_upload_results()
if snapshot.fs_migration_needed:
run_snapshot_maintenance(str(snapshot.id))
snapshot.refresh_from_db()
if snapshot.status != Snapshot.StatusChoices.SEALED:
continue
claimed_snapshot_ids.append(str(snapshot.id))
_runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
if not claimed_snapshot_ids or selected_plugins is None:
return True
run_crawl(
root_crawl_id,
snapshot_ids=claimed_snapshot_ids,
selected_plugins=selected_plugins,
process_discovered_snapshots_inline=True,
interactive_interrupts=interactive_interrupts,
config_overrides={
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": QUEUED_PLUGIN_RESULT_BATCH_SIZE,
},
)
if all(plugin.startswith("search_backend_") for plugin in selected_plugins):
queued_results = ArchiveResult.objects.filter(
snapshot_id=OuterRef("pk"),
status=ArchiveResult.StatusChoices.QUEUED,
)
Snapshot.objects.filter(
id__in=claimed_snapshot_ids,
status=Snapshot.StatusChoices.SEALED,
).annotate(
has_queued_results=Exists(queued_results),
).filter(
has_queued_results=False,
).update(
retry_at=None,
modified_at=timezone.now(),
)
return True
def _run_due_binary() -> bool:
@ -1836,6 +1899,7 @@ def run_pending_crawls(
analyze_sweep_started_at = 0.0
orchestrator_started_at = time.monotonic()
while True:
raise_if_shutdown_requested()
now_monotonic = time.monotonic()
if now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
for model in (ArchiveResult, Snapshot, Crawl, Process):
@ -1867,29 +1931,6 @@ def run_pending_crawls(
if _fast_forward_same_path_snapshot_fs_versions():
continue
# Final-state snapshot maintenance comes before normal crawl work:
# filesystem/index maintenance and upload finalization should drain
# promptly, but pure search backend backfills are deferred below so
# they do not starve live crawls.
sealed_snapshots = Snapshot.objects.filter(
retry_at__lte=timezone.now(),
status=Snapshot.StatusChoices.SEALED,
)
if search_plugin_names:
sealed_snapshots = sealed_snapshots.exclude(
archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
archiveresult__plugin__in=search_plugin_names,
)
if crawl_id:
sealed_snapshots = sealed_snapshots.filter(crawl_id=crawl_id)
if _run_due_snapshot_query(
sealed_snapshots,
lock_seconds=60,
interactive_interrupts=interactive_interrupts,
runtime_config=runtime_config,
):
continue
if not maintenance_only:
active_snapshots = Snapshot.objects.filter(
retry_at__lte=timezone.now(),
@ -1987,6 +2028,32 @@ def run_pending_crawls(
):
continue
# Broad final-state maintenance is intentionally a fallback. Specific
# queued plugin work above can use ArchiveResult's scheduler indexes;
# this branch may need to prove that no due sealed snapshot remains, so
# avoid paying that scan while targeted work is already available.
sealed_snapshots = Snapshot.objects.filter(
retry_at__lte=timezone.now(),
status=Snapshot.StatusChoices.SEALED,
)
if search_plugin_names:
queued_search_snapshot_ids = ArchiveResult.objects.filter(
status=ArchiveResult.StatusChoices.QUEUED,
plugin__in=search_plugin_names,
).values("snapshot_id")
sealed_snapshots = sealed_snapshots.exclude(
id__in=queued_search_snapshot_ids,
)
if crawl_id:
sealed_snapshots = sealed_snapshots.filter(crawl_id=crawl_id)
if _run_due_snapshot_query(
sealed_snapshots,
lock_seconds=60,
interactive_interrupts=interactive_interrupts,
runtime_config=runtime_config,
):
continue
if not maintenance_only:
if _run_due_crawl_status(
Crawl.StatusChoices.SEALED,

18
archivebox/services/snapshot_service.py
View File

@ -11,7 +11,12 @@ from abx_dl.limits import CrawlLimitState
from abx_dl.services.base import BaseService
def finalize_completed_snapshot(snapshot_id: str) -> None:
def finalize_completed_snapshot(
snapshot_id: str,
*,
output_dir=None,
crawl_limit_stop_reason: str | None = None,
) -> None:
from archivebox.core.models import Snapshot
snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
@ -22,7 +27,7 @@ def finalize_completed_snapshot(snapshot_id: str) -> None:
snapshot.downloaded_at = timezone.now()
snapshot.save(update_fields=["downloaded_at", "modified_at"])
stop_reason = _crawl_limit_stop_reason(snapshot.crawl)
stop_reason = crawl_limit_stop_reason if crawl_limit_stop_reason is not None else _crawl_limit_stop_reason(snapshot.crawl)
if snapshot.crawl_id and stop_reason in ("crawl_max_size", "crawl_timeout"):
Snapshot.objects.filter(
crawl_id=snapshot.crawl_id,
@ -40,14 +45,17 @@ def finalize_completed_snapshot(snapshot_id: str) -> None:
snapshot.sm.seal()
snapshot.refresh_from_db()
snapshot.write_index_jsonl()
snapshot.write_index_jsonl(output_dir=output_dir)
def _crawl_limit_stop_reason(crawl) -> str:
from archivebox.config.common import get_config
config = get_config(crawl=crawl, include_machine=False)
config["CRAWL_DIR"] = str(crawl.output_dir)
config_model = get_config(crawl=crawl)
config = config_model.for_crawl_runtime(
crawl=crawl,
persona=crawl.resolve_persona(),
)
return CrawlLimitState.from_config(config).get_stop_reason()

2
archivebox/templates/core/add.html
View File

@ -304,7 +304,7 @@
<div class="error">{{ form.config.errors }}</div>
{% endif %}
<div class="help-text">
Override any config option for this crawl (e.g., TIMEOUT, USER_AGENT, CHROME_BINARY, etc.). <code>URL_ALLOWLIST</code>, <code>URL_DENYLIST</code>, and <code>PLUGINS</code> are updated automatically from the fields above.
Override crawl-scoped config options (e.g., TIMEOUT, USER_AGENT, URL_ALLOWLIST, URL_DENYLIST, PLUGINS). <code>*_BINARY</code> paths are managed on Persona or Machine config.
</div>
</div>
</details>

39
archivebox/tests/conftest.py
View File

@ -25,19 +25,10 @@ pytest_plugins = ["archivebox.tests.fixtures"]
REPO_ROOT = Path(__file__).resolve().parents[2]
PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
SESSION_DATA_DIR = Path(
os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
).resolve()
# Force ArchiveBox imports to see a temp DATA_DIR during test collection.
os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve()
(SESSION_DATA_DIR / "tests").mkdir(parents=True, exist_ok=True)
os.chdir(SESSION_DATA_DIR)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.core.settings")
os.environ.pop("ARCHIVE_DIR", None)
os.environ.pop("USERS_DIR", None)
os.environ.pop("CRAWL_DIR", None)
os.environ.pop("SNAP_DIR", None)
def _is_repo_path(path: Path) -> bool:
@ -56,7 +47,7 @@ def _assert_safe_runtime_paths(*, cwd: Path | None = None, env: dict[str, str] |
if cwd is not None:
_assert_not_repo_path(cwd, label="cwd")
for key in ("DATA_DIR", "ARCHIVE_DIR", "USERS_DIR", "CRAWL_DIR", "SNAP_DIR"):
for key in ("CRAWL_DIR", "SNAP_DIR"):
value = (env or {}).get(key)
if value:
_assert_not_repo_path(Path(value), label=key)
@ -89,9 +80,8 @@ def run_archivebox_cmd(
"""
cmd = [sys.executable, "-m", "archivebox"] + args
_assert_not_repo_path(data_dir, label="DATA_DIR")
_assert_not_repo_path(data_dir, label="cwd")
base_env = os.environ.copy()
base_env["DATA_DIR"] = str(data_dir)
base_env["USE_COLOR"] = "False"
base_env["SHOW_PROGRESS"] = "False"
# Disable slow extractors for faster tests
@ -143,9 +133,8 @@ def isolate_test_runtime(tmp_path, monkeypatch):
contract is that every test starts in its own temp directory and any
in-process ``os.environ`` edits are rolled back afterwards.
Each in-process test gets an explicit temp ``DATA_DIR`` so ArchiveBox code
never falls back to the repo cwd. Subprocess helpers that intentionally test
cwd-based behavior remove ``DATA_DIR`` for the child process themselves.
ArchiveBox derives DATA_DIR from cwd, so subprocess helpers pass the target
collection as cwd instead of using DATA_DIR as an override.
"""
_assert_not_repo_path(tmp_path, label="tmp_path")
original_cwd = Path.cwd()
@ -176,11 +165,6 @@ def isolate_test_runtime(tmp_path, monkeypatch):
monkeypatch.setattr(os, "chdir", guarded_chdir)
monkeypatch.setattr(subprocess, "Popen", guarded_popen)
os.environ["DATA_DIR"] = str(tmp_path)
os.environ.pop("ARCHIVE_DIR", None)
os.environ.pop("USERS_DIR", None)
os.environ.pop("CRAWL_DIR", None)
os.environ.pop("SNAP_DIR", None)
reset_machine_model_caches()
try:
_assert_safe_runtime_paths(cwd=Path.cwd(), env=os.environ)
@ -330,11 +314,6 @@ def run_archivebox_cmd_cwd(
_assert_not_repo_path(cwd, label="cwd")
base_env = os.environ.copy()
base_env.pop("DATA_DIR", None)
base_env.pop("ARCHIVE_DIR", None)
base_env.pop("USERS_DIR", None)
base_env.pop("CRAWL_DIR", None)
base_env.pop("SNAP_DIR", None)
base_env["USE_COLOR"] = "False"
base_env["SHOW_PROGRESS"] = "False"
@ -437,11 +416,6 @@ def run_python_cwd(
) -> tuple[str, str, int]:
_assert_not_repo_path(cwd, label="cwd")
base_env = os.environ.copy()
base_env.pop("DATA_DIR", None)
base_env.pop("ARCHIVE_DIR", None)
base_env.pop("USERS_DIR", None)
base_env.pop("CRAWL_DIR", None)
base_env.pop("SNAP_DIR", None)
_assert_safe_runtime_paths(cwd=cwd, env=base_env)
result = subprocess.run(
[sys.executable, "-"],
@ -473,7 +447,6 @@ def init_archive(cwd: Path) -> None:
def build_test_env(port: int, **extra: str) -> dict[str, str]:
env = os.environ.copy()
env.pop("DATA_DIR", None)
env.update(
{
"PLUGINS": "wget",
@ -874,7 +847,7 @@ def real_archive_with_example(tmp_path_factory, request):
Uses cwd for DATA_DIR.
"""
tmp_path = tmp_path_factory.mktemp("archivebox_data")
if getattr(request, "cls", None) is not None:
if request.cls is not None:
request.cls.data_dir = tmp_path
stdout, stderr, returncode = run_archivebox_cmd_cwd(

1
archivebox/tests/migrations_helpers.py
View File

@ -1023,7 +1023,6 @@ def seed_0_8_data(db_path: Path) -> dict[str, list[dict]]:
def run_archivebox(data_dir: Path, args: list, timeout: int = 60, env: dict | None = None) -> subprocess.CompletedProcess:
"""Run archivebox command in subprocess with given data directory."""
base_env = os.environ.copy()
base_env["DATA_DIR"] = str(data_dir)
base_env["USE_COLOR"] = "False"
base_env["SHOW_PROGRESS"] = "False"
# Disable ALL extractors for faster tests (can be overridden by env parameter)

44
archivebox/tests/test_archive_result_service.py
View File

@ -22,15 +22,19 @@ def _create_snapshot():
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
crawl = Crawl.objects.create(
crawl = Crawl(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
)
return Snapshot.objects.create(
crawl.save()
snapshot = Snapshot(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.STARTED,
)
snapshot.save()
return snapshot
def test_process_completed_projects_inline_archiveresult():
@ -236,6 +240,42 @@ def test_snapshot_resolved_title_ignores_failed_title_output_str():
_cleanup_machine_process_rows()
def test_snapshot_title_ignores_noresults_title_output_str():
from archivebox.core.models import ArchiveResult
from archivebox.services.archive_result_service import ArchiveResultService
import asyncio
snapshot = _create_snapshot()
plugin_dir = Path(snapshot.output_dir) / "title"
plugin_dir.mkdir(parents=True, exist_ok=True)
bus = create_bus(name="test_noresults_title_does_not_update_snapshot")
service = ArchiveResultService(bus)
event = ArchiveResultEvent(
snapshot_id=str(snapshot.id),
plugin="title",
hook_name="on_Snapshot__54_title.js",
status="noresults",
output_str="TimeoutError: Navigation timeout of 54172 ms exceeded",
start_ts="2026-03-22T12:00:00+00:00",
end_ts="2026-03-22T12:00:01+00:00",
)
async def emit_event() -> None:
await service.on_ArchiveResultEvent__save_to_db(event)
asyncio.run(emit_event())
result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", hook_name="on_Snapshot__54_title.js")
assert result.status == ArchiveResult.StatusChoices.NORESULTS
assert result.output_str == "TimeoutError: Navigation timeout of 54172 ms exceeded"
snapshot.refresh_from_db()
assert snapshot.title in (None, "")
assert snapshot.resolved_title == ""
_cleanup_machine_process_rows()
def test_snapshot_save_normalizes_url_title_to_none():
from archivebox.core.models import Snapshot

4
archivebox/tests/test_auth_ldap.py
View File

@ -117,14 +117,14 @@ class TestLDAPAuthBackend:
"""Test that ArchiveBoxLDAPBackend class is defined."""
from archivebox.ldap.auth import ArchiveBoxLDAPBackend
assert hasattr(ArchiveBoxLDAPBackend, "authenticate_ldap_user")
assert ArchiveBoxLDAPBackend.authenticate_ldap_user is not None
def test_ldap_backend_inherits_correctly(self):
"""Test that ArchiveBoxLDAPBackend has correct inheritance."""
from archivebox.ldap.auth import ArchiveBoxLDAPBackend
# Should have authenticate_ldap_user method (from base or overridden)
assert callable(getattr(ArchiveBoxLDAPBackend, "authenticate_ldap_user", None))
assert callable(ArchiveBoxLDAPBackend.authenticate_ldap_user)
class TestArchiveBoxWithLDAP:

2
archivebox/tests/test_cli_add.py
View File

@ -269,7 +269,7 @@ def test_add_records_selected_persona_on_crawl(tmp_path, process, disable_extrac
crawl = Crawl.objects.get()
assert crawl.persona_id
assert crawl.config["ACTIVE_PERSONA"] == "Default"
assert "ACTIVE_PERSONA" not in crawl.config
assert (tmp_path / "personas" / "Default" / "chrome_profile").is_dir()

16
archivebox/tests/test_cli_init.py
View File

@ -46,21 +46,15 @@ def test_init_creates_archive_directory(tmp_path):
assert archive_dir.is_dir()
def test_init_respects_configured_archive_and_users_dirs(tmp_path):
"""Test that init creates configured archive/users storage roots."""
def test_init_uses_cwd_archive_and_users_dirs(tmp_path):
"""Test that init creates archive/users storage roots under cwd."""
os.chdir(tmp_path)
archive_dir = tmp_path / "mounted_archive"
users_dir = archive_dir / "custom_users"
env = os.environ.copy()
env["ARCHIVE_DIR"] = str(archive_dir)
env["USERS_DIR"] = str(users_dir)
result = subprocess.run(["archivebox", "init"], env=env, capture_output=True)
result = subprocess.run(["archivebox", "init"], capture_output=True)
assert result.returncode == 0
assert archive_dir.is_dir()
assert users_dir.is_dir()
assert not (tmp_path / "archive").exists()
assert (tmp_path / "archive").is_dir()
assert (tmp_path / "archive" / "users").is_dir()
def test_init_creates_sources_directory(tmp_path):

2
archivebox/tests/test_cli_run.py
View File

@ -402,7 +402,7 @@ class TestRunDaemonMode:
stdout = proc.stdout.read()
stderr = proc.stderr.read()
assert proc.returncode == 0, stdout + stderr
assert proc.returncode == 143, stdout + stderr
assert "No records to process" not in stderr
def test_run_daemon_takeover_has_single_active_runner_gate(self, initialized_archive, db):

24
archivebox/tests/test_cli_server.py
View File

@ -11,6 +11,7 @@ import signal
import socket
import subprocess
import sys
import time
from datetime import datetime
from types import SimpleNamespace
@ -128,6 +129,7 @@ def test_runner_worker_uses_current_interpreter():
from archivebox.workers.supervisord_util import RUNNER_WORKER
assert RUNNER_WORKER["command"] == f"{sys.executable} -m archivebox run --daemon"
assert 'ARCHIVEBOX_RUNNER_DAEMON="1"' in RUNNER_WORKER["environment"]
def test_daphne_worker_uses_default_application_close_timeout():
@ -175,6 +177,28 @@ def test_server_daemon_starts_real_plugin_owned_sonic_worker(archivebox_daemon_s
assert "sonic" in state["worker_sonic"]["name"]
def test_server_daemon_restarts_runner_killed_by_signal(archivebox_daemon_server):
server = archivebox_daemon_server(
SEARCH_BACKEND_ENGINE="sqlite",
)
state = server.wait_for_workers(("worker_daphne", "worker_runner"))
old_runner_pid = state["worker_runner"]["pid"]
os.kill(old_runner_pid, signal.SIGTERM)
deadline = time.time() + 30
while time.time() < deadline:
state = server.worker_state()
runner = state.get("worker_runner", {})
if runner.get("statename") == "RUNNING" and runner.get("pid") and runner.get("pid") != old_runner_pid:
break
time.sleep(0.5)
else:
raise AssertionError(f"worker_runner did not restart after SIGTERM: {state}")
assert state["worker_daphne"]["statename"] == "RUNNING", state
def test_sonic_worker_is_disabled_when_sonic_disabled_and_engine_not_sonic(tmp_path):
from archivebox.workers.supervisord_util import get_sonic_supervisord_worker_from_plugin

18
archivebox/tests/test_crawl_runner.py
View File

@ -129,6 +129,7 @@ def test_snapshot_payload_uses_crawl_persona_runtime_dirs():
assert Path(config["CHROME_DOWNLOADS_DIR"]).is_relative_to(crawl.output_dir)
assert Path(config["CHROME_USER_DATA_DIR"]).name == "chrome_profile"
assert Path(config["CHROME_DOWNLOADS_DIR"]).name == "chrome_downloads"
assert config["ACTIVE_PERSONA"] == "RuntimePersona"
assert Path(config["CRAWL_DIR"]) == crawl.output_dir
assert Path(config["SNAP_DIR"]) == snapshot.output_dir
@ -163,6 +164,18 @@ def test_ensure_background_runner_skips_with_real_running_orchestrator_record():
assert process.status == Process.StatusChoices.RUNNING
@pytest.mark.django_db(transaction=True)
def test_ensure_background_runner_does_not_spawn_runner_without_supervisord():
from archivebox.services.runner import ensure_background_runner
from archivebox.workers.supervisord_util import get_existing_supervisord_process, stop_existing_supervisord_process
stop_existing_supervisord_process()
assert get_existing_supervisord_process(quiet=True) is None
assert ensure_background_runner(allow_under_pytest=True) is False
assert get_existing_supervisord_process(quiet=True) is None
def test_runner_task_context_clears_inherited_abxbus_handler_context(tmp_path):
from abx_dl.events import CrawlEvent, MachineEvent
from abx_dl.orchestrator import create_bus
@ -174,7 +187,7 @@ def test_runner_task_context_clears_inherited_abxbus_handler_context(tmp_path):
async def emit_from_runner_task():
observations.append(("in_handler_context", in_handler_context()))
machine_event = bus.emit(MachineEvent(config={"ABX_RUNTIME": "archivebox"}, config_type="user"))
machine_event = bus.emit(MachineEvent(config={"TIMEOUT": "30"}, config_type="user"))
await machine_event.now()
observations.append(("machine_event_path", bool(machine_event.event_path)))
@ -405,7 +418,6 @@ def test_machine_service_persists_only_derived_config_events(tmp_path, hermetic_
config={
"CHROME_ISOLATION": "snapshot",
"CHROME_USER_DATA_DIR": "/tmp/stale-profile",
"ABX_RUNTIME": "archivebox",
},
config_type="user",
),
@ -569,7 +581,7 @@ def test_crawl_runner_empty_plugin_selection_emits_lifecycle_and_seals_crawl(tmp
assert runner.bus.event_is_child_of(completed_events[0], crawl_events[0])
assert runner.bus.event_is_child_of(snapshot_events[0], start_events[0])
assert runner.bus.event_is_child_of(snapshot_completed_events[0], snapshot_events[0])
assert any(event.config_type == "user" and event.config.get("ABX_RUNTIME") == "archivebox" for event in machine_events)
assert any(event.config_type == "user" for event in machine_events)
crawl.refresh_from_db()
snapshot = Snapshot.objects.get(crawl=crawl)

147
archivebox/tests/test_frozen_crawl_config.py
View File

@ -1,3 +1,6 @@
import time
from types import SimpleNamespace
import pytest
from django.test import RequestFactory
from django.utils import timezone
@ -65,6 +68,8 @@ def test_crawl_save_freezes_full_raw_persona_config_and_redacts_public_serializa
assert crawl.config["USER_AGENT"] == "Frozen UA"
assert crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET
assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 3
assert "ACTIVE_PERSONA" not in crawl.config
assert "DEFAULT_PERSONA" not in crawl.config
assert "CRAWL_DIR" not in crawl.config
assert "SNAP_DIR" not in crawl.config
assert "DEBUG" not in crawl.config
@ -82,9 +87,10 @@ def test_crawl_save_freezes_full_raw_persona_config_and_redacts_public_serializa
redacted_runtime_config = get_config(crawl=crawl, redact_sensitive=True)
assert redacted_runtime_config.USER_AGENT == "Frozen UA"
assert redacted_runtime_config.TWOCAPTCHA_API_KEY == SENSITIVE_CONFIG_VALUE_REDACTED
execution_config = runtime_config.for_crawl_execution()
execution_config = runtime_config.for_crawl()
assert execution_config["DEBUG"] is False
assert execution_config["CRAWL_DIR"] == str(crawl.output_dir)
assert "CRAWL_DIR" not in execution_config
assert "SNAP_DIR" not in execution_config
assert "SECRET_KEY" not in execution_config
assert "PUBLIC_ADD_VIEW" not in execution_config
assert "DATABASE_NAME" not in execution_config
@ -123,11 +129,110 @@ def test_config_scopes_are_derived_from_section_and_field_metadata():
assert ArchiveBoxConfig.scope_for_key("TIMEOUT") == "crawl_frozen"
assert ArchiveBoxConfig.scope_for_key("DEBUG") == "crawl_execution"
assert ArchiveBoxConfig.scope_for_key("CRAWL_DIR") == "crawl_execution"
assert ArchiveBoxConfig.scope_for_key("DEFAULT_PERSONA") == "crawl_execution"
assert ArchiveBoxConfig.scope_for_key("WGET_ENABLED") == "crawl_execution"
assert ArchiveBoxConfig.scope_for_key("WGET_WARC_ENABLED") == "crawl_frozen"
assert ArchiveBoxConfig.scope_for_key("SECRET_KEY") == "server"
assert ArchiveBoxConfig.scope_for_key("DATABASE_NAME") == "server"
def test_plugin_selection_enabled_keys_are_derived_from_plugins_not_frozen_or_env_overridden(archivebox_db, monkeypatch):
from archivebox.config.common import get_config
from archivebox.crawls.models import Crawl
from archivebox.plugins.discovery import get_plugin_special_config
monkeypatch.setenv("ARCHIVEDOTORG_ENABLED", "False")
user = _user("frozen-config-enabled-admin")
persona = _persona(user, name="Enabled Persona")
env_default_crawl = Crawl(
urls="https://example.com/env-enabled",
persona=persona,
created_by=user,
config={},
status=Crawl.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
env_default_crawl.save()
env_default_config = get_config(crawl=env_default_crawl, include_machine=False)
assert env_default_config.ARCHIVEDOTORG_ENABLED is False
crawl = Crawl(
urls="https://example.com/enabled",
persona=persona,
created_by=user,
config={"PLUGINS": "archivedotorg", "ARCHIVEDOTORG_ENABLED": True, "DEFAULT_PERSONA": "Other"},
status=Crawl.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
crawl.save()
assert crawl.config["PLUGINS"] == "archivedotorg"
assert "ARCHIVEDOTORG_ENABLED" not in crawl.config
assert "DEFAULT_PERSONA" not in crawl.config
runtime_config = get_config(crawl=crawl, include_machine=False)
assert runtime_config.ARCHIVEDOTORG_ENABLED is True
assert runtime_config.WGET_ENABLED is False
assert get_plugin_special_config("archivedotorg", runtime_config)["enabled"] is True
assert get_plugin_special_config("wget", runtime_config)["enabled"] is False
monkeypatch.delenv("ARCHIVEDOTORG_ENABLED")
Crawl.objects.filter(id=crawl.id).update(
config={
**crawl.config,
"ARCHIVEDOTORG_ENABLED": False,
"YTDLP_ENABLED": True,
"WGET_ENABLED": True,
},
)
crawl.refresh_from_db()
stale_runtime_config = get_config(crawl=crawl, include_machine=False)
assert stale_runtime_config.ARCHIVEDOTORG_ENABLED is True
assert stale_runtime_config.YTDLP_ENABLED is False
assert stale_runtime_config.WGET_ENABLED is False
def test_crawl_config_projections_stay_under_hot_path_budget():
from archivebox.config.common import ArchiveBoxConfig
config = ArchiveBoxConfig(TIMEOUT=12, USER_AGENT="Perf UA", CHROME_BINARY="perf-chrome")
persona = SimpleNamespace(
config={"USER_AGENT": "Persona UA"},
get_derived_config=lambda: {
"ACTIVE_PERSONA": "Perf Persona",
"USER_AGENT": "Persona UA",
"CHROME_USER_DATA_DIR": "/tmp/persona/chrome",
},
)
crawl = SimpleNamespace(config={"TIMEOUT": 13, "CHROME_BINARY": "crawl-chrome"})
snapshot = SimpleNamespace(config={"TIMEOUT": 14, "CHROME_BINARY": "snapshot-chrome"})
runtime_kwargs = {
"crawl": crawl,
"snapshot": snapshot,
"persona": persona,
"crawl_output_dir": "/tmp/archivebox/crawls/perf",
"snapshot_output_dir": "/tmp/archivebox/crawls/perf/snapshots/example",
"extra_context": {"snapshot_id": "perf"},
}
methods = {
"for_crawl": config.for_crawl,
"for_crawl_frozen": lambda: config.for_crawl_frozen(persona=persona),
"for_crawl_runtime": lambda: config.for_crawl_runtime(**runtime_kwargs),
}
iterations = 250
max_average_seconds = 0.020
for name, method in methods.items():
method()
started_at = time.perf_counter()
for _ in range(iterations):
method()
average_seconds = (time.perf_counter() - started_at) / iterations
assert average_seconds < max_average_seconds, f"{name} averaged {average_seconds * 1000:.3f}ms"
def test_api_create_and_cli_add_store_full_frozen_config(archivebox_db):
from archivebox.api.v1_crawls import CrawlCreateSchema, CrawlSchema, create_crawl
from archivebox.cli.archivebox_add import add
@ -201,3 +306,39 @@ def test_schedule_enqueue_refreezes_using_current_template_persona_defaults(arch
assert "PUBLIC_ADD_VIEW" not in child.config
assert template.config["USER_AGENT"] == "Initial schedule UA"
assert template.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET
def test_crawl_config_backfill_migration_uses_frozen_config_helper(archivebox_db):
import importlib
from django.apps import apps
from django.db import connection
from archivebox.crawls.models import Crawl
migration = importlib.import_module("archivebox.crawls.migrations.0018_freeze_crawl_config_snapshots")
user = _user("frozen-config-migration-admin")
persona = _persona(user, name="Migration Persona", user_agent="Migration UA")
crawl = Crawl(
urls="https://example.com/migration",
persona=persona,
created_by=user,
config={"TIMEOUT": 44, "CHROME_BINARY": "migration-chrome"},
status=Crawl.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
crawl.save()
Crawl.objects.filter(id=crawl.id).update(config={"TIMEOUT": 44, "CHROME_BINARY": "migration-chrome"})
migration.freeze_existing_crawl_configs(apps, SimpleNamespace(connection=connection))
crawl.refresh_from_db()
assert crawl.config["TIMEOUT"] == 44
assert "CHROME_BINARY" not in crawl.config
assert crawl.config["USER_AGENT"] == "Migration UA"
assert crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET
assert "ACTIVE_PERSONA" not in crawl.config
assert "DEFAULT_PERSONA" not in crawl.config
assert "CRAWL_DIR" not in crawl.config
assert "SNAP_DIR" not in crawl.config
assert "SECRET_KEY" not in crawl.config

8
archivebox/tests/test_hooks.py
View File

@ -45,10 +45,11 @@ def create_test_plugin_structure(plugins_dir: Path) -> None:
def run_plugin_discovery_subprocess(tmp_path: Path, plugins_dir: Path, script: str):
env = os.environ.copy()
env["ARCHIVEBOX_USER_PLUGINS_DIR"] = str(plugins_dir)
data_dir = tmp_path / "data"
data_dir.mkdir()
env["DATA_DIR"] = str(data_dir)
cwd_plugins_dir = data_dir / "custom_plugins"
if plugins_dir != cwd_plugins_dir:
shutil.copytree(plugins_dir, cwd_plugins_dir)
env["PYTHONPATH"] = str(REPO_ROOT) + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "")
subprocess_script = "\n".join(
[
@ -205,7 +206,7 @@ class TestRequiredBinaryConfigHandling:
assert binary_name == "wget2"
def test_binary_env_var_empty_default(self):
"""Empty configured values should fall back to config defaults."""
"""Empty configured binary values should keep the schema default."""
configured_binary = ""
if configured_binary:
binary_name = configured_binary
@ -736,7 +737,6 @@ print(json.dumps({
hook_path,
output_dir,
config={
"DATA_DIR": str(tmp_path),
"LIB_DIR": str(lib_dir),
"NODE_PATH": configured_node_path,
},

2
archivebox/tests/test_machine_models.py
View File

@ -90,7 +90,6 @@ class TestMachineModel:
def test_machine_config_save_heals_json_encoded_string_values(self, machine):
machine.config = {
"PERSONAS_DIR": '"/data/personas"',
"EXTRA_CONTEXT": 'prefix "inner" suffix',
"USER_AGENT": '"ArchiveBox \\"Quoted\\" Agent"',
}
@ -98,7 +97,6 @@ class TestMachineModel:
machine.refresh_from_db()
assert machine.config["PERSONAS_DIR"] == "/data/personas"
assert machine.config["EXTRA_CONTEXT"] == 'prefix "inner" suffix'
assert machine.config["USER_AGENT"] == 'ArchiveBox "Quoted" Agent'

17
archivebox/tests/test_misc_checks.py
View File

@ -1,6 +1,10 @@
import os
import signal
import pytest
from archivebox.core.shutdown_util import foreground_shutdown_signals
from archivebox.core.shutdown_util import raise_if_shutdown_requested
from archivebox.misc.checks import _migration_interrupt_message
from archivebox.misc.checks import _exit_on_migration_interrupt
@ -38,3 +42,16 @@ def test_migration_interrupt_handler_exits_for_sigint_and_sigterm(monkeypatch):
else:
raise AssertionError(f"{sig.name} should exit during migration auto-apply")
assert signal.getsignal(sig) == previous_handler
def test_nested_foreground_signal_state_propagates_to_outer_context():
with foreground_shutdown_signals(first_signal_message=None) as outer_state:
try:
with foreground_shutdown_signals(first_signal_message=None):
os.kill(os.getpid(), signal.SIGTERM)
except KeyboardInterrupt:
pass
assert outer_state.signal_name == "SIGTERM"
with pytest.raises(KeyboardInterrupt):
raise_if_shutdown_requested()

13
archivebox/tests/test_orm_helpers.py
View File

@ -17,23 +17,20 @@ def use_archivebox_db(path: str | Path = ".") -> Iterator[None]:
original_name = connection.settings_dict["NAME"]
original_database_name = connections.databases["default"]["NAME"]
original_setting_name = settings.DATABASES["default"]["NAME"]
original_connection = getattr(connections._connections, "default", None)
original_connection = connections._connections.default
db_path = str(archivebox_db_path(path))
connection.close()
connection.settings_dict["NAME"] = db_path
connections.databases["default"]["NAME"] = db_path
settings.DATABASES["default"]["NAME"] = db_path
if original_connection is not None:
delattr(connections._connections, "default")
del connections._connections.default
try:
yield
finally:
connections["default"].close()
connections.databases["default"]["NAME"] = original_database_name
settings.DATABASES["default"]["NAME"] = original_setting_name
if hasattr(connections._connections, "default"):
delattr(connections._connections, "default")
if original_connection is not None:
original_connection.settings_dict["NAME"] = original_name
setattr(connections._connections, "default", original_connection)
del connections._connections.default
original_connection.settings_dict["NAME"] = original_name
connections._connections.default = original_connection

25
archivebox/tests/test_persona_admin.py
View File

@ -5,6 +5,7 @@ from django.contrib.auth import get_user_model
from django.contrib.auth.models import UserManager
from django.urls import reverse
from archivebox.config.common import ArchiveBoxConfig
from archivebox.personas.importers import (
discover_persona_template_profiles,
import_persona_from_source,
@ -175,20 +176,20 @@ def test_persona_admin_add_post_runs_shared_importer(client, admin_user):
def test_persona_admin_saves_typed_plugin_config(client, admin_user):
from archivebox.config.common import get_config
client.login(username="personaadmin", password="testpassword")
add_response = client.get(reverse("admin:personas_persona_add"), HTTP_HOST=ADMIN_HOST)
add_form = add_response.context["adminform"].form
personas_dir_fields = [
field
for group in add_form.plugin_groups
for card in group["plugins"]
for field in card["config_fields"]
if field["key"] == "PERSONAS_DIR"
]
assert personas_dir_fields
assert {field["value"] for field in personas_dir_fields} == {str(get_config().PERSONAS_DIR)}
exposed_config_keys = {field["key"] for group in add_form.plugin_groups for card in group["plugins"] for field in card["config_fields"]}
assert not {key for key in exposed_config_keys if ArchiveBoxConfig.scope_for_key(key) == "crawl_execution"}
assert (
not {
"ARCHIVE_DIR",
"USERS_DIR",
"PERSONAS_DIR",
"CUSTOM_TEMPLATES_DIR",
}
& exposed_config_keys
)
response = client.post(
reverse("admin:personas_persona_add"),
@ -216,7 +217,6 @@ def test_persona_config_save_heals_json_encoded_string_values(admin_user):
name="QuotedConfigPersona",
created_by=admin_user,
config={
"PERSONAS_DIR": '"/data/personas"',
"EXTRA_CONTEXT": 'prefix "inner" suffix',
"USER_AGENT": '"ArchiveBox \\"Quoted\\" Agent"',
},
@ -224,6 +224,5 @@ def test_persona_config_save_heals_json_encoded_string_values(admin_user):
persona.refresh_from_db()
assert persona.config["PERSONAS_DIR"] == "/data/personas"
assert persona.config["EXTRA_CONTEXT"] == 'prefix "inner" suffix'
assert persona.config["USER_AGENT"] == 'ArchiveBox "Quoted" Agent'

28
archivebox/tests/test_persona_runtime.py
View File

@ -218,7 +218,7 @@ def test_get_config_treats_missing_persona_id_as_null(initialized_archive):
assert payload["persona_id"] == "None"
def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(initialized_archive):
def test_get_config_resolves_parent_scopes_for_snapshot_runtime(initialized_archive):
script = textwrap.dedent(
"""
import json
@ -237,6 +237,7 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Machine
from archivebox.personas.models import Persona
from archivebox.services.runner import CrawlRunner
CONSTANTS.CONFIG_FILE.write_text('[ARCHIVING_CONFIG]\\nTIMEOUT=11\\nCHROME_BINARY=file-chrome\\n')
@ -254,6 +255,8 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
persona_id=persona.id,
config={'TIMEOUT': 44, 'CHROME_BINARY': 'crawl-chrome'},
)
persona.config = {'TIMEOUT': 99, 'CHROME_BINARY': 'persona-chrome-updated'}
persona.save(update_fields=['config'])
snapshot = Snapshot.objects.create(
url='https://example.com',
crawl=crawl,
@ -262,7 +265,6 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
result = ArchiveResult.objects.create(
snapshot=snapshot,
plugin='title',
config={'TIMEOUT': 66, 'CHROME_BINARY': 'archiveresult-chrome'},
)
env_config = get_config(include_machine=False)
@ -270,8 +272,10 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
persona_config = get_config(persona=persona)
crawl_config = get_config(crawl=crawl)
snapshot_config = get_config(snapshot=snapshot)
result_config = get_config(archiveresult=result)
override_config = get_config(archiveresult=result, overrides={'TIMEOUT': 77, 'CHROME_BINARY': 'override-chrome'})
override_config = get_config(snapshot=snapshot, overrides={'TIMEOUT': 77, 'CHROME_BINARY': 'override-chrome'})
runner = CrawlRunner(crawl, selected_plugins=['title'], show_progress=False)
runner.load_run_state()
runtime_config = runner.load_snapshot_payload(str(snapshot.id))['config']
print(json.dumps({
'env': [env_config.TIMEOUT, env_config.CHROME_BINARY],
@ -279,13 +283,12 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
'persona': [persona_config.TIMEOUT, persona_config.CHROME_BINARY],
'crawl': [crawl_config.TIMEOUT, crawl_config.CHROME_BINARY],
'snapshot': [snapshot_config.TIMEOUT, snapshot_config.CHROME_BINARY],
'archiveresult': [result_config.TIMEOUT, result_config.CHROME_BINARY],
'override': [override_config.TIMEOUT, override_config.CHROME_BINARY],
'snap_dir': str(result_config.SNAP_DIR),
'snap_dir': str(runtime_config['SNAP_DIR']),
'expected_snap_dir': str(snapshot.output_dir),
'crawl_dir': str(result_config.CRAWL_DIR),
'crawl_dir': str(runtime_config['CRAWL_DIR']),
'expected_crawl_dir': str(crawl.output_dir),
'active_persona': result_config.ACTIVE_PERSONA,
'active_persona': runtime_config['ACTIVE_PERSONA'],
}, default=str))
""",
)
@ -296,11 +299,10 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
payload = json.loads(stdout.strip().splitlines()[-1])
assert payload["env"] == [22, "env-chrome"]
assert payload["machine"] == [22, "machine-chrome"]
assert payload["persona"] == [33, "persona-chrome"]
assert payload["crawl"] == [44, "crawl-chrome"]
assert payload["snapshot"] == [55, "snapshot-chrome"]
assert payload["archiveresult"] == [66, "archiveresult-chrome"]
assert payload["override"] == [77, "override-chrome"]
assert payload["persona"] == [99, "persona-chrome-updated"]
assert payload["crawl"] == [44, "persona-chrome-updated"]
assert payload["snapshot"] == [55, "persona-chrome-updated"]
assert payload["override"] == [77, "persona-chrome-updated"]
assert payload["snap_dir"] == payload["expected_snap_dir"]
assert payload["crawl_dir"] == payload["expected_crawl_dir"]
assert payload["active_persona"] == "StackPersona"

7
archivebox/tests/test_search.py
View File

@ -4,7 +4,6 @@ import socket
import subprocess
import sys
import time
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import urlencode
@ -97,8 +96,6 @@ def test_search_backend_env_exposes_resolved_runtime_config(tmp_path):
os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] = "old-host"
config = AttrDict(
{
"DATA_DIR": tmp_path,
"USERS_DIR": tmp_path / "archive" / "users",
"SEARCH_BACKEND_ENGINE": "sonic",
"SEARCH_BACKEND_SONIC_HOST_NAME": "sonic",
"SEARCH_BACKEND_SONIC_PORT": 1491,
@ -109,8 +106,6 @@ def test_search_backend_env_exposes_resolved_runtime_config(tmp_path):
try:
with search_backend_env(config=config):
assert os.environ["DATA_DIR"] == str(tmp_path)
assert os.environ["SNAP_DIR"] == str(Path(tmp_path) / "archive" / "users")
assert os.environ["SEARCH_BACKEND_ENGINE"] == "sonic"
assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "sonic"
assert os.environ["SEARCH_BACKEND_SONIC_PORT"] == "1491"
@ -465,7 +460,6 @@ class TestSearchBackendsE2E:
merged_env = os.environ.copy()
merged_env.update(
{
"DATA_DIR": str(data_dir),
"USE_COLOR": "False",
"SHOW_PROGRESS": "False",
"SAVE_WARC": "False",
@ -505,7 +499,6 @@ class TestSearchBackendsE2E:
sonic_env = os.environ.copy()
sonic_env.update(
{
"DATA_DIR": str(data_dir),
"USE_COLOR": "False",
"SHOW_PROGRESS": "False",
"SEARCH_BACKEND_ENGINE": "sonic",

76
archivebox/tests/test_shutdown_util.py Normal file
View File

@ -0,0 +1,76 @@
import signal
import pytest
from archivebox.cli import archivebox_run
from archivebox.core import shutdown_util
def test_foreground_shutdown_second_signal_exits_immediately(monkeypatch):
def fake_exit(code):
raise SystemExit(code)
monkeypatch.setattr(shutdown_util.os, "_exit", fake_exit)
with shutdown_util.foreground_shutdown_signals() as state:
handler = signal.getsignal(signal.SIGTERM)
with pytest.raises(KeyboardInterrupt):
handler(signal.SIGTERM, None)
assert state.signal_name == "SIGTERM"
with pytest.raises(SystemExit) as err:
handler(signal.SIGTERM, None)
assert err.value.code == 130
def test_foreground_shutdown_can_request_cooperative_shutdown_without_raising(monkeypatch):
def fake_exit(code):
raise SystemExit(code)
seen = []
monkeypatch.setattr(shutdown_util.os, "_exit", fake_exit)
with shutdown_util.foreground_shutdown_signals(
first_signal_message=None,
on_signal=seen.append,
raise_on_first_signal=False,
) as state:
handler = signal.getsignal(signal.SIGTERM)
handler(signal.SIGTERM, None)
assert state.signal_name == "SIGTERM"
assert seen == [signal.SIGTERM]
with pytest.raises(SystemExit) as err:
handler(signal.SIGTERM, None)
assert err.value.code == 130
def test_daemon_runner_signal_exit_is_unexpected_for_supervisor(monkeypatch):
def fake_exit(code):
raise SystemExit(code)
monkeypatch.setattr(archivebox_run.os, "_exit", fake_exit)
with pytest.raises(SystemExit) as err:
archivebox_run._exit_daemon_runner_on_signal(signal.SIGTERM)
assert err.value.code == 143
def test_crawl_runner_daemon_signal_exits_before_async_cleanup(monkeypatch):
from archivebox.services import runner as runner_module
from archivebox.services.runner import CrawlRunner
def fake_exit(code):
raise SystemExit(code)
runner = object.__new__(CrawlRunner)
monkeypatch.setenv("ARCHIVEBOX_RUNNER_DAEMON", "1")
monkeypatch.setattr(runner_module.os, "_exit", fake_exit)
with pytest.raises(SystemExit) as err:
runner._request_abort_from_signal(signal.SIGTERM)
assert err.value.code == 143

10
archivebox/tests/test_test_harness.py
View File

@ -18,15 +18,17 @@ def test_in_process_archivebox_config_uses_temp_data_dir():
from archivebox.config import CONSTANTS
from archivebox.config.common import get_config
data_dir = Path(os.environ["DATA_DIR"]).resolve()
data_dir = CONSTANTS.DATA_DIR.resolve()
assert data_dir == Path.cwd().resolve()
assert test_harness.REPO_ROOT not in data_dir.parents
assert CONSTANTS.DATA_DIR != test_harness.REPO_ROOT
config = get_config(include_machine=False)
assert config.DATA_DIR == data_dir
assert config.ARCHIVE_DIR == data_dir / "archive"
assert config.USERS_DIR == data_dir / "archive" / "users"
assert "DATA_DIR" not in config
assert "ARCHIVE_DIR" not in config
assert "USERS_DIR" not in config
assert CONSTANTS.ARCHIVE_DIR == data_dir / "archive"
assert CONSTANTS.USERS_DIR == data_dir / "archive" / "users"
def test_cli_helpers_reject_repo_root_runtime_paths():

60
archivebox/tests/test_ui_add_view.py
View File

@ -3,10 +3,11 @@ import pytest
from django.contrib.auth import get_user_model
from django.urls import reverse
from archivebox.config.common import get_config
from archivebox.config.common import ArchiveBoxConfig
from archivebox.core.models import Snapshot, Tag
from archivebox.crawls.models import Crawl
from archivebox.personas.models import Persona
from archivebox.services.runner import CrawlRunner
from archivebox.workers.models import RETRY_AT_MAX
@ -74,15 +75,28 @@ def test_add_view_admin_renders_plugin_config_grid(client, admin_user, monkeypat
assert b">Docs</a>" in response.content
assert b"https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/" in response.content
assert b"https://archivebox.github.io/abx-plugins/#" in response.content
personas_dir_fields = [
field
assert not any(
field["key"].endswith("_BINARY") for group in form.plugin_groups for card in group["plugins"] for field in card["config_fields"]
)
assert not any(
field["key"] == f"{card['name'].upper()}_ENABLED"
for group in form.plugin_groups
for card in group["plugins"]
for field in card["config_fields"]
if field["key"] == "PERSONAS_DIR"
]
assert personas_dir_fields
assert {field["value"] for field in personas_dir_fields} == {str(get_config().PERSONAS_DIR)}
)
exposed_config_keys = {field["key"] for group in form.plugin_groups for card in group["plugins"] for field in card["config_fields"]}
assert not {key for key in exposed_config_keys if ArchiveBoxConfig.scope_for_key(key) == "crawl_execution"}
assert (
not {
"ARCHIVE_DIR",
"USERS_DIR",
"PERSONAS_DIR",
"CUSTOM_TEMPLATES_DIR",
}
& exposed_config_keys
)
assert b"plugin_config__chrome__CHROME_BINARY" not in response.content
assert b"plugin_config__wget__WGET_ENABLED" not in response.content
def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_user, monkeypatch):
@ -127,6 +141,20 @@ def test_add_view_public_only_lists_public_personas(client, admin_user, monkeypa
assert {"NODE_BINARY", "TWOCAPTCHA_API_KEY"}.isdisjoint(persona_config_map["Default"]["effective_config"])
def test_persona_config_grid_allows_binary_fields(client, admin_user):
client.force_login(admin_user)
persona = Persona.objects.create(name="Binary Persona", created_by=admin_user)
response = client.get(
reverse("admin:personas_persona_change", args=[persona.pk]),
HTTP_HOST=ADMIN_HOST,
)
assert response.status_code == 200
assert b"plugin_config__chrome__CHROME_BINARY" in response.content
assert b"plugin_config__wget__WGET_ENABLED" in response.content
def test_add_view_hides_search_backend_plugins(client, monkeypatch):
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "sqlite")
@ -253,11 +281,14 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin
crawl = Crawl.objects.order_by("-created_at").first()
assert crawl is not None
assert crawl.persona_id == private_persona.id
assert crawl.config["ACTIVE_PERSONA"] == "Private"
assert "ACTIVE_PERSONA" not in crawl.config
assert crawl.resolve_persona() == private_persona
runtime_config = get_config(crawl=crawl)
assert runtime_config.ACTIVE_PERSONA == "Private"
assert runtime_config.COOKIES_FILE == private_cookies_file
snapshot = Snapshot.objects.create(url="https://example.com/private", crawl=crawl)
runner = CrawlRunner(crawl, selected_plugins=["title"], show_progress=False)
runner.load_run_state()
runtime_config = runner.load_snapshot_payload(str(snapshot.id))["config"]
assert runtime_config["ACTIVE_PERSONA"] == "Private"
assert runtime_config["COOKIES_FILE"] == str(private_cookies_file)
def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatch):
@ -282,9 +313,11 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatc
"permissions": "public",
"start_paused": "",
"main_plugins": ["wget"],
"plugin_config__wget__WGET_ENABLED": "false",
"plugin_config__wget__WGET_TIMEOUT": "77",
"plugin_config__wget__WGET_WARC_ENABLED": "false",
"config": "{}",
"plugin_config__chrome__CHROME_BINARY": "/tmp/malicious-chrome",
"config": '{"NODE_BINARY": "/tmp/malicious-node", "WGET_ENABLED": false}',
},
HTTP_HOST=ADMIN_HOST,
)
@ -296,6 +329,9 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatc
assert crawl.config["PLUGINS"] == "wget"
assert crawl.config["WGET_TIMEOUT"] == 77
assert crawl.config["WGET_WARC_ENABLED"] is False
assert "WGET_ENABLED" not in crawl.config
assert "CHROME_BINARY" not in crawl.config
assert "NODE_BINARY" not in crawl.config
def test_add_view_public_submission_ignores_plugin_and_custom_config(client, admin_user, monkeypatch):

11
archivebox/tests/test_urls.py
View File

@ -73,7 +73,7 @@ def _build_script(body: str) -> str:
from archivebox.core.middleware import ADMIN_LOGIN_HINT_COOKIE
def response_body(resp):
if getattr(resp, "streaming", False):
if resp.streaming:
return b"".join(resp.streaming_content)
return resp.content
@ -761,13 +761,16 @@ class TestUrlRouting:
},
)
def test_subdomain_replay_assets_fall_back_to_chromewebstore_lib_dir(self) -> None:
def test_subdomain_replay_assets_use_derived_chromewebstore_extensions_dir(self) -> None:
lib_dir = self.data_dir / "test-lib"
self._run(
"""
snapshot = get_snapshot()
snapshot_host = get_snapshot_host(str(snapshot.id))
extension_dir = Path(SERVER_CONFIG.LIB_DIR) / "chromewebstore" / "extensions" / "test__archivewebpage"
expected_extensions_dir = Path(SERVER_CONFIG.LIB_DIR) / "chromewebstore" / "extensions"
assert Path(SERVER_CONFIG.CHROME_EXTENSIONS_DIR).resolve() == expected_extensions_dir.resolve()
extension_dir = Path(SERVER_CONFIG.CHROME_EXTENSIONS_DIR) / "test__archivewebpage"
extension_dir.mkdir(parents=True, exist_ok=True)
(extension_dir / "ui.js").write_text("window.__archivebox_replay_ui_from_lib__ = true;\\n", encoding="utf-8")
(extension_dir / "sw.js").write_text("self.__archivebox_replay_sw_from_lib__ = true;\\n", encoding="utf-8")
@ -776,7 +779,6 @@ class TestUrlRouting:
resp = client.get("/replay/ui.js", HTTP_HOST=snapshot_host)
body = response_body(resp).decode("utf-8", "ignore")
assert SERVER_CONFIG.CHROME_EXTENSIONS_DIR == ""
assert resp.status_code == 200
assert resp["Content-Type"].startswith("application/javascript")
assert "window.__archivebox_replay_ui_from_lib__" in body
@ -788,7 +790,6 @@ class TestUrlRouting:
"BIND_ADDR": "127.0.0.1:8766",
"BASE_URL": "",
"LIB_DIR": str(lib_dir),
"CHROME_EXTENSIONS_DIR": "",
},
)

44
archivebox/workers/supervisord_util.py
View File

@ -159,7 +159,7 @@ RUNNER_WORKER = {
"command": _shell_join([sys.executable, "-m", "archivebox", "run", "--daemon"]),
"autostart": "false",
"autorestart": "unexpected",
"environment": 'PYTHONUNBUFFERED="1",COLUMNS="200"',
"environment": 'PYTHONUNBUFFERED="1",COLUMNS="200",ARCHIVEBOX_RUNNER_DAEMON="1"',
"stopasgroup": "true",
"killasgroup": "true",
"stopwaitsecs": "30",
@ -171,6 +171,7 @@ RUNNER_ONCE_WORKER = lambda args, name="worker_runner_once": {
**RUNNER_WORKER,
"name": name,
"command": _shell_join([sys.executable, "-m", "archivebox", "run", *args]),
"environment": 'PYTHONUNBUFFERED="1",COLUMNS="200"',
"autorestart": "false",
"stopwaitsecs": "1",
"stdout_logfile": f"logs/{name}.log",
@ -671,7 +672,7 @@ def stop_existing_supervisord_process():
pass
def stop_own_supervisord_process():
def stop_own_supervisord_process(*, record_exit: bool = True):
"""Stop only the supervisord child started by this Python process."""
global _supervisord_proc
@ -702,19 +703,20 @@ def stop_own_supervisord_process():
except psutil.NoSuchProcess:
pass
wait_popen_and_kill_children(_supervisord_proc, children, timeout=2.0, kill_timeout=1.0)
try:
from archivebox.machine.models import Machine, Process
if record_exit:
try:
from archivebox.machine.models import Machine, Process
for process in Process.objects.filter(
machine=Machine.current(),
process_type=Process.TypeChoices.SUPERVISORD,
status=Process.StatusChoices.RUNNING,
pwd=str(CONSTANTS.DATA_DIR),
pid=stopped_pid,
).iterator(chunk_size=10):
process.mark_exited(exit_code=0)
except Exception:
pass
for process in Process.objects.filter(
machine=Machine.current(),
process_type=Process.TypeChoices.SUPERVISORD,
status=Process.StatusChoices.RUNNING,
pwd=str(CONSTANTS.DATA_DIR),
pid=stopped_pid,
).iterator(chunk_size=10):
process.mark_exited(exit_code=0)
except Exception:
pass
except (BrokenPipeError, OSError, psutil.TimeoutExpired):
pass
finally:
@ -970,10 +972,10 @@ def build_server_worker_plan(*, config, host: str, port: str, debug: bool, reloa
except Exception:
current_sonic = None
supervisor_pid = None
sonic_host = str(getattr(config, "SEARCH_BACKEND_SONIC_HOST_NAME", "127.0.0.1") or "127.0.0.1")
sonic_host = str(config.SEARCH_BACKEND_SONIC_HOST_NAME or "127.0.0.1")
if sonic_host.strip().lower() == "localhost":
sonic_host = "127.0.0.1"
sonic_port = int(getattr(config, "SEARCH_BACKEND_SONIC_PORT"))
sonic_port = int(config.SEARCH_BACKEND_SONIC_PORT)
if not (isinstance(current_sonic, dict) and current_sonic.get("statename") in ("STARTING", "RUNNING")):
stop_stale_sonic_processes(sonic_worker, supervisor_pid=supervisor_pid, host=sonic_host, port=sonic_port)
if not (isinstance(current_sonic, dict) and current_sonic.get("statename") in ("STARTING", "RUNNING")) and is_port_in_use(
@ -1304,11 +1306,15 @@ def start_server_workers(
raise
STDERR.print(f"\n[🛑] Got {e.__class__.__name__} exception, stopping gracefully...")
finally:
if not daemonize and (should_stop_supervisord is None or should_stop_supervisord()):
signal_shutdown_requested = bool(shutdown_state and shutdown_state.signal_name)
if not daemonize and (signal_shutdown_requested or should_stop_supervisord is None or should_stop_supervisord()):
# Ensure supervisord and all children are stopped only while this
# foreground parent is still the active server parent. Standby
# parents must not tear down a newer leader's services.
stop_own_supervisord_process()
# parents must not tear down a newer leader's services. If this
# foreground parent itself received an OS shutdown signal, always
# stop the supervisord child it owns; stop_own_supervisord_process()
# does not target supervisord processes owned by other parents.
stop_own_supervisord_process(record_exit=not signal_shutdown_requested)
return tail_result

7
docs/Configuration.md
View File

@ -1,6 +1,6 @@
# Configuration
Configuration of ArchiveBox is done by using the `archivebox config` command, modifying the `ArchiveBox.conf` file in the data folder, or by using environment variables. All three methods work equivalently when using Docker as well.
Configuration of ArchiveBox is done by using the `archivebox config` command, modifying the `ArchiveBox.conf` file in the data folder, or by setting environment variables as process defaults. All three methods work in Docker as well.
*Some equivalent examples of setting some configuration options:*
```bash
@ -11,7 +11,7 @@ echo "TIMEOUT=120" >> ArchiveBox.conf
env TIMEOUT=120 archivebox add ~/Downloads/bookmarks_export.html
```
Environment variables take precedence over the config file, which is useful if you only want to use a certain option temporarily during a single run. For more examples see [Usage: Configuration](Usage#run-archivebox-with-configuration-options)...
Environment variables seed process-level defaults. Persisted Machine, Persona, Crawl, and Snapshot settings can override them depending on scope, and existing Crawl config is not silently overwritten by later environment changes. Runtime-derived values like crawl/snapshot output dirs are resolved fresh for each run instead of being stored in frozen crawl config. For more examples see [Usage: Configuration](Usage#run-archivebox-with-configuration-options)...
<br/>
@ -136,7 +136,7 @@ archivebox add --persona=personal https://members.example.com/feed
<a id="active_persona"></a>
#### `DEFAULT_PERSONA`
**Possible Values:** [`Default`]/`personal`/`work`/...
The persona profile used when no explicit persona is selected for a crawl. Personas bundle a Chrome user-data-dir, a `cookies.txt`, auth state, a user-agent, and any other per-identity config into a single named profile, letting you swap between archiving contexts (logged-out vs. signed-into-work-account vs. signed-into-personal-account) without manually juggling files.
The persona profile used when no explicit persona is selected for a new crawl. The selected persona is stored on the Crawl row; `DEFAULT_PERSONA` is not duplicated into `Crawl.config`. Personas bundle a Chrome user-data-dir, a `cookies.txt`, auth state, a user-agent, and any other per-identity config into a single named profile, letting you swap between archiving contexts (logged-out vs. signed-into-work-account vs. signed-into-personal-account) without manually juggling files.
ArchiveBox auto-creates the named persona on disk if it doesn't already exist. See the [Personas wiki page](https://github.com/ArchiveBox/ArchiveBox/wiki/Personas) for the full directory layout.
@ -1061,7 +1061,6 @@ A handful of *core* options (documented above on this page) act as the **fallbac
| [`USER_AGENT`](#user_agent) | [`WGET_USER_AGENT`](https://archivebox.github.io/abx-plugins/#wget), [`CHROME_USER_AGENT`](https://archivebox.github.io/abx-plugins/#chrome), [`SINGLEFILE_USER_AGENT`](https://archivebox.github.io/abx-plugins/#singlefile), ... |
| [`COOKIES_FILE`](#cookies_file) | [`WGET_COOKIES_FILE`](https://archivebox.github.io/abx-plugins/#wget), [`YTDLP_COOKIES_FILE`](https://archivebox.github.io/abx-plugins/#ytdlp), [`GALLERYDL_COOKIES_FILE`](https://archivebox.github.io/abx-plugins/#gallerydl), [`SINGLEFILE_COOKIES_FILE`](https://archivebox.github.io/abx-plugins/#singlefile), ... |
| [`RESOLUTION`](#resolution) | [`SCREENSHOT_RESOLUTION`](https://archivebox.github.io/abx-plugins/#screenshot), [`PDF_RESOLUTION`](https://archivebox.github.io/abx-plugins/#pdf), [`CHROME_RESOLUTION`](https://archivebox.github.io/abx-plugins/#chrome) |
| [`DEFAULT_PERSONA`](#default_persona) | per-plugin persona scoping (browser profile / cookie jar selection) |
> [!TIP]
> The resolution order for any plugin-tunable option is always:

2
docs/apidocs/archivebox/archivebox.config.common.md
View File

@ -1357,7 +1357,7 @@ Bases: {py:obj}`archivebox.config.common.ShellConfig`, {py:obj}`archivebox.confi
````
````{py:function} get_config(defaults: archivebox.config.common.ConfigOverrides | None = None, overrides: archivebox.config.common.ConfigOverrides | None = None, base_config: archivebox.config.common.ArchiveBoxBaseConfig | collections.abc.Mapping[str, object] | None = None, persona: typing.Any = None, user: typing.Any = None, crawl: typing.Any = None, snapshot: typing.Any = None, archiveresult: typing.Any = None, machine: typing.Any = None, include_machine: bool = True, resolve_plugins: bool = True) -> archivebox.config.common.ArchiveBoxBaseConfig
````{py:function} get_config(defaults: archivebox.config.common.ConfigOverrides | None = None, overrides: archivebox.config.common.ConfigOverrides | None = None, base_config: archivebox.config.common.ArchiveBoxBaseConfig | collections.abc.Mapping[str, object] | None = None, persona: typing.Any = None, crawl: typing.Any = None, snapshot: typing.Any = None, machine: typing.Any = None, include_machine: bool = True, resolve_plugins: bool = True) -> archivebox.config.common.ArchiveBoxBaseConfig
:canonical: archivebox.config.common.get_config
```{autodoc2-docstring} archivebox.config.common.get_config

2
pyproject.toml
View File

@ -78,7 +78,7 @@ dependencies = [
"w3lib>=2.2.1", # used for parsing content-type encoding from http response headers & html tags
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus==2.5.8", # EventBus API
"abxbus==2.5.9", # EventBus API
"abxpkg>=1.11.141", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.144", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.144", # shared ArchiveBox downloader package with blocking install preflight