The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Update ArchiveBox for abx dependency releases

Browse Source
This commit is contained in:
Nick Sweeting 2026-05-16 23:50:16 -07:00
parent 6ff3d344ea
commit 0cc4251c8a
No known key found for this signature in database
89 changed files with 2903 additions and 1925 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.dockerignore
View File

@ -49,6 +49,9 @@ typings/
tmp/
data/
data*/
-
personas/
sources/
output/
index.sqlite3
index.sqlite3-wal

3
.gitignore vendored
View File

@ -43,6 +43,9 @@ archivebox/tests/data/
archive/
output/
logs/
/-
/personas/
/sources/
index.sqlite3
queue.sqlite3
*.sqlite*

1
Dockerfile
View File

@ -310,6 +310,7 @@ RUN --mount=type=cache,target=/home/archivebox/.npm_cache,sharing=locked,id=npm-
"@postlight/parser@^2.2.3" \
"readability-extractor@github:ArchiveBox/readability-extractor" \
"single-file-cli@^1.1.54" \
"abxbus@^2.5.4" \
"puppeteer-core@^23.5.0" \
"puppeteer@^23.5.0" \
"@puppeteer/browsers@^2.4.0" \

2
archivebox/__init__.py
View File

@ -81,7 +81,7 @@ LOADED_PLUGINS = ALL_PLUGINS
# Setup basic config, constants, paths, and version
from .config.constants import CONSTANTS # noqa
from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR # noqa
from .config.paths import PACKAGE_DIR, DATA_DIR # noqa
from .config.version import VERSION # noqa
# Set MACHINE_ID env var so hook scripts can use it

7
archivebox/api/v1_cli.py
View File

@ -8,9 +8,10 @@ from enum import Enum
from django.http import HttpRequest
from ninja import Router, Schema
from pydantic import Field
from archivebox.misc.util import ansi_to_html
from archivebox.config.common import ARCHIVING_CONFIG
from archivebox.config.common import get_config
# from .auth import API_AUTH_METHODS
@ -61,7 +62,7 @@ class AddCommandSchema(Schema):
depth: int = 0
parser: str = "auto"
plugins: str = ""
update: bool = not ARCHIVING_CONFIG.ONLY_NEW # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
overwrite: bool = False
index_only: bool = False
@ -87,7 +88,7 @@ class ScheduleCommandSchema(Schema):
tag: str = ""
depth: int = 0
overwrite: bool = False
update: bool = not ARCHIVING_CONFIG.ONLY_NEW
update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
clear: bool = False

11
archivebox/api/v1_core.py
View File

@ -8,7 +8,6 @@ from datetime import datetime
from django.db.models import Model, Q, Sum
from django.db.models.functions import Coalesce
from django.conf import settings
from django.http import HttpRequest, HttpResponse
from django.core.exceptions import ValidationError
from django.contrib.auth import get_user_model
@ -22,7 +21,7 @@ from ninja.errors import HttpError
from archivebox.core.models import Snapshot, ArchiveResult, Tag
from archivebox.api.auth import auth_using_token
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.common import get_config
from archivebox.core.tag_utils import (
build_tag_cards,
delete_tag as delete_tag_record,
@ -632,10 +631,10 @@ def search_tags(
def _public_tag_listing_enabled() -> bool:
explicit = getattr(settings, "PUBLIC_SNAPSHOTS_LIST", None)
if explicit is not None:
return bool(explicit)
return bool(getattr(settings, "PUBLIC_INDEX", SERVER_CONFIG.PUBLIC_INDEX))
config = get_config()
if config.PUBLIC_SNAPSHOTS_LIST is not None:
return config.PUBLIC_SNAPSHOTS_LIST
return config.PUBLIC_INDEX
def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:

11
archivebox/api/v1_crawls.py
View File

@ -173,17 +173,22 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
crawl.retry_at = payload["retry_at"]
update_fields.append("retry_at")
crawl.save(update_fields=update_fields)
if payload.get("status") == Crawl.StatusChoices.SEALED:
cancelled_at = timezone.now()
crawl.retry_at = None
if "retry_at" not in update_fields:
update_fields.append("retry_at")
crawl.save(update_fields=update_fields)
Snapshot.objects.filter(
crawl=crawl,
status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
).update(
status=Snapshot.StatusChoices.SEALED,
retry_at=None,
modified_at=timezone.now(),
modified_at=cancelled_at,
)
else:
crawl.save(update_fields=update_fields)
return crawl

4
archivebox/cli/__init__.py
View File

@ -165,9 +165,9 @@ def cli(ctx, help=False):
os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
if "--reload" in sys.argv:
os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
from archivebox.config.common import STORAGE_CONFIG
from archivebox.config.common import get_config
os.environ["ARCHIVEBOX_RUNSERVER_PIDFILE"] = str(STORAGE_CONFIG.TMP_DIR / "runserver.pid")
os.environ["ARCHIVEBOX_RUNSERVER_PIDFILE"] = str(get_config().TMP_DIR / "runserver.pid")
from archivebox.config.django import setup_django
from archivebox.misc.checks import check_data_folder

10
archivebox/cli/archivebox_add.py
View File

@ -16,8 +16,8 @@ from django.db.models import QuerySet
from archivebox.misc.util import enforce_types, docstring
from archivebox.misc.util import parse_filesize_to_bytes
from archivebox import CONSTANTS
from archivebox.config.common import ARCHIVING_CONFIG, SERVER_CONFIG
from archivebox.config.permissions import USER, HOSTNAME
from archivebox.config.common import get_config
if TYPE_CHECKING:
@ -94,13 +94,13 @@ def add(
from archivebox.personas.models import Persona
from archivebox.misc.logging_util import printable_filesize
from archivebox.misc.system import get_dir_size
from archivebox.config.configset import get_config
from archivebox.services.runner import run_crawl
config = get_config()
created_by_id = created_by_id or get_or_create_system_user_pk()
started_at = timezone.now()
if update is None:
update = not ARCHIVING_CONFIG.ONLY_NEW
update = not config.ONLY_NEW
# 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
@ -118,7 +118,7 @@ def add(
# Read URLs directly into crawl
urls_content = sources_file.read_text()
persona_name = (persona or "Default").strip() or "Default"
plugins = plugins or str(get_config().get("PLUGINS") or "")
plugins = plugins or str(config.get("PLUGINS") or "")
persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
persona_obj.ensure_dirs()
@ -221,7 +221,7 @@ def add(
except Exception:
rel_output_str = str(crawl.output_dir)
bind_addr = SERVER_CONFIG.BIND_ADDR or "127.0.0.1:8000"
bind_addr = config.BIND_ADDR or "127.0.0.1:8000"
if bind_addr.startswith("http://") or bind_addr.startswith("https://"):
base_url = bind_addr
else:

2
archivebox/cli/archivebox_archiveresult.py
View File

@ -73,7 +73,7 @@ def create_archiveresults(
0: Success
1: Failure
"""
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
from archivebox.hooks import discover_hooks
from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
from archivebox.core.models import Snapshot

39
archivebox/cli/archivebox_config.py
View File

@ -28,13 +28,24 @@ def config(
from archivebox.misc.checks import check_data_folder
from archivebox.misc.logging_util import printable_config
from archivebox.config.collection import load_all_config, write_config_file, get_real_name
from archivebox.config.configset import get_flat_config, get_all_configs
from abx_plugins.plugins.base.utils import resolve_alias
from archivebox.config.collection import write_config_file
from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
from archivebox.hooks import discover_plugin_configs
check_data_folder()
FLAT_CONFIG = get_flat_config()
FLAT_CONFIG = get_config().as_dict()
CONFIGS = get_all_configs()
plugin_schemas = {
plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
}
core_config_aliases = {
alias.upper(): field_name
for field_name, field in ArchiveBoxConfig.model_fields.items()
for alias in (field_name, str(field.alias or ""))
if alias
}
config_options: list[str] = list(kwargs.pop("key=value", []) or keys or [f"{key}={val}" for key, val in kwargs.items()])
no_args = not (get or set or reset or config_options)
@ -42,7 +53,9 @@ def config(
matching_config = {}
if search:
if config_options:
config_options = [get_real_name(key) for key in config_options]
config_options = [
core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
]
matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
for config_section in CONFIGS.values():
aliases = getattr(config_section, "aliases", {})
@ -63,7 +76,9 @@ def config(
elif get or no_args:
if config_options:
config_options = [get_real_name(key) for key in config_options]
config_options = [
core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
]
matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
failed_config = [key for key in config_options if key not in FLAT_CONFIG]
if failed_config:
@ -85,17 +100,11 @@ def config(
print(_format_toml(kv_in_section))
print("[grey53]################################################################[/grey53]")
# Display plugin config section
from archivebox.hooks import discover_plugin_configs
plugin_configs = discover_plugin_configs()
plugin_keys = {}
# Collect all plugin config keys
for plugin_name, schema in plugin_configs.items():
if "properties" not in schema:
continue
for key in schema["properties"].keys():
for schema in plugin_schemas.values():
for key in schema.keys():
if key in matching_config:
plugin_keys[key] = matching_config[key]
@ -120,7 +129,7 @@ def config(
raw_key, val = line.split("=", 1)
raw_key = raw_key.upper().strip()
key = get_real_name(raw_key)
key = core_config_aliases.get(raw_key) or resolve_alias(raw_key, plugin_schemas)
if key != raw_key:
print(
f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
@ -134,7 +143,7 @@ def config(
if new_config:
before = FLAT_CONFIG
matching_config = write_config_file(new_config)
after = {**load_all_config(), **get_flat_config()}
after = get_config().as_dict()
print(printable_config(matching_config))
side_effect_changes = {}

4
archivebox/cli/archivebox_help.py
View File

@ -15,6 +15,7 @@ def help() -> None:
from archivebox.cli import ArchiveBoxGroup
from archivebox.config import CONSTANTS
from archivebox.config.common import get_config
from archivebox.config.permissions import IN_DOCKER
from archivebox.misc.logging_util import log_cli_command
@ -67,7 +68,8 @@ def help() -> None:
[link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
""")
if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
config = get_config()
if os.access(config.ARCHIVE_DIR, os.R_OK) and config.ARCHIVE_DIR.is_dir():
pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
EXAMPLE_USAGE = f"""
[light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]

116
archivebox/cli/archivebox_init.py
View File

@ -5,7 +5,6 @@ __package__ = "archivebox.cli"
import os
import sys
from pathlib import Path
from collections.abc import Mapping
from rich import print
import rich_click as click
@ -13,17 +12,13 @@ import rich_click as click
from archivebox.misc.util import docstring, enforce_types
def _normalize_snapshot_record(link_dict: Mapping[str, object]) -> tuple[str, dict[str, object]] | None:
url = link_dict.get("url")
if not isinstance(url, str) or not url:
return None
record: dict[str, object] = {"url": url}
for key in ("timestamp", "title", "tags", "sources"):
value = link_dict.get(key)
if value is not None:
record[key] = value
return url, record
def _display_data_path(path: Path, data_dir: Path) -> str:
path = Path(path).resolve()
data_dir = Path(data_dir).resolve()
try:
return f"./{path.relative_to(data_dir)}"
except ValueError:
return str(path)
@enforce_types
@ -31,11 +26,12 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
"""Initialize a new ArchiveBox collection in the current directory"""
from archivebox.config import CONSTANTS, VERSION, DATA_DIR
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.common import get_config
from archivebox.config.collection import write_config_file
from archivebox.misc.legacy import parse_json_main_index, parse_json_links_details
from archivebox.misc.db import apply_migrations
config = get_config()
# if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
# print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
# print("[red] You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
@ -68,14 +64,16 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
else:
print("\n[green][+] Building archive folder structure...[/green]")
print(
f" + ./{CONSTANTS.ARCHIVE_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.SOURCES_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.LOGS_DIR.relative_to(DATA_DIR)}...",
)
archive_path = _display_data_path(config.ARCHIVE_DIR, DATA_DIR)
sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, DATA_DIR)
logs_path = _display_data_path(CONSTANTS.LOGS_DIR, DATA_DIR)
print(f" + {archive_path}, {sources_path}, {logs_path}...")
Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
Path(CONSTANTS.ARCHIVE_DIR).mkdir(exist_ok=True)
config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
config.USERS_DIR.mkdir(parents=True, exist_ok=True)
Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
print(f" + ./{CONSTANTS.CONFIG_FILE.relative_to(DATA_DIR)}...")
print(f" + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
# create the .archivebox_id file with a unique ID for this collection
from archivebox.config.paths import _get_collection_id
@ -83,7 +81,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
_get_collection_id(DATA_DIR, force_create=True)
# create the ArchiveBox.conf file
write_config_file({"SECRET_KEY": SERVER_CONFIG.SECRET_KEY})
write_config_file({"SECRET_KEY": config.SECRET_KEY})
if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
@ -99,11 +97,9 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
print()
print(f" √ ./{CONSTANTS.DATABASE_FILE.relative_to(DATA_DIR)}")
print(f" √ {_display_data_path(CONSTANTS.DATABASE_FILE, DATA_DIR)}")
# from django.contrib.auth.models import User
# if SHELL_CONFIG.IS_TTY and not User.objects.filter(is_superuser=True).exclude(username='system').exists():
# print('{green}[+] Creating admin user account...{reset}'.format(**SHELL_CONFIG.ANSI))
# call_command("createsuperuser", interactive=True)
print()
@ -111,86 +107,44 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
from archivebox.core.models import Snapshot
all_links = Snapshot.objects.none()
pending_links: dict[str, dict[str, object]] = {}
snapshot_count = 0
if existing_index:
all_links = Snapshot.objects.all()
print(f" √ Loaded {all_links.count()} links from existing main index.")
snapshot_count = Snapshot.objects.count()
print(f" √ Loaded {snapshot_count} links from existing main index.")
if quick:
print(" > Skipping orphan snapshot import (quick mode)")
else:
try:
# Import orphaned links from legacy JSON indexes
orphaned_json_links: dict[str, dict[str, object]] = {}
for link_dict in parse_json_main_index(DATA_DIR):
normalized = _normalize_snapshot_record(link_dict)
if normalized is None:
continue
url, record = normalized
if not all_links.filter(url=url).exists():
orphaned_json_links[url] = record
if orphaned_json_links:
pending_links.update(orphaned_json_links)
print(f" [yellow]√ Added {len(orphaned_json_links)} orphaned links from existing JSON index...[/yellow]")
orphaned_data_dir_links: dict[str, dict[str, object]] = {}
for link_dict in parse_json_links_details(DATA_DIR):
normalized = _normalize_snapshot_record(link_dict)
if normalized is None:
continue
url, record = normalized
if not all_links.filter(url=url).exists():
orphaned_data_dir_links[url] = record
if orphaned_data_dir_links:
pending_links.update(orphaned_data_dir_links)
print(f" [yellow]√ Added {len(orphaned_data_dir_links)} orphaned links from existing archive directories.[/yellow]")
if pending_links:
for link_dict in pending_links.values():
Snapshot.from_json(link_dict)
# Hint for orphaned snapshot directories
print()
print(" [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
print(" archivebox update")
except (KeyboardInterrupt, SystemExit):
print(file=sys.stderr)
print("[yellow]:stop_sign: Stopped checking archive directories due to Ctrl-C/SIGTERM[/yellow]", file=sys.stderr)
print(" Your archive data is safe, but you should re-run `archivebox init` to finish the process later.", file=sys.stderr)
print(file=sys.stderr)
print(" [violet]Hint:[/violet] In the future you can run a quick init without checking dirs like so:", file=sys.stderr)
print(" archivebox init --quick", file=sys.stderr)
raise SystemExit(1)
print(" > Skipping orphan snapshot import during init.")
print()
print(" [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
print(" archivebox update")
print("\n[green]----------------------------------------------------------------------[/green]")
from django.contrib.auth.models import User
if (SERVER_CONFIG.ADMIN_USERNAME and SERVER_CONFIG.ADMIN_PASSWORD) and not User.objects.filter(
username=SERVER_CONFIG.ADMIN_USERNAME,
config = get_config()
if (config.ADMIN_USERNAME and config.ADMIN_PASSWORD) and not User.objects.filter(
username=config.ADMIN_USERNAME,
).exists():
print("[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]")
User.objects.create_superuser(username=SERVER_CONFIG.ADMIN_USERNAME, password=SERVER_CONFIG.ADMIN_PASSWORD)
User.objects.create_superuser(username=config.ADMIN_USERNAME, password=config.ADMIN_PASSWORD)
if existing_index:
print("[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]")
else:
print(f"[green][√] Done. A new ArchiveBox collection was initialized ({len(all_links) + len(pending_links)} links).[/green]")
print(f"[green][√] Done. A new ArchiveBox collection was initialized ({snapshot_count} links).[/green]")
CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
CONSTANTS.DEFAULT_LIB_DIR.mkdir(parents=True, exist_ok=True)
(CONSTANTS.DEFAULT_LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
from archivebox.config.common import STORAGE_CONFIG
from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
STORAGE_CONFIG.TMP_DIR.mkdir(parents=True, exist_ok=True)
STORAGE_CONFIG.LIB_DIR.mkdir(parents=True, exist_ok=True)
(STORAGE_CONFIG.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
config = get_config()
config.TMP_DIR.mkdir(parents=True, exist_ok=True)
config.LIB_DIR.mkdir(parents=True, exist_ok=True)
(config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
if working_tmp_dir:

6
archivebox/cli/archivebox_install.py
View File

@ -22,11 +22,13 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
"""
from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.config.paths import ARCHIVE_DIR
from archivebox.config.common import get_config
from archivebox.misc.logging import stderr
from archivebox.cli.archivebox_init import init
if not (os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()):
config = get_config()
archive_dir = config.ARCHIVE_DIR
if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()):
init() # must init full index because we need a db to store Binary entries in
# Show what we're installing

5
archivebox/cli/archivebox_manage.py
View File

@ -10,10 +10,11 @@ from archivebox.misc.util import docstring, enforce_types
def manage(args: list[str] | None = None) -> None:
"""Run an ArchiveBox Django management command"""
from archivebox.config.common import SHELL_CONFIG
from archivebox.config.common import get_config
from archivebox.misc.logging import stderr
if (args and "createsuperuser" in args) and (SHELL_CONFIG.IN_DOCKER and not SHELL_CONFIG.IS_TTY):
config = get_config()
if (args and "createsuperuser" in args) and (config.IN_DOCKER and not config.IS_TTY):
stderr("[!] Warning: you need to pass -it to use interactive commands in docker", color="lightyellow")
stderr(" docker run -it archivebox manage {}".format(" ".join(args or ["..."])), color="lightyellow")
stderr("")

4
archivebox/cli/archivebox_persona.py
View File

@ -267,7 +267,7 @@ def extract_cookies_via_cdp(
Returns True if successful, False otherwise.
"""
from archivebox.config.common import STORAGE_CONFIG
from archivebox.config.common import get_config
# Find the cookie extraction script
chrome_plugin_dir = Path(__file__).parent.parent / "plugins" / "chrome"
@ -278,7 +278,7 @@ def extract_cookies_via_cdp(
return False
# Get node modules dir
node_modules_dir = STORAGE_CONFIG.LIB_DIR / "npm" / "node_modules"
node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules"
# Set up environment
env = os.environ.copy()

4
archivebox/cli/archivebox_remove.py
View File

@ -12,7 +12,7 @@ import rich_click as click
from django.db.models import QuerySet
from archivebox.config import DATA_DIR
from archivebox.config.constants import CONSTANTS
from archivebox.config.common import get_config
from archivebox.config.django import setup_django
from archivebox.misc.util import enforce_types, docstring
from archivebox.misc.checks import check_data_folder
@ -70,7 +70,7 @@ def remove(
for snapshot in snapshots:
if delete:
shutil.rmtree(snapshot.output_dir, ignore_errors=True)
legacy_path = CONSTANTS.ARCHIVE_DIR / snapshot.timestamp
legacy_path = get_config().ARCHIVE_DIR / snapshot.timestamp
if legacy_path.is_symlink():
legacy_path.unlink(missing_ok=True)
finally:

4
archivebox/cli/archivebox_run.py
View File

@ -333,10 +333,6 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
sys.exit(1)
if daemon:
if not sys.stdin.isatty():
exit_code = process_stdin_records()
if exit_code != 0:
sys.exit(exit_code)
sys.exit(run_runner(daemon=True))
if not sys.stdin.isatty():

7
archivebox/cli/archivebox_schedule.py
View File

@ -6,7 +6,7 @@ import rich_click as click
from rich import print
from archivebox.misc.util import enforce_types, docstring
from archivebox.config.common import ARCHIVING_CONFIG
from archivebox.config.common import get_config
@enforce_types
@ -21,7 +21,7 @@ def schedule(
tag: str = "",
depth: int | str = 0,
overwrite: bool = False,
update: bool = not ARCHIVING_CONFIG.ONLY_NEW,
update: bool | None = None,
import_path: str | None = None,
):
"""Manage database-backed scheduled crawls processed by the crawl runner."""
@ -33,6 +33,9 @@ def schedule(
from archivebox.crawls.schedule_utils import validate_schedule
from archivebox.services.runner import run_pending_crawls
if update is None:
update = not get_config().ONLY_NEW
depth = int(depth)
result: dict[str, object] = {
"created_schedule_ids": [],

9
archivebox/cli/archivebox_search.py
View File

@ -13,6 +13,7 @@ import rich_click as click
from django.db.models import Q, QuerySet
from archivebox.config import DATA_DIR
from archivebox.config.common import get_config
from archivebox.misc.logging import stderr
from archivebox.misc.util import enforce_types, docstring
@ -59,14 +60,14 @@ def _snapshots_to_json(
from datetime import datetime, timezone as tz
from archivebox.config import VERSION
from archivebox.config.common import SERVER_CONFIG
from archivebox.misc.util import to_json
config = get_config()
main_index_header = (
{
"info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
"schema": "archivebox.index.json",
"copyright_info": SERVER_CONFIG.FOOTER_INFO,
"copyright_info": config.FOOTER_INFO,
"meta": {
"project": "ArchiveBox",
"version": VERSION,
@ -119,9 +120,9 @@ def _snapshots_to_html(
from django.template.loader import render_to_string
from archivebox.config import VERSION
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.version import get_COMMIT_HASH
config = get_config()
template = "static_index.html" if with_headers else "minimal_index.html"
snapshot_list = list(snapshots.iterator(chunk_size=500))
@ -134,7 +135,7 @@ def _snapshots_to_html(
"date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
"time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
"links": snapshot_list,
"FOOTER_INFO": SERVER_CONFIG.FOOTER_INFO,
"FOOTER_INFO": config.FOOTER_INFO,
},
)

16
archivebox/cli/archivebox_server.py
View File

@ -2,14 +2,15 @@
__package__ = "archivebox.cli"
from collections.abc import Iterable
import sys
import os
from collections.abc import Iterable
import rich_click as click
from rich import print
from archivebox.misc.util import docstring, enforce_types
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.common import get_config
def stop_existing_background_runner(*, machine, process_model, supervisor=None, stop_worker_fn=None, log=print) -> int:
@ -95,7 +96,7 @@ def stop_existing_server_workers(*, supervisor, stop_worker_fn, host: str, port:
@enforce_types
def server(
runserver_args: Iterable[str] = (SERVER_CONFIG.BIND_ADDR,),
runserver_args: Iterable[str] | None = None,
reload: bool = False,
init: bool = False,
debug: bool = False,
@ -104,7 +105,8 @@ def server(
) -> None:
"""Run the ArchiveBox HTTP server"""
runserver_args = list(runserver_args)
config = get_config()
runserver_args = list(runserver_args or (config.BIND_ADDR,))
if init:
from archivebox.cli.archivebox_init import init as archivebox_init
@ -116,11 +118,9 @@ def server(
check_data_folder()
from archivebox.config.common import SHELL_CONFIG
run_in_debug = SHELL_CONFIG.DEBUG or debug or reload
run_in_debug = config.DEBUG or debug or reload
if debug or reload:
SHELL_CONFIG.DEBUG = True
os.environ["DEBUG"] = "True"
from django.contrib.auth.models import User

28
archivebox/cli/archivebox_status.py
View File

@ -8,8 +8,8 @@ import rich_click as click
from rich import print
from archivebox.misc.util import enforce_types, docstring
from archivebox.config import DATA_DIR, CONSTANTS, ARCHIVE_DIR
from archivebox.config.common import SHELL_CONFIG
from archivebox.config import DATA_DIR, CONSTANTS
from archivebox.config.common import get_config
from archivebox.misc.legacy import parse_json_links_details
from archivebox.misc.system import get_dir_size
from archivebox.misc.logging_util import printable_filesize
@ -24,6 +24,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
from django.db.models.functions import Coalesce
from archivebox.core.models import Snapshot
config = get_config()
User = get_user_model()
print("[green]\\[*] Scanning archive main index...[/green]")
@ -36,13 +37,14 @@ def status(out_dir: Path = DATA_DIR) -> None:
links = list(Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)))
num_sql_links = len(links)
num_link_details = sum(1 for link in parse_json_links_details(out_dir=out_dir))
archive_dir = config.ARCHIVE_DIR
print(f" > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {CONSTANTS.SQL_INDEX_FILENAME})")
print(f" > JSON Link Details: {num_link_details} links".ljust(36), f"(found in {ARCHIVE_DIR.name}/*/index.json)")
print(f" > JSON Link Details: {num_link_details} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
print()
print("[green]\\[*] Scanning archive data directories...[/green]")
users_dir = out_dir / "users"
scan_roots = [root for root in (ARCHIVE_DIR, users_dir) if root.exists()]
scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(ARCHIVE_DIR)
users_dir = config.USERS_DIR
scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
print(f"[yellow] {scan_roots_display}[/yellow]")
num_bytes = num_dirs = num_files = 0
for root in scan_roots:
@ -65,11 +67,17 @@ def status(out_dir: Path = DATA_DIR) -> None:
expected_snapshot_dirs = {str(Path(snapshot.output_dir).resolve()) for snapshot in links if Path(snapshot.output_dir).exists()}
discovered_snapshot_dirs = set()
if ARCHIVE_DIR.exists():
discovered_snapshot_dirs.update(str(entry.resolve()) for entry in ARCHIVE_DIR.iterdir() if entry.is_dir())
if archive_dir.exists():
discovered_snapshot_dirs.update(
str(entry.resolve())
for entry in archive_dir.iterdir()
if entry.is_dir() and not entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry)
)
if users_dir.exists():
discovered_snapshot_dirs.update(str(entry.resolve()) for entry in users_dir.glob("*/snapshots/*/*/*") if entry.is_dir())
discovered_snapshot_dirs.update(
str(entry.resolve()) for entry in users_dir.glob(f"*/{CONSTANTS.SNAPSHOTS_DIR_NAME}/*/*/*") if entry.is_dir()
)
orphaned_dirs = sorted(discovered_snapshot_dirs - expected_snapshot_dirs)
num_present = len(discovered_snapshot_dirs)
@ -123,7 +131,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
f"[{snapshot.num_outputs} {('X', '√')[snapshot.is_archived]} {printable_filesize(snapshot.archive_size)}] "
f'"{snapshot.title}": {snapshot.url}'
"[/grey53]"
)[: SHELL_CONFIG.TERM_WIDTH],
)[: config.TERM_WIDTH],
)
print("[grey53] ...")

207
archivebox/cli/archivebox_update.py
View File

@ -260,34 +260,50 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
Only processes real directories (skips symlinks - those are already migrated).
For each old dir found in archive/:
1. Load or create DB snapshot
2. Trigger fs migration on save() to move to data/users/{user}/...
2. Trigger fs migration on save() to move to data/archive/users/{user}/...
3. Leave symlink in archive/ pointing to new location
After this drains, archive/ should only contain symlinks and we can trust
1:1 mapping between DB and filesystem.
"""
from archivebox.core.models import Snapshot
from archivebox.config import CONSTANTS
from archivebox.config.common import get_config
from archivebox.crawls.models import Crawl
from django.db import transaction
from django.utils import timezone
stats = {"processed": 0, "migrated": 0, "skipped": 0, "invalid": 0}
crawl_output_dirs: dict[str, Path] = {}
crawl_url_lines: dict[str, list[str]] = {}
crawl_url_sets: dict[str, set[str]] = {}
dirty_crawl_ids: set[str] = set()
archive_dir = CONSTANTS.ARCHIVE_DIR
runtime_config = get_config()
archive_dir = runtime_config.ARCHIVE_DIR
if not archive_dir.exists():
return stats
print("[DEBUG Phase1] Scanning for old directories in archive/...")
for crawl in Crawl.objects.filter(label__startswith="[migration] orphaned").iterator():
url_entries = crawl._iter_url_lines()
existing_urls = {url for _raw_line, url in url_entries if url}
lines = (crawl.urls or "").splitlines()
changed = False
for url in crawl.snapshot_set.order_by("timestamp").values_list("url", flat=True):
if url not in existing_urls:
lines.append(url)
existing_urls.add(url)
changed = True
if changed:
Crawl.objects.filter(pk=crawl.pk).update(urls="\n".join(lines), modified_at=timezone.now())
# Scan for real directories only (skip symlinks - they're already migrated)
all_entries = list(os.scandir(archive_dir))
print(f"[DEBUG Phase1] Total entries in archive/: {len(all_entries)}")
entries = [
(e.stat().st_mtime, e.path)
for e in all_entries
if e.is_dir(follow_symlinks=False) # Skip symlinks
if e.is_dir(follow_symlinks=False) and Snapshot.is_legacy_archive_dir(Path(e.path)) # Skip symlinks and 0.9.x roots
]
entries.sort(reverse=True) # Newest first
print(f"[DEBUG Phase1] Real directories (not symlinks): {len(entries)}")
print(f"[*] Found {len(entries)} old directories to drain")
for mtime, entry_path in entries:
@ -313,7 +329,42 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
continue
try:
snapshot.save()
Snapshot.objects.bulk_create([snapshot])
snapshot.migrate_filesystem_to_current_version(source_dir=entry_path, config=runtime_config)
Snapshot.objects.filter(pk=snapshot.pk).update(
fs_version=snapshot.fs_version,
modified_at=timezone.now(),
)
migration_cleanup = getattr(snapshot, "_pending_fs_migration_cleanup", None)
new_dir = None
if migration_cleanup:
old_dir, new_dir = migration_cleanup
transaction.on_commit(
lambda old_dir=old_dir, new_dir=new_dir, snapshot=snapshot: snapshot._cleanup_old_migration_dir(old_dir, new_dir),
)
delattr(snapshot, "_pending_fs_migration_cleanup")
crawl = _get_snapshot_crawl(snapshot)
crawl_dir = None
if crawl is not None:
crawl_cache_key = str(crawl.id)
crawl_dir = crawl_output_dirs.get(crawl_cache_key)
if crawl_dir is None:
crawl_dir = Path(crawl.output_dir)
crawl_output_dirs[crawl_cache_key] = crawl_dir
existing_urls = crawl_url_sets.get(crawl_cache_key)
if existing_urls is None:
url_entries = crawl._iter_url_lines()
existing_urls = {url for _raw_line, url in url_entries if url}
crawl_url_sets[crawl_cache_key] = existing_urls
crawl_url_lines[crawl_cache_key] = (crawl.urls or "").splitlines()
if snapshot.url not in existing_urls:
crawl_url_lines[crawl_cache_key].append(snapshot.url)
existing_urls.add(snapshot.url)
dirty_crawl_ids.add(crawl_cache_key)
snapshot.ensure_crawl_symlink(crawl_dir=crawl_dir, snapshot_dir=new_dir)
stats["migrated"] += 1
print(f" [{stats['processed']}] Imported orphaned snapshot: {entry_path.name}")
except Exception as e:
@ -326,8 +377,6 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
if not has_valid_crawl:
# Create a new crawl (created_by will default to system user)
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(urls=snapshot.url)
# Use queryset update to avoid triggering save() hooks
from archivebox.core.models import Snapshot as SnapshotModel
@ -335,59 +384,57 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
SnapshotModel.objects.filter(pk=snapshot.pk).update(crawl=crawl)
# Refresh the instance
snapshot.crawl = crawl
print(f"[DEBUG Phase1] Created missing crawl for snapshot {str(snapshot.id)[:8]}")
# Check if needs migration (0.8.x → 0.9.x)
print(
f"[DEBUG Phase1] Snapshot {str(snapshot.id)[:8]}: fs_version={snapshot.fs_version}, needs_migration={snapshot.fs_migration_needed}",
)
if snapshot.fs_migration_needed:
try:
# Calculate paths using actual directory (entry_path), not snapshot.timestamp
# because snapshot.timestamp might be truncated
old_dir = entry_path
new_dir = snapshot.get_storage_path_for_version("0.9.0")
print(f"[DEBUG Phase1] Migrating {old_dir.name} → {new_dir}")
# Manually migrate files
if not new_dir.exists() and old_dir.exists():
new_dir.mkdir(parents=True, exist_ok=True)
import shutil
file_count = 0
for old_file in old_dir.rglob("*"):
if old_file.is_file():
rel_path = old_file.relative_to(old_dir)
new_file = new_dir / rel_path
if not new_file.exists():
new_file.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(old_file, new_file)
file_count += 1
print(f"[DEBUG Phase1] Copied {file_count} files")
# Update only fs_version field using queryset update (bypasses validation)
from archivebox.core.models import Snapshot as SnapshotModel
SnapshotModel.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
# Commit the transaction
transaction.commit()
# Cleanup: delete old dir and create symlink
if old_dir.exists() and old_dir != new_dir:
snapshot._cleanup_old_migration_dir(old_dir, new_dir)
try:
old_version = snapshot.fs_version
snapshot.migrate_filesystem_to_current_version(source_dir=entry_path, config=runtime_config)
if snapshot.fs_version != old_version or getattr(snapshot, "_pending_fs_migration_cleanup", None):
Snapshot.objects.filter(pk=snapshot.pk).update(
fs_version=snapshot.fs_version,
modified_at=timezone.now(),
)
migration_cleanup = getattr(snapshot, "_pending_fs_migration_cleanup", None)
new_dir = None
if migration_cleanup:
old_dir, new_dir = migration_cleanup
transaction.on_commit(
lambda old_dir=old_dir, new_dir=new_dir, snapshot=snapshot: snapshot._cleanup_old_migration_dir(old_dir, new_dir),
)
delattr(snapshot, "_pending_fs_migration_cleanup")
crawl_dir = None
if snapshot.crawl_id:
crawl_cache_key = str(snapshot.crawl_id)
crawl_dir = crawl_output_dirs.get(crawl_cache_key)
if crawl_dir is None:
crawl = _get_snapshot_crawl(snapshot)
if crawl is not None:
crawl_dir = Path(crawl.output_dir)
crawl_output_dirs[crawl_cache_key] = crawl_dir
snapshot.ensure_crawl_symlink(crawl_dir=crawl_dir, snapshot_dir=new_dir)
stats["migrated"] += 1
print(f" [{stats['processed']}] Migrated: {entry_path.name}")
except Exception as e:
else:
stats["skipped"] += 1
print(f" [{stats['processed']}] Skipped (error: {e}): {entry_path.name}")
else:
except Exception as e:
stats["skipped"] += 1
print(f" [{stats['processed']}] Skipped (error: {e}): {entry_path.name}")
if stats["processed"] % batch_size == 0:
for crawl_id in tuple(dirty_crawl_ids):
Crawl.objects.filter(pk=crawl_id).update(
urls="\n".join(crawl_url_lines[crawl_id]),
modified_at=timezone.now(),
)
dirty_crawl_ids.clear()
transaction.commit()
for crawl_id in tuple(dirty_crawl_ids):
Crawl.objects.filter(pk=crawl_id).update(
urls="\n".join(crawl_url_lines[crawl_id]),
modified_at=timezone.now(),
)
dirty_crawl_ids.clear()
transaction.commit()
return stats
@ -404,10 +451,12 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -
after Phase 1 has drained all old archive/ directories.
"""
from archivebox.core.models import Snapshot
from archivebox.config.common import get_config
from django.db import transaction
from django.utils import timezone
stats = {"processed": 0, "reconciled": 0, "queued": 0}
runtime_config = get_config()
queryset = Snapshot.objects.all()
if resume:
@ -416,7 +465,7 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -
print(f"[*] Processing {total} snapshots from database (most recent first)...")
# Process from most recent to least recent
for snapshot in queryset.select_related("crawl").order_by("-bookmarked_at").iterator(chunk_size=batch_size):
for snapshot in queryset.select_related("crawl__created_by").order_by("-bookmarked_at").iterator(chunk_size=batch_size):
stats["processed"] += 1
# Skip snapshots with missing crawl references (orphaned by migration errors)
@ -424,41 +473,47 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -
continue
try:
print(
f"[DEBUG Phase2] Snapshot {str(snapshot.id)[:8]}: fs_version={snapshot.fs_version}, needs_migration={snapshot.fs_migration_needed}",
)
# Check if snapshot has a directory on disk
from pathlib import Path
output_dir = Path(snapshot.output_dir)
output_dir = Path(snapshot.get_storage_path_for_version(snapshot.fs_version, config=runtime_config))
has_directory = output_dir.exists() and output_dir.is_dir()
current_fs_version = Snapshot._fs_current_version()
update_values = {
"status": Snapshot.StatusChoices.QUEUED,
"retry_at": timezone.now(),
"modified_at": timezone.now(),
}
# Only reconcile if directory exists (don't create empty directories for orphans)
if has_directory:
snapshot.reconcile_with_index_json()
json_path = output_dir / "index.json"
jsonl_path = output_dir / "index.jsonl"
if json_path.exists() or not jsonl_path.exists():
old_title = snapshot.title
snapshot.reconcile_with_index_json(output_dir=output_dir)
if snapshot.title != old_title:
update_values["title"] = snapshot.title
# Clean up invalid field values from old migrations
if not isinstance(snapshot.current_step, int):
snapshot.current_step = 0
update_values["current_step"] = 0
# If still needs migration, it's an orphan (no directory on disk)
# Mark it as migrated to prevent save() from triggering filesystem migration
if snapshot.fs_migration_needed:
if has_directory:
print(f"[DEBUG Phase2] WARNING: Snapshot {str(snapshot.id)[:8]} has directory but still needs migration")
legacy_dir = snapshot.get_storage_path_for_version("0.8.0", config=runtime_config)
current_dir = snapshot.get_storage_path_for_version(current_fs_version, config=runtime_config)
if legacy_dir.exists() or current_dir.exists():
snapshot.migrate_filesystem_to_current_version(config=runtime_config)
snapshot.status = update_values["status"]
snapshot.retry_at = update_values["retry_at"]
if "current_step" in update_values:
snapshot.current_step = update_values["current_step"]
snapshot.save(update_fields=tuple([*update_values.keys(), "fs_version"]))
else:
print(f"[DEBUG Phase2] Orphan snapshot {str(snapshot.id)[:8]} - marking as migrated without filesystem operation")
# Use queryset update to set fs_version without triggering save() hooks
from archivebox.core.models import Snapshot as SnapshotModel
SnapshotModel.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
snapshot.fs_version = "0.9.0"
# Queue for archiving (state machine will handle it)
snapshot.status = Snapshot.StatusChoices.QUEUED
snapshot.retry_at = timezone.now()
snapshot.save()
update_values["fs_version"] = current_fs_version
Snapshot.objects.filter(pk=snapshot.pk).update(**update_values)
else:
Snapshot.objects.filter(pk=snapshot.pk).update(**update_values)
stats["reconciled"] += 1 if has_directory else 0
stats["queued"] += 1

21
archivebox/cli/archivebox_version.py
View File

@ -107,9 +107,8 @@ def version(
from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER
from archivebox.config.paths import get_data_locations, get_code_locations
from archivebox.config.common import SHELL_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG
from archivebox.misc.logging_util import printable_folder_status
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
console = Console()
prnt = console.print
@ -127,7 +126,7 @@ def version(
)
prnt(
f"IN_DOCKER={IN_DOCKER}",
f"IN_QEMU={SHELL_CONFIG.IN_QEMU}",
f"IN_QEMU={config.IN_QEMU}",
f"ARCH={p.machine}",
f"OS={p.system}",
f"PLATFORM={platform.platform()}",
@ -144,8 +143,8 @@ def version(
prnt(
f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
f"FS_PERMS={STORAGE_CONFIG.OUTPUT_PERMISSIONS}",
f"FS_ATOMIC={STORAGE_CONFIG.ENFORCE_ATOMIC_WRITES}",
f"FS_PERMS={config.OUTPUT_PERMISSIONS}",
f"FS_ATOMIC={config.ENFORCE_ATOMIC_WRITES}",
f"FS_REMOTE={OUTPUT_IS_REMOTE_FS}",
)
except Exception:
@ -154,16 +153,16 @@ def version(
)
prnt(
f"DEBUG={SHELL_CONFIG.DEBUG}",
f"IS_TTY={SHELL_CONFIG.IS_TTY}",
f"DEBUG={config.DEBUG}",
f"IS_TTY={config.IS_TTY}",
f"SUDO={CONSTANTS.IS_ROOT}",
f"ID={CONSTANTS.MACHINE_ID}:{CONSTANTS.COLLECTION_ID}",
f"SEARCH_BACKEND={SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}",
f"SEARCH_BACKEND={config.SEARCH_BACKEND_ENGINE}",
f"LDAP={LDAP_ENABLED}",
)
prnt()
if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
if not (os.access(config.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
PANEL_TEXT = "\n".join(
(
"",
@ -226,7 +225,7 @@ def version(
_format_binary_abspath(
installed.abspath,
pwd=Path.cwd(),
lib_dir=STORAGE_CONFIG.LIB_DIR,
lib_dir=config.LIB_DIR,
personas_dir=Path.home() / ".config" / "abx" / "personas",
home=Path.home(),
)
@ -283,7 +282,7 @@ def version(
prnt(f" [red]Error getting code locations: {e}[/red]")
prnt()
if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
if os.access(config.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
try:
for name, path in get_data_locations().items():

98
archivebox/config/__init__.py
View File

@ -1,9 +1,4 @@
"""
ArchiveBox config exports.
This module provides backwards-compatible config exports for extractors
and other modules that expect to import config values directly.
"""
"""Minimal import-time config exports."""
__package__ = "archivebox.config"
__order__ = 200
@ -11,95 +6,6 @@ __order__ = 200
from .paths import (
PACKAGE_DIR,
DATA_DIR,
ARCHIVE_DIR,
)
from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR # noqa
from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR # noqa
from .version import VERSION # noqa
###############################################################################
# Config value exports for extractors
# These provide backwards compatibility with extractors that import from ..config
###############################################################################
def _get_config():
"""Lazy import to avoid circular imports."""
from .common import ARCHIVING_CONFIG, STORAGE_CONFIG
return ARCHIVING_CONFIG, STORAGE_CONFIG
# Direct exports (evaluated at import time for backwards compat)
# These are recalculated each time the module attribute is accessed
def __getattr__(name: str):
"""
Module-level __getattr__ for lazy config loading.
Only provides backwards compatibility for GENERIC/SHARED config.
Plugin-specific config (binaries, args, toggles) should come from plugin config.json files.
"""
# Generic timeout settings (used by multiple plugins)
if name == "TIMEOUT":
cfg, _ = _get_config()
return cfg.TIMEOUT
# Generic SSL/Security settings (used by multiple plugins)
if name == "CHECK_SSL_VALIDITY":
cfg, _ = _get_config()
return cfg.CHECK_SSL_VALIDITY
# Generic storage settings (used by multiple plugins)
if name == "RESTRICT_FILE_NAMES":
_, storage = _get_config()
return storage.RESTRICT_FILE_NAMES
# Generic user agent / cookies (used by multiple plugins)
if name == "COOKIES_FILE":
cfg, _ = _get_config()
return cfg.COOKIES_FILE
if name == "USER_AGENT":
cfg, _ = _get_config()
return cfg.USER_AGENT
# Generic resolution settings (used by multiple plugins)
if name == "RESOLUTION":
cfg, _ = _get_config()
return cfg.RESOLUTION
# Allowlist/Denylist patterns (compiled regexes)
if name == "SAVE_ALLOWLIST_PTN":
cfg, _ = _get_config()
return cfg.SAVE_ALLOWLIST_PTNS
if name == "SAVE_DENYLIST_PTN":
cfg, _ = _get_config()
return cfg.SAVE_DENYLIST_PTNS
raise AttributeError(f"module 'archivebox.config' has no attribute '{name}'")
# Re-export common config classes for direct imports
def get_CONFIG():
"""Get all config sections as a dict."""
from .common import (
SHELL_CONFIG,
STORAGE_CONFIG,
GENERAL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
)
from .ldap import LDAP_CONFIG
return {
"SHELL_CONFIG": SHELL_CONFIG,
"STORAGE_CONFIG": STORAGE_CONFIG,
"GENERAL_CONFIG": GENERAL_CONFIG,
"SERVER_CONFIG": SERVER_CONFIG,
"ARCHIVING_CONFIG": ARCHIVING_CONFIG,
"SEARCHBACKEND_CONFIG": SEARCH_BACKEND_CONFIG,
"LDAP_CONFIG": LDAP_CONFIG,
}

257
archivebox/config/collection.py
View File

@ -1,176 +1,18 @@
__package__ = "archivebox.config"
import os
import json
from typing import Any
from pathlib import Path
from configparser import ConfigParser
from benedict import benedict
from archivebox.config.constants import CONSTANTS
from archivebox.misc.logging import stderr
class CaseConfigParser(ConfigParser):
def optionxform(self, optionstr: str) -> str:
return optionstr
def get_real_name(key: str) -> str:
"""get the up-to-date canonical name for a given old alias or current key"""
# Config aliases are no longer used with the simplified config system
# Just return the key as-is since we no longer have a complex alias mapping
return key
def load_config_val(
key: str,
default: Any = None,
type: type | None = None,
aliases: tuple[str, ...] | None = None,
config: benedict | None = None,
env_vars: os._Environ | None = None,
config_file_vars: dict[str, str] | None = None,
) -> Any:
"""parse bool, int, and str key=value pairs from env"""
assert isinstance(config, dict)
is_read_only = type is None
if is_read_only:
if callable(default):
return default(config)
return default
# get value from environment variables or config files
config_keys_to_check = (key, *(aliases or ()))
val = None
for key in config_keys_to_check:
if env_vars:
val = env_vars.get(key)
if val:
break
if config_file_vars:
val = config_file_vars.get(key)
if val:
break
is_unset = val is None
if is_unset:
if callable(default):
return default(config)
return default
assert isinstance(val, str)
# calculate value based on expected type
BOOL_TRUEIES = ("true", "yes", "1")
BOOL_FALSEIES = ("false", "no", "0")
if type is bool:
if val.lower() in BOOL_TRUEIES:
return True
elif val.lower() in BOOL_FALSEIES:
return False
else:
raise ValueError(f"Invalid configuration option {key}={val} (expected a boolean: True/False)")
elif type is str:
if val.lower() in (*BOOL_TRUEIES, *BOOL_FALSEIES):
raise ValueError(f"Invalid configuration option {key}={val} (expected a string, but value looks like a boolean)")
return val.strip()
elif type is int:
if not val.strip().isdigit():
raise ValueError(f"Invalid configuration option {key}={val} (expected an integer)")
return int(val.strip())
elif type is list or type is dict:
return json.loads(val)
elif type is Path:
return Path(val)
raise Exception("Config values can only be str, bool, int, or json")
def load_config_file() -> benedict | None:
"""load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
config_path = CONSTANTS.CONFIG_FILE
if os.access(config_path, os.R_OK):
config_file = CaseConfigParser()
config_file.read(config_path)
# flatten into one namespace
config_file_vars = benedict({key.upper(): val for section, options in config_file.items() for key, val in options.items()})
# print('[i] Loaded config file', os.path.abspath(config_path))
# print(config_file_vars)
return config_file_vars
return None
class PluginConfigSection:
"""Pseudo-section for all plugin config keys written to [PLUGINS] section in ArchiveBox.conf"""
toml_section_header = "PLUGINS"
def __init__(self, key: str):
self._key = key
def __getattr__(self, name: str) -> Any:
# Allow hasattr checks to pass for the key
if name == self._key:
return None
raise AttributeError(f"PluginConfigSection has no attribute '{name}'")
def update_in_place(self, warn: bool = True, persist: bool = False, **kwargs):
"""No-op update since plugins read config dynamically via get_config()."""
pass
def section_for_key(key: str) -> Any:
"""Find the config section containing a given key."""
from archivebox.config.common import (
SHELL_CONFIG,
STORAGE_CONFIG,
GENERAL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
)
# First check core config sections
for section in [
SHELL_CONFIG,
STORAGE_CONFIG,
GENERAL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
]:
if hasattr(section, key):
return section
# Check if this is a plugin config key
from archivebox.hooks import discover_plugin_configs
plugin_configs = discover_plugin_configs()
for plugin_name, schema in plugin_configs.items():
if "properties" in schema and key in schema["properties"]:
# All plugin config goes to [PLUGINS] section
return PluginConfigSection(key)
raise ValueError(f"No config section found for key: {key}")
from archivebox.config.configset import CaseConfigParser
def write_config_file(config: dict[str, str]) -> benedict:
"""load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
from archivebox.config.common import get_all_configs
from archivebox.hooks import discover_plugin_configs
from archivebox.misc.system import atomic_write
CONFIG_HEADER = """# This is the config file for your ArchiveBox collection.
@ -197,15 +39,25 @@ def write_config_file(config: dict[str, str]) -> benedict:
with open(config_path, encoding="utf-8") as old:
atomic_write(f"{config_path}.bak", old.read())
config_sections = get_all_configs()
plugin_configs = discover_plugin_configs()
# Set up sections in empty config file
for key, val in config.items():
section = section_for_key(key)
assert section is not None
section_name = None
for section in config_sections.values():
if key in type(section).model_fields:
section_name = section.toml_section_header
break
if not hasattr(section, "toml_section_header"):
raise ValueError(f"{key} is read-only (defined in {type(section).__module__}.{type(section).__name__}). Refusing to set.")
if section_name is None:
for schema in plugin_configs.values():
if "properties" in schema and key in schema["properties"]:
section_name = "PLUGINS"
break
section_name = section.toml_section_header
if section_name is None:
raise ValueError(f"No config section found for key: {key}")
if section_name in config_file:
existing_config = dict(config_file[section_name])
@ -213,7 +65,6 @@ def write_config_file(config: dict[str, str]) -> benedict:
existing_config = {}
config_file[section_name] = benedict({**existing_config, key: val})
section.update_in_place(warn=False, persist=False, **{key: val})
with open(config_path, "w+", encoding="utf-8") as new:
config_file.write(new)
@ -221,9 +72,9 @@ def write_config_file(config: dict[str, str]) -> benedict:
updated_config = {}
try:
# validate the updated_config by attempting to re-parse it
from archivebox.config.configset import get_flat_config
from archivebox.config.common import get_config
updated_config = {**load_all_config(), **get_flat_config()}
updated_config = get_config().as_dict()
except BaseException: # lgtm [py/catch-base-exception]
# something went horribly wrong, revert to the previous version
with open(f"{config_path}.bak", encoding="utf-8") as old:
@ -235,71 +86,3 @@ def write_config_file(config: dict[str, str]) -> benedict:
os.remove(f"{config_path}.bak")
return benedict({key.upper(): updated_config.get(key.upper()) for key in config.keys()})
def load_config(
defaults: dict[str, Any],
config: benedict | None = None,
out_dir: str | None = None,
env_vars: os._Environ | None = None,
config_file_vars: dict[str, str] | None = None,
) -> benedict:
env_vars = env_vars or os.environ
config_file_vars = config_file_vars or load_config_file()
extended_config = benedict(config.copy() if config else {})
for key, default in defaults.items():
try:
# print('LOADING CONFIG KEY:', key, 'DEFAULT=', default)
extended_config[key] = load_config_val(
key,
default=default["default"],
type=default.get("type"),
aliases=default.get("aliases"),
config=extended_config,
env_vars=env_vars,
config_file_vars=config_file_vars,
)
except KeyboardInterrupt:
raise SystemExit(0)
except Exception as e:
stderr()
stderr(f"[X] Error while loading configuration value: {key}", color="red", config=extended_config)
stderr(f" {e.__class__.__name__}: {e}")
stderr()
stderr(" Check your config for mistakes and try again (your archive data is unaffected).")
stderr()
stderr(" For config documentation and examples see:")
stderr(" https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration")
stderr()
# raise
# raise SystemExit(2)
return benedict(extended_config)
def load_all_config():
"""Load all config sections and return as a flat dict."""
from archivebox.config.common import (
SHELL_CONFIG,
STORAGE_CONFIG,
GENERAL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
)
flat_config = benedict()
for config_section in [
SHELL_CONFIG,
STORAGE_CONFIG,
GENERAL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
]:
flat_config.update(dict(config_section))
return flat_config

313
archivebox/config/common.py
View File

@ -1,25 +1,37 @@
__package__ = "archivebox.config"
import json
import re
import secrets
import sys
import shutil
from typing import ClassVar
from collections.abc import Mapping
from typing import Any, ClassVar, cast
from pathlib import Path
from rich.console import Console
from pydantic import Field, field_validator
from pydantic import BaseModel, Field, create_model, field_validator, model_validator
from pydantic_settings import SettingsConfigDict
from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
from archivebox.config.configset import BaseConfigSet
from archivebox.config.configset import COMPUTED_CONFIG_KEYS
from .constants import CONSTANTS
from .ldap import LDAPConfig
from .version import get_COMMIT_HASH, get_BUILD_TIME, VERSION
from .permissions import IN_DOCKER
ConfigOverrides = Mapping[str, object]
ConfigPayload = dict[str, object]
PluginSchemaDocuments = dict[str, dict[str, Any]]
###################### Config ##########################
_STDOUT_CONSOLE = Console()
_STDERR_CONSOLE = Console(stderr=True)
_WARNED_SERVER_SECURITY_MODES: set[str] = set()
_WARNED_ARCHIVING_CONFIGS: set[tuple[int, bool]] = set()
def rprint(*args, file=None, **kwargs):
@ -58,12 +70,13 @@ class ShellConfig(BaseConfigSet):
return get_BUILD_TIME()
SHELL_CONFIG = ShellConfig()
class StorageConfig(BaseConfigSet):
toml_section_header: str = "STORAGE_CONFIG"
# ARCHIVE_DIR / USERS_DIR are resolved dynamically via get_config().
ARCHIVE_DIR: Path = Field(default=CONSTANTS.ARCHIVE_DIR)
USERS_DIR: Path = Field(default=CONSTANTS.USERS_DIR)
# TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
# must be a short path due to unix path length restrictions for socket files (<100 chars)
# must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
@ -90,18 +103,12 @@ class StorageConfig(BaseConfigSet):
DIR_OUTPUT_PERMISSIONS: str = Field(default="755") # computed from OUTPUT_PERMISSIONS
STORAGE_CONFIG = StorageConfig()
class GeneralConfig(BaseConfigSet):
toml_section_header: str = "GENERAL_CONFIG"
TAG_SEPARATOR_PATTERN: str = Field(default=r"[,]")
GENERAL_CONFIG = GeneralConfig()
class ServerConfig(BaseConfigSet):
toml_section_header: str = "SERVER_CONFIG"
@ -130,6 +137,7 @@ class ServerConfig(BaseConfigSet):
PUBLIC_INDEX: bool = Field(default=True)
PUBLIC_SNAPSHOTS: bool = Field(default=True)
PUBLIC_SNAPSHOTS_LIST: bool | None = Field(default=None)
PUBLIC_ADD_VIEW: bool = Field(default=False)
ADMIN_USERNAME: str | None = Field(default=None)
@ -186,15 +194,14 @@ class ServerConfig(BaseConfigSet):
)
SERVER_CONFIG = ServerConfig()
def _print_server_security_mode_warning() -> None:
if not SERVER_CONFIG.IS_LOWER_SECURITY_MODE:
def _print_server_security_mode_warning(config: ServerConfig) -> None:
if not config.IS_LOWER_SECURITY_MODE:
return
if config.SERVER_SECURITY_MODE in _WARNED_SERVER_SECURITY_MODES:
return
rprint(
f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={SERVER_CONFIG.SERVER_SECURITY_MODE}[/yellow]",
f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={config.SERVER_SECURITY_MODE}[/yellow]",
file=sys.stderr,
)
rprint(
@ -217,19 +224,33 @@ def _print_server_security_mode_warning() -> None:
"[yellow] 3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]",
file=sys.stderr,
)
_print_server_security_mode_warning()
_WARNED_SERVER_SECURITY_MODES.add(config.SERVER_SECURITY_MODE)
class ArchivingConfig(BaseConfigSet):
toml_section_header: str = "ARCHIVING_CONFIG"
PLUGINS: str = Field(
default="",
description="Comma-separated plugin selection for this run. Empty means use enabled plugin defaults.",
)
ENABLED_PLUGINS: str = Field(
default="",
description="Comma-separated plugin selection override used by the UI and API.",
)
ENABLED_EXTRACTORS: str = Field(
default="",
description="Legacy comma-separated plugin selection override.",
)
ONLY_NEW: bool = Field(default=True)
OVERWRITE: bool = Field(default=False)
TIMEOUT: int = Field(default=60)
MAX_URL_ATTEMPTS: int = Field(default=50)
MAX_DEPTH: int = Field(default=0)
MAX_URLS: int = Field(default=0)
MAX_SIZE: int = Field(default=0)
RESOLUTION: str = Field(default="1440,2000")
CHECK_SSL_VALIDITY: bool = Field(default=True)
@ -298,10 +319,6 @@ class ArchivingConfig(BaseConfigSet):
)
ARCHIVING_CONFIG = ArchivingConfig()
ARCHIVING_CONFIG.warn_if_invalid()
class SearchBackendConfig(BaseConfigSet):
toml_section_header: str = "SEARCH_BACKEND_CONFIG"
@ -312,4 +329,250 @@ class SearchBackendConfig(BaseConfigSet):
SEARCH_PROCESS_HTML: bool = Field(default=True)
SEARCH_BACKEND_CONFIG = SearchBackendConfig()
def _plugin_user_config_value(value: Any) -> str:
if isinstance(value, Path):
return str(value)
if isinstance(value, (dict, list, bool, int, float)) or value is None:
return json.dumps(value)
return str(value)
def _plugin_user_config(config: Mapping[str, object]) -> dict[str, str]:
return {key: _plugin_user_config_value(value) for key, value in config.items()}
def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
from archivebox.hooks import discover_plugin_configs
schemas: PluginSchemaDocuments = {}
if BASE_CONFIG_PATH.exists():
schemas["base"] = {
"properties": json.loads(BASE_CONFIG_PATH.read_text()).get("properties", {}),
}
schemas.update(discover_plugin_configs())
return schemas
def _plugin_config_properties(plugin_schemas: PluginSchemaDocuments) -> dict[str, dict[str, Any]]:
properties: dict[str, dict[str, Any]] = {}
for schema in plugin_schemas.values():
schema_properties = schema.get("properties") or {}
if isinstance(schema_properties, dict):
properties.update(schema_properties)
return properties
def _plugin_config_model(plugin_schemas: PluginSchemaDocuments) -> type[BaseModel]:
return build_config_model("ArchiveBoxPluginConfig", _plugin_config_properties(plugin_schemas))
def _archivebox_config_input_names() -> set[str]:
names = set(ArchiveBoxConfig.model_fields)
for field in ArchiveBoxConfig.model_fields.values():
if isinstance(field.alias, str):
names.add(field.alias)
return names
class ArchiveBoxBaseConfig(
ShellConfig,
StorageConfig,
GeneralConfig,
ServerConfig,
ArchivingConfig,
SearchBackendConfig,
LDAPConfig,
):
"""Merged, typed ArchiveBox config.
Core ArchiveBox fields are declared above. Plugin-owned fields are added to
the concrete ArchiveBoxConfig model from plugin JSONSchema below, so
ArchiveBox does not hardcode any individual plugin config names.
"""
model_config = SettingsConfigDict(
env_prefix="",
extra="ignore",
validate_default=True,
use_enum_values=True,
arbitrary_types_allowed=True,
populate_by_name=True,
)
DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR)
ABX_RUNTIME: str = Field(default="archivebox")
CRAWL_DIR: Path | None = Field(default=None)
CRAWL_OUTPUT_DIR: Path | None = Field(default=None)
SNAP_DIR: Path | None = Field(default=None)
computed_config_keys: ClassVar[tuple[str, ...]] = COMPUTED_CONFIG_KEYS
@model_validator(mode="after")
def resolve_runtime_paths(self):
self.DATA_DIR = self.DATA_DIR.expanduser().resolve()
archive_dir = self.ARCHIVE_DIR.expanduser()
if archive_dir == (CONSTANTS.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME) and self.DATA_DIR != CONSTANTS.DATA_DIR:
archive_dir = self.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME
if not archive_dir.is_absolute():
archive_dir = self.DATA_DIR / archive_dir
self.ARCHIVE_DIR = archive_dir.resolve()
users_dir = self.USERS_DIR.expanduser()
if users_dir == (CONSTANTS.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME):
users_dir = self.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME
if not users_dir.is_absolute():
users_dir = self.ARCHIVE_DIR / users_dir
self.USERS_DIR = users_dir.resolve()
return self
def _build_archivebox_config_model(plugin_schemas: PluginSchemaDocuments) -> type[ArchiveBoxBaseConfig]:
core_fields = set(ArchiveBoxBaseConfig.model_fields)
plugin_fields: dict[str, Any] = {
key: (field.annotation, field) for key, field in _plugin_config_model(plugin_schemas).model_fields.items() if key not in core_fields
}
return cast(
type[ArchiveBoxBaseConfig],
create_model(
"ArchiveBoxConfig",
__base__=ArchiveBoxBaseConfig,
__module__=__name__,
**plugin_fields,
),
)
PLUGIN_CONFIG_SCHEMAS = _discover_plugin_config_schemas()
ArchiveBoxConfig = _build_archivebox_config_model(PLUGIN_CONFIG_SCHEMAS)
def get_config(
defaults: ConfigOverrides | None = None,
overrides: ConfigOverrides | None = None,
persona: Any = None,
user: Any = None,
crawl: Any = None,
snapshot: Any = None,
archiveresult: Any = None,
machine: Any = None,
) -> ArchiveBoxBaseConfig:
"""
Get merged config from all sources.
Priority (highest to lowest):
1. Explicit overrides
2. Per-snapshot config and output path
3. Per-crawl config and output path
4. Per-user config
5. Per-persona derived config
6. Current machine derived config
7. Environment variables
8. Config file (ArchiveBox.conf)
9. Plugin schema defaults
10. Core config defaults
"""
if snapshot is None and archiveresult is not None:
snapshot = archiveresult.snapshot
if crawl is None and snapshot is not None:
crawl = snapshot.crawl
if machine is None:
try:
from django.apps import apps
if apps.ready:
from archivebox.machine.models import Machine
machine = Machine.current()
except Exception:
machine = None
if persona is None and crawl is not None:
from archivebox.personas.models import Persona
persona_id = crawl.persona_id
if persona_id:
persona = Persona.objects.filter(id=persona_id).first()
if persona is None:
raise Persona.DoesNotExist(f"Crawl {crawl.id} references missing Persona {persona_id}")
if persona is None:
crawl_config = crawl.config or {}
default_persona_name = str(crawl_config.get("DEFAULT_PERSONA") or "").strip()
if default_persona_name:
persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default")
persona.ensure_dirs()
config_data: ConfigPayload = dict(defaults or {})
config_data.update(ArchiveBoxConfig().model_dump(mode="json"))
plugin_schemas = {
plugin_name: schema.get("properties", {}) for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)
}
scope_overrides: ConfigPayload = {}
if machine is not None and machine.config:
from archivebox.machine.models import _sanitize_machine_config
scope_overrides.update(_sanitize_machine_config(machine.config))
if persona is not None:
scope_overrides.update(persona.get_derived_config())
if user is not None and user.config:
scope_overrides.update(user.config)
if crawl is not None and crawl.config:
scope_overrides.update(crawl.config)
if crawl is not None:
scope_overrides["CRAWL_OUTPUT_DIR"] = crawl.output_dir
scope_overrides["CRAWL_DIR"] = crawl.output_dir
if snapshot is not None and snapshot.config:
scope_overrides.update(snapshot.config)
if snapshot is not None:
scope_overrides["SNAP_DIR"] = snapshot.output_dir
if overrides:
scope_overrides.update(overrides)
archivebox_scope_overrides = {key: value for key, value in scope_overrides.items() if key in _archivebox_config_input_names()}
config_data.update(archivebox_scope_overrides)
plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
plugin_sections = resolve_plugin_configs(
plugin_schemas,
global_config=plugin_global_config,
user_config={**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **_plugin_user_config(scope_overrides)},
)
for plugin_config in plugin_sections.values():
config_data.update(plugin_config)
config_data.update(archivebox_scope_overrides)
config_data["ABX_RUNTIME"] = "archivebox"
config = ArchiveBoxConfig.model_validate(config_data)
archiving_warning_key = (config.TIMEOUT, config.USE_COLOR)
if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:
config.warn_if_invalid()
_WARNED_ARCHIVING_CONFIGS.add(archiving_warning_key)
_print_server_security_mode_warning(config)
return config
def get_all_configs() -> dict[str, BaseConfigSet]:
"""Get all config section objects as a dictionary."""
return {
"SHELL_CONFIG": ShellConfig(),
"STORAGE_CONFIG": StorageConfig(),
"GENERAL_CONFIG": GeneralConfig(),
"SERVER_CONFIG": ServerConfig(),
"ARCHIVING_CONFIG": ArchivingConfig(),
"SEARCH_BACKEND_CONFIG": SearchBackendConfig(),
"LDAP_CONFIG": LDAPConfig(),
}

368
archivebox/config/configset.py
View File

@ -1,21 +1,31 @@
"""
Simplified config system for ArchiveBox.
This replaces the complex abx_spec_config/base_configset.py with a simpler
approach that still supports environment variables, config files, and
per-object overrides.
"""
"""Pydantic-backed config loading for ArchiveBox."""
__package__ = "archivebox.config"
import os
import json
from pathlib import Path
from typing import Any
from typing import Any, ClassVar
from configparser import ConfigParser
from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
COMPUTED_CONFIG_KEYS = (
"TERM_WIDTH",
"COMMIT_HASH",
"BUILD_TIME",
"USES_SUBDOMAIN_ROUTING",
"ENABLES_FULL_JS_REPLAY",
"CONTROL_PLANE_ENABLED",
"BLOCK_UNSAFE_METHODS",
"SHOULD_NEUTER_RISKY_REPLAY",
"IS_UNSAFE_MODE",
"IS_DANGEROUS_MODE",
"IS_LOWER_SECURITY_MODE",
"URL_ALLOWLIST_PTN",
"URL_DENYLIST_PTN",
"SAVE_ALLOWLIST_PTNS",
"SAVE_DENYLIST_PTNS",
)
class CaseConfigParser(ConfigParser):
def optionxform(self, optionstr: str) -> str:
@ -74,7 +84,9 @@ class BaseConfigSet(BaseSettings):
env_prefix="",
extra="ignore",
validate_default=True,
populate_by_name=True,
)
computed_config_keys: ClassVar[tuple[str, ...]] = ()
@classmethod
def settings_customise_sources(
@ -108,301 +120,57 @@ class BaseConfigSet(BaseSettings):
# Flatten all sections into single namespace
return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
def update_in_place(self, warn: bool = True, persist: bool = False, **kwargs) -> None:
"""
Update config values in place.
def __getitem__(self, key: str) -> Any:
if key in type(self).model_fields:
return getattr(self, key)
if self.__pydantic_extra__ and key in self.__pydantic_extra__:
return self.__pydantic_extra__[key]
if key in self.computed_config_keys:
return getattr(self, key)
raise KeyError(key)
This allows runtime updates to config without reloading.
"""
for key, value in kwargs.items():
if hasattr(self, key):
# Use object.__setattr__ to bypass pydantic's frozen model
object.__setattr__(self, key, value)
def __setitem__(self, key: str, value: Any) -> None:
if key in type(self).model_fields:
object.__setattr__(self, key, value)
return
if key in self.computed_config_keys:
raise KeyError(f"{key} is computed and cannot be set")
if self.model_config.get("extra") != "allow":
raise KeyError(f"Unknown config key: {key}")
extra = self.__pydantic_extra__
if extra is None:
extra = {}
object.__setattr__(self, "__pydantic_extra__", extra)
extra[key] = value
def update(self, *args, **kwargs) -> None:
values = dict(*args, **kwargs)
for key, value in values.items():
if key in self.computed_config_keys:
continue
self[key] = value
def get_config(
defaults: dict | None = None,
persona: Any = None,
user: Any = None,
crawl: Any = None,
snapshot: Any = None,
archiveresult: Any = None,
machine: Any = None,
) -> dict[str, Any]:
"""
Get merged config from all sources.
def __contains__(self, key: str) -> bool:
return (
key in type(self).model_fields
or bool(self.__pydantic_extra__ and key in self.__pydantic_extra__)
or key in self.computed_config_keys
)
Priority (highest to lowest):
1. Per-snapshot config (snapshot.config JSON field)
2. Per-crawl config (crawl.config JSON field)
3. Per-user config (user.config JSON field)
4. Per-persona config (persona.get_derived_config() - includes CHROME_USER_DATA_DIR etc.)
5. Environment variables
6. Config file (ArchiveBox.conf)
7. Plugin schema defaults (config.json)
8. Core config defaults
def get(self, key: str, default: Any = None) -> Any:
return self[key] if key in self else default
Args:
defaults: Default values to start with
persona: Persona object (provides derived paths like CHROME_USER_DATA_DIR)
user: User object with config JSON field
crawl: Crawl object with config JSON field
snapshot: Snapshot object with config JSON field
archiveresult: ArchiveResult object (auto-fetches snapshot)
machine: Unused legacy argument kept for call compatibility
def as_dict(self) -> dict[str, Any]:
data = self.model_dump()
for key in self.computed_config_keys:
data[key] = getattr(self, key)
return data
Note: Objects are auto-fetched from relationships if not provided:
- snapshot auto-fetched from archiveresult.snapshot
- crawl auto-fetched from snapshot.crawl
- user auto-fetched from crawl.created_by
def items(self):
return self.as_dict().items()
Returns:
Merged config dict
"""
# Auto-fetch related objects from relationships
if snapshot is None and archiveresult and hasattr(archiveresult, "snapshot"):
snapshot = archiveresult.snapshot
def keys(self):
return self.as_dict().keys()
if crawl is None and snapshot and hasattr(snapshot, "crawl"):
crawl = snapshot.crawl
if user is None and crawl and hasattr(crawl, "created_by"):
user = crawl.created_by
if persona is None and crawl is not None:
from archivebox.personas.models import Persona
persona_id = getattr(crawl, "persona_id", None)
if persona_id:
persona = Persona.objects.filter(id=persona_id).first()
if persona is None:
raise Persona.DoesNotExist(f"Crawl {getattr(crawl, 'id', None)} references missing Persona {persona_id}")
if persona is None:
crawl_config = getattr(crawl, "config", None) or {}
default_persona_name = str(crawl_config.get("DEFAULT_PERSONA") or "").strip()
if default_persona_name:
persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default")
persona.ensure_dirs()
from archivebox.config.constants import CONSTANTS
from archivebox.config.common import (
SHELL_CONFIG,
STORAGE_CONFIG,
GENERAL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
)
# Start with defaults
config = dict(defaults or {})
# Add plugin config defaults from JSONSchema config.json files
try:
from archivebox.hooks import get_config_defaults_from_plugins
plugin_defaults = get_config_defaults_from_plugins()
config.update(plugin_defaults)
except ImportError:
pass # hooks not available yet during early startup
# Add all core config sections
config.update(dict(SHELL_CONFIG))
config.update(dict(STORAGE_CONFIG))
config.update(dict(GENERAL_CONFIG))
config.update(dict(SERVER_CONFIG))
config.update(dict(ARCHIVING_CONFIG))
config.update(dict(SEARCH_BACKEND_CONFIG))
# Load from archivebox.config.file
config_file = CONSTANTS.CONFIG_FILE
if config_file.exists():
file_config = BaseConfigSet.load_from_file(config_file)
config.update(file_config)
# Override with environment variables (for keys that exist in config)
for key in config:
env_val = os.environ.get(key)
if env_val is not None:
config[key] = _parse_env_value(env_val, config.get(key))
# Also add NEW environment variables (not yet in config)
# This is important for worker subprocesses that receive config via Process.env
for key, value in os.environ.items():
if key.isupper() and key not in config: # Only uppercase keys (config convention)
config[key] = _parse_env_value(value, None)
# Also check plugin config aliases in environment
try:
from archivebox.hooks import discover_plugin_configs
plugin_configs = discover_plugin_configs()
for plugin_name, schema in plugin_configs.items():
for key, prop_schema in schema.get("properties", {}).items():
# Check x-aliases
for alias in prop_schema.get("x-aliases", []):
if alias in os.environ and key not in os.environ:
config[key] = _parse_env_value(os.environ[alias], config.get(key))
break
# Check x-fallback
fallback = prop_schema.get("x-fallback")
if fallback and fallback in config and key not in config:
config[key] = config[fallback]
except ImportError:
pass
# Apply persona config overrides (includes derived paths like CHROME_USER_DATA_DIR)
if persona and hasattr(persona, "get_derived_config"):
config.update(persona.get_derived_config())
# Apply user config overrides
if user and hasattr(user, "config") and user.config:
config.update(user.config)
# Apply crawl config overrides
if crawl and hasattr(crawl, "config") and crawl.config:
config.update(crawl.config)
# Add crawl path aliases for hooks that need shared crawl state.
if crawl and hasattr(crawl, "output_dir"):
config["CRAWL_OUTPUT_DIR"] = str(crawl.output_dir)
config["CRAWL_DIR"] = str(crawl.output_dir)
# Apply snapshot config overrides (highest priority)
if snapshot and hasattr(snapshot, "config") and snapshot.config:
config.update(snapshot.config)
if snapshot and hasattr(snapshot, "output_dir"):
config["SNAP_DIR"] = str(snapshot.output_dir)
# Normalize all aliases to canonical names (after all sources merged)
# This handles aliases that came from user/crawl/snapshot configs, not just env
try:
from archivebox.hooks import discover_plugin_configs
plugin_configs = discover_plugin_configs()
aliases_to_normalize = {} # {alias_key: canonical_key}
# Build alias mapping from all plugin schemas
for plugin_name, schema in plugin_configs.items():
for canonical_key, prop_schema in schema.get("properties", {}).items():
for alias in prop_schema.get("x-aliases", []):
aliases_to_normalize[alias] = canonical_key
# Normalize: copy alias values to canonical keys (aliases take precedence)
for alias_key, canonical_key in aliases_to_normalize.items():
if alias_key in config:
# Alias exists - copy to canonical key (overwriting any default)
config[canonical_key] = config[alias_key]
# Remove alias from config to keep it clean
del config[alias_key]
except ImportError:
pass
if not config.get("DATA_DIR"):
config["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
config["ABX_RUNTIME"] = "archivebox"
return config
def get_flat_config() -> dict[str, Any]:
"""
Get a flat dictionary of all config values.
Replaces abx.pm.hook.get_FLAT_CONFIG()
"""
return get_config()
def get_all_configs() -> dict[str, BaseConfigSet]:
"""
Get all config section objects as a dictionary.
Replaces abx.pm.hook.get_CONFIGS()
"""
from archivebox.config.common import (
SHELL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
)
return {
"SHELL_CONFIG": SHELL_CONFIG,
"SERVER_CONFIG": SERVER_CONFIG,
"ARCHIVING_CONFIG": ARCHIVING_CONFIG,
"SEARCH_BACKEND_CONFIG": SEARCH_BACKEND_CONFIG,
}
def _parse_env_value(value: str, default: Any = None) -> Any:
"""Parse an environment variable value based on expected type."""
if default is None:
# Try to guess the type
if value.lower() in ("true", "false", "yes", "no", "1", "0"):
return value.lower() in ("true", "yes", "1")
try:
return int(value)
except ValueError:
pass
try:
return json.loads(value)
except (json.JSONDecodeError, ValueError):
pass
return value
# Parse based on default's type
if isinstance(default, bool):
return value.lower() in ("true", "yes", "1")
elif isinstance(default, int):
return int(value)
elif isinstance(default, float):
return float(value)
elif isinstance(default, (list, dict)):
return json.loads(value)
elif isinstance(default, Path):
return Path(value)
else:
return value
# Default worker concurrency settings
DEFAULT_WORKER_CONCURRENCY = {
"crawl": 2,
"snapshot": 3,
"wget": 2,
"ytdlp": 2,
"screenshot": 3,
"singlefile": 2,
"title": 5,
"favicon": 5,
"headers": 5,
"archivedotorg": 2,
"readability": 3,
"mercury": 3,
"git": 2,
"pdf": 2,
"dom": 3,
}
def get_worker_concurrency() -> dict[str, int]:
"""
Get worker concurrency settings.
Can be configured via WORKER_CONCURRENCY env var as JSON dict.
"""
config = get_config()
# Start with defaults
concurrency = DEFAULT_WORKER_CONCURRENCY.copy()
# Override with config
if "WORKER_CONCURRENCY" in config:
custom = config["WORKER_CONCURRENCY"]
if isinstance(custom, str):
custom = json.loads(custom)
concurrency.update(custom)
return concurrency
def values(self):
return self.as_dict().values()

18
archivebox/config/constants.py
View File

@ -25,6 +25,7 @@ from .paths import (
PACKAGE_DIR,
DATA_DIR,
ARCHIVE_DIR,
USERS_DIR,
get_collection_id,
get_machine_id,
get_machine_type,
@ -48,6 +49,7 @@ class ConstantsDict:
PACKAGE_DIR: Path = PACKAGE_DIR
DATA_DIR: Path = DATA_DIR
ARCHIVE_DIR: Path = ARCHIVE_DIR
USERS_DIR: Path = USERS_DIR
MACHINE_TYPE: str = get_machine_type()
MACHINE_ID: str = get_machine_id()
@ -76,13 +78,17 @@ class ConstantsDict:
# Data dirs
ARCHIVE_DIR_NAME: str = "archive"
USERS_DIR_NAME: str = "users"
SNAPSHOTS_DIR_NAME: str = "snapshots"
CRAWLS_DIR_NAME: str = "crawls"
SOURCES_DIR_NAME: str = "sources"
PERSONAS_DIR_NAME: str = "personas"
CACHE_DIR_NAME: str = "cache"
LOGS_DIR_NAME: str = "logs"
CUSTOM_PLUGINS_DIR_NAME: str = "custom_plugins"
CUSTOM_TEMPLATES_DIR_NAME: str = "custom_templates"
ARCHIVE_DIR: Path = DATA_DIR / ARCHIVE_DIR_NAME
ARCHIVE_DIR: Path = ARCHIVE_DIR
USERS_DIR: Path = USERS_DIR
SOURCES_DIR: Path = DATA_DIR / SOURCES_DIR_NAME
PERSONAS_DIR: Path = DATA_DIR / PERSONAS_DIR_NAME
LOGS_DIR: Path = DATA_DIR / LOGS_DIR_NAME
@ -110,6 +116,16 @@ class ConstantsDict:
DEFAULT_LIB_DIR: Path = DATA_DIR / LIB_DIR_NAME / MACHINE_TYPE # ./data/lib/arm64-linux-docker
DEFAULT_LIB_BIN_DIR: Path = DEFAULT_LIB_DIR / "bin" # ./data/lib/arm64-linux-docker/bin
RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
(
USERS_DIR_NAME,
SNAPSHOTS_DIR_NAME,
CRAWLS_DIR_NAME,
"invalid",
".DS_Store",
),
)
# Config constants
TIMEZONE: str = "UTC"
DEFAULT_CLI_COLORS: dict[str, str] = DEFAULT_CLI_COLORS

16
archivebox/config/django.py
View File

@ -12,16 +12,16 @@ import django.db
from archivebox.misc import logging
from . import CONSTANTS
from .common import SHELL_CONFIG
from .constants import CONSTANTS
from .common import get_config
CONFIG = get_config()
if not SHELL_CONFIG.USE_COLOR:
if not CONFIG.USE_COLOR:
os.environ["NO_COLOR"] = "1"
if not SHELL_CONFIG.SHOW_PROGRESS:
if not CONFIG.SHOW_PROGRESS:
os.environ["TERM"] = "dumb"
# recreate rich console obj based on new config values
STDOUT = CONSOLE = Console()
STDERR = Console(stderr=True)
logging.CONSOLE = CONSOLE
@ -107,15 +107,15 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
traceback.print_exc()
return
from django.conf import settings
from archivebox.core.settings_logging import ERROR_LOG as DEFAULT_ERROR_LOG
# log startup message to the error log
error_log = getattr(settings, "ERROR_LOG", DEFAULT_ERROR_LOG)
error_log = DEFAULT_ERROR_LOG
with open(error_log, "a", encoding="utf-8") as f:
command = " ".join(sys.argv)
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d__%H:%M:%S")
f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={SHELL_CONFIG.IN_DOCKER} IS_TTY={SHELL_CONFIG.IS_TTY}\n")
config = get_config()
f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={config.IN_DOCKER} IS_TTY={config.IS_TTY}\n")
if check_db:
# make sure the data dir is owned by a non-root user

4
archivebox/config/ldap.py
View File

@ -50,7 +50,3 @@ class LDAPConfig(BaseConfigSet):
return False, f"LDAP_* config options must all be set if LDAP_ENABLED=True\nMissing: {', '.join(missing)}"
return True, ""
# Singleton instance
LDAP_CONFIG = LDAPConfig()

95
archivebox/config/paths.py
View File

@ -8,16 +8,30 @@ import platform
from pathlib import Path
from functools import cache
from datetime import datetime
from typing import TYPE_CHECKING
from benedict import benedict
from .permissions import SudoPermission, IS_ROOT, ARCHIVEBOX_USER
if TYPE_CHECKING:
from archivebox.config.common import ArchiveBoxConfig
#############################################################################################
PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent # archivebox source code dir
DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve() # archivebox user data dir
ARCHIVE_DIR: Path = DATA_DIR / "archive" # archivebox snapshot data dir
def _env_path(key: str, default: Path) -> Path:
path = Path(os.environ.get(key) or default).expanduser()
if not path.is_absolute():
path = DATA_DIR / path
return path.resolve()
ARCHIVE_DIR: Path = _env_path("ARCHIVE_DIR", DATA_DIR / "archive") # archivebox snapshot data dir
USERS_DIR: Path = _env_path("USERS_DIR", ARCHIVE_DIR / "users") # archivebox user-scoped crawl/snapshot data dir
IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
@ -154,15 +168,15 @@ def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
return len(f"file://{socket_file}") <= 96
@cache
def get_or_create_working_tmp_dir(autofix=True, quiet=True):
from archivebox import CONSTANTS
from archivebox.config.common import STORAGE_CONFIG
def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
from archivebox.config.constants import CONSTANTS
from archivebox.config.common import get_config
from archivebox.misc.checks import check_tmp_dir
config = config or get_config(**config_kwargs)
# try a few potential directories in order of preference
CANDIDATES = [
STORAGE_CONFIG.TMP_DIR, # <user-specified>
config.TMP_DIR, # <user-specified>
CONSTANTS.DEFAULT_TMP_DIR, # ./data/tmp/<machine_id>
Path("/var/run/archivebox") / get_collection_id(), # /var/run/archivebox/abc5d8512
Path("/tmp") / "archivebox" / get_collection_id(), # /tmp/archivebox/abc5d8512
@ -182,8 +196,8 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True):
except Exception:
pass
if check_tmp_dir(candidate, throw=False, quiet=True, must_exist=True):
if autofix and STORAGE_CONFIG.TMP_DIR != candidate:
STORAGE_CONFIG.update_in_place(TMP_DIR=candidate)
if autofix and config.TMP_DIR != candidate:
os.environ["TMP_DIR"] = str(candidate)
return candidate
try:
if (
@ -200,23 +214,23 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True):
# Fall back to the shortest writable path so read-only CLI commands can still run,
# and let later permission checks surface the missing socket support if needed.
if fallback_candidate:
if autofix and STORAGE_CONFIG.TMP_DIR != fallback_candidate:
STORAGE_CONFIG.update_in_place(TMP_DIR=fallback_candidate)
if autofix and config.TMP_DIR != fallback_candidate:
os.environ["TMP_DIR"] = str(fallback_candidate)
return fallback_candidate
if not quiet:
raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!")
@cache
def get_or_create_working_lib_dir(autofix=True, quiet=False):
from archivebox import CONSTANTS
from archivebox.config.common import STORAGE_CONFIG
def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
from archivebox.config.constants import CONSTANTS
from archivebox.config.common import get_config
from archivebox.misc.checks import check_lib_dir
config = config or get_config(**config_kwargs)
# try a few potential directories in order of preference
CANDIDATES = [
STORAGE_CONFIG.LIB_DIR, # <user-specified>
config.LIB_DIR, # <user-specified>
CONSTANTS.DEFAULT_LIB_DIR, # ./data/lib/arm64-linux-docker
Path("/usr/local/share/archivebox") / get_collection_id(), # /usr/local/share/archivebox/abc5
*(
@ -231,23 +245,23 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False):
except Exception:
pass
if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
if autofix and STORAGE_CONFIG.LIB_DIR != candidate:
STORAGE_CONFIG.update_in_place(LIB_DIR=candidate)
if autofix and config.LIB_DIR != candidate:
os.environ["LIB_DIR"] = str(candidate)
return candidate
if not quiet:
raise OSError(f"ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!")
@cache
def get_data_locations():
from archivebox.config import CONSTANTS
from archivebox.config.common import STORAGE_CONFIG
def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
from archivebox.config.constants import CONSTANTS
from archivebox.config.common import get_config
config = config or get_config(**config_kwargs)
try:
tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True) or STORAGE_CONFIG.TMP_DIR
tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True, config=config) or config.TMP_DIR
except Exception:
tmp_dir = STORAGE_CONFIG.TMP_DIR
tmp_dir = config.TMP_DIR
return benedict(
{
@ -271,10 +285,20 @@ def get_data_locations():
"is_mount": os.path.ismount(DATABASE_FILE.resolve()),
},
"ARCHIVE_DIR": {
"path": ARCHIVE_DIR.resolve(),
"path": config.ARCHIVE_DIR.resolve(),
"enabled": True,
"is_valid": os.path.isdir(ARCHIVE_DIR) and os.access(ARCHIVE_DIR, os.R_OK) and os.access(ARCHIVE_DIR, os.W_OK),
"is_mount": os.path.ismount(ARCHIVE_DIR.resolve()),
"is_valid": os.path.isdir(config.ARCHIVE_DIR)
and os.access(config.ARCHIVE_DIR, os.R_OK)
and os.access(config.ARCHIVE_DIR, os.W_OK),
"is_mount": os.path.ismount(config.ARCHIVE_DIR.resolve()),
},
"USERS_DIR": {
"path": config.USERS_DIR.resolve(),
"enabled": os.path.isdir(config.USERS_DIR),
"is_valid": os.path.isdir(config.USERS_DIR)
and os.access(config.USERS_DIR, os.R_OK)
and os.access(config.USERS_DIR, os.W_OK),
"is_mount": os.path.ismount(config.USERS_DIR.resolve()),
},
"SOURCES_DIR": {
"path": CONSTANTS.SOURCES_DIR.resolve(),
@ -311,15 +335,15 @@ def get_data_locations():
)
@cache
def get_code_locations():
from archivebox.config import CONSTANTS
from archivebox.config.common import STORAGE_CONFIG
def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
from archivebox.config.constants import CONSTANTS
from archivebox.config.common import get_config
config = config or get_config(**config_kwargs)
try:
lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True) or STORAGE_CONFIG.LIB_DIR
lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
except Exception:
lib_dir = STORAGE_CONFIG.LIB_DIR
lib_dir = config.LIB_DIR
lib_bin_dir = lib_dir / "bin"
@ -336,10 +360,9 @@ def get_code_locations():
"is_valid": os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK), # read + list
},
"CUSTOM_TEMPLATES_DIR": {
"path": STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR.resolve(),
"enabled": os.path.isdir(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR),
"is_valid": os.path.isdir(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR)
and os.access(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK), # read
"path": config.CUSTOM_TEMPLATES_DIR.resolve(),
"enabled": os.path.isdir(config.CUSTOM_TEMPLATES_DIR),
"is_valid": os.path.isdir(config.CUSTOM_TEMPLATES_DIR) and os.access(config.CUSTOM_TEMPLATES_DIR, os.R_OK), # read
},
"USER_PLUGINS_DIR": {
"path": CONSTANTS.USER_PLUGINS_DIR.resolve(),

2
archivebox/config/version.py
View File

@ -12,8 +12,6 @@ from datetime import datetime
IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent # archivebox source code dir
DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve() # archivebox user data dir
ARCHIVE_DIR: Path = DATA_DIR / "archive" # archivebox snapshot data dir
#############################################################################################

20
archivebox/core/__init__.py
View File

@ -7,23 +7,3 @@ def register_admin(admin_site):
from archivebox.core.admin import register_admin as do_register
do_register(admin_site)
def get_CONFIG():
from archivebox.config.common import (
SHELL_CONFIG,
STORAGE_CONFIG,
GENERAL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
)
return {
"SHELL_CONFIG": SHELL_CONFIG,
"STORAGE_CONFIG": STORAGE_CONFIG,
"GENERAL_CONFIG": GENERAL_CONFIG,
"SERVER_CONFIG": SERVER_CONFIG,
"ARCHIVING_CONFIG": ARCHIVING_CONFIG,
"SEARCHBACKEND_CONFIG": SEARCH_BACKEND_CONFIG,
}

4
archivebox/core/admin_archiveresults.py
View File

@ -20,7 +20,7 @@ from django.utils import timezone
from django.utils.text import smart_split
from archivebox.config import DATA_DIR
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.common import get_config
from archivebox.misc.paginators import AcceleratedPaginator
from archivebox.base_models.admin import BaseModelAdmin
from archivebox.hooks import get_plugin_icon
@ -469,7 +469,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
list_filter = ("status", "plugin", "start_ts")
ordering = ["-start_ts"]
list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE
list_per_page = get_config().SNAPSHOTS_PER_PAGE
paginator = AcceleratedPaginator
save_on_top = True

8
archivebox/core/admin_snapshots.py
View File

@ -15,7 +15,7 @@ from django.template import Template, RequestContext
from django.contrib.admin.helpers import ActionForm
from archivebox.config import DATA_DIR
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.common import get_config
from archivebox.misc.util import htmldecode, urldecode
from archivebox.misc.paginators import AcceleratedPaginator
from archivebox.misc.logging_util import printable_filesize
@ -248,7 +248,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
ordering = ["-created_at"]
actions = ["add_tags", "remove_tags", "resnapshot_snapshot", "update_snapshots", "overwrite_snapshots", "delete_snapshots"]
inlines = [] # Removed TagInline, using TagEditorWidget instead
list_per_page = min(max(5, SERVER_CONFIG.SNAPSHOTS_PER_PAGE), 5000)
list_per_page = min(max(5, get_config().SNAPSHOTS_PER_PAGE), 5000)
action_form = SnapshotActionForm
paginator = AcceleratedPaginator
@ -897,7 +897,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
return None
def _get_expected_hook_total(self, obj) -> int:
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
try:
config = get_config(crawl=obj.crawl, snapshot=obj)
@ -976,7 +976,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
# Monkey patch here plus core_tags.py
admin_cls.change_list_template = "private_index_grid.html"
admin_cls.list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE
admin_cls.list_per_page = get_config().SNAPSHOTS_PER_PAGE
admin_cls.list_max_show_all = admin_cls.list_per_page
# Call monkey patched view

4
archivebox/core/forms.py
View File

@ -7,7 +7,7 @@ from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_byt
from taggit.utils import edit_string_for_tags, parse_tags
from archivebox.base_models.admin import KeyValueWidget
from archivebox.crawls.schedule_utils import validate_schedule
from archivebox.config.common import SEARCH_BACKEND_CONFIG
from archivebox.config.common import get_config
from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
from archivebox.hooks import get_plugins, discover_plugin_configs, get_plugin_icon
from archivebox.personas.models import Persona
@ -259,7 +259,7 @@ class AddLinkForm(forms.Form):
(p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in extensions
]
required_search_plugin = f"search_backend_{SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}".strip()
required_search_plugin = f"search_backend_{get_config().SEARCH_BACKEND_ENGINE}".strip()
search_choices = [choice[0] for choice in get_choice_field(self, "search_plugins").choices]
if required_search_plugin in search_choices:
get_choice_field(self, "search_plugins").initial = [required_search_plugin]

165
archivebox/core/host_utils.py
View File

@ -1,9 +1,10 @@
from __future__ import annotations
import re
from typing import Any
from urllib.parse import urlparse
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.common import get_config
_SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
@ -35,16 +36,19 @@ def normalize_base_url(value: str | None) -> str:
return _normalize_base_url(value)
def get_listen_host() -> str:
return (SERVER_CONFIG.LISTEN_HOST or "").strip()
def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
return (config.LISTEN_HOST or "").strip()
def get_listen_parts() -> tuple[str, str | None]:
return split_host_port(get_listen_host())
def get_listen_parts(config: dict[str, Any] | None = None, **config_kwargs: Any) -> tuple[str, str | None]:
config = config or get_config(**config_kwargs)
return split_host_port(get_listen_host(config=config))
def _build_listen_host(subdomain: str | None) -> str:
host, port = get_listen_parts()
def _build_listen_host(subdomain: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
host, port = get_listen_parts(config=config)
if not host:
return ""
full_host = f"{subdomain}.{host}" if subdomain else host
@ -53,34 +57,38 @@ def _build_listen_host(subdomain: str | None) -> str:
return full_host
def get_admin_host() -> str:
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
return get_listen_host().lower()
override = _normalize_base_url(SERVER_CONFIG.ADMIN_BASE_URL)
def get_admin_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return get_listen_host(config=config).lower()
override = _normalize_base_url(config.ADMIN_BASE_URL)
if override:
return urlparse(override).netloc.lower()
return _build_listen_host("admin")
return _build_listen_host("admin", config=config)
def get_web_host() -> str:
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
return get_listen_host().lower()
override = _normalize_base_url(SERVER_CONFIG.ARCHIVE_BASE_URL)
def get_web_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return get_listen_host(config=config).lower()
override = _normalize_base_url(config.ARCHIVE_BASE_URL)
if override:
return urlparse(override).netloc.lower()
return _build_listen_host("web")
return _build_listen_host("web", config=config)
def get_api_host() -> str:
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
return get_listen_host().lower()
return _build_listen_host("api")
def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return get_listen_host(config=config).lower()
return _build_listen_host("api", config=config)
def get_public_host() -> str:
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
return get_listen_host().lower()
return _build_listen_host("public")
def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return get_listen_host(config=config).lower()
return _build_listen_host("public", config=config)
def get_snapshot_subdomain(snapshot_id: str) -> str:
@ -89,16 +97,18 @@ def get_snapshot_subdomain(snapshot_id: str) -> str:
return f"snap-{suffix}"
def get_snapshot_host(snapshot_id: str) -> str:
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
return get_listen_host().lower()
return _build_listen_host(get_snapshot_subdomain(snapshot_id))
def get_snapshot_host(snapshot_id: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return get_listen_host(config=config).lower()
return _build_listen_host(get_snapshot_subdomain(snapshot_id), config=config)
def get_original_host(domain: str) -> str:
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
return get_listen_host().lower()
return _build_listen_host(domain)
def get_original_host(domain: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return get_listen_host(config=config).lower()
return _build_listen_host(domain, config=config)
def is_snapshot_subdomain(subdomain: str) -> bool:
@ -114,11 +124,12 @@ def get_snapshot_lookup_key(snapshot_ref: str) -> str:
return value
def get_listen_subdomain(request_host: str) -> str:
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
def get_listen_subdomain(request_host: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return ""
req_host, req_port = split_host_port(request_host)
listen_host, listen_port = get_listen_parts()
listen_host, listen_port = get_listen_parts(config=config)
if not listen_host:
return ""
if listen_port and req_port and listen_port != req_port:
@ -156,73 +167,79 @@ def _build_base_url_for_host(host: str, request=None) -> str:
return f"{scheme}://{host}"
def get_admin_base_url(request=None) -> str:
override = _normalize_base_url(SERVER_CONFIG.ADMIN_BASE_URL)
def get_admin_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
override = _normalize_base_url(config.ADMIN_BASE_URL)
if override:
return override
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
return _build_base_url_for_host(get_listen_host(), request=request)
return _build_base_url_for_host(get_admin_host(), request=request)
if not config.USES_SUBDOMAIN_ROUTING:
return _build_base_url_for_host(get_listen_host(config=config), request=request)
return _build_base_url_for_host(get_admin_host(config=config), request=request)
def get_web_base_url(request=None) -> str:
override = _normalize_base_url(SERVER_CONFIG.ARCHIVE_BASE_URL)
def get_web_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
override = _normalize_base_url(config.ARCHIVE_BASE_URL)
if override:
return override
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
return _build_base_url_for_host(get_listen_host(), request=request)
return _build_base_url_for_host(get_web_host(), request=request)
if not config.USES_SUBDOMAIN_ROUTING:
return _build_base_url_for_host(get_listen_host(config=config), request=request)
return _build_base_url_for_host(get_web_host(config=config), request=request)
def get_api_base_url(request=None) -> str:
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
return _build_base_url_for_host(get_listen_host(), request=request)
return _build_base_url_for_host(get_api_host(), request=request)
def get_api_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return _build_base_url_for_host(get_listen_host(config=config), request=request)
return _build_base_url_for_host(get_api_host(config=config), request=request)
def get_public_base_url(request=None) -> str:
return _build_base_url_for_host(get_public_host(), request=request)
def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
return _build_base_url_for_host(get_public_host(config=config), request=request)
# Backwards-compat aliases (archive == web)
def get_archive_base_url(request=None) -> str:
return get_web_base_url(request=request)
def get_archive_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
return get_web_base_url(request=request, config=config, **config_kwargs)
def get_snapshot_base_url(snapshot_id: str, request=None) -> str:
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
return _build_url(get_web_base_url(request=request), f"/snapshot/{snapshot_id}")
return _build_base_url_for_host(get_snapshot_host(snapshot_id), request=request)
def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return _build_url(get_web_base_url(request=request, config=config), f"/snapshot/{snapshot_id}")
return _build_base_url_for_host(get_snapshot_host(snapshot_id, config=config), request=request)
def get_original_base_url(domain: str, request=None) -> str:
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
return _build_url(get_web_base_url(request=request), f"/original/{domain}")
return _build_base_url_for_host(get_original_host(domain), request=request)
def get_original_base_url(domain: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
if not config.USES_SUBDOMAIN_ROUTING:
return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}")
return _build_base_url_for_host(get_original_host(domain, config=config), request=request)
def build_admin_url(path: str = "", request=None) -> str:
return _build_url(get_admin_base_url(request), path)
def build_admin_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
return _build_url(get_admin_base_url(request, config=config, **config_kwargs), path)
def build_web_url(path: str = "", request=None) -> str:
return _build_url(get_web_base_url(request), path)
def build_web_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
return _build_url(get_web_base_url(request, config=config, **config_kwargs), path)
def build_api_url(path: str = "", request=None) -> str:
return _build_url(get_api_base_url(request), path)
def build_api_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
return _build_url(get_api_base_url(request, config=config, **config_kwargs), path)
def build_archive_url(path: str = "", request=None) -> str:
return _build_url(get_archive_base_url(request), path)
def build_archive_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
return _build_url(get_archive_base_url(request, config=config, **config_kwargs), path)
def build_snapshot_url(snapshot_id: str, path: str = "", request=None) -> str:
return _build_url(get_snapshot_base_url(snapshot_id, request=request), path)
def build_snapshot_url(snapshot_id: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
return _build_url(get_snapshot_base_url(snapshot_id, request=request, config=config, **config_kwargs), path)
def build_original_url(domain: str, path: str = "", request=None) -> str:
return _build_url(get_original_base_url(domain, request=request), path)
def build_original_url(domain: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
return _build_url(get_original_base_url(domain, request=request, config=config, **config_kwargs), path)
def _build_url(base_url: str, path: str) -> str:

33
archivebox/core/middleware.py
View File

@ -12,7 +12,7 @@ from django.contrib.staticfiles import finders
from django.utils.http import http_date
from django.http import HttpResponseForbidden, HttpResponseNotModified
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.common import get_config
from archivebox.config import VERSION
from archivebox.config.version import get_COMMIT_HASH
from archivebox.core.host_utils import (
@ -85,7 +85,7 @@ def CacheControlMiddleware(get_response):
if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
if not response.get("Cache-Control"):
policy = "public" if SERVER_CONFIG.PUBLIC_SNAPSHOTS else "private"
policy = "public" if get_config().PUBLIC_SNAPSHOTS else "private"
response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
# print('Set Cache-Control header to', response['Cache-Control'])
return response
@ -98,7 +98,7 @@ def ServerSecurityModeMiddleware(get_response):
allowed_methods = {"GET", "HEAD", "OPTIONS"}
def middleware(request):
if SERVER_CONFIG.CONTROL_PLANE_ENABLED:
if get_config().CONTROL_PLANE_ENABLED:
return get_response(request)
request.user = AnonymousUser()
@ -123,19 +123,20 @@ def HostRoutingMiddleware(get_response):
def middleware(request):
request_host = (request.get_host() or "").lower()
admin_host = get_admin_host()
web_host = get_web_host()
api_host = get_api_host()
public_host = get_public_host()
listen_host = get_listen_host()
subdomain = get_listen_subdomain(request_host)
config = get_config()
admin_host = get_admin_host(config=config)
web_host = get_web_host(config=config)
api_host = get_api_host(config=config)
public_host = get_public_host(config=config)
listen_host = get_listen_host(config=config)
subdomain = get_listen_subdomain(request_host, config=config)
# Framework-owned assets must bypass snapshot/original-domain replay routing.
# Otherwise pages on snapshot subdomains can receive HTML for JS/CSS requests.
if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}:
return get_response(request)
if SERVER_CONFIG.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
if (
request.path == "/admin"
or request.path.startswith("/admin/")
@ -147,7 +148,7 @@ def HostRoutingMiddleware(get_response):
target = f"{target}?{request.META['QUERY_STRING']}"
return redirect(target)
if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING:
if not config.USES_SUBDOMAIN_ROUTING:
if host_matches(request_host, listen_host):
return get_response(request)
@ -164,7 +165,7 @@ def HostRoutingMiddleware(get_response):
if host_matches(request_host, admin_host):
snapshot_match = snapshot_path_re.match(request.path)
if SERVER_CONFIG.USES_SUBDOMAIN_ROUTING and snapshot_match:
if config.USES_SUBDOMAIN_ROUTING and snapshot_match:
snapshot_id = snapshot_match.group("snapshot_id")
replay_path = (snapshot_match.group("path") or "").strip("/")
if replay_path == "index.html":
@ -224,17 +225,19 @@ def HostRoutingMiddleware(get_response):
class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
header = "HTTP_{normalized}".format(normalized=SERVER_CONFIG.REVERSE_PROXY_USER_HEADER.replace("-", "_").upper())
header = "HTTP_REMOTE_USER"
def process_request(self, request):
if SERVER_CONFIG.REVERSE_PROXY_WHITELIST == "":
config = get_config()
self.header = "HTTP_{normalized}".format(normalized=config.REVERSE_PROXY_USER_HEADER.replace("-", "_").upper())
if config.REVERSE_PROXY_WHITELIST == "":
return
ip = request.META.get("REMOTE_ADDR")
if not isinstance(ip, str):
return
for cidr in SERVER_CONFIG.REVERSE_PROXY_WHITELIST.split(","):
for cidr in config.REVERSE_PROXY_WHITELIST.split(","):
try:
network = ipaddress.ip_network(cidr)
except ValueError:

7
archivebox/core/migrations/0007_archiveresult.py
View File

@ -12,12 +12,7 @@ try:
ARCHIVE_DIR = CONSTANTS.ARCHIVE_DIR
except ImportError:
try:
from archivebox.config import CONFIG
ARCHIVE_DIR = Path(CONFIG.get("ARCHIVE_DIR", "./archive"))
except ImportError:
ARCHIVE_DIR = Path("./archive")
ARCHIVE_DIR = Path("./archive")
try:
from archivebox.misc.util import to_json

150
archivebox/core/migrations/0023_upgrade_to_0_9_0.py
View File

@ -4,6 +4,9 @@
from django.db import migrations, models, connection
import django.utils.timezone
from uuid import UUID
from datetime import datetime
import json
def get_table_columns(table_name):
@ -13,6 +16,18 @@ def get_table_columns(table_name):
return {row[1] for row in cursor.fetchall()}
def normalize_cmd(cmd):
if not cmd:
return "[]"
try:
parsed = json.loads(cmd)
if isinstance(parsed, list):
return json.dumps([str(part) for part in parsed])
except (TypeError, json.JSONDecodeError):
pass
return json.dumps(str(cmd).split())
def upgrade_core_tables(apps, schema_editor):
"""Upgrade core tables from v0.7.2 or v0.8.6rc0 to v0.9.0."""
from archivebox.uuid_compat import uuid7
@ -64,29 +79,72 @@ def upgrade_core_tables(apps, schema_editor):
if has_uuid and not has_abid:
# Migrating from v0.7.2+ (has uuid column)
print("Migrating ArchiveResult from v0.7.2+ schema (with uuid)...")
cursor.execute("""
INSERT OR IGNORE INTO core_archiveresult_new (
id, uuid, snapshot_id, cmd, pwd, cmd_version,
start_ts, end_ts, status, extractor, output
cursor.execute(
"SELECT id, uuid, snapshot_id, cmd, pwd, cmd_version, start_ts, end_ts, status, extractor, output FROM core_archiveresult",
)
old_records = cursor.fetchall()
for record in old_records:
try:
new_uuid = UUID(str(record[1])).hex
except (TypeError, ValueError):
new_uuid = uuid7().hex
start_ts = record[6] or datetime.now().isoformat()
end_ts = record[7] or start_ts
cursor.execute(
"""
INSERT OR IGNORE INTO core_archiveresult_new (
id, uuid, snapshot_id, cmd, pwd, cmd_version,
start_ts, end_ts, status, extractor, output
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
record[0],
new_uuid,
record[2],
normalize_cmd(record[3]),
record[4] or "",
record[5] or "",
start_ts,
end_ts,
"succeeded" if record[8] == "success" else (record[8] or "queued"),
record[9] or "",
record[10] or "",
),
)
SELECT
id, uuid, snapshot_id, cmd, pwd, cmd_version,
start_ts, end_ts, status, extractor, output
FROM core_archiveresult;
""")
elif has_abid and not has_uuid:
# Migrating from v0.8.6rc0 (has abid instead of uuid)
print("Migrating ArchiveResult from v0.8.6rc0 schema...")
cursor.execute("""
INSERT OR IGNORE INTO core_archiveresult_new (
id, uuid, snapshot_id, cmd, pwd, cmd_version,
start_ts, end_ts, status, extractor, output
cursor.execute(
"SELECT id, snapshot_id, cmd, pwd, cmd_version, start_ts, end_ts, status, extractor, output FROM core_archiveresult",
)
old_records = cursor.fetchall()
for record in old_records:
try:
new_uuid = UUID(str(record[0])).hex
except (TypeError, ValueError):
new_uuid = uuid7().hex
start_ts = record[5] or datetime.now().isoformat()
end_ts = record[6] or start_ts
cursor.execute(
"""
INSERT OR IGNORE INTO core_archiveresult_new (
uuid, snapshot_id, cmd, pwd, cmd_version,
start_ts, end_ts, status, extractor, output
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
new_uuid,
record[1],
normalize_cmd(record[2]),
record[3] or "",
record[4] or "",
start_ts,
end_ts,
"succeeded" if record[7] == "success" else (record[7] or "queued"),
record[8] or "",
record[9] or "",
),
)
SELECT
id, abid as uuid, snapshot_id, cmd, pwd, cmd_version,
start_ts, end_ts, status, extractor, output
FROM core_archiveresult;
""")
else:
# Migrating from v0.7.2 (no uuid or abid column - generate fresh UUIDs)
print("Migrating ArchiveResult from v0.7.2 schema (no uuid - generating UUIDs)...")
@ -96,6 +154,8 @@ def upgrade_core_tables(apps, schema_editor):
old_records = cursor.fetchall()
for record in old_records:
new_uuid = uuid7().hex
start_ts = record[5] or datetime.now().isoformat()
end_ts = record[6] or start_ts
cursor.execute(
"""
INSERT OR IGNORE INTO core_archiveresult_new (
@ -107,14 +167,14 @@ def upgrade_core_tables(apps, schema_editor):
record[0],
new_uuid,
record[1],
record[2],
record[3],
record[4],
record[5],
record[6],
record[7],
record[8],
record[9],
normalize_cmd(record[2]),
record[3] or "",
record[4] or "",
start_ts,
end_ts,
"succeeded" if record[7] == "success" else (record[7] or "queued"),
record[8] or "",
record[9] or "",
),
)
@ -174,18 +234,27 @@ def upgrade_core_tables(apps, schema_editor):
if has_added and not has_bookmarked_at:
# Migrating from v0.7.2 (has added/updated fields)
print("Migrating Snapshot from v0.7.2 schema...")
# Transform added→bookmarked_at/created_at and updated→modified_at
# timestamp is the legacy bookmark/import timestamp and archive/{timestamp} identity.
# added is the DB row creation/import time, and updated was renamed to downloaded_at in 0.8.x.
cursor.execute("""
INSERT OR IGNORE INTO core_snapshot_new (
id, url, timestamp, title,
bookmarked_at, created_at, modified_at,
bookmarked_at, created_at, modified_at, downloaded_at,
status
)
SELECT
id, url, timestamp, title,
COALESCE(added, CURRENT_TIMESTAMP) as bookmarked_at,
COALESCE(
CASE
WHEN CAST(timestamp AS REAL) BETWEEN 788918400 AND 2082758400
THEN datetime(CAST(timestamp AS REAL), 'unixepoch')
END,
added,
CURRENT_TIMESTAMP
) as bookmarked_at,
COALESCE(added, CURRENT_TIMESTAMP) as created_at,
COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at,
updated as downloaded_at,
'queued' as status
FROM core_snapshot;
""")
@ -198,17 +267,21 @@ def upgrade_core_tables(apps, schema_editor):
has_crawl_id = "crawl_id" in snapshot_cols
# Build column list based on what exists
cols = ["id", "url", "timestamp", "title", "bookmarked_at", "created_at", "modified_at", "downloaded_at"]
insert_cols = ["id", "url", "timestamp", "title", "bookmarked_at", "created_at", "modified_at", "downloaded_at"]
select_cols = ["id", "url", "timestamp", "title", "bookmarked_at", "created_at", "modified_at", "downloaded_at"]
if has_crawl_id:
cols.append("crawl_id")
insert_cols.append("crawl_id")
select_cols.append("REPLACE(crawl_id, '-', '')")
if has_status:
cols.append("status")
insert_cols.append("status")
select_cols.append("status")
if has_retry_at:
cols.append("retry_at")
insert_cols.append("retry_at")
select_cols.append("retry_at")
cursor.execute(f"""
INSERT OR IGNORE INTO core_snapshot_new ({", ".join(cols)})
SELECT {", ".join(cols)}
INSERT OR IGNORE INTO core_snapshot_new ({", ".join(insert_cols)})
SELECT {", ".join(select_cols)}
FROM core_snapshot;
""")
else:
@ -324,7 +397,7 @@ def upgrade_core_tables(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
("core", "0022_auto_20231023_2008"),
("crawls", "0001_initial"),
("crawls", "0002_upgrade_from_0_8_6"),
("auth", "0012_alter_user_first_name_max_length"),
]
@ -359,6 +432,11 @@ class Migration(migrations.Migration):
name="modified_at",
field=models.DateTimeField(auto_now=True),
),
migrations.AddField(
model_name="snapshot",
name="downloaded_at",
field=models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True),
),
# Declare fs_version (already created in database with DEFAULT '0.8.0')
migrations.AddField(
model_name="snapshot",

11
archivebox/core/migrations/0024_assign_default_crawl.py
View File

@ -23,6 +23,9 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
print("✓ Fresh install or all snapshots already have crawls")
return
cursor.execute("SELECT url FROM core_snapshot WHERE crawl_id IS NULL ORDER BY bookmarked_at, timestamp")
crawl_urls = "\n".join(url for (url,) in cursor.fetchall() if url)
# Get or create system user (pk=1)
cursor.execute("SELECT id FROM auth_user WHERE id = 1")
if not cursor.fetchone():
@ -36,7 +39,7 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
# Create a default crawl for migrated snapshots
# At this point crawls_crawl is guaranteed to have v0.9.0 schema (crawls/0002 ran first)
crawl_id = str(uuid_lib.uuid4())
crawl_id = uuid_lib.uuid4().hex
now = datetime.now().isoformat()
cursor.execute(
@ -45,11 +48,11 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
id, created_at, modified_at, num_uses_succeeded, num_uses_failed,
urls, max_depth, tags_str, label, notes, output_dir,
status, retry_at, created_by_id, schedule_id, config, persona_id
) VALUES (?, ?, ?, 0, 0, '', 0, '', 'Migrated from v0.7.2/v0.8.6',
) VALUES (?, ?, ?, 0, 0, ?, 0, '', 'Migrated from v0.7.2/v0.8.6',
'Auto-created crawl for migrated snapshots', '',
'sealed', ?, 1, NULL, '{}', NULL)
""",
[crawl_id, now, now, now],
[crawl_id, now, now, crawl_urls, now],
)
# Assign all snapshots without a crawl to the default crawl
@ -118,7 +121,7 @@ class Migration(migrations.Migration):
SELECT
id, url, timestamp, title,
bookmarked_at, created_at, modified_at,
crawl_id, parent_snapshot_id,
REPLACE(crawl_id, '-', ''), REPLACE(parent_snapshot_id, '-', ''),
downloaded_at, depth, fs_version,
COALESCE(config, '{}'), COALESCE(notes, ''),
num_uses_succeeded, num_uses_failed,

9
archivebox/core/migrations/0025_alter_archiveresult_options_alter_snapshot_options_and_more.py
View File

@ -140,7 +140,7 @@ class Migration(migrations.Migration):
name="retry_at",
field=models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True),
),
# NOTE: bookmarked_at and created_at already added by migration 0023
# NOTE: bookmarked_at, created_at, and downloaded_at already added by migration 0023
migrations.AddField(
model_name="snapshot",
name="config",
@ -160,11 +160,8 @@ class Migration(migrations.Migration):
name="depth",
field=models.PositiveSmallIntegerField(db_index=True, default=0),
),
migrations.AddField(
model_name="snapshot",
name="downloaded_at",
field=models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True),
),
# NOTE: downloaded_at already added by migration 0023 so it can preserve
# v0.7.x updated / v0.8.x downloaded_at values without a duplicate table rebuild.
# NOTE: fs_version already added by migration 0023 with default='0.8.0'
# NOTE: modified_at already added by migration 0023
migrations.AddField(

629
archivebox/core/models.py
View File

File diff suppressed because it is too large Load Diff

48
archivebox/core/settings.py
View File

@ -13,8 +13,8 @@ from django.utils.crypto import get_random_string
import archivebox
from archivebox.config import DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS # noqa
from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG, STORAGE_CONFIG # noqa
from archivebox.config.constants import CONSTANTS
from archivebox.config.common import get_config
from archivebox.core.host_utils import normalize_base_url, get_admin_base_url, get_api_base_url
from .settings_logging import SETTINGS_LOGGING
@ -23,6 +23,8 @@ IS_MIGRATING = "makemigrations" in sys.argv[:3] or "migrate" in sys.argv[:3]
IS_TESTING = "test" in sys.argv[:3] or "PYTEST_CURRENT_TEST" in os.environ
IS_SHELL = "shell" in sys.argv[:3] or "shell_plus" in sys.argv[:3]
IS_GETTING_VERSION_OR_HELP = "version" in sys.argv or "help" in sys.argv or "--version" in sys.argv or "--help" in sys.argv
CONFIG = get_config()
PACKAGE_DIR = CONSTANTS.PACKAGE_DIR
################################################################################
### ArchiveBox Plugin Settings
@ -45,7 +47,7 @@ LOGOUT_REDIRECT_URL = os.environ.get("LOGOUT_REDIRECT_URL", "/")
PASSWORD_RESET_URL = "/accounts/password_reset/"
APPEND_SLASH = True
DEBUG = SHELL_CONFIG.DEBUG or ("--debug" in sys.argv)
DEBUG = CONFIG.DEBUG or ("--debug" in sys.argv)
INSTALLED_APPS = [
@ -110,11 +112,9 @@ AUTHENTICATION_BACKENDS = [
# LDAP Authentication Configuration
# Conditionally loaded if LDAP_ENABLED=True and django-auth-ldap is installed
try:
from archivebox.config.ldap import LDAP_CONFIG
if LDAP_CONFIG.LDAP_ENABLED:
if CONFIG.LDAP_ENABLED:
# Validate LDAP configuration
is_valid, error_msg = LDAP_CONFIG.validate_ldap_config()
is_valid, error_msg = CONFIG.validate_ldap_config()
if not is_valid:
from rich import print
@ -127,23 +127,23 @@ try:
ldap = importlib.import_module("ldap")
# Configure LDAP authentication
AUTH_LDAP_SERVER_URI = LDAP_CONFIG.LDAP_SERVER_URI
AUTH_LDAP_BIND_DN = LDAP_CONFIG.LDAP_BIND_DN
AUTH_LDAP_BIND_PASSWORD = LDAP_CONFIG.LDAP_BIND_PASSWORD
AUTH_LDAP_SERVER_URI = CONFIG.LDAP_SERVER_URI
AUTH_LDAP_BIND_DN = CONFIG.LDAP_BIND_DN
AUTH_LDAP_BIND_PASSWORD = CONFIG.LDAP_BIND_PASSWORD
# Configure user search
AUTH_LDAP_USER_SEARCH = LDAPSearch(
LDAP_CONFIG.LDAP_USER_BASE,
CONFIG.LDAP_USER_BASE,
getattr(ldap, "SCOPE_SUBTREE", 2),
LDAP_CONFIG.LDAP_USER_FILTER,
CONFIG.LDAP_USER_FILTER,
)
# Map LDAP attributes to Django user model fields
AUTH_LDAP_USER_ATTR_MAP = {
"username": LDAP_CONFIG.LDAP_USERNAME_ATTR,
"first_name": LDAP_CONFIG.LDAP_FIRSTNAME_ATTR,
"last_name": LDAP_CONFIG.LDAP_LASTNAME_ATTR,
"email": LDAP_CONFIG.LDAP_EMAIL_ATTR,
"username": CONFIG.LDAP_USERNAME_ATTR,
"first_name": CONFIG.LDAP_FIRSTNAME_ATTR,
"last_name": CONFIG.LDAP_LASTNAME_ATTR,
"email": CONFIG.LDAP_EMAIL_ATTR,
}
# Use custom LDAP backend that supports LDAP_CREATE_SUPERUSER
@ -175,9 +175,9 @@ except ImportError:
STATIC_URL = "/static/"
TEMPLATES_DIR_NAME = "templates"
CUSTOM_TEMPLATES_ENABLED = os.path.isdir(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR) and os.access(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK)
CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONFIG.CUSTOM_TEMPLATES_DIR) and os.access(CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK)
STATICFILES_DIRS = [
*([str(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
*([str(CONFIG.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
# *[
# str(plugin_dir / 'static')
# for plugin_dir in PLUGIN_DIRS.values()
@ -188,7 +188,7 @@ STATICFILES_DIRS = [
]
TEMPLATE_DIRS = [
*([str(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
*([str(CONFIG.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
# *[
# str(plugin_dir / 'templates')
# for plugin_dir in PLUGIN_DIRS.values()
@ -328,14 +328,14 @@ STORAGES = {
"BACKEND": "django.core.files.storage.FileSystemStorage",
"OPTIONS": {
"base_url": "/archive/",
"location": ARCHIVE_DIR,
"location": CONFIG.ARCHIVE_DIR,
},
},
# "snapshots": {
# "BACKEND": "django.core.files.storage.FileSystemStorage",
# "OPTIONS": {
# "base_url": "/snapshots/",
# "location": CONSTANTS.SNAPSHOTS_DIR,
# "location": CONSTANTS.USERS_DIR,
# },
# },
# "personas": {
@ -353,10 +353,10 @@ CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}
### Security Settings
################################################################################
SECRET_KEY = SERVER_CONFIG.SECRET_KEY or get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789_")
SECRET_KEY = CONFIG.SECRET_KEY or get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789_")
ALLOWED_HOSTS = SERVER_CONFIG.ALLOWED_HOSTS.split(",")
CSRF_TRUSTED_ORIGINS = list(set(SERVER_CONFIG.CSRF_TRUSTED_ORIGINS.split(",")))
ALLOWED_HOSTS = CONFIG.ALLOWED_HOSTS.split(",")
CSRF_TRUSTED_ORIGINS = list(set(CONFIG.CSRF_TRUSTED_ORIGINS.split(",")))
admin_base_url = normalize_base_url(get_admin_base_url())
if admin_base_url and admin_base_url not in CSRF_TRUSTED_ORIGINS:

8
archivebox/core/templatetags/config_tags.py
View File

@ -4,19 +4,19 @@ from typing import Any
from django import template
from archivebox.config.configset import get_config as _get_config
from archivebox.config.common import get_config
register = template.Library()
@register.simple_tag
def get_config(key: str) -> Any:
@register.simple_tag(name="get_config")
def get_config_tag(key: str) -> Any:
"""
Get a config value by key.
Usage: {% get_config "ARCHIVEDOTORG_ENABLED" as enabled %}
"""
try:
return _get_config().get(key)
return get_config().get(key)
except (KeyError, AttributeError):
return None

18
archivebox/core/urls.py
View File

@ -1,11 +1,15 @@
__package__ = "archivebox.core"
import sys
from importlib.util import find_spec
from django.urls import path, re_path, include
from django.views import static
from django.conf import settings
from django.views.generic.base import RedirectView
from django.http import HttpRequest
from archivebox.config.constants import CONSTANTS
from archivebox.config.common import get_config
from archivebox.misc.serve_static import serve_static
from archivebox.core.admin_site import archivebox_admin
@ -28,13 +32,13 @@ from archivebox.core.views import (
# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
# print('DEBUG', settings.DEBUG)
CONFIG = get_config()
DEBUG = CONFIG.DEBUG or ("--debug" in sys.argv)
urlpatterns = [
re_path(r"^static/(?P<path>.*)$", serve_static),
# re_path(r"^media/(?P<path>.*)$", static.serve, {"document_root": settings.MEDIA_ROOT}),
path("robots.txt", static.serve, {"document_root": settings.STATICFILES_DIRS[0], "path": "robots.txt"}),
path("favicon.ico", static.serve, {"document_root": settings.STATICFILES_DIRS[0], "path": "favicon.ico"}),
path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
path("public/", PublicIndexView.as_view(), name="public-index"),
path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),
@ -79,10 +83,10 @@ def _raise_test_error(_request: HttpRequest):
raise ZeroDivisionError("Intentional test error route")
if settings.DEBUG_TOOLBAR:
if DEBUG and ("--nothreading" in sys.argv) and ("--reload" not in sys.argv) and find_spec("debug_toolbar"):
urlpatterns += [path("__debug__/", include("debug_toolbar.urls"))]
if settings.DEBUG_REQUESTS_TRACKER:
if DEBUG and find_spec("requests_tracker"):
urlpatterns += [path("__requests_tracker__/", include("requests_tracker.urls"))]

120
archivebox/core/views.py
View File

@ -6,7 +6,7 @@ import posixpath
from glob import glob, escape
from django.utils import timezone
import inspect
from typing import cast, get_type_hints
from typing import cast
from collections.abc import Callable
from pathlib import Path
from urllib.parse import quote, urlparse
@ -27,9 +27,9 @@ from django.utils.decorators import method_decorator
from admin_data_views.typing import TableContext, ItemContext, SectionData
from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, DATA_DIR, VERSION
from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG, SEARCH_BACKEND_CONFIG
from archivebox.config.configset import get_flat_config, get_config, get_all_configs
from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, VERSION
from archivebox.config.common import get_config, get_all_configs
from archivebox.config.configset import BaseConfigSet
from archivebox.misc.util import base_url, htmlencode, ts_to_date_str, urldecode, without_fragment
from archivebox.misc.serve_static import serve_static_with_byterange_support
from archivebox.misc.logging_util import printable_filesize
@ -94,17 +94,17 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
def _admin_login_redirect_or_forbidden(request: HttpRequest):
if SERVER_CONFIG.CONTROL_PLANE_ENABLED:
if get_config().CONTROL_PLANE_ENABLED:
return redirect(f"/admin/login/?next={request.path}")
return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.")
class HomepageView(View):
def get(self, request):
if request.user.is_authenticated and SERVER_CONFIG.CONTROL_PLANE_ENABLED:
if request.user.is_authenticated and get_config().CONTROL_PLANE_ENABLED:
return redirect("/admin/core/snapshot/")
if SERVER_CONFIG.PUBLIC_INDEX:
if get_config().PUBLIC_INDEX:
return redirect("/public")
return _admin_login_redirect_or_forbidden(request)
@ -251,7 +251,7 @@ class SnapshotView(View):
"num_failures": snapshot.num_failures,
"oldest_archive_date": ts_to_date_str(snapshot.oldest_archive_date),
"warc_path": warc_path,
"PREVIEW_ORIGINALS": SERVER_CONFIG.PREVIEW_ORIGINALS,
"PREVIEW_ORIGINALS": get_config().PREVIEW_ORIGINALS,
"archiveresults": [*non_compact_outputs, *compact_outputs],
"best_result": best_result,
"snapshot": snapshot, # Pass the snapshot object for template tags
@ -264,7 +264,7 @@ class SnapshotView(View):
return render(template_name="core/snapshot.html", request=request, context=context)
def get(self, request, path):
if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS:
if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS:
return _admin_login_redirect_or_forbidden(request)
snapshot = None
@ -466,7 +466,7 @@ class SnapshotPathView(View):
path: str = "",
url: str | None = None,
):
if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS:
if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS:
return _admin_login_redirect_or_forbidden(request)
if username == "system":
@ -501,20 +501,20 @@ class SnapshotPathView(View):
if date:
try:
if len(date) == 4:
qs = qs.filter(created_at__year=int(date))
qs = qs.filter(bookmarked_at__year=int(date))
elif len(date) == 6:
qs = qs.filter(created_at__year=int(date[:4]), created_at__month=int(date[4:6]))
qs = qs.filter(bookmarked_at__year=int(date[:4]), bookmarked_at__month=int(date[4:6]))
elif len(date) == 8:
qs = qs.filter(
created_at__year=int(date[:4]),
created_at__month=int(date[4:6]),
created_at__day=int(date[6:8]),
bookmarked_at__year=int(date[:4]),
bookmarked_at__month=int(date[4:6]),
bookmarked_at__day=int(date[6:8]),
)
except ValueError:
pass
if requested_url:
snapshot = qs.order_by("-created_at", "-bookmarked_at", "-timestamp").first()
snapshot = qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first()
else:
requested_domain = domain or ""
if requested_domain.startswith(("http://", "https://")):
@ -524,9 +524,9 @@ class SnapshotPathView(View):
# Prefer exact domain matches
matches = [
s for s in qs.order_by("-created_at", "-bookmarked_at") if Snapshot.extract_domain_from_url(s.url) == requested_domain
s for s in qs.order_by("-bookmarked_at", "-created_at") if Snapshot.extract_domain_from_url(s.url) == requested_domain
]
snapshot = matches[0] if matches else qs.order_by("-created_at", "-bookmarked_at", "-timestamp").first()
snapshot = matches[0] if matches else qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first()
if not snapshot:
return HttpResponse(
@ -633,7 +633,7 @@ def _latest_response_match(domain: str, rel_path: str) -> tuple[Path, Path] | No
return None
domain = domain.split(":", 1)[0].lower()
# TODO: optimize by querying output_files in DB instead of globbing filesystem
data_root = DATA_DIR / "users"
data_root = get_config().USERS_DIR
escaped_domain = escape(domain)
escaped_path = escape(rel_path)
pattern = str(data_root / "*" / "snapshots" / "*" / escaped_domain / "*" / "responses" / escaped_domain / escaped_path)
@ -658,7 +658,7 @@ def _latest_responses_root(domain: str) -> Path | None:
if not domain:
return None
domain = domain.split(":", 1)[0].lower()
data_root = DATA_DIR / "users"
data_root = get_config().USERS_DIR
escaped_domain = escape(domain)
pattern = str(data_root / "*" / "snapshots" / "*" / escaped_domain / "*" / "responses" / escaped_domain)
matches = glob(pattern)
@ -675,7 +675,7 @@ def _latest_snapshot_for_domain(domain: str) -> Snapshot | None:
return None
requested_domain = domain.split(":", 1)[0].lower()
snapshots = SnapshotView.find_snapshots_for_url(f"https://{requested_domain}").order_by("-created_at", "-bookmarked_at", "-timestamp")
snapshots = SnapshotView.find_snapshots_for_url(f"https://{requested_domain}").order_by("-bookmarked_at", "-created_at", "-timestamp")
for snapshot in snapshots:
if Snapshot.extract_domain_from_url(snapshot.url).lower() == requested_domain:
return snapshot
@ -734,7 +734,7 @@ def _serve_responses_path(request, responses_root: Path, rel_path: str, show_ind
def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
rel_path = path or ""
is_directory_request = bool(path) and path.endswith("/")
show_indexes = bool(request.GET.get("files")) or (SERVER_CONFIG.USES_SUBDOMAIN_ROUTING and is_directory_request)
show_indexes = bool(request.GET.get("files")) or (get_config().USES_SUBDOMAIN_ROUTING and is_directory_request)
if not show_indexes and (not rel_path or rel_path == "index.html"):
return SnapshotView.render_live_index(request, snapshot)
@ -804,7 +804,7 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
if snapshot:
return SnapshotView.render_live_index(request, snapshot)
if SERVER_CONFIG.PUBLIC_ADD_VIEW or request.user.is_authenticated:
if get_config().PUBLIC_ADD_VIEW or request.user.is_authenticated:
target_url = _original_request_url(domain, path, request.META.get("QUERY_STRING", ""))
return redirect(build_web_url(f"/web/{quote(target_url, safe=':/')}"))
@ -815,7 +815,7 @@ class SnapshotHostView(View):
"""Serve snapshot directory contents on <snapshot-subdomain>.<listen_host>/<path>."""
def get(self, request, snapshot_id: str, path: str = ""):
if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS:
if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS:
return _admin_login_redirect_or_forbidden(request)
snapshot = _find_snapshot_by_ref(snapshot_id)
@ -836,7 +836,7 @@ class SnapshotReplayView(View):
"""Serve snapshot directory contents on a one-domain replay path."""
def get(self, request, snapshot_id: str, path: str = ""):
if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS:
if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS:
return _admin_login_redirect_or_forbidden(request)
snapshot = _find_snapshot_by_ref(snapshot_id)
@ -850,7 +850,7 @@ class OriginalDomainHostView(View):
"""Serve responses from the most recent snapshot when using <domain>.<listen_host>/<path>."""
def get(self, request, domain: str, path: str = ""):
if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS:
if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS:
return _admin_login_redirect_or_forbidden(request)
return _serve_original_domain_replay(request, domain, path)
@ -859,7 +859,7 @@ class OriginalDomainReplayView(View):
"""Serve original-domain replay content on a one-domain replay path."""
def get(self, request, domain: str, path: str = ""):
if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS:
if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS:
return _admin_login_redirect_or_forbidden(request)
return _serve_original_domain_replay(request, domain, path)
@ -867,15 +867,15 @@ class OriginalDomainReplayView(View):
class PublicIndexView(ListView):
template_name = "public_index.html"
model = Snapshot
paginate_by = SERVER_CONFIG.SNAPSHOTS_PER_PAGE
paginate_by = get_config().SNAPSHOTS_PER_PAGE
ordering = ["-bookmarked_at", "-created_at"]
def get_context_data(self, **kwargs):
return {
**super().get_context_data(**kwargs),
"VERSION": VERSION,
"COMMIT_HASH": SHELL_CONFIG.COMMIT_HASH,
"FOOTER_INFO": SERVER_CONFIG.FOOTER_INFO,
"COMMIT_HASH": get_config().COMMIT_HASH,
"FOOTER_INFO": get_config().FOOTER_INFO,
"search_mode": get_search_mode(self.request.GET.get("search_mode")),
}
@ -935,7 +935,7 @@ class PublicIndexView(ListView):
def get(self, *args, **kwargs):
if self.request.user.is_authenticated:
return redirect("/admin/core/snapshot/")
if SERVER_CONFIG.PUBLIC_INDEX:
if get_config().PUBLIC_INDEX:
response = super().get(*args, **kwargs)
return response
else:
@ -957,7 +957,7 @@ class AddView(UserPassesTestMixin, FormView):
return super().get_initial()
def test_func(self):
return SERVER_CONFIG.PUBLIC_ADD_VIEW or self.request.user.is_authenticated
return get_config().PUBLIC_ADD_VIEW or self.request.user.is_authenticated
def _can_override_crawl_config(self) -> bool:
user = self.request.user
@ -975,7 +975,7 @@ class AddView(UserPassesTestMixin, FormView):
return custom_config
def get_context_data(self, **kwargs):
required_search_plugin = f"search_backend_{SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}".strip()
required_search_plugin = f"search_backend_{get_config().SEARCH_BACKEND_ENGINE}".strip()
plugin_configs = discover_plugin_configs()
plugin_dependency_map = {
plugin_name: [
@ -990,7 +990,7 @@ class AddView(UserPassesTestMixin, FormView):
# We can't just call request.build_absolute_uri in the template, because it would include query parameters
"absolute_add_path": self.request.build_absolute_uri(self.request.path),
"VERSION": VERSION,
"FOOTER_INFO": SERVER_CONFIG.FOOTER_INFO,
"FOOTER_INFO": get_config().FOOTER_INFO,
"required_search_plugin": required_search_plugin,
"plugin_dependency_map_json": json.dumps(plugin_dependency_map, sort_keys=True),
"stdout": "",
@ -1112,7 +1112,7 @@ class AddView(UserPassesTestMixin, FormView):
class WebAddView(AddView):
def _latest_snapshot_for_url(self, requested_url: str):
return SnapshotView.find_snapshots_for_url(requested_url).order_by("-created_at", "-bookmarked_at", "-timestamp").first()
return SnapshotView.find_snapshots_for_url(requested_url).order_by("-bookmarked_at", "-created_at", "-timestamp").first()
def _normalize_add_url(self, requested_url: str) -> str:
if requested_url.startswith(("http://", "https://")):
@ -1671,8 +1671,7 @@ def find_config_default(key: str) -> str:
for config in CONFIGS.values():
if key in dict(config):
default_field = getattr(config, "model_fields", dict(config))[key]
default_val = default_field.default if hasattr(default_field, "default") else default_field
default_val = type(config).model_fields[key].default
break
if isinstance(default_val, Callable):
@ -1686,41 +1685,12 @@ def find_config_default(key: str) -> str:
def find_config_type(key: str) -> str:
from typing import ClassVar
CONFIGS = get_all_configs()
for config in CONFIGS.values():
if hasattr(config, key):
# Try to get from pydantic model_fields first (more reliable)
if hasattr(config, "model_fields") and key in config.model_fields:
field = config.model_fields[key]
if hasattr(field, "annotation") and field.annotation is not None:
try:
return str(field.annotation.__name__)
except AttributeError:
return str(field.annotation)
# Fallback to get_type_hints with proper namespace
try:
import typing
namespace = {
"ClassVar": ClassVar,
"Optional": typing.Optional,
"Union": typing.Union,
"List": list,
"Dict": dict,
"Path": Path,
}
type_hints = get_type_hints(config, globalns=namespace, localns=namespace)
try:
return str(type_hints[key].__name__)
except AttributeError:
return str(type_hints[key])
except Exception:
# If all else fails, return str
pass
if key in type(config).model_fields:
annotation = type(config).model_fields[key].annotation
return getattr(annotation, "__name__", str(annotation))
return "str"
@ -1748,8 +1718,6 @@ def find_config_source(key: str, merged_config: dict) -> str:
pass
# Check if it's from archivebox.config.file
from archivebox.config.configset import BaseConfigSet
file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
if key in file_config:
return "Config File"
@ -1869,10 +1837,8 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
@render_with_item_view
def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
from archivebox.machine.models import Machine
from archivebox.config.configset import BaseConfigSet
CONFIGS = get_all_configs()
FLAT_CONFIG = get_flat_config()
assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings."
@ -1909,7 +1875,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
sources_info.append(("Default", default_val, "gray"))
# Final computed value
final_value = merged_config.get(key, FLAT_CONFIG.get(key, CONFIGS.get(key, None)))
final_value = merged_config.get(key, CONFIGS.get(key, None))
if not key_is_safe(key):
final_value = "********"
@ -1923,7 +1889,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
section_header = mark_safe(
f'[CONSTANTS] &nbsp; <b><code style="color: lightgray">{key}</code></b> &nbsp; <small>(read-only, hardcoded by ArchiveBox)</small>',
)
elif key in FLAT_CONFIG:
elif key in merged_config:
section_header = mark_safe(
f'data / ArchiveBox.conf &nbsp; [{find_config_section(key)}] &nbsp; <b><code style="color: lightgray">{key}</code></b>',
)
@ -1967,13 +1933,13 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
<b>Configuration Sources (highest priority first):</b><br/><br/>
{sources_html}
<br/><br/>
<p style="display: {"block" if key in FLAT_CONFIG and key not in CONSTANTS_CONFIG else "none"}">
<p style="display: {"block" if key in merged_config and key not in CONSTANTS_CONFIG else "none"}">
<i>To change this value, edit <code>data/ArchiveBox.conf</code> or run:</i>
<br/><br/>
<code>archivebox config --set {key}="{
val.strip("'")
if (val := find_config_default(key))
else (str(FLAT_CONFIG[key] if key_is_safe(key) else "********")).strip("'")
else (str(final_value if key_is_safe(key) else "********")).strip("'")
}"</code>
</p>
'''),

64
archivebox/crawls/migrations/0002_upgrade_from_0_8_6.py
View File

@ -19,6 +19,64 @@ def upgrade_crawl_table_from_v086(apps, schema_editor):
has_seed_id = "seed_id" in crawl_cols
has_urls = "urls" in crawl_cols
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='crawls_crawlschedule'")
if cursor.fetchone():
cursor.execute("PRAGMA table_info(crawls_crawlschedule)")
schedule_cols = {row[1] for row in cursor.fetchall()}
schedule_sets = []
if "id" in schedule_cols:
schedule_sets.append("id = REPLACE(id, '-', '')")
if "template_id" in schedule_cols:
schedule_sets.append(
"template_id = CASE "
"WHEN template_id IS NOT NULL "
"AND LENGTH(REPLACE(template_id, '-', '')) = 32 "
"AND REPLACE(template_id, '-', '') NOT GLOB '*[^0-9A-Fa-f]*' "
"THEN REPLACE(template_id, '-', '') ELSE NULL END",
)
if schedule_sets:
cursor.execute(f"UPDATE crawls_crawlschedule SET {', '.join(schedule_sets)}")
crawl_sets = []
if "id" in crawl_cols:
crawl_sets.append("id = REPLACE(id, '-', '')")
if "persona_id" in crawl_cols:
crawl_sets.append(
"persona_id = CASE "
"WHEN persona_id IS NOT NULL "
"AND LENGTH(REPLACE(persona_id, '-', '')) = 32 "
"AND REPLACE(persona_id, '-', '') NOT GLOB '*[^0-9A-Fa-f]*' "
"THEN REPLACE(persona_id, '-', '') ELSE NULL END",
)
if "schedule_id" in crawl_cols:
crawl_sets.append(
"schedule_id = CASE "
"WHEN schedule_id IS NOT NULL "
"AND LENGTH(REPLACE(schedule_id, '-', '')) = 32 "
"AND REPLACE(schedule_id, '-', '') NOT GLOB '*[^0-9A-Fa-f]*' "
"THEN REPLACE(schedule_id, '-', '') ELSE NULL END",
)
if crawl_sets:
cursor.execute(f"UPDATE crawls_crawl SET {', '.join(crawl_sets)}")
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_snapshot'")
if cursor.fetchone():
cursor.execute("PRAGMA table_info(core_snapshot)")
snapshot_cols = {row[1] for row in cursor.fetchall()}
if "crawl_id" in snapshot_cols:
cursor.execute(
"""
UPDATE core_snapshot
SET crawl_id = CASE
WHEN crawl_id IS NOT NULL
AND LENGTH(REPLACE(crawl_id, '-', '')) = 32
AND REPLACE(crawl_id, '-', '') NOT GLOB '*[^0-9A-Fa-f]*'
THEN REPLACE(crawl_id, '-', '')
ELSE NULL
END
""",
)
# Only upgrade if we have v0.8.6rc0 schema
if not (has_seed_id and not has_urls):
return
@ -66,9 +124,9 @@ def upgrade_crawl_table_from_v086(apps, schema_editor):
status, retry_at, created_by_id, schedule_id
)
SELECT
id, created_at, modified_at, num_uses_succeeded, num_uses_failed,
'', config, max_depth, tags_str, NULL, '', '', '',
status, retry_at, created_by_id, schedule_id
REPLACE(id, '-', ''), created_at, modified_at, num_uses_succeeded, num_uses_failed,
'', config, max_depth, tags_str, REPLACE(persona_id, '-', ''), '', '', '',
status, retry_at, created_by_id, REPLACE(schedule_id, '-', '')
FROM crawls_crawl;
""")

31
archivebox/crawls/models.py
View File

@ -1,6 +1,7 @@
__package__ = "archivebox.crawls"
from typing import TYPE_CHECKING
from io import StringIO
import uuid
import json
import re
@ -272,17 +273,23 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
@property
def output_dir(self) -> Path:
"""
Construct output directory: users/{username}/crawls/{YYYYMMDD}/{domain}/{crawl-id}
Construct output directory: archive/users/{username}/crawls/{YYYYMMDD}/{domain}/{crawl-id}
Domain is extracted from the first URL in the crawl.
"""
from archivebox import DATA_DIR
from archivebox.config import CONSTANTS
from archivebox.config.common import get_config
from archivebox.core.models import Snapshot
date_str = self.created_at.strftime("%Y%m%d")
urls = self.get_urls_list()
domain = Snapshot.extract_domain_from_url(urls[0]) if urls else "unknown"
first_url = ""
for raw_line in StringIO(self.urls or ""):
candidate = raw_line.strip()
if candidate and not candidate.startswith("#"):
first_url = candidate
break
domain = Snapshot.extract_domain_from_url(first_url) if first_url else "unknown"
return DATA_DIR / "users" / self.created_by.username / "crawls" / date_str / domain / str(self.id)
return get_config().USERS_DIR / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
def get_urls_list(self) -> list[str]:
"""Get list of URLs from urls field, filtering out comments and empty lines."""
@ -359,7 +366,7 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
def get_url_allowlist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
if use_effective_config:
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
config = get_config(crawl=self, snapshot=snapshot)
else:
@ -368,7 +375,7 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
def get_url_denylist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]:
if use_effective_config:
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
config = get_config(crawl=self, snapshot=snapshot)
else:
@ -616,6 +623,9 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
from archivebox.core.models import Snapshot
from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
if self.status == self.StatusChoices.SEALED:
return []
created_snapshots = []
for line in self.urls.splitlines():
@ -692,6 +702,9 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
from archivebox.core.models import Snapshot
from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
if self.status == self.StatusChoices.SEALED:
return None
url = sanitize_extracted_url(fix_url_from_markdown(str(url or "").strip()))
if not url:
return None
@ -825,7 +838,7 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
import time
from pathlib import Path
from archivebox.hooks import run_hook, discover_hooks, process_hook_records, is_finite_background_hook
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
from archivebox.machine.models import Binary, Machine
# Debug logging to file (since stdout/stderr redirected to /dev/null in progress mode)
@ -1050,7 +1063,7 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
persona.cleanup_runtime_for_crawl(self)
# Run on_CrawlEnd hooks
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
config = get_config(crawl=self)

131
archivebox/hooks.py
View File

@ -45,12 +45,12 @@ __package__ = "archivebox"
import os
import json
from collections.abc import Iterable, Mapping
from functools import lru_cache
from pathlib import Path
from typing import TYPE_CHECKING, Any, Optional, TypedDict
from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, TypedDict
from abx_plugins import get_plugins_dir
from django.conf import settings
from django.utils.safestring import mark_safe
from archivebox.config.constants import CONSTANTS
from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
@ -59,10 +59,38 @@ if TYPE_CHECKING:
from archivebox.machine.models import Process
class ConfigLookup(Protocol):
def get(self, key: str, default: Any = None) -> Any: ...
def items(self) -> Iterable[tuple[str, Any]]: ...
class PluginSpecialConfig(TypedDict):
enabled: bool
timeout: int
binary: str
class ConfigDump(Protocol):
def as_dict(self) -> dict[str, Any]: ...
def _has_config_dump(config: object) -> TypeGuard[ConfigDump]:
return callable(getattr(config, "as_dict", None))
def _config_to_overrides(config: ConfigLookup | Mapping[str, Any] | None) -> dict[str, Any]:
if config is None:
return {}
if _has_config_dump(config):
return dict(config.as_dict())
return dict(config.items())
# Plugin directories
BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
USER_PLUGINS_DIR = Path(
os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or getattr(settings, "USER_PLUGINS_DIR", "") or str(CONSTANTS.USER_PLUGINS_DIR),
os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or str(CONSTANTS.USER_PLUGINS_DIR),
).expanduser()
@ -147,10 +175,30 @@ class HookResult(TypedDict, total=False):
records: list[dict[str, Any]] # Parsed JSONL records with 'type' field
def _model_output_dir_from_child_path(path: Path, marker: str) -> Path | None:
"""
Infer the model output dir from a model dir or one of its plugin subdirs.
Current ArchiveBox snapshot/crawl dirs are:
.../{snapshots,crawls}/YYYYMMDD/domain/uuid[/plugin]
"""
parts = path.resolve().parts
try:
marker_index = parts.index(marker)
except ValueError:
return None
model_end_index = marker_index + 4
if len(parts) < model_end_index:
return None
return Path(*parts[:model_end_index])
def discover_hooks(
event_name: str,
filter_disabled: bool = True,
config: dict[str, Any] | None = None,
config: ConfigLookup | None = None,
**config_kwargs: Any,
) -> list[Path]:
"""
Find all hook scripts for an event family.
@ -170,15 +218,15 @@ def discover_hooks(
Event names are normalized by stripping a trailing `Event`.
If no matching `on_{EventFamily}__*` scripts exist, returns [].
filter_disabled: If True, skip hooks from disabled plugins (default: True)
config: Optional config dict from get_config() (merges file, env, machine, crawl, snapshot)
If None, will call get_config() with global scope
config: Optional pre-merged config dict from get_config().
**config_kwargs: Scope/override args forwarded to get_config() when config is not supplied.
Returns:
Sorted list of hook script paths from enabled plugins only.
Examples:
# With proper config context (recommended):
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
config = get_config(crawl=my_crawl, snapshot=my_snapshot)
discover_hooks('Snapshot', config=config)
# Returns: [Path('.../on_Snapshot__10_title.py'), ...] (wget excluded if SAVE_WGET=False)
@ -217,9 +265,9 @@ def discover_hooks(
if filter_disabled and hook_event_name != "BinaryRequest":
# Get merged config if not provided (lazy import to avoid circular dependency)
if config is None:
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
config = get_config()
config = get_config(**config_kwargs)
enabled_hooks = []
@ -250,7 +298,7 @@ def discover_hooks(
def run_hook(
script: Path,
output_dir: Path,
config: dict[str, Any],
config: ConfigLookup | Mapping[str, Any] | None = None,
timeout: int | None = None,
parent: Optional["Process"] = None,
**kwargs: Any,
@ -267,7 +315,8 @@ def run_hook(
Args:
script: Path to the hook script (.sh, .py, or .js)
output_dir: Working directory for the script (where output files go)
config: Merged config dict from get_config(crawl=..., snapshot=...) - REQUIRED
config: Optional pre-merged config dict from get_config(crawl=..., snapshot=...).
If omitted, pass scope/override args using kwargs prefixed with config_.
timeout: Maximum execution time in seconds
If None, auto-detects from PLUGINNAME_TIMEOUT config (fallback to TIMEOUT, default 300)
parent: Optional parent Process (for tracking worker->hook hierarchy)
@ -277,20 +326,24 @@ def run_hook(
Process model instance (use process.exit_code, process.stdout, process.get_records())
Example:
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
config = get_config(crawl=my_crawl, snapshot=my_snapshot)
process = run_hook(hook_path, output_dir, config=config, url=url, snapshot_id=id)
if process.status == 'exited':
records = process.get_records() # Get parsed JSONL output
"""
from archivebox.machine.models import Process, Machine, NetworkInterface
from archivebox.config.common import get_config
from archivebox.config.constants import CONSTANTS
import sys
config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
resolved_config = get_config(overrides=_config_to_overrides(config), **config_scope)
# Auto-detect timeout from plugin config if not explicitly provided
if timeout is None:
plugin_name = script.parent.name
plugin_config = get_plugin_special_config(plugin_name, config)
plugin_config = get_plugin_special_config(plugin_name, resolved_config)
timeout = plugin_config["timeout"]
if timeout:
timeout = min(int(timeout), int(CONSTANTS.MAX_HOOK_RUNTIME_SECONDS))
@ -360,17 +413,18 @@ def run_hook(
# Set up environment with base paths
env = os.environ.copy()
env["DATA_DIR"] = str(config.get("DATA_DIR") or getattr(settings, "DATA_DIR", Path.cwd()))
env["ARCHIVE_DIR"] = str(config.get("ARCHIVE_DIR") or getattr(settings, "ARCHIVE_DIR", Path.cwd() / "archive"))
env["DATA_DIR"] = str(resolved_config.DATA_DIR)
env["ARCHIVE_DIR"] = str(resolved_config.ARCHIVE_DIR)
env["ABX_RUNTIME"] = "archivebox"
env.setdefault("MACHINE_ID", getattr(settings, "MACHINE_ID", "") or os.environ.get("MACHINE_ID", ""))
env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
resolved_output_dir = output_dir.resolve()
output_parts = set(resolved_output_dir.parts)
if "snapshots" in output_parts:
env["SNAP_DIR"] = str(resolved_output_dir.parent)
if "crawls" in output_parts:
env["CRAWL_DIR"] = str(resolved_output_dir.parent)
snap_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
crawl_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.CRAWLS_DIR_NAME)
if snap_dir:
env["SNAP_DIR"] = str(snap_dir)
if crawl_dir:
env["CRAWL_DIR"] = str(crawl_dir)
crawl_id = kwargs.get("_crawl_id") or kwargs.get("crawl_id")
if crawl_id:
@ -384,8 +438,8 @@ def run_hook(
pass
# Get LIB_DIR and LIB_BIN_DIR from config
lib_dir = config.get("LIB_DIR", getattr(settings, "LIB_DIR", None))
lib_bin_dir = config.get("LIB_BIN_DIR", getattr(settings, "LIB_BIN_DIR", None))
lib_dir = resolved_config.LIB_DIR
lib_bin_dir = resolved_config.LIB_BIN_DIR
if lib_dir:
env["LIB_DIR"] = str(lib_dir)
if not lib_bin_dir and lib_dir:
@ -394,11 +448,11 @@ def run_hook(
# Set Node.js module resolution paths.
# NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory.
node_modules_dir = config.get("NODE_MODULES_DIR")
node_modules_dir = resolved_config.get("NODE_MODULES_DIR")
if not node_modules_dir and lib_dir:
node_modules_dir = Path(lib_dir) / "npm" / "node_modules"
node_path_parts = [part for part in str(config.get("NODE_PATH") or "").split(os.pathsep) if part]
node_path_parts = [part for part in str(resolved_config.get("NODE_PATH") or "").split(os.pathsep) if part]
if node_modules_dir:
node_modules_dir = Path(node_modules_dir)
node_modules_dir.mkdir(parents=True, exist_ok=True)
@ -425,7 +479,7 @@ def run_hook(
"SNAP_DIR",
"CRAWL_DIR",
}
for key, value in config.items():
for key, value in resolved_config.items():
if key in SKIP_KEYS:
continue # Already handled specially above, don't overwrite
if value is None:
@ -632,28 +686,29 @@ def get_plugin_name(plugin: str) -> str:
return plugin
def get_enabled_plugins(config: dict[str, Any] | None = None) -> list[str]:
def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any) -> list[str]:
"""
Get the list of enabled plugins based on config and available hooks.
Filters plugins by USE_/SAVE_ flags. Only returns plugins that are enabled.
Args:
config: Merged config dict from get_config() - if None, uses global config
config: Optional pre-merged config dict from get_config().
**config_kwargs: Scope/override args forwarded to get_config() when config is not supplied.
Returns:
Plugin names sorted alphabetically (numeric prefix controls order).
Example:
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
config = get_config(crawl=my_crawl, snapshot=my_snapshot)
enabled = get_enabled_plugins(config) # ['wget', 'media', 'chrome', ...]
"""
# Get merged config if not provided
if config is None:
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
config = get_config()
config = get_config(**config_kwargs)
def normalize_enabled_plugins(value: Any) -> list[str]:
if value is None:
@ -675,10 +730,12 @@ def get_enabled_plugins(config: dict[str, Any] | None = None) -> list[str]:
return [str(value).strip()] if str(value).strip() else []
# Support explicit ENABLED_PLUGINS override (legacy)
if "ENABLED_PLUGINS" in config:
return normalize_enabled_plugins(config["ENABLED_PLUGINS"])
if "ENABLED_EXTRACTORS" in config:
return normalize_enabled_plugins(config["ENABLED_EXTRACTORS"])
enabled_plugins = config.get("ENABLED_PLUGINS")
if enabled_plugins:
return normalize_enabled_plugins(enabled_plugins)
enabled_extractors = config.get("ENABLED_EXTRACTORS")
if enabled_extractors:
return normalize_enabled_plugins(enabled_extractors)
# Filter all plugins by enabled status
all_plugins = get_plugins()
@ -870,7 +927,7 @@ def get_config_defaults_from_plugins() -> dict[str, Any]:
return defaults
def get_plugin_special_config(plugin_name: str, config: dict[str, Any], _visited: set[str] | None = None) -> dict[str, Any]:
def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited: set[str] | None = None) -> PluginSpecialConfig:
"""
Extract special config keys for a plugin following naming conventions.
@ -897,7 +954,7 @@ def get_plugin_special_config(plugin_name: str, config: dict[str, Any], _visited
}
Examples:
>>> from archivebox.config.configset import get_config
>>> from archivebox.config.common import get_config
>>> config = get_config(crawl=my_crawl, snapshot=my_snapshot)
>>> get_plugin_special_config('wget', config)
{'enabled': True, 'timeout': 120, 'binary': '/usr/bin/wget'}

4
archivebox/ldap/auth.py
View File

@ -32,7 +32,7 @@ class ArchiveBoxLDAPBackend(BaseLDAPBackend):
This method is called by django-auth-ldap after successful LDAP authentication.
"""
from archivebox.config.ldap import LDAP_CONFIG
from archivebox.config.common import get_config
base_authenticate = getattr(super(), "authenticate_ldap_user", None)
if base_authenticate is None:
@ -40,7 +40,7 @@ class ArchiveBoxLDAPBackend(BaseLDAPBackend):
user = base_authenticate(ldap_user, password)
if user and LDAP_CONFIG.LDAP_CREATE_SUPERUSER:
if user and get_config().LDAP_CREATE_SUPERUSER:
# Grant superuser privileges to all LDAP-authenticated users
if not user.is_superuser:
user.is_superuser = True

22
archivebox/machine/migrations/0012_add_machine_config_if_missing.py Normal file
View File

@ -0,0 +1,22 @@
from django.db import migrations
def add_machine_config_if_missing(apps, schema_editor):
cursor = schema_editor.connection.cursor()
cursor.execute("PRAGMA table_info(machine_machine)")
columns = {row[1] for row in cursor.fetchall()}
if "config" not in columns:
cursor.execute("ALTER TABLE machine_machine ADD COLUMN config TEXT")
class Migration(migrations.Migration):
dependencies = [
("machine", "0011_remove_binary_output_dir"),
]
operations = [
migrations.RunPython(
add_machine_config_if_missing,
reverse_code=migrations.RunPython.noop,
),
]

56
archivebox/machine/models.py
View File

@ -179,10 +179,44 @@ class Machine(ModelWithHealthStats):
return cls._sanitize_config(_CURRENT_MACHINE)
else:
_CURRENT_MACHINE = None
_CURRENT_MACHINE, _ = cls.objects.update_or_create(
guid=get_host_guid(),
defaults={"hostname": socket.gethostname(), **get_os_info(), **get_vm_info(), "stats": get_host_stats()},
)
host_guid = get_host_guid()
try:
_CURRENT_MACHINE = cls.objects.get(guid=host_guid)
except cls.DoesNotExist:
_CURRENT_MACHINE = cls.objects.create(
guid=host_guid,
hostname=socket.gethostname(),
**get_os_info(),
**get_vm_info(),
stats=get_host_stats(),
)
else:
if timezone.now() >= _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
for key, value in {
"hostname": socket.gethostname(),
**get_os_info(),
**get_vm_info(),
"stats": get_host_stats(),
}.items():
setattr(_CURRENT_MACHINE, key, value)
_CURRENT_MACHINE.save(
update_fields=[
"hostname",
"hw_in_docker",
"hw_in_vm",
"hw_manufacturer",
"hw_product",
"hw_uuid",
"os_arch",
"os_family",
"os_platform",
"os_release",
"os_kernel",
"stats",
"modified_at",
],
)
return cls._sanitize_config(_CURRENT_MACHINE)
@classmethod
@ -427,9 +461,9 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
Get output directory for this binary's hook logs.
Path: data/machines/{machine_uuid}/binaries/{binary_name}/{binary_uuid}
"""
from django.conf import settings
from archivebox.config.common import get_config
return Path(settings.DATA_DIR) / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
return get_config().DATA_DIR / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
def to_json(self) -> dict:
"""
@ -582,7 +616,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
"""
import json
from archivebox.hooks import discover_hooks, run_hook
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
# Get merged config (Binary doesn't have crawl/snapshot context).
config = get_config()
@ -658,9 +692,9 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
self.save()
# Symlink binary into LIB_BIN_DIR if configured
from django.conf import settings
from archivebox.config.common import get_config
lib_bin_dir = getattr(settings, "LIB_BIN_DIR", None)
lib_bin_dir = get_config().LIB_BIN_DIR
if lib_bin_dir:
self.symlink_to_lib_bin(lib_bin_dir)
@ -2303,7 +2337,7 @@ class Process(models.Model):
"""
import subprocess
from pathlib import Path
from django.conf import settings
from archivebox.config.common import get_config
chrome_utils = Path(__file__).parent.parent / "plugins" / "chrome" / "chrome_utils.js"
if not chrome_utils.exists():
@ -2311,7 +2345,7 @@ class Process(models.Model):
try:
result = subprocess.run(
["node", str(chrome_utils), "killZombieChrome", str(settings.DATA_DIR)],
["node", str(chrome_utils), "killZombieChrome", str(get_config().DATA_DIR)],
capture_output=True,
timeout=30,
text=True,

57
archivebox/misc/checks.py
View File

@ -19,12 +19,15 @@ from rich.panel import Panel
# that the check is called after django.setup() has been called
def check_data_folder() -> None:
from archivebox import DATA_DIR, ARCHIVE_DIR
def check_data_folder(config=None, **config_kwargs) -> None:
from archivebox import DATA_DIR
from archivebox.config import CONSTANTS
from archivebox.config.common import get_config
from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir
archive_dir_exists = os.path.isdir(ARCHIVE_DIR)
config = config or get_config(**config_kwargs)
archive_dir = config.ARCHIVE_DIR
archive_dir_exists = os.path.isdir(archive_dir)
if not archive_dir_exists:
print("[red][X] No archivebox index found in the current directory.[/red]", file=sys.stderr)
print(f" {DATA_DIR}", file=sys.stderr)
@ -39,16 +42,17 @@ def check_data_folder() -> None:
# Create data dir subdirs
create_and_chown_dir(CONSTANTS.SOURCES_DIR)
create_and_chown_dir(config.USERS_DIR)
create_and_chown_dir(CONSTANTS.PERSONAS_DIR / "Default")
create_and_chown_dir(CONSTANTS.LOGS_DIR)
# create_and_chown_dir(CONSTANTS.CACHE_DIR)
# Create /tmp and /lib dirs if they don't exist
get_or_create_working_tmp_dir(autofix=True, quiet=False)
get_or_create_working_lib_dir(autofix=True, quiet=False)
get_or_create_working_tmp_dir(autofix=True, quiet=False, config=config)
get_or_create_working_lib_dir(autofix=True, quiet=False, config=config)
# Check data dir permissions, /tmp, and /lib permissions
check_data_dir_permissions()
check_data_dir_permissions(config=config)
def check_migrations():
@ -141,7 +145,7 @@ def check_not_inside_source_dir():
raise SystemExit("[!] Cannot run from source dir, set DATA_DIR or cd to a data folder first")
def check_data_dir_permissions():
def check_data_dir_permissions(config=None, **config_kwargs):
from archivebox import DATA_DIR
from archivebox.misc.logging import STDERR
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, DEFAULT_PUID, DEFAULT_PGID, IS_ROOT, USER
@ -183,35 +187,37 @@ def check_data_dir_permissions():
" [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]",
)
from archivebox.config.common import STORAGE_CONFIG
from archivebox.config.common import get_config
config = config or get_config(**config_kwargs)
try:
tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True, config=config) or config.TMP_DIR
except Exception:
tmp_dir = config.TMP_DIR
try:
tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True) or STORAGE_CONFIG.TMP_DIR
lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
except Exception:
tmp_dir = STORAGE_CONFIG.TMP_DIR
try:
lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True) or STORAGE_CONFIG.LIB_DIR
except Exception:
lib_dir = STORAGE_CONFIG.LIB_DIR
lib_dir = config.LIB_DIR
# Check /tmp dir permissions
check_tmp_dir(tmp_dir, throw=False, must_exist=True)
check_tmp_dir(tmp_dir, throw=False, must_exist=True, config=config)
# Check /lib dir permissions
check_lib_dir(lib_dir, throw=False, must_exist=True)
check_lib_dir(lib_dir, throw=False, must_exist=True, config=config)
os.umask(0o777 - int(STORAGE_CONFIG.DIR_OUTPUT_PERMISSIONS, base=8))
os.umask(0o777 - int(config.DIR_OUTPUT_PERMISSIONS, base=8))
def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True):
def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs):
from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir
from archivebox.misc.logging import STDERR
from archivebox.misc.logging_util import pretty_path
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.config.common import STORAGE_CONFIG
from archivebox.config.common import get_config
tmp_dir = tmp_dir or STORAGE_CONFIG.TMP_DIR
config = config or get_config(**config_kwargs)
tmp_dir = tmp_dir or config.TMP_DIR
socket_file = tmp_dir.absolute().resolve() / "supervisord.sock"
if not must_exist and not os.path.isdir(tmp_dir):
@ -264,16 +270,15 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True):
return False
def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_exist=True):
def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs):
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.misc.logging import STDERR
from archivebox.misc.logging_util import pretty_path
from archivebox.config.paths import dir_is_writable, get_or_create_working_lib_dir
from archivebox.config.common import STORAGE_CONFIG
from archivebox.config.common import get_config
lib_dir = lib_dir or STORAGE_CONFIG.LIB_DIR
# assert lib_dir == STORAGE_CONFIG.LIB_DIR, "lib_dir is not the same as the one in the flat config"
config = config or get_config(**config_kwargs)
lib_dir = lib_dir or config.LIB_DIR
if not must_exist and not os.path.isdir(lib_dir):
return True

16
archivebox/misc/folders.py
View File

@ -13,11 +13,12 @@ import shutil
from pathlib import Path
from archivebox.config import DATA_DIR, CONSTANTS
from archivebox.config.common import get_config
from archivebox.misc.util import enforce_types
@enforce_types
def fix_invalid_folder_locations(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str]]:
def fix_invalid_folder_locations(out_dir: Path = DATA_DIR, config=None, **config_kwargs) -> tuple[list[str], list[str]]:
"""
Legacy cleanup: Move folders to their correct timestamp-named locations based on index.json.
@ -26,9 +27,16 @@ def fix_invalid_folder_locations(out_dir: Path = DATA_DIR) -> tuple[list[str], l
"""
fixed = []
cant_fix = []
for entry in os.scandir(out_dir / CONSTANTS.ARCHIVE_DIR_NAME):
config = config or get_config(**config_kwargs)
archive_dir = config.ARCHIVE_DIR if Path(out_dir).resolve() == DATA_DIR.resolve() else out_dir / CONSTANTS.ARCHIVE_DIR_NAME
if not archive_dir.exists():
return fixed, cant_fix
for entry in os.scandir(archive_dir):
entry_path = Path(entry.path)
if entry_path.name in CONSTANTS.RESERVED_ARCHIVE_DIR_NAMES or entry_path.name.startswith("."):
continue
if entry.is_dir(follow_symlinks=True):
index_path = Path(entry.path) / "index.json"
index_path = entry_path / "index.json"
if index_path.exists():
try:
with open(index_path) as f:
@ -41,7 +49,7 @@ def fix_invalid_folder_locations(out_dir: Path = DATA_DIR) -> tuple[list[str], l
continue
if not entry.path.endswith(f"/{timestamp}"):
dest = out_dir / CONSTANTS.ARCHIVE_DIR_NAME / timestamp
dest = archive_dir / timestamp
if dest.exists():
cant_fix.append(entry.path)
else:

19
archivebox/misc/legacy.py
View File

@ -58,7 +58,7 @@ def parse_json_main_index(out_dir: Path) -> Iterator[SnapshotDict]:
return
def parse_json_links_details(out_dir: Path) -> Iterator[SnapshotDict]:
def parse_json_links_details(out_dir: Path, config=None, **config_kwargs) -> Iterator[SnapshotDict]:
"""
Parse links from individual snapshot index.jsonl/index.json files in archive directories.
@ -66,18 +66,29 @@ def parse_json_links_details(out_dir: Path) -> Iterator[SnapshotDict]:
Prefers index.jsonl (new format) over index.json (legacy format).
"""
from archivebox.config import CONSTANTS
from archivebox.config.common import get_config
archive_dir = out_dir / CONSTANTS.ARCHIVE_DIR_NAME
config = config or get_config(**config_kwargs)
archive_dir = config.ARCHIVE_DIR if Path(out_dir).resolve() == CONSTANTS.DATA_DIR.resolve() else out_dir / CONSTANTS.ARCHIVE_DIR_NAME
if not archive_dir.exists():
return
for entry in os.scandir(archive_dir):
if not entry.is_dir():
continue
entry_path = Path(entry.path)
if entry_path.name in CONSTANTS.RESERVED_ARCHIVE_DIR_NAMES or entry_path.name.startswith("."):
continue
try:
ts_int = int(float(entry_path.name))
except (TypeError, ValueError, OverflowError):
continue
if not 788918400 <= ts_int <= 2082758400:
continue
# Try index.jsonl first (new format)
jsonl_file = Path(entry.path) / CONSTANTS.JSONL_INDEX_FILENAME
json_file = Path(entry.path) / CONSTANTS.JSON_INDEX_FILENAME
jsonl_file = entry_path / CONSTANTS.JSONL_INDEX_FILENAME
json_file = entry_path / CONSTANTS.JSON_INDEX_FILENAME
link = None

38
archivebox/misc/logging_util.py
View File

@ -24,7 +24,7 @@ from rich import print
from rich.panel import Panel
from archivebox.config import CONSTANTS, DATA_DIR, VERSION
from archivebox.config.common import SHELL_CONFIG
from archivebox.config.common import get_config
from archivebox.misc.system import get_dir_size
from archivebox.misc.util import enforce_types
from archivebox.misc.logging import ANSI
@ -55,10 +55,12 @@ _LAST_RUN_STATS = RuntimeStats()
class TimedProgress:
"""Show a progress bar and measure elapsed time until .end() is called"""
def __init__(self, seconds, prefix=""):
def __init__(self, seconds, prefix="", config=None, **config_kwargs):
self.SHOW_PROGRESS = SHELL_CONFIG.SHOW_PROGRESS
self.ANSI = SHELL_CONFIG.ANSI
config = config or get_config(**config_kwargs)
self.SHOW_PROGRESS = config.SHOW_PROGRESS
self.ANSI = config.ANSI
self.TERM_WIDTH = config.TERM_WIDTH
if self.SHOW_PROGRESS:
self.p = Process(target=progress_bar, args=(seconds, prefix, self.ANSI))
@ -94,7 +96,7 @@ class TimedProgress:
# clear whole terminal line
try:
sys.stdout.write("\r{}{}\r".format((" " * SHELL_CONFIG.TERM_WIDTH), self.ANSI["reset"]))
sys.stdout.write("\r{}{}\r".format((" " * self.TERM_WIDTH), self.ANSI["reset"]))
except (OSError, BrokenPipeError):
# ignore when the parent proc has stopped listening to our stdout
pass
@ -103,15 +105,16 @@ class TimedProgress:
@enforce_types
def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI) -> None:
def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI, config=None, **config_kwargs) -> None:
"""show timer in the form of progress bar, with percentage and seconds remaining"""
output_buf = sys.stdout or sys.__stdout__ or sys.stderr or sys.__stderr__
chunk = "█" if output_buf and output_buf.encoding.upper() == "UTF-8" else "#"
last_width = SHELL_CONFIG.TERM_WIDTH
config = config or get_config(**config_kwargs)
last_width = config.TERM_WIDTH
chunks = last_width - len(prefix) - 20 # number of progress chunks to show (aka max bar width)
try:
for s in range(seconds * chunks):
max_width = SHELL_CONFIG.TERM_WIDTH
max_width = config.TERM_WIDTH
if max_width < last_width:
# when the terminal size is shrunk, we have to write a newline
# otherwise the progress bar will keep wrapping incorrectly
@ -153,7 +156,7 @@ def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI) ->
sys.stdout.flush()
# uncomment to have it disappear when it hits 100% instead of staying full red:
# time.sleep(0.5)
# sys.stdout.write('\r{}{}\r'.format((' ' * SHELL_CONFIG.TERM_WIDTH), ANSI['reset']))
# sys.stdout.write('\r{}{}\r'.format((' ' * get_config().TERM_WIDTH), ANSI['reset']))
# sys.stdout.flush()
except (KeyboardInterrupt, BrokenPipeError):
print()
@ -226,13 +229,22 @@ def log_indexing_process_finished():
_LAST_RUN_STATS.index_end_ts = end_ts
def log_indexing_started(out_path: str):
if SHELL_CONFIG.IS_TTY:
sys.stdout.write(f" > ./{Path(out_path).relative_to(DATA_DIR)}")
def _display_data_path(out_path: str) -> str:
path = Path(out_path).resolve()
try:
return f"./{path.relative_to(DATA_DIR)}"
except ValueError:
return str(path)
def log_indexing_started(out_path: str, config=None, **config_kwargs):
config = config or get_config(**config_kwargs)
if config.IS_TTY:
sys.stdout.write(f" > {_display_data_path(out_path)}")
def log_indexing_finished(out_path: str):
print(f"\r √ ./{Path(out_path).relative_to(DATA_DIR)}")
print(f"\r √ {_display_data_path(out_path)}")
### Archiving Stage

22
archivebox/misc/serve_static.py
View File

@ -23,7 +23,7 @@ from django.http import StreamingHttpResponse, Http404, HttpResponse, HttpRespon
from django.utils._os import safe_join
from django.utils.http import http_date
from django.utils.translation import gettext as _
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.common import get_config
from archivebox.misc.logging_util import printable_filesize
@ -60,8 +60,9 @@ def _hash_for_path(document_root: Path, rel_path: str) -> str | None:
return file_map.get(rel_path)
def _cache_policy() -> str:
return "public" if SERVER_CONFIG.PUBLIC_SNAPSHOTS else "private"
def _cache_policy(config=None, **config_kwargs) -> str:
config = config or get_config(**config_kwargs)
return "public" if config.PUBLIC_SNAPSHOTS else "private"
def _format_direntry_timestamp(stat_result: os.stat_result) -> str:
@ -604,14 +605,23 @@ def _is_risky_replay_document(fullpath: Path, content_type: str) -> bool:
return any(marker in head for marker in RISKY_REPLAY_MARKERS)
def _apply_archive_replay_headers(response: HttpResponse, *, fullpath: Path, content_type: str, is_archive_replay: bool) -> HttpResponse:
def _apply_archive_replay_headers(
response: HttpResponse,
*,
fullpath: Path,
content_type: str,
is_archive_replay: bool,
config=None,
**config_kwargs,
) -> HttpResponse:
if not is_archive_replay:
return response
response.headers.setdefault("X-Content-Type-Options", "nosniff")
response.headers.setdefault("X-ArchiveBox-Security-Mode", SERVER_CONFIG.SERVER_SECURITY_MODE)
config = config or get_config(**config_kwargs)
response.headers.setdefault("X-ArchiveBox-Security-Mode", config.SERVER_SECURITY_MODE)
if SERVER_CONFIG.SHOULD_NEUTER_RISKY_REPLAY and _is_risky_replay_document(fullpath, content_type):
if config.SHOULD_NEUTER_RISKY_REPLAY and _is_risky_replay_document(fullpath, content_type):
response.headers["Content-Security-Policy"] = (
"sandbox; "
"default-src 'self' data: blob:; "

6
archivebox/misc/shell_welcome_message.py
View File

@ -16,15 +16,15 @@ import archivebox
from benedict import benedict # noqa
from django.utils import timezone # noqa
from datetime import datetime, timedelta # noqa
from django.conf import settings # noqa
from archivebox import CONSTANTS # noqa
from archivebox.cli import * # noqa
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
CONFIG = get_config()
if __name__ == "__main__":
CONFIG = get_config()
# load the rich extension for ipython for pretty printing
# https://rich.readthedocs.io/en/stable/introduction.html#ipython-extension
get_ipython().run_line_magic("load_ext", "rich") # type: ignore # noqa

20
archivebox/misc/system.py
View File

@ -12,7 +12,7 @@ from subprocess import PIPE, Popen, CalledProcessError, CompletedProcess, Timeou
from atomicwrites import atomic_write as lib_atomic_write
from archivebox.config.common import STORAGE_CONFIG
from archivebox.config.common import get_config
from archivebox.misc.util import enforce_types, ExtendedEncoder
IS_WINDOWS = os.name == "nt"
@ -87,7 +87,7 @@ def run(cmd, *args, input=None, capture_output=True, timeout=None, check=False,
@enforce_types
def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool = True) -> None:
def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool = True, config=None, **config_kwargs) -> None:
"""Safe atomic write to filesystem by writing to temp file + atomic rename"""
mode = "wb+" if isinstance(contents, bytes) else "w"
@ -101,7 +101,8 @@ def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool
elif isinstance(contents, (bytes, str)):
f.write(contents)
except OSError as e:
if STORAGE_CONFIG.ENFORCE_ATOMIC_WRITES:
config = config or get_config(**config_kwargs)
if config.ENFORCE_ATOMIC_WRITES:
print(f"[X] OSError: Failed to write {path} with fcntl.F_FULLFSYNC. ({e})")
print(
" You can store the archive/ subfolder on a hard drive or network share that doesn't support support synchronous writes,",
@ -119,11 +120,12 @@ def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool
f.write(contents)
# set file permissions
os.chmod(path, int(STORAGE_CONFIG.OUTPUT_PERMISSIONS, base=8))
config = config or get_config(**config_kwargs)
os.chmod(path, int(config.OUTPUT_PERMISSIONS, base=8))
@enforce_types
def chmod_file(path: str, cwd: str = "") -> None:
def chmod_file(path: str, cwd: str = "", config=None, **config_kwargs) -> None:
"""chmod -R <permissions> <cwd>/<path>"""
root = Path(cwd or os.getcwd()) / path
@ -132,14 +134,16 @@ def chmod_file(path: str, cwd: str = "") -> None:
if not root.is_dir():
# path is just a plain file
os.chmod(root, int(STORAGE_CONFIG.OUTPUT_PERMISSIONS, base=8))
config = config or get_config(**config_kwargs)
os.chmod(root, int(config.OUTPUT_PERMISSIONS, base=8))
else:
config = config or get_config(**config_kwargs)
for subpath in Path(path).glob("**/*"):
if subpath.is_dir():
# directories need execute permissions to be able to list contents
os.chmod(subpath, int(STORAGE_CONFIG.DIR_OUTPUT_PERMISSIONS, base=8))
os.chmod(subpath, int(config.DIR_OUTPUT_PERMISSIONS, base=8))
else:
os.chmod(subpath, int(STORAGE_CONFIG.OUTPUT_PERMISSIONS, base=8))
os.chmod(subpath, int(config.OUTPUT_PERMISSIONS, base=8))
@enforce_types

36
archivebox/misc/util.py
View File

@ -404,16 +404,17 @@ def parse_date(date: Any) -> datetime | None:
@enforce_types
def download_url(url: str, timeout: int | None = None) -> str:
def download_url(url: str, timeout: int | None = None, config=None, **config_kwargs) -> str:
"""Download the contents of a remote url and return the text"""
from archivebox.config.common import ARCHIVING_CONFIG
from archivebox.config.common import get_config
timeout = timeout or ARCHIVING_CONFIG.TIMEOUT
config = config or get_config(**config_kwargs)
timeout = timeout or config.TIMEOUT
session = requests.Session()
if ARCHIVING_CONFIG.COOKIES_FILE and Path(ARCHIVING_CONFIG.COOKIES_FILE).is_file():
cookie_jar = http.cookiejar.MozillaCookieJar(ARCHIVING_CONFIG.COOKIES_FILE)
if config.COOKIES_FILE and Path(config.COOKIES_FILE).is_file():
cookie_jar = http.cookiejar.MozillaCookieJar(config.COOKIES_FILE)
cookie_jar.load(ignore_discard=True, ignore_expires=True)
for cookie in cookie_jar:
if cookie.value is not None:
@ -421,8 +422,8 @@ def download_url(url: str, timeout: int | None = None) -> str:
response = session.get(
url,
headers={"User-Agent": ARCHIVING_CONFIG.USER_AGENT},
verify=ARCHIVING_CONFIG.CHECK_SSL_VALIDITY,
headers={"User-Agent": config.USER_AGENT},
verify=config.CHECK_SSL_VALIDITY,
timeout=timeout,
)
@ -440,19 +441,20 @@ def download_url(url: str, timeout: int | None = None) -> str:
@enforce_types
def get_headers(url: str, timeout: int | None = None) -> str:
def get_headers(url: str, timeout: int | None = None, config=None, **config_kwargs) -> str:
"""Download the contents of a remote url and return the headers"""
# TODO: get rid of this and use an abx pluggy hook instead
from archivebox.config.common import ARCHIVING_CONFIG
from archivebox.config.common import get_config
timeout = timeout or ARCHIVING_CONFIG.TIMEOUT
config = config or get_config(**config_kwargs)
timeout = timeout or config.TIMEOUT
try:
response = requests.head(
url,
headers={"User-Agent": ARCHIVING_CONFIG.USER_AGENT},
verify=ARCHIVING_CONFIG.CHECK_SSL_VALIDITY,
headers={"User-Agent": config.USER_AGENT},
verify=config.CHECK_SSL_VALIDITY,
timeout=timeout,
allow_redirects=True,
)
@ -463,8 +465,8 @@ def get_headers(url: str, timeout: int | None = None) -> str:
except RequestException:
response = requests.get(
url,
headers={"User-Agent": ARCHIVING_CONFIG.USER_AGENT},
verify=ARCHIVING_CONFIG.CHECK_SSL_VALIDITY,
headers={"User-Agent": config.USER_AGENT},
verify=config.CHECK_SSL_VALIDITY,
timeout=timeout,
stream=True,
)
@ -692,7 +694,7 @@ for url_str, num_urls in _test_url_strs.items():
### Chrome Helpers
def chrome_cleanup():
def chrome_cleanup(config=None, **config_kwargs):
"""
Cleans up any state or runtime files that Chrome leaves behind when killed by
a timeout or other error. Handles:
@ -713,9 +715,9 @@ def chrome_cleanup():
# Also clean up the active persona's explicit CHROME_USER_DATA_DIR if set
# (in case it's a custom path not under PERSONAS_DIR)
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
config = get_config()
config = config or get_config(**config_kwargs)
chrome_user_data_dir = config.get("CHROME_USER_DATA_DIR")
if chrome_user_data_dir:
singleton_lock = Path(chrome_user_data_dir) / "SingletonLock"

4
archivebox/personas/importers.py
View File

@ -603,13 +603,13 @@ def export_browser_state(
return False, None, "Missing browser source."
from abx_plugins import get_plugins_dir
from archivebox.config.common import STORAGE_CONFIG
from archivebox.config.common import get_config
state_script = Path(__file__).with_name("export_browser_state.js")
if not state_script.exists():
return False, None, f"Browser state export script not found at {state_script}"
node_modules_dir = STORAGE_CONFIG.LIB_DIR / "npm" / "node_modules"
node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules"
chrome_plugin_dir = Path(get_plugins_dir()).resolve()
env = os.environ.copy()

66
archivebox/search/__init__.py
View File

@ -14,13 +14,15 @@ Search backends must provide a search.py module with:
__package__ = "archivebox.search"
import os
from contextlib import contextmanager
from typing import Any
from django.db.models import Case, IntegerField, QuerySet, Value, When
from archivebox.misc.util import enforce_types
from archivebox.misc.logging import stderr
from archivebox.config.common import SEARCH_BACKEND_CONFIG
from archivebox.config.common import get_config
# Cache discovered backends to avoid repeated filesystem scans
@ -28,13 +30,34 @@ _search_backends_cache: dict | None = None
SEARCH_MODES = ("meta", "contents", "deep")
def get_default_search_mode() -> str:
return "meta" if SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE == "ripgrep" else "contents"
@contextmanager
def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
"""Expose ArchiveBox collection roots to in-process search backends."""
config = config or get_config(**config_kwargs)
updates = {
"DATA_DIR": str(config.DATA_DIR),
"SNAP_DIR": str(config.USERS_DIR),
}
previous = {key: os.environ.get(key) for key in updates}
os.environ.update(updates)
try:
yield
finally:
for key, value in previous.items():
if value is None:
os.environ.pop(key, None)
else:
os.environ[key] = value
def get_search_mode(search_mode: str | None) -> str:
def get_default_search_mode(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
config = config or get_config(**config_kwargs)
return "meta" if config.SEARCH_BACKEND_ENGINE == "ripgrep" else "contents"
def get_search_mode(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
normalized = (search_mode or "").strip().lower()
return normalized if normalized in SEARCH_MODES else get_default_search_mode()
return normalized if normalized in SEARCH_MODES else get_default_search_mode(config=config, **config_kwargs)
def prioritize_metadata_matches(
@ -90,7 +113,7 @@ def get_available_backends() -> dict:
return _search_backends_cache
def get_backend() -> Any:
def get_backend(config: dict[str, Any] | None = None, **config_kwargs: Any) -> Any:
"""
Get the configured search backend module.
@ -99,7 +122,8 @@ def get_backend() -> Any:
Falls back to 'ripgrep' if configured backend is not found.
"""
backend_name = SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE
config = config or get_config(**config_kwargs)
backend_name = config.SEARCH_BACKEND_ENGINE
backends = get_available_backends()
if backend_name in backends:
@ -117,7 +141,7 @@ def get_backend() -> Any:
@enforce_types
def query_search_index(query: str, search_mode: str | None = None) -> QuerySet:
def query_search_index(query: str, search_mode: str | None = None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> QuerySet:
"""
Search for snapshots matching the query.
@ -125,16 +149,17 @@ def query_search_index(query: str, search_mode: str | None = None) -> QuerySet:
"""
from archivebox.core.models import Snapshot
if not SEARCH_BACKEND_CONFIG.USE_SEARCHING_BACKEND:
config = config or get_config(**config_kwargs)
if not config.USE_SEARCHING_BACKEND:
return Snapshot.objects.none()
search_mode = "contents" if search_mode is None else get_search_mode(search_mode)
search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config)
if search_mode == "meta":
return Snapshot.objects.none()
backends = get_available_backends()
backend_names: list[str] = []
configured_backend = SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE
configured_backend = config.SEARCH_BACKEND_ENGINE
if search_mode == "deep":
if "ripgrep" in backends:
backend_names.append("ripgrep")
@ -154,10 +179,11 @@ def query_search_index(query: str, search_mode: str | None = None) -> QuerySet:
for backend_name in backend_names:
backend = backends[backend_name]
try:
if backend_name == "ripgrep":
snapshot_pks.extend(backend.search(query, search_mode=search_mode))
else:
snapshot_pks.extend(backend.search(query))
with search_backend_env(config=config):
if backend_name == "ripgrep":
snapshot_pks.extend(backend.search(query, search_mode=search_mode))
else:
snapshot_pks.extend(backend.search(query))
successful_backends += 1
except Exception as err:
errors.append(err)
@ -177,18 +203,20 @@ def query_search_index(query: str, search_mode: str | None = None) -> QuerySet:
@enforce_types
def flush_search_index(snapshots: QuerySet) -> None:
def flush_search_index(snapshots: QuerySet, config: dict[str, Any] | None = None, **config_kwargs: Any) -> None:
"""
Remove snapshots from the search index.
"""
if not SEARCH_BACKEND_CONFIG.USE_INDEXING_BACKEND or not snapshots:
config = config or get_config(**config_kwargs)
if not config.USE_INDEXING_BACKEND or not snapshots:
return
backend = get_backend()
backend = get_backend(config=config)
snapshot_pks = [str(pk) for pk in snapshots.values_list("pk", flat=True)]
try:
backend.flush(snapshot_pks)
with search_backend_env(config=config):
backend.flush(snapshot_pks)
except Exception as err:
stderr()
stderr(

37
archivebox/services/archive_result_service.py
View File

@ -242,29 +242,30 @@ class ArchiveResultService(BaseService):
process_query = process_query.filter(pid=process_started.pid)
process = await process_query.order_by("-modified_at").afirst()
result, _created = await ArchiveResult.objects.aget_or_create(
start_ts = parse_event_datetime(event.start_ts)
end_ts = parse_event_datetime(event.end_ts) or timezone.now()
defaults = {
"status": _normalize_status(event.status),
"output_str": event.output_str,
"output_json": event.output_json,
"output_files": output_files,
"output_size": output_size,
"output_mimetypes": output_mimetypes,
"start_ts": start_ts or timezone.now(),
"end_ts": end_ts,
}
if process is not None:
defaults["process"] = process
if event.error:
defaults["notes"] = event.error
result, _created = await ArchiveResult.objects.aupdate_or_create(
snapshot=snapshot,
plugin=event.plugin,
hook_name=event.hook_name,
defaults={
"status": ArchiveResult.StatusChoices.STARTED,
"process": process,
},
defaults=defaults,
)
result.process = process or result.process
result.status = _normalize_status(event.status)
result.output_str = event.output_str
result.output_json = event.output_json
result.output_files = output_files
result.output_size = output_size
result.output_mimetypes = output_mimetypes
result.start_ts = parse_event_datetime(event.start_ts) or result.start_ts or timezone.now()
result.end_ts = parse_event_datetime(event.end_ts) or timezone.now()
if event.error:
result.notes = event.error
await result.asave()
if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
next_title = _extract_snapshot_title(str(snapshot.output_dir), event.plugin, result.output_str, snapshot_url=snapshot.url)
if next_title and _should_update_snapshot_title(snapshot.title or "", next_title, snapshot_url=snapshot.url):

11
archivebox/services/crawl_service.py
View File

@ -1,6 +1,8 @@
from __future__ import annotations
from asgiref.sync import sync_to_async
from django.utils import timezone
from abx_dl.events import CrawlCleanupEvent, CrawlCompletedEvent, CrawlSetupEvent, CrawlStartEvent
from abx_dl.services.base import BaseService
@ -42,9 +44,12 @@ class CrawlService(BaseService):
is_finished = await sync_to_async(crawl.is_finished, thread_sensitive=True)()
if is_finished:
crawl.status = Crawl.StatusChoices.SEALED
crawl.retry_at = None
elif crawl.status != Crawl.StatusChoices.SEALED:
crawl.status = Crawl.StatusChoices.STARTED
crawl.retry_at = None
crawl.retry_at = timezone.now()
else:
crawl.retry_at = None
await crawl.asave(update_fields=["status", "retry_at", "modified_at"])
async def on_CrawlCompletedEvent__save_to_db(self, event: CrawlCompletedEvent) -> None:
@ -55,7 +60,9 @@ class CrawlService(BaseService):
if not is_finished:
if crawl.status != Crawl.StatusChoices.SEALED:
crawl.status = Crawl.StatusChoices.STARTED
crawl.retry_at = None
crawl.retry_at = timezone.now()
else:
crawl.retry_at = None
await crawl.asave(update_fields=["status", "retry_at", "modified_at"])
return

466
archivebox/services/runner.py
View File

@ -8,6 +8,7 @@ import shutil
import subprocess
import sys
import time
from collections.abc import Mapping
from contextlib import nullcontext
from pathlib import Path
from tempfile import TemporaryDirectory
@ -19,8 +20,11 @@ from rich.console import Console
from abx_dl.events import (
BinaryRequestEvent,
CrawlAbortEvent,
CrawlCleanupEvent,
CrawlCompletedEvent,
CrawlEvent,
CrawlSetupEvent,
CrawlStartEvent,
InstallEvent,
MachineEvent,
@ -44,7 +48,11 @@ from abx_dl.orchestrator import (
from abx_dl.services.process_service import ProcessService as HookProcessService
from abx_dl.services.binary_service import BinaryService as HookBinaryService
from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService
from abxbus.event_bus import EventBus
from abxbus import BaseEvent
from abxbus.event_bus import EventBus, get_current_event, in_handler_context
from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError
from archivebox.config.configset import BaseConfigSet
from .archive_result_service import ArchiveResultService
from .binary_service import BinaryService
@ -66,7 +74,15 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str
return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
def _normalize_runtime_config(config: dict[str, Any]) -> dict[str, Any]:
def _normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | None) -> dict[str, Any]:
if config is None:
return {}
if isinstance(config, BaseConfigSet):
config = config.model_dump(mode="json")
elif isinstance(config, str):
config = json.loads(config)
else:
config = dict(config)
return {key: value for key, value in json.loads(json.dumps(config, default=str)).items() if value is not None}
@ -78,22 +94,28 @@ def _runner_task_context() -> contextvars.Context:
return context
def _is_external_task_cancelled(error: asyncio.CancelledError) -> bool:
return not isinstance(error, (EventHandlerAbortedError, EventHandlerCancelledError))
async def _emit_machine_config(
bus,
*,
config: dict[str, Any],
derived_config: dict[str, Any],
parent_event=None,
) -> None:
user_config = _normalize_runtime_config(config)
derived_machine_config = _normalize_runtime_config(derived_config)
await bus.emit(
emitter = parent_event.emit if parent_event is not None else bus.emit
await emitter(
MachineEvent(
config=user_config,
config_type="user",
),
).now()
if derived_machine_config:
await bus.emit(
await emitter(
MachineEvent(
config=derived_machine_config,
config_type="derived",
@ -185,6 +207,22 @@ class CrawlRunner:
self.crawl_output_dir = ""
self._live_stream = None
self.root_crawl_event_id: str | None = None
self.root_crawl_start_event_id: str | None = None
self._skip_wait_until_idle = False
async def crawl_is_cancelled(self) -> bool:
from archivebox.crawls.models import Crawl
return await Crawl.objects.filter(id=self.crawl.id, status=Crawl.StatusChoices.SEALED).aexists()
async def watch_for_cancelled_crawl(self, parent_event: BaseEvent, *, poll_interval: float = 1.0) -> None:
while True:
await asyncio.sleep(poll_interval)
if not await self.crawl_is_cancelled():
continue
abort_event = parent_event.emit(CrawlAbortEvent())
await _run_event_now(abort_event, abort_event.event_timeout)
return
def runtime_plugins(self) -> dict[str, Plugin]:
return filter_plugins(self.plugins, self.selected_plugins, include_providers=True) if self.selected_plugins else self.plugins
@ -195,6 +233,7 @@ class CrawlRunner:
runtime="archivebox",
crawl_id=str(self.crawl.id),
)
root_snapshot_id: str | None = None
try:
snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)()
live_ui = self._create_live_ui()
@ -210,14 +249,11 @@ class CrawlRunner:
)
if snapshot_ids:
root_snapshot_id = snapshot_ids[0]
await self.run_crawl_setup(root_snapshot_id)
for snapshot_id in snapshot_ids:
await self.enqueue_snapshot(snapshot_id)
await self.wait_for_snapshot_tasks()
await self.run_crawl_cleanup(root_snapshot_id)
await self.run_crawl(root_snapshot_id, snapshot_ids)
finally:
await heartbeat.stop()
await self.bus.wait_until_idle()
if not self._skip_wait_until_idle:
await self.bus.wait_until_idle()
if self._live_stream is not None:
try:
self._live_stream.close()
@ -230,27 +266,82 @@ class CrawlRunner:
task = self.snapshot_tasks.get(snapshot_id)
if task is not None and not task.done():
return
task = asyncio.create_task(self.run_snapshot(snapshot_id), context=_runner_task_context())
current_event = get_current_event()
if isinstance(current_event, CrawlStartEvent):
task = asyncio.create_task(self.run_snapshot(snapshot_id))
elif in_handler_context():
return
else:
task = asyncio.create_task(self.run_snapshot(snapshot_id), context=_runner_task_context())
self.snapshot_tasks[snapshot_id] = task
async def wait_for_snapshot_tasks(self) -> None:
task_errors: list[Exception] = []
while True:
pending_tasks: list[asyncio.Task[None]] = []
for snapshot_id, task in list(self.snapshot_tasks.items()):
if task.done():
if self.snapshot_tasks.get(snapshot_id) is task:
self.snapshot_tasks.pop(snapshot_id, None)
task.result()
try:
task.result()
except asyncio.CancelledError as err:
if _is_external_task_cancelled(err):
raise
await sync_to_async(recover_orphaned_snapshots, thread_sensitive=True)()
await sync_to_async(recover_orphaned_crawls, thread_sensitive=True)()
except Exception as err:
task_errors.append(err)
continue
pending_tasks.append(task)
if not pending_tasks:
return
await self.enqueue_pending_snapshots_from_projection()
if not self.snapshot_tasks:
if task_errors:
if len(task_errors) == 1:
raise task_errors[0]
raise ExceptionGroup("One or more snapshot tasks failed", task_errors)
return
continue
done, _pending = await asyncio.wait(pending_tasks, return_when=asyncio.FIRST_COMPLETED)
for task in done:
task.result()
for snapshot_id, tracked_task in list(self.snapshot_tasks.items()):
if tracked_task is task:
self.snapshot_tasks.pop(snapshot_id, None)
break
try:
task.result()
except asyncio.CancelledError as err:
if _is_external_task_cancelled(err):
raise
await sync_to_async(recover_orphaned_snapshots, thread_sensitive=True)()
await sync_to_async(recover_orphaned_crawls, thread_sensitive=True)()
except Exception as err:
task_errors.append(err)
await self.enqueue_pending_snapshots_from_projection()
async def enqueue_pending_snapshots_from_projection(self) -> None:
from archivebox.core.models import Snapshot
if not isinstance(get_current_event(), CrawlStartEvent):
return
pending_snapshot_ids = await sync_to_async(
lambda: [
str(snapshot_id)
for snapshot_id in self.crawl.snapshot_set.exclude(status=Snapshot.StatusChoices.SEALED)
.filter(retry_at__lte=timezone.now())
.order_by("depth", "created_at")
.values_list("id", flat=True)
],
thread_sensitive=True,
)()
for snapshot_id in pending_snapshot_ids:
if snapshot_id not in self.snapshot_tasks:
await self.enqueue_snapshot(snapshot_id)
def load_run_state(self) -> list[str]:
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
from archivebox.hooks import discover_hooks
from archivebox.machine.models import Machine, NetworkInterface, Process, _sanitize_machine_config
@ -361,7 +452,7 @@ class CrawlRunner:
def load_snapshot_payload(self, snapshot_id: str) -> dict[str, Any]:
from archivebox.core.models import Snapshot
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
config = get_config(crawl=self.crawl, snapshot=snapshot)
@ -388,7 +479,7 @@ class CrawlRunner:
"depth": snapshot.depth,
"status": snapshot.status,
"output_dir": str(snapshot.output_dir),
"config": config,
"config": _normalize_runtime_config(config),
"_snapshot": snapshot,
}
@ -431,11 +522,11 @@ class CrawlRunner:
if has_capacity:
continue
break
if self.process_discovered_snapshots_inline:
if self.process_discovered_snapshots_inline and isinstance(get_current_event(), CrawlStartEvent):
await self.enqueue_snapshot(str(child_snapshot.id))
async def run_crawl_setup(self, snapshot_id: str) -> None:
snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(snapshot_id)
async def run_crawl(self, root_snapshot_id: str, snapshot_ids: list[str]) -> None:
snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(root_snapshot_id)
config = _normalize_runtime_config(snapshot["config"])
derived_config = _normalize_runtime_config(self.derived_config)
output_dir = Path(self.crawl_output_dir)
@ -449,32 +540,19 @@ class CrawlRunner:
setup_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("CrawlSetup")]
crawl_setup_phase_timeout = compute_phase_timeout(setup_hooks, config)
install_phase_timeout = compute_install_phase_timeout(get_install_plugins(plugins), config)
await _emit_machine_config(self.bus, config=config, derived_config=derived_config)
setup_abx_services(
self.bus,
plugins=plugins,
url=snapshot["url"],
snapshot=abx_snapshot,
output_dir=output_dir,
install_enabled=False,
crawl_setup_enabled=True,
crawl_start_enabled=False,
snapshot_cleanup_enabled=False,
crawl_cleanup_enabled=False,
crawl_setup_phase_timeout=crawl_setup_phase_timeout,
snapshot_phase_timeout=0.0,
snapshot_cleanup_phase_timeout=0.0,
crawl_cleanup_phase_timeout=crawl_setup_phase_timeout,
persist_derived=False,
auto_install=True,
emit_jsonl=False,
MachineService=None,
BinaryService=HookBinaryService,
ProcessService=None,
ArchiveResultService=None,
TagService=None,
SnapshotService=None,
snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")]
max_snapshot_count = max(1, int(self.crawl.max_urls or len(snapshot_ids) or 1))
snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config) * max_snapshot_count
crawl_cleanup_phase_timeout = crawl_setup_phase_timeout
crawl_lifecycle_timeout = (
crawl_setup_phase_timeout
+ snapshot_phase_timeout
+ crawl_cleanup_phase_timeout
+ CrawlCompletedEvent.model_fields["event_timeout"].default
+ 30.0
)
await _emit_machine_config(self.bus, config=config, derived_config=derived_config)
install_cancel_watcher: asyncio.Task[None] | None = None
install_event = self.bus.emit(
InstallEvent(
url=snapshot["url"],
@ -484,16 +562,135 @@ class CrawlRunner:
event_handler_slow_timeout=slow_warning_timeout(install_phase_timeout),
),
)
await _run_event_now(install_event, install_phase_timeout)
async def on_archivebox_InstallEvent(event: InstallEvent) -> None:
nonlocal install_cancel_watcher
if event.event_id != install_event.event_id:
return
install_cancel_watcher = asyncio.create_task(self.watch_for_cancelled_crawl(event))
on_archivebox_InstallEvent.__name__ = "on_archivebox_InstallEvent__cancel_watcher"
self.bus.on(InstallEvent, on_archivebox_InstallEvent)
setup_abx_services(
self.bus,
plugins=plugins,
url=snapshot["url"],
snapshot=abx_snapshot,
output_dir=output_dir,
install_enabled=False,
crawl_setup_enabled=True,
crawl_event_enabled=False,
crawl_start_enabled=False,
snapshot_cleanup_enabled=False,
crawl_cleanup_enabled=True,
crawl_completed_enabled=False,
crawl_setup_phase_timeout=crawl_setup_phase_timeout,
snapshot_phase_timeout=0.0,
snapshot_cleanup_phase_timeout=0.0,
crawl_cleanup_phase_timeout=crawl_setup_phase_timeout,
persist_derived=False,
auto_install=True,
emit_jsonl=False,
abort_requested=self.crawl_is_cancelled,
MachineService=None,
BinaryService=HookBinaryService,
ProcessService=None,
ArchiveResultService=None,
TagService=None,
SnapshotService=None,
)
try:
await _run_event_now(install_event, install_phase_timeout)
finally:
if install_cancel_watcher is not None:
install_cancel_watcher.cancel()
await asyncio.gather(install_cancel_watcher, return_exceptions=True)
async def on_archivebox_CrawlStartEvent(event: CrawlStartEvent) -> None:
if event.event_id != self.root_crawl_start_event_id:
return
for snapshot_id in snapshot_ids:
await self.enqueue_snapshot(snapshot_id)
await self.wait_for_snapshot_tasks()
async def on_archivebox_CrawlEvent(event: CrawlEvent) -> None:
if event.event_id != self.root_crawl_event_id:
return
cancel_watcher = asyncio.create_task(self.watch_for_cancelled_crawl(event))
try:
try:
if not await self.crawl_is_cancelled():
await _run_event_now(
event.emit(
CrawlSetupEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
output_dir=str(output_dir),
event_timeout=crawl_setup_phase_timeout,
event_handler_slow_timeout=slow_warning_timeout(crawl_setup_phase_timeout),
),
),
crawl_setup_phase_timeout,
)
if not await self.crawl_is_cancelled():
crawl_start_event = event.emit(
CrawlStartEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
output_dir=str(output_dir),
event_timeout=0,
event_handler_timeout=0,
event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout),
),
)
self.root_crawl_start_event_id = crawl_start_event.event_id
await _run_event_now(crawl_start_event, None)
finally:
await _run_event_now(
event.emit(
CrawlCleanupEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
output_dir=str(output_dir),
event_timeout=crawl_setup_phase_timeout,
event_handler_slow_timeout=slow_warning_timeout(crawl_setup_phase_timeout),
),
),
crawl_setup_phase_timeout,
)
finally:
cancel_watcher.cancel()
await asyncio.gather(cancel_watcher, return_exceptions=True)
await _run_event_now(
event.emit(
CrawlCompletedEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
output_dir=str(output_dir),
),
),
CrawlCompletedEvent.model_fields["event_timeout"].default,
)
on_archivebox_CrawlStartEvent.__name__ = "on_archivebox_CrawlStartEvent__run_snapshots"
on_archivebox_CrawlEvent.__name__ = "on_archivebox_CrawlEvent__run_recursive_crawl"
self.bus.on(CrawlStartEvent, on_archivebox_CrawlStartEvent)
self.bus.on(CrawlEvent, on_archivebox_CrawlEvent)
crawl_event = CrawlEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
output_dir=str(output_dir),
event_timeout=crawl_setup_phase_timeout,
event_handler_slow_timeout=slow_warning_timeout(crawl_setup_phase_timeout),
event_timeout=0,
event_handler_timeout=0,
event_handler_slow_timeout=slow_warning_timeout(crawl_lifecycle_timeout),
)
self.root_crawl_event_id = crawl_event.event_id
await _run_event_now(self.bus.emit(crawl_event), crawl_setup_phase_timeout)
emitted_crawl_event = self.bus.emit(crawl_event)
await _run_event_now(emitted_crawl_event, None)
if await self.crawl_is_cancelled():
self._skip_wait_until_idle = True
return
for plugin, hook in setup_hooks:
if hook.is_background:
continue
@ -523,96 +720,62 @@ class CrawlRunner:
if completed_process.status == "failed":
raise RuntimeError(f"Crawl setup hook {plugin.name}:{hook.name} failed")
async def run_crawl_cleanup(self, snapshot_id: str) -> None:
snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(snapshot_id)
if self.root_crawl_event_id is None:
return
config = _normalize_runtime_config(snapshot["config"])
output_dir = Path(self.crawl_output_dir)
plugins = self.runtime_plugins()
setup_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("CrawlSetup")]
crawl_cleanup_phase_timeout = compute_phase_timeout(setup_hooks, config)
await _run_event_now(
self.bus.emit(
CrawlCleanupEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
output_dir=str(output_dir),
event_parent_id=self.root_crawl_event_id,
event_timeout=crawl_cleanup_phase_timeout,
event_handler_slow_timeout=slow_warning_timeout(crawl_cleanup_phase_timeout),
),
),
crawl_cleanup_phase_timeout,
)
async def run_snapshot(self, snapshot_id: str) -> None:
async with self.snapshot_semaphore:
crawl_start_event = get_current_event()
if not isinstance(crawl_start_event, CrawlStartEvent):
raise RuntimeError("Snapshot events must be emitted from a CrawlStartEvent handler")
snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(snapshot_id)
if snapshot["status"] == "sealed":
return
if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() == "max_size":
await sync_to_async(self.seal_snapshot_due_to_limit, thread_sensitive=True)(snapshot_id)
return
try:
config = _normalize_runtime_config(snapshot["config"])
derived_config = _normalize_runtime_config(self.derived_config)
output_dir = Path(snapshot["output_dir"])
plugins = self.runtime_plugins()
abx_snapshot = AbxSnapshot(
id=snapshot["id"],
url=snapshot["url"],
depth=int(snapshot["depth"]),
crawl_id=str(self.crawl.id),
)
snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")]
snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config)
await _emit_machine_config(self.bus, config=config, derived_config=derived_config)
HookSnapshotService(
self.bus,
url=snapshot["url"],
snapshot=abx_snapshot,
output_dir=output_dir,
plugins=plugins,
snapshot_phase_timeout=snapshot_phase_timeout,
snapshot_cleanup_enabled=True,
snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
)
crawl_start_event = CrawlStartEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
output_dir=str(output_dir),
event_timeout=snapshot_phase_timeout,
event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout),
)
await _run_event_now(self.bus.emit(crawl_start_event), snapshot_phase_timeout)
snapshot_event = SnapshotEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
output_dir=str(output_dir),
depth=int(snapshot["depth"]),
event_parent_id=crawl_start_event.event_id,
event_timeout=snapshot_phase_timeout,
event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout),
)
emitted_snapshot_event = self.bus.emit(snapshot_event)
await _run_event_now(emitted_snapshot_event, snapshot_phase_timeout)
completed_snapshot = await self.bus.find(
SnapshotCompletedEvent,
child_of=emitted_snapshot_event,
past=True,
future=snapshot_phase_timeout,
)
if completed_snapshot is None:
raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
await completed_snapshot.now(timeout=snapshot_phase_timeout)
await completed_snapshot.wait(timeout=snapshot_phase_timeout)
await completed_snapshot.event_results_list()
await self.enqueue_discovered_snapshots_from_outputs(snapshot)
finally:
current_task = asyncio.current_task()
if current_task is not None and self.snapshot_tasks.get(snapshot_id) is current_task:
self.snapshot_tasks.pop(snapshot_id, None)
config = _normalize_runtime_config(snapshot["config"])
derived_config = _normalize_runtime_config(self.derived_config)
output_dir = Path(snapshot["output_dir"])
plugins = self.runtime_plugins()
abx_snapshot = AbxSnapshot(
id=snapshot["id"],
url=snapshot["url"],
depth=int(snapshot["depth"]),
crawl_id=str(self.crawl.id),
)
snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")]
snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config)
await _emit_machine_config(self.bus, config=config, derived_config=derived_config, parent_event=crawl_start_event)
HookSnapshotService(
self.bus,
url=snapshot["url"],
snapshot=abx_snapshot,
output_dir=output_dir,
plugins=plugins,
snapshot_phase_timeout=snapshot_phase_timeout,
snapshot_cleanup_enabled=True,
snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
abort_requested=self.crawl_is_cancelled,
)
snapshot_event = SnapshotEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
output_dir=str(output_dir),
depth=int(snapshot["depth"]),
event_timeout=snapshot_phase_timeout,
event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout),
)
emitted_snapshot_event = crawl_start_event.emit(snapshot_event)
await _run_event_now(emitted_snapshot_event, snapshot_phase_timeout)
completed_snapshot = await self.bus.find(
SnapshotCompletedEvent,
child_of=emitted_snapshot_event,
past=True,
future=snapshot_phase_timeout,
)
if completed_snapshot is None:
raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
await completed_snapshot.wait(timeout=snapshot_phase_timeout)
await completed_snapshot.event_results_list()
await self.enqueue_discovered_snapshots_from_outputs(snapshot)
def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None:
from archivebox.core.models import Snapshot
@ -646,7 +809,7 @@ def run_crawl(
async def _run_binary(binary_id: str) -> None:
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
from archivebox.machine.models import Binary, Machine
binary = await Binary.objects.aget(id=binary_id)
@ -698,7 +861,7 @@ def run_binary(binary_id: str) -> None:
async def _run_install(plugin_names: list[str] | None = None) -> None:
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
from archivebox.machine.models import Machine
plugins = discover_plugins()
@ -948,10 +1111,26 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) ->
run_crawl(str(queued_crawl.id), process_discovered_snapshots_inline=True)
continue
pending = Crawl.objects.filter(
retry_at__lte=timezone.now(),
status=Crawl.StatusChoices.STARTED,
)
if crawl_id:
pending = pending.filter(id=crawl_id)
pending = pending.order_by("retry_at", "created_at")
crawl = pending.first()
if crawl is not None:
if not crawl.claim_processing_lock(lock_seconds=60):
continue
run_crawl(str(crawl.id), process_discovered_snapshots_inline=True)
continue
if crawl_id is None:
snapshot = (
Snapshot.objects.filter(retry_at__lte=timezone.now())
.exclude(status=Snapshot.StatusChoices.SEALED)
.exclude(crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED])
.select_related("crawl")
.order_by("retry_at", "created_at")
.first()
@ -981,22 +1160,7 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) ->
run_binary(str(binary.id))
continue
pending = Crawl.objects.filter(
retry_at__lte=timezone.now(),
status=Crawl.StatusChoices.STARTED,
)
if crawl_id:
pending = pending.filter(id=crawl_id)
pending = pending.order_by("retry_at", "created_at")
crawl = pending.first()
if crawl is None:
if daemon:
time.sleep(2.0)
continue
return 0
if not crawl.claim_processing_lock(lock_seconds=60):
if daemon:
time.sleep(2.0)
continue
run_crawl(str(crawl.id), process_discovered_snapshots_inline=True)
return 0

10
archivebox/tests/conftest.py
View File

@ -20,6 +20,8 @@ PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve()
# Force ArchiveBox imports to see a temp DATA_DIR during test collection.
os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
os.environ.pop("ARCHIVE_DIR", None)
os.environ.pop("USERS_DIR", None)
os.environ.pop("CRAWL_DIR", None)
os.environ.pop("SNAP_DIR", None)
@ -40,7 +42,7 @@ def _assert_safe_runtime_paths(*, cwd: Path | None = None, env: dict[str, str] |
if cwd is not None:
_assert_not_repo_path(cwd, label="cwd")
for key in ("DATA_DIR", "CRAWL_DIR", "SNAP_DIR"):
for key in ("DATA_DIR", "ARCHIVE_DIR", "USERS_DIR", "CRAWL_DIR", "SNAP_DIR"):
value = (env or {}).get(key)
if value:
_assert_not_repo_path(Path(value), label=key)
@ -139,6 +141,8 @@ def isolate_test_runtime(tmp_path, monkeypatch):
original_popen = subprocess.Popen
os.chdir(tmp_path)
os.environ.pop("DATA_DIR", None)
os.environ.pop("ARCHIVE_DIR", None)
os.environ.pop("USERS_DIR", None)
os.environ.pop("CRAWL_DIR", None)
os.environ.pop("SNAP_DIR", None)
@ -218,6 +222,8 @@ def run_archivebox_cmd_cwd(
_assert_not_repo_path(cwd, label="cwd")
base_env = os.environ.copy()
base_env.pop("DATA_DIR", None)
base_env.pop("ARCHIVE_DIR", None)
base_env.pop("USERS_DIR", None)
base_env.pop("CRAWL_DIR", None)
base_env.pop("SNAP_DIR", None)
base_env["USE_COLOR"] = "False"
@ -258,6 +264,8 @@ def run_python_cwd(
_assert_not_repo_path(cwd, label="cwd")
base_env = os.environ.copy()
base_env.pop("DATA_DIR", None)
base_env.pop("ARCHIVE_DIR", None)
base_env.pop("USERS_DIR", None)
base_env.pop("CRAWL_DIR", None)
base_env.pop("SNAP_DIR", None)
_assert_safe_runtime_paths(cwd=cwd, env=base_env)

35
archivebox/tests/test_add_view.py
View File

@ -4,7 +4,6 @@ import pytest
from django.contrib.auth import get_user_model
from django.urls import reverse
from archivebox.config.common import SERVER_CONFIG, SEARCH_BACKEND_CONFIG
from archivebox.core.models import Tag
from archivebox.crawls.models import Crawl
@ -26,7 +25,7 @@ def admin_user(db):
def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, monkeypatch):
monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True)
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
response = client.get(reverse("add"), HTTP_HOST=WEB_HOST)
body = response.content.decode()
@ -52,8 +51,8 @@ def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, m
def test_add_view_checks_configured_search_backend_by_default(client, monkeypatch):
monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True)
monkeypatch.setattr(SEARCH_BACKEND_CONFIG, "SEARCH_BACKEND_ENGINE", "sqlite")
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "sqlite")
response = client.get(reverse("add"), HTTP_HOST=WEB_HOST)
body = response.content.decode()
@ -67,7 +66,7 @@ def test_add_view_checks_configured_search_backend_by_default(client, monkeypatc
def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_user, monkeypatch):
monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True)
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
client.force_login(admin_user)
response = client.post(
@ -107,7 +106,7 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
def test_add_view_starts_background_runner_after_creating_crawl(client, admin_user, monkeypatch):
monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True)
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
client.force_login(admin_user)
runner_calls = []
@ -137,7 +136,7 @@ def test_add_view_starts_background_runner_after_creating_crawl(client, admin_us
def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkeypatch):
monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True)
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
client.force_login(admin_user)
response = client.post(
@ -185,7 +184,7 @@ def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkey
def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_user, monkeypatch):
monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True)
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
client.force_login(admin_user)
response = client.post(
@ -225,7 +224,7 @@ def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_us
def test_add_view_exposes_api_token_for_tag_widget_autocomplete(client, admin_user, monkeypatch):
monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True)
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
client.force_login(admin_user)
response = client.get(reverse("add"), HTTP_HOST=WEB_HOST)
@ -234,9 +233,9 @@ def test_add_view_exposes_api_token_for_tag_widget_autocomplete(client, admin_us
assert b"window.ARCHIVEBOX_API_KEY" in response.content
def test_tags_autocomplete_requires_auth_when_public_snapshots_list_disabled(client, settings):
settings.PUBLIC_SNAPSHOTS_LIST = False
settings.PUBLIC_INDEX = False
def test_tags_autocomplete_requires_auth_when_public_snapshots_list_disabled(client, monkeypatch):
monkeypatch.setenv("PUBLIC_SNAPSHOTS_LIST", "false")
monkeypatch.setenv("PUBLIC_INDEX", "false")
Tag.objects.create(name="archive")
response = client.get(
@ -248,9 +247,9 @@ def test_tags_autocomplete_requires_auth_when_public_snapshots_list_disabled(cli
assert response.status_code == 401
def test_tags_autocomplete_allows_public_access_when_public_snapshots_list_enabled(client, settings):
settings.PUBLIC_SNAPSHOTS_LIST = True
settings.PUBLIC_INDEX = False
def test_tags_autocomplete_allows_public_access_when_public_snapshots_list_enabled(client, monkeypatch):
monkeypatch.setenv("PUBLIC_SNAPSHOTS_LIST", "true")
monkeypatch.setenv("PUBLIC_INDEX", "false")
Tag.objects.create(name="archive")
response = client.get(
@ -263,9 +262,9 @@ def test_tags_autocomplete_allows_public_access_when_public_snapshots_list_enabl
assert response.json()["tags"][0]["name"] == "archive"
def test_tags_autocomplete_allows_authenticated_user_when_public_snapshots_list_disabled(client, admin_user, settings):
settings.PUBLIC_SNAPSHOTS_LIST = False
settings.PUBLIC_INDEX = False
def test_tags_autocomplete_allows_authenticated_user_when_public_snapshots_list_disabled(client, admin_user, monkeypatch):
monkeypatch.setenv("PUBLIC_SNAPSHOTS_LIST", "false")
monkeypatch.setenv("PUBLIC_INDEX", "false")
Tag.objects.create(name="archive")
client.force_login(admin_user)

12
archivebox/tests/test_admin_views.py
View File

@ -22,7 +22,6 @@ from django.contrib.auth import get_user_model
from django.contrib.auth.models import UserManager
from django.utils import timezone
from archivebox.config.common import SEARCH_BACKEND_CONFIG
pytestmark = pytest.mark.django_db
@ -924,16 +923,15 @@ class TestAdminSnapshotListView:
assert f"/admin/core/snapshot/{snapshot.pk}/redo-failed/".encode() in response.content
def test_snapshot_view_url_uses_canonical_replay_url_for_mode(self, snapshot, monkeypatch):
from archivebox.config.common import SERVER_CONFIG
from archivebox.core.admin_site import archivebox_admin
from archivebox.core.admin_snapshots import SnapshotAdmin
admin = SnapshotAdmin(snapshot.__class__, archivebox_admin)
monkeypatch.setattr(SERVER_CONFIG, "SERVER_SECURITY_MODE", "safe-subdomains-fullreplay")
monkeypatch.setenv("SERVER_SECURITY_MODE", "safe-subdomains-fullreplay")
assert admin.get_snapshot_view_url(snapshot) == f"http://snap-{str(snapshot.pk).replace('-', '')[-12:]}.archivebox.localhost:8000"
monkeypatch.setattr(SERVER_CONFIG, "SERVER_SECURITY_MODE", "safe-onedomain-nojsreplay")
monkeypatch.setenv("SERVER_SECURITY_MODE", "safe-onedomain-nojsreplay")
assert admin.get_snapshot_view_url(snapshot) == f"http://archivebox.localhost:8000/snapshot/{snapshot.pk}"
def test_find_snapshots_for_url_matches_fragment_suffixed_variants(self, crawl, db):
@ -1381,7 +1379,7 @@ class TestAdminSnapshotSearch:
"""Tests for admin snapshot search functionality."""
def test_admin_search_mode_selector_defaults_to_meta_for_ripgrep(self, client, admin_user, monkeypatch):
monkeypatch.setattr(SEARCH_BACKEND_CONFIG, "SEARCH_BACKEND_ENGINE", "ripgrep")
monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep")
client.login(username="testadmin", password="testpassword")
response = client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_HOST)
@ -1392,7 +1390,7 @@ class TestAdminSnapshotSearch:
assert b'name="search_mode" value="deep"' in response.content
def test_admin_search_mode_selector_defaults_to_contents_for_non_ripgrep(self, client, admin_user, monkeypatch):
monkeypatch.setattr(SEARCH_BACKEND_CONFIG, "SEARCH_BACKEND_ENGINE", "sqlite")
monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "sqlite")
client.login(username="testadmin", password="testpassword")
response = client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_HOST)
@ -1632,7 +1630,7 @@ class TestPublicIndexSearch:
@override_settings(PUBLIC_INDEX=True)
def test_public_search_mode_selector_defaults_to_meta_for_ripgrep(self, client, monkeypatch):
monkeypatch.setattr(SEARCH_BACKEND_CONFIG, "SEARCH_BACKEND_ENGINE", "ripgrep")
monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep")
response = client.get("/public/", HTTP_HOST=PUBLIC_HOST)

31
archivebox/tests/test_auth_ldap.py
View File

@ -17,20 +17,21 @@ class TestLDAPConfig(unittest.TestCase):
def test_ldap_config_defaults(self):
"""Test that LDAP config loads with correct defaults."""
from archivebox.config.ldap import LDAP_CONFIG
from archivebox.config.common import get_config
# Check default values
self.assertFalse(LDAP_CONFIG.LDAP_ENABLED)
self.assertIsNone(LDAP_CONFIG.LDAP_SERVER_URI)
self.assertIsNone(LDAP_CONFIG.LDAP_BIND_DN)
self.assertIsNone(LDAP_CONFIG.LDAP_BIND_PASSWORD)
self.assertIsNone(LDAP_CONFIG.LDAP_USER_BASE)
self.assertEqual(LDAP_CONFIG.LDAP_USER_FILTER, "(uid=%(user)s)")
self.assertEqual(LDAP_CONFIG.LDAP_USERNAME_ATTR, "username")
self.assertEqual(LDAP_CONFIG.LDAP_FIRSTNAME_ATTR, "givenName")
self.assertEqual(LDAP_CONFIG.LDAP_LASTNAME_ATTR, "sn")
self.assertEqual(LDAP_CONFIG.LDAP_EMAIL_ATTR, "mail")
self.assertFalse(LDAP_CONFIG.LDAP_CREATE_SUPERUSER)
config = get_config()
self.assertFalse(config.LDAP_ENABLED)
self.assertIsNone(config.LDAP_SERVER_URI)
self.assertIsNone(config.LDAP_BIND_DN)
self.assertIsNone(config.LDAP_BIND_PASSWORD)
self.assertIsNone(config.LDAP_USER_BASE)
self.assertEqual(config.LDAP_USER_FILTER, "(uid=%(user)s)")
self.assertEqual(config.LDAP_USERNAME_ATTR, "username")
self.assertEqual(config.LDAP_FIRSTNAME_ATTR, "givenName")
self.assertEqual(config.LDAP_LASTNAME_ATTR, "sn")
self.assertEqual(config.LDAP_EMAIL_ATTR, "mail")
self.assertFalse(config.LDAP_CREATE_SUPERUSER)
def test_ldap_config_validation_disabled(self):
"""Test that validation passes when LDAP is disabled."""
@ -74,10 +75,10 @@ class TestLDAPConfig(unittest.TestCase):
self.assertEqual(error_msg, "")
def test_ldap_config_in_get_config(self):
"""Test that LDAP_CONFIG is included in get_CONFIG()."""
from archivebox.config import get_CONFIG
"""Test that LDAP_CONFIG is included in the typed config sections."""
from archivebox.config.common import get_all_configs
all_config = get_CONFIG()
all_config = get_all_configs()
self.assertIn("LDAP_CONFIG", all_config)
self.assertEqual(all_config["LDAP_CONFIG"].__class__.__name__, "LDAPConfig")

25
archivebox/tests/test_cli_init.py
View File

@ -8,10 +8,10 @@ import os
import sqlite3
import subprocess
from archivebox.config.common import STORAGE_CONFIG
from archivebox.config.common import get_config
DIR_PERMISSIONS = STORAGE_CONFIG.OUTPUT_PERMISSIONS.replace("6", "7").replace("4", "5")
DIR_PERMISSIONS = get_config().OUTPUT_PERMISSIONS.replace("6", "7").replace("4", "5")
def test_init_creates_database_file(tmp_path):
@ -35,6 +35,23 @@ def test_init_creates_archive_directory(tmp_path):
assert archive_dir.is_dir()
def test_init_respects_configured_archive_and_users_dirs(tmp_path):
"""Test that init creates configured archive/users storage roots."""
os.chdir(tmp_path)
archive_dir = tmp_path / "mounted_archive"
users_dir = archive_dir / "custom_users"
env = os.environ.copy()
env["ARCHIVE_DIR"] = str(archive_dir)
env["USERS_DIR"] = str(users_dir)
result = subprocess.run(["archivebox", "init"], env=env, capture_output=True)
assert result.returncode == 0
assert archive_dir.is_dir()
assert users_dir.is_dir()
assert not (tmp_path / "archive").exists()
def test_init_creates_sources_directory(tmp_path):
"""Test that init creates sources directory."""
os.chdir(tmp_path)
@ -145,11 +162,11 @@ def test_init_sets_correct_file_permissions(tmp_path):
# Check database permissions
db_path = tmp_path / "index.sqlite3"
assert oct(db_path.stat().st_mode)[-3:] in (STORAGE_CONFIG.OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
assert oct(db_path.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
# Check directory permissions
archive_dir = tmp_path / "archive"
assert oct(archive_dir.stat().st_mode)[-3:] in (STORAGE_CONFIG.OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
assert oct(archive_dir.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
def test_init_is_idempotent(tmp_path):

4
archivebox/tests/test_cli_real_flows.py
View File

@ -158,7 +158,7 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(tmp_path, pro
failed_results = [(url, plugin, output) for url, plugin, status, _files, _size, output in archive_results if status == "failed"]
assert len(failed_results) <= 2, failed_results
snapshot_root = tmp_path / "users/system/snapshots"
snapshot_root = tmp_path / "archive/users/system/snapshots"
html_outputs = [path for path in snapshot_root.rglob("wget/**/*.html") if path.is_file()]
header_outputs = [path for path in snapshot_root.rglob("headers/**/headers.json") if path.is_file() and path.stat().st_size > 0]
title_outputs = [path for path in snapshot_root.rglob("title/title.txt") if path.is_file() and path.stat().st_size > 0]
@ -257,6 +257,6 @@ def test_cli_recursive_crawl_processes_discovered_html_urls(tmp_path, process):
assert by_url_plugin[("https://example.com", "parse_html_urls")] == "succeeded"
assert by_url_plugin[("https://iana.org/domains/example", "wget")] == "succeeded"
urls_outputs = list((tmp_path / "users/system/snapshots").rglob("parse_html_urls/urls.jsonl"))
urls_outputs = list((tmp_path / "archive/users/system/snapshots").rglob("parse_html_urls/urls.jsonl"))
assert urls_outputs
assert any("https://iana.org/domains/example" in path.read_text() for path in urls_outputs)

26
archivebox/tests/test_cli_run.py
View File

@ -274,7 +274,7 @@ class TestRunEmpty:
class TestRunDaemonMode:
def test_run_daemon_processes_stdin_before_runner(self, monkeypatch):
def test_run_daemon_starts_runner_without_reading_stdin(self, monkeypatch):
from archivebox.cli import archivebox_run
class FakeStdin:
@ -286,7 +286,7 @@ class TestRunDaemonMode:
monkeypatch.setattr(
archivebox_run,
"process_stdin_records",
lambda: calls.append("stdin") or 0,
lambda: (_ for _ in ()).throw(AssertionError("daemon mode must not block on stdin")),
)
monkeypatch.setattr(
archivebox_run,
@ -298,27 +298,7 @@ class TestRunDaemonMode:
archivebox_run.main.callback(daemon=True, crawl_id=None, snapshot_id=None, binary_id=None)
assert exit_info.value.code == 0
assert calls == ["stdin", "runner:True"]
def test_run_daemon_skips_runner_if_stdin_processing_fails(self, monkeypatch):
from archivebox.cli import archivebox_run
class FakeStdin:
def isatty(self):
return False
monkeypatch.setattr(sys, "stdin", FakeStdin())
monkeypatch.setattr(archivebox_run, "process_stdin_records", lambda: 1)
monkeypatch.setattr(
archivebox_run,
"run_runner",
lambda daemon=False: (_ for _ in ()).throw(AssertionError("runner should not start after stdin failure")),
)
with pytest.raises(SystemExit) as exit_info:
archivebox_run.main.callback(daemon=True, crawl_id=None, snapshot_id=None, binary_id=None)
assert exit_info.value.code == 1
assert calls == ["runner:True"]
@pytest.mark.django_db

24
archivebox/tests/test_config.py
View File

@ -106,6 +106,30 @@ def test_config_invalid_key_fails(tmp_path, process):
assert result.returncode != 0 or "failed" in result.stdout.lower()
def test_config_ignores_legacy_unknown_keys(tmp_path, process):
"""Old ArchiveBox.conf keys should not prevent startup during upgrades."""
os.chdir(tmp_path)
(tmp_path / "ArchiveBox.conf").write_text(
"""
[ARCHIVING_CONFIG]
MAX_MEDIA_SIZE = "750m"
[SEARCH_BACKEND_CONFIG]
SEARCH_BACKEND_HOST_NAME = "sonic"
SEARCH_BACKEND_PASSWORD = "SecretPassword"
""",
)
result = subprocess.run(
["archivebox", "version"],
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
assert "Extra inputs are not permitted" not in result.stderr
def test_config_set_requires_equals_sign(tmp_path, process):
"""Test that --set requires KEY=VALUE format."""
os.chdir(tmp_path)

2
archivebox/tests/test_config_views.py
View File

@ -252,7 +252,6 @@ def test_live_config_value_view_renames_source_field_and_uses_plugin_definition_
request.user = SimpleNamespace(is_superuser=True)
monkeypatch.setattr(core_views, "get_all_configs", lambda: {})
monkeypatch.setattr(core_views, "get_flat_config", lambda: {})
monkeypatch.setattr(core_views, "get_config", lambda: {"PARSE_DOM_OUTLINKS_ENABLED": True})
monkeypatch.setattr(core_views, "find_config_default", lambda key: "True")
monkeypatch.setattr(core_views, "find_config_type", lambda key: "bool")
@ -308,7 +307,6 @@ def test_live_config_value_view_priority_text_matches_runtime_precedence(monkeyp
request.user = SimpleNamespace(is_superuser=True)
monkeypatch.setattr(core_views, "get_all_configs", lambda: {})
monkeypatch.setattr(core_views, "get_flat_config", lambda: {"CHECK_SSL_VALIDITY": True})
monkeypatch.setattr(core_views, "get_config", lambda: {"CHECK_SSL_VALIDITY": False})
monkeypatch.setattr(core_views, "find_config_default", lambda key: "True")
monkeypatch.setattr(core_views, "find_config_type", lambda key: "bool")

2
archivebox/tests/test_hooks.py
View File

@ -697,7 +697,7 @@ print(json.dumps({
)
hook_path.chmod(0o755)
output_dir = tmp_path / "users" / "system" / "snapshots" / "20260513" / "example.com" / "test" / "envprobe"
output_dir = tmp_path / "archive" / "users" / "system" / "snapshots" / "20260513" / "example.com" / "test" / "envprobe"
process = run_hook(
hook_path,
output_dir,

20
archivebox/tests/test_machine_models.py
View File

@ -154,6 +154,26 @@ class TestMachineModel(TestCase):
self.assertNotIn("CHROME_USER_DATA_DIR", refreshed.config)
self.assertNotIn("CHROMIUM_VERSION", refreshed.config)
def test_get_config_auto_applies_current_machine_config(self):
"""get_config() should include sanitized Machine.current() config by default."""
import archivebox.machine.models as models
from archivebox.config.common import get_config
machine = Machine.current()
machine.config = {
"CHROME_BINARY": "/tmp/chromium",
"ABX_INSTALL_CACHE": {"chrome": "2026-03-24T00:00:00+00:00"},
"CHROME_ISOLATION": "snapshot",
}
machine.save(update_fields=["config"])
models._CURRENT_MACHINE = machine
config = get_config()
self.assertEqual(config.CHROME_BINARY, "/tmp/chromium")
self.assertEqual(config["ABX_INSTALL_CACHE"], {"chrome": "2026-03-24T00:00:00+00:00"})
self.assertEqual(config.CHROME_ISOLATION, "crawl")
def test_machine_manager_current(self):
"""Machine.objects.current() should return current machine."""
machine = Machine.current()

69
archivebox/tests/test_migrations_07_to_09.py
View File

@ -93,8 +93,12 @@ class TestMigrationFrom07x(unittest.TestCase):
self.assertTrue(ok, msg)
def test_migration_preserves_archiveresults(self):
"""Migration should preserve all archive results."""
"""Migration should preserve ArchiveResult rows and link each one to a Process."""
expected_count = len(self.original_data["archiveresults"])
expected_counts = {}
for result in self.original_data["archiveresults"]:
key = (result["extractor"], result["status"])
expected_counts[key] = expected_counts.get(key, 0) + 1
result = run_archivebox(self.work_dir, ["init"], timeout=45)
self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
@ -102,6 +106,20 @@ class TestMigrationFrom07x(unittest.TestCase):
ok, msg = verify_archiveresult_count(self.db_path, expected_count)
self.assertTrue(ok, msg)
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute("SELECT plugin, status, COUNT(*) FROM core_archiveresult GROUP BY plugin, status")
migrated_counts = {(plugin, status): count for plugin, status, count in cursor.fetchall()}
cursor.execute("SELECT COUNT(*) FROM core_archiveresult WHERE process_id IS NULL")
missing_process_count = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM machine_process")
process_count = cursor.fetchone()[0]
conn.close()
self.assertEqual(migrated_counts, expected_counts)
self.assertEqual(missing_process_count, 0)
self.assertEqual(process_count, expected_count)
def test_migration_preserves_foreign_keys(self):
"""Migration should maintain foreign key relationships."""
result = run_archivebox(self.work_dir, ["init"], timeout=45)
@ -110,6 +128,55 @@ class TestMigrationFrom07x(unittest.TestCase):
ok, msg = verify_foreign_keys(self.db_path)
self.assertTrue(ok, msg)
def test_migration_preserves_legacy_timestamp_meanings(self):
"""0.7.x timestamp is bookmark identity; added is row creation; updated is downloaded."""
snapshot = self.original_data["snapshots"][0]
legacy_bookmark_ts = "1609459200.123456"
legacy_added = "2024-08-28 09:40:00"
legacy_updated = "2024-08-29 10:41:00"
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute(
"""
UPDATE core_snapshot
SET timestamp = ?, added = ?, updated = ?
WHERE id = ?
""",
(legacy_bookmark_ts, legacy_added, legacy_updated, snapshot["id"]),
)
conn.commit()
conn.close()
result = run_archivebox(self.work_dir, ["init"], timeout=45)
self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute(
"SELECT timestamp, bookmarked_at, created_at, modified_at, downloaded_at FROM core_snapshot WHERE id = ?",
(snapshot["id"],),
)
timestamp, bookmarked_at, created_at, modified_at, downloaded_at = cursor.fetchone()
conn.close()
self.assertEqual(timestamp, legacy_bookmark_ts)
self.assertTrue(bookmarked_at.startswith("2021-01-01"), bookmarked_at)
self.assertTrue(created_at.startswith("2024-08-28"), created_at)
self.assertTrue(modified_at.startswith("2024-08-29"), modified_at)
self.assertTrue(downloaded_at.startswith("2024-08-29"), downloaded_at)
def test_update_saves_migrated_snapshots_without_foreign_key_errors(self):
"""Migrated 0.7.x snapshots should be writable through the current ORM."""
result = run_archivebox(self.work_dir, ["init"], timeout=45)
self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
result = run_archivebox(self.work_dir, ["update"], timeout=60)
output = result.stdout + result.stderr
self.assertEqual(result.returncode, 0, f"Update failed after migration: {result.stderr}")
self.assertNotIn("FOREIGN KEY constraint failed", output)
self.assertNotIn("Skipping snapshot", output)
def test_status_works_after_migration(self):
"""Status command should work after migration."""
result = run_archivebox(self.work_dir, ["init"], timeout=45)

264
archivebox/tests/test_migrations_08_to_09.py
View File

@ -15,6 +15,7 @@ import sqlite3
import tempfile
import unittest
import json
import uuid
from pathlib import Path
from .migrations_helpers import (
@ -134,8 +135,13 @@ class TestMigrationFrom08x(unittest.TestCase):
self.assertTrue(ok, msg)
def test_migration_preserves_archiveresults(self):
"""Migration should preserve all archive results."""
"""Migration should preserve ArchiveResult rows and link each one to a Process."""
expected_count = len(self.original_data["archiveresults"])
expected_counts = {}
for result in self.original_data["archiveresults"]:
status = "succeeded" if result["status"] == "success" else result["status"]
key = (result["extractor"], status)
expected_counts[key] = expected_counts.get(key, 0) + 1
result = run_archivebox(self.work_dir, ["init"], timeout=45)
self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
@ -143,6 +149,20 @@ class TestMigrationFrom08x(unittest.TestCase):
ok, msg = verify_archiveresult_count(self.db_path, expected_count)
self.assertTrue(ok, msg)
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute("SELECT plugin, status, COUNT(*) FROM core_archiveresult GROUP BY plugin, status")
migrated_counts = {(plugin, status): count for plugin, status, count in cursor.fetchall()}
cursor.execute("SELECT COUNT(*) FROM core_archiveresult WHERE process_id IS NULL")
missing_process_count = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM machine_process")
process_count = cursor.fetchone()[0]
conn.close()
self.assertEqual(migrated_counts, expected_counts)
self.assertEqual(missing_process_count, 0)
self.assertEqual(process_count, expected_count)
def test_migration_preserves_archiveresult_status(self):
"""Migration should preserve archive result status values."""
result = run_archivebox(self.work_dir, ["init"], timeout=45)
@ -213,6 +233,77 @@ class TestMigrationFrom08x(unittest.TestCase):
ok, msg = verify_foreign_keys(self.db_path)
self.assertTrue(ok, msg)
def test_migration_preserves_08_timestamp_meanings(self):
"""0.8.x already has separated timestamp/bookmarked_at/created_at/downloaded_at fields."""
snapshot = self.original_data["snapshots"][0]
legacy_timestamp = "1609459200.123456"
bookmarked_at = "2021-01-01 00:00:00"
created_at = "2024-08-28 09:40:00"
modified_at = "2024-08-29 10:41:00"
downloaded_at = "2024-08-30 11:42:00"
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute(
"""
UPDATE core_snapshot
SET timestamp = ?, bookmarked_at = ?, created_at = ?, modified_at = ?, downloaded_at = ?
WHERE id = ?
""",
(legacy_timestamp, bookmarked_at, created_at, modified_at, downloaded_at, snapshot["id"]),
)
conn.commit()
conn.close()
result = run_archivebox(self.work_dir, ["init"], timeout=45)
self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute(
"SELECT timestamp, bookmarked_at, created_at, modified_at, downloaded_at FROM core_snapshot WHERE id = ?",
(snapshot["id"],),
)
migrated = cursor.fetchone()
conn.close()
self.assertEqual(migrated[0], legacy_timestamp)
self.assertTrue(migrated[1].startswith("2021-01-01"), migrated[1])
self.assertTrue(migrated[2].startswith("2024-08-28"), migrated[2])
self.assertTrue(migrated[3].startswith("2024-08-29"), migrated[3])
self.assertTrue(migrated[4].startswith("2024-08-30"), migrated[4])
def test_hyphenated_crawl_ids_are_normalized_before_snapshot_saves(self):
"""0.8.x crawl UUIDs with dashes should migrate to Django's SQLite UUID format."""
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
for crawl in self.original_data["crawls"]:
hyphenated = str(uuid.UUID(hex=crawl["id"]))
cursor.execute("UPDATE crawls_crawl SET id = ? WHERE id = ?", (hyphenated, crawl["id"]))
cursor.execute("UPDATE core_snapshot SET crawl_id = ? WHERE crawl_id = ?", (hyphenated, crawl["id"]))
crawl["id"] = hyphenated
conn.commit()
conn.close()
result = run_archivebox(self.work_dir, ["init"], timeout=45)
self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute("SELECT COUNT(*) FROM crawls_crawl WHERE id LIKE '%-%'")
hyphenated_crawls = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM core_snapshot WHERE crawl_id LIKE '%-%'")
hyphenated_snapshot_refs = cursor.fetchone()[0]
conn.close()
self.assertEqual(hyphenated_crawls, 0)
self.assertEqual(hyphenated_snapshot_refs, 0)
result = run_archivebox(self.work_dir, ["update"], timeout=60)
output = result.stdout + result.stderr
self.assertEqual(result.returncode, 0, f"Update failed after migration: {result.stderr}")
self.assertNotIn("FOREIGN KEY constraint failed", output)
def test_migration_removes_seed_id_column(self):
"""Migration should remove seed_id column from archivebox.crawls.crawl."""
result = run_archivebox(self.work_dir, ["init"], timeout=45)
@ -517,6 +608,140 @@ class TestFilesystemMigration08to09(unittest.TestCase):
"""Clean up temporary directory."""
shutil.rmtree(self.work_dir, ignore_errors=True)
def test_update_migrates_db_snapshot_when_legacy_index_missing(self):
"""A legacy folder with no index file should still migrate if its timestamp exists in DB."""
create_data_dir_structure(self.work_dir)
conn = sqlite3.connect(str(self.db_path))
conn.executescript(SCHEMA_0_7)
conn.close()
original_data = seed_0_7_data(self.db_path)
snapshot = original_data["snapshots"][0]
snapshot_dir = self.work_dir / "archive" / snapshot["timestamp"]
snapshot_dir.mkdir(parents=True, exist_ok=True)
(snapshot_dir / "screenshot.png").write_text("existing-db-snapshot")
result = run_archivebox(self.work_dir, ["init"], timeout=60)
self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
result = run_archivebox(self.work_dir, ["update"], timeout=120)
self.assertEqual(result.returncode, 0, f"Update failed: {result.stderr}")
migrated_files = list((self.work_dir / "archive" / "users").glob("*/snapshots/*/*/*/screenshot.png"))
self.assertEqual(len(migrated_files), 1)
self.assertEqual(migrated_files[0].read_text(), "existing-db-snapshot")
self.assertFalse((self.work_dir / "invalid").exists())
def test_update_recovers_orphan_with_corrupt_index_from_archive_org_url(self):
"""A corrupt legacy index can be imported when archive.org.txt has the original URL."""
create_data_dir_structure(self.work_dir)
conn = sqlite3.connect(str(self.db_path))
conn.executescript(SCHEMA_0_7)
conn.close()
seed_0_7_data(self.db_path)
timestamp = "1339747993"
original_url = "http://www.wired.com/wiredenterprise/2012/01/seamicro-and-google/all/1"
snapshot_dir = self.work_dir / "archive" / timestamp
snapshot_dir.mkdir(parents=True, exist_ok=True)
(snapshot_dir / "index.json").write_text("")
(snapshot_dir / "archive.org.txt").write_text(f"https://web.archive.org/web/20170531210128/{original_url}\n")
(snapshot_dir / "output.pdf").write_text("orphan-output")
result = run_archivebox(self.work_dir, ["init"], timeout=60)
self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
result = run_archivebox(self.work_dir, ["update"], timeout=120)
self.assertEqual(result.returncode, 0, f"Update failed: {result.stderr}")
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute("SELECT url, timestamp FROM core_snapshot WHERE timestamp = ?", (timestamp,))
row = cursor.fetchone()
conn.close()
self.assertEqual(row, (original_url, timestamp))
migrated_files = list((self.work_dir / "archive" / "users").glob("*/snapshots/*/*/*/output.pdf"))
self.assertEqual(len(migrated_files), 1)
self.assertEqual(migrated_files[0].read_text(), "orphan-output")
self.assertFalse((self.work_dir / "invalid").exists())
def test_update_preserves_legacy_folder_timestamp_over_index_float_variant(self):
"""Legacy folder timestamp is the on-disk identity even if index.json has a .0 variant."""
create_data_dir_structure(self.work_dir)
conn = sqlite3.connect(str(self.db_path))
conn.executescript(SCHEMA_0_7)
conn.close()
seed_0_7_data(self.db_path)
timestamp = "1508259732"
url = "https://example.com/folder-timestamp"
snapshot_dir = self.work_dir / "archive" / timestamp
snapshot_dir.mkdir(parents=True, exist_ok=True)
(snapshot_dir / "index.json").write_text(
json.dumps(
{
"url": url,
"timestamp": "1508259732.0",
"title": "Folder Timestamp",
},
),
)
(snapshot_dir / "output.html").write_text("folder timestamp output")
result = run_archivebox(self.work_dir, ["init"], timeout=60)
self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
result = run_archivebox(self.work_dir, ["update"], timeout=120)
self.assertEqual(result.returncode, 0, f"Update failed: {result.stderr}")
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute("SELECT timestamp FROM core_snapshot WHERE url = ?", (url,))
row = cursor.fetchone()
conn.close()
self.assertEqual(row, (timestamp,))
self.assertTrue((self.work_dir / "archive" / timestamp).is_symlink())
self.assertFalse((self.work_dir / "archive" / f"{timestamp}.0").exists())
self.assertFalse((self.work_dir / "invalid").exists())
def test_update_preserves_distinct_legacy_dirs_with_integer_and_float_timestamps(self):
"""Sibling legacy dirs like 1508259732 and 1508259732.0 must not fuzzy-merge."""
create_data_dir_structure(self.work_dir)
conn = sqlite3.connect(str(self.db_path))
conn.executescript(SCHEMA_0_7)
conn.close()
seed_0_7_data(self.db_path)
url = "https://example.com/duplicate-timestamp"
for timestamp, payload in [("1508259732.0", "float-dir"), ("1508259732", "int-dir")]:
snapshot_dir = self.work_dir / "archive" / timestamp
snapshot_dir.mkdir(parents=True, exist_ok=True)
(snapshot_dir / "index.json").write_text(
json.dumps(
{
"url": url,
"timestamp": timestamp,
"title": payload,
},
),
)
(snapshot_dir / f"{payload}.txt").write_text(payload)
result = run_archivebox(self.work_dir, ["init"], timeout=60)
self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
result = run_archivebox(self.work_dir, ["update"], timeout=120)
self.assertEqual(result.returncode, 0, f"Update failed: {result.stderr}")
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute("SELECT timestamp FROM core_snapshot WHERE url = ? ORDER BY timestamp", (url,))
rows = cursor.fetchall()
conn.close()
self.assertEqual(rows, [("1508259732",), ("1508259732.0",)])
self.assertTrue((self.work_dir / "archive" / "1508259732").is_symlink())
self.assertTrue((self.work_dir / "archive" / "1508259732.0").is_symlink())
self.assertFalse((self.work_dir / "invalid").exists())
def test_archiveresult_files_preserved_after_migration(self):
"""
Test that ArchiveResult output files are reorganized into new structure.
@ -524,7 +749,7 @@ class TestFilesystemMigration08to09(unittest.TestCase):
This test verifies that:
1. Migration preserves ArchiveResult data in Process/Binary records
2. Running `archivebox update` reorganizes files into new structure
3. New structure: users/username/snapshots/YYYYMMDD/example.com/snap-uuid-here/output.ext
3. New structure: archive/users/username/snapshots/YYYYMMDD/example.com/snap-uuid-here/output.ext
4. All files are moved (no data loss)
5. Old archive/timestamp/ directories are cleaned up
"""
@ -536,7 +761,7 @@ class TestFilesystemMigration08to09(unittest.TestCase):
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
for i, snapshot in enumerate(original_data["snapshots"]):
legacy_timestamp = str(1704110400 + (i * 86400))
legacy_timestamp = "1609459200.123456" if i == 0 else str(1704110400 + (i * 86400))
cursor.execute(
"UPDATE core_snapshot SET timestamp = ? WHERE id = ?",
(legacy_timestamp, snapshot["id"]),
@ -572,7 +797,7 @@ class TestFilesystemMigration08to09(unittest.TestCase):
# Count archive directories and files BEFORE migration
archive_dir = self.work_dir / "archive"
dirs_before = list(archive_dir.glob("*")) if archive_dir.exists() else []
dirs_before = [d for d in archive_dir.glob("*") if d.name.replace(".", "").isdigit()] if archive_dir.exists() else []
dirs_before_count = len([d for d in dirs_before if d.is_dir()])
# Count total files in all archive directories
@ -600,7 +825,7 @@ class TestFilesystemMigration08to09(unittest.TestCase):
self.assertEqual(result.returncode, 0, f"Init (migration) failed: {result.stderr}")
# Count archive directories and files AFTER migration
dirs_after = list(archive_dir.glob("*")) if archive_dir.exists() else []
dirs_after = [d for d in archive_dir.glob("*") if d.name.replace(".", "").isdigit()] if archive_dir.exists() else []
dirs_after_count = len([d for d in dirs_after if d.is_dir()])
files_after = []
@ -640,8 +865,8 @@ class TestFilesystemMigration08to09(unittest.TestCase):
self.assertEqual(result.returncode, 0, f"Update failed: {result.stderr}")
# Check new filesystem structure
# New structure: users/username/snapshots/YYYYMMDD/example.com/snap-uuid-here/output.ext
users_dir = self.work_dir / "users"
# New structure: archive/users/username/snapshots/YYYYMMDD/example.com/snap-uuid-here/output.ext
users_dir = self.work_dir / "archive" / "users"
snapshots_base = None
if users_dir.exists():
@ -656,7 +881,7 @@ class TestFilesystemMigration08to09(unittest.TestCase):
print(f"[*] New structure base: {snapshots_base}")
# Count files in new structure
# Structure: users/{username}/snapshots/YYYYMMDD/{domain}/{uuid}/files...
# Structure: archive/users/{username}/snapshots/YYYYMMDD/{domain}/{uuid}/files...
files_new_structure = []
new_sample_files = {}
@ -679,6 +904,18 @@ class TestFilesystemMigration08to09(unittest.TestCase):
print(f"[*] Files in new structure: {files_new_count}")
print(f"[*] Sample files in new structure: {len(new_sample_files)}")
migrated_2021_files = list(users_dir.glob("*/snapshots/20210101/*/*/favicon.ico"))
self.assertGreater(
len(migrated_2021_files),
0,
"Legacy snapshot should be bucketed by normalized bookmarked_at, not created_at/import time",
)
crawl_snapshot_links = list(users_dir.glob("*/crawls/*/*/*/snapshots/*/*"))
crawl_snapshot_symlinks = [path for path in crawl_snapshot_links if path.is_symlink()]
crawl_dirs = list(users_dir.glob("*/crawls/*/*/*"))
print(f"[*] Crawl snapshot symlinks: {len(crawl_snapshot_symlinks)}")
# Check old structure (should be gone or empty)
old_archive_dir = self.work_dir / "archive"
old_files_remaining = []
@ -705,6 +942,17 @@ class TestFilesystemMigration08to09(unittest.TestCase):
"No files found in new structure after update",
)
self.assertGreater(
len(crawl_snapshot_symlinks),
0,
"No crawl snapshot symlinks created for migrated snapshots",
)
self.assertFalse(
any((crawl_dir / "index.jsonl").exists() for crawl_dir in crawl_dirs),
"Migrated crawl dirs should match normal 0.9 crawl dirs and not add crawl index.jsonl files",
)
# CRITICAL: Verify old structure is cleaned up
self.assertEqual(
old_files_count,

2
archivebox/tests/test_persona_runtime.py
View File

@ -196,7 +196,7 @@ def test_get_config_raises_for_missing_persona_id(initialized_archive):
import django
django.setup()
from archivebox.config.configset import get_config
from archivebox.config.common import get_config
from archivebox.crawls.models import Crawl
from archivebox.personas.models import Persona

6
archivebox/tests/test_recursive_crawl.py
View File

@ -185,7 +185,7 @@ def test_parser_extractors_emit_snapshot_jsonl(tmp_path, process, recursive_test
if status == "succeeded" and output:
assert "parsed" in output.lower(), "Parser summary should report parsed URLs"
urls_jsonl_files = list(Path("users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl"))
urls_jsonl_files = list(Path("archive/users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl"))
assert urls_jsonl_files, "parse_html_urls should write urls.jsonl output"
records = []
@ -485,7 +485,7 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p
assert len([row for row in parser_results if row[3] == "failed"]) <= 2
assert len([row for row in wget_results if row[2] == "failed"]) <= 2
urls_jsonl_files = list(Path("users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl"))
urls_jsonl_files = list(Path("archive/users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl"))
assert urls_jsonl_files, "parse_html_urls should write urls.jsonl files"
parsed_urls = set()
for path in urls_jsonl_files:
@ -495,7 +495,7 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p
assert set(recursive_test_site["child_urls"]).issubset(parsed_urls)
assert set(recursive_test_site["deep_urls"]).issubset(parsed_urls)
snapshot_dirs = [path.parent for path in Path("users/system/snapshots").rglob("index.jsonl")]
snapshot_dirs = [path.parent for path in Path("archive/users/system/snapshots").rglob("index.jsonl")]
assert snapshot_dirs
for snapshot_dir in snapshot_dirs:
assert (snapshot_dir / "index.jsonl").exists()

238
archivebox/tests/test_runner.py
View File

@ -6,6 +6,7 @@ from pathlib import Path
from types import SimpleNamespace
import pytest
from asgiref.sync import sync_to_async
from django.test import RequestFactory
@ -84,6 +85,7 @@ class _DummyService:
pass
@pytest.mark.django_db(transaction=True)
def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
@ -107,8 +109,10 @@ def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch):
created_buses: list[_DummyBus] = []
original_create_bus = runner_module.create_bus
def fake_create_bus(*, name, total_timeout=3600.0, **kwargs):
bus = _DummyBus(name)
bus = original_create_bus(name=name, total_timeout=total_timeout, **kwargs)
created_buses.append(bus)
return bus
@ -119,7 +123,6 @@ def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch):
monkeypatch.setattr(runner_module, "BinaryService", _DummyService)
monkeypatch.setattr(runner_module, "TagService", _DummyService)
monkeypatch.setattr(runner_module, "CrawlService", _DummyService)
monkeypatch.setattr(runner_module, "SnapshotService", _DummyService)
monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService)
monkeypatch.setattr(runner_module, "_emit_machine_config", lambda *args, **kwargs: asyncio.sleep(0))
monkeypatch.setattr(runner_module, "setup_abx_services", lambda *args, **kwargs: None)
@ -156,17 +159,11 @@ def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch):
monkeypatch.setattr(crawl_runner, "load_snapshot_payload", lambda snapshot_id: snapshot_data[snapshot_id])
monkeypatch.setattr(crawl_runner, "enqueue_discovered_snapshots_from_outputs", lambda snapshot: asyncio.sleep(0))
async def run_both():
await asyncio.gather(
crawl_runner.run_snapshot(str(snapshot_a.id)),
crawl_runner.run_snapshot(str(snapshot_b.id)),
)
asyncio.run(run_both())
asyncio.run(crawl_runner.run_crawl(str(snapshot_a.id), [str(snapshot_a.id), str(snapshot_b.id)]))
from abx_dl.events import SnapshotEvent
snapshot_events = [event for event in crawl_runner.bus.emitted if isinstance(event, SnapshotEvent)]
snapshot_events = asyncio.run(crawl_runner.bus.filter(SnapshotEvent, past=True))
assert len(snapshot_events) == 2
assert {event.snapshot_id for event in snapshot_events} == {str(snapshot_a.id), str(snapshot_b.id)}
assert {event.url for event in snapshot_events} == {snapshot_a.url, snapshot_b.url}
@ -174,6 +171,7 @@ def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch):
assert len(created_buses) == 1
@pytest.mark.django_db(transaction=True)
def test_run_snapshot_does_not_wait_for_crawl_background_daemons(monkeypatch):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
@ -190,14 +188,12 @@ def test_run_snapshot_does_not_wait_for_crawl_background_daemons(monkeypatch):
status=Snapshot.StatusChoices.QUEUED,
)
monkeypatch.setattr(runner_module, "create_bus", lambda **kwargs: _NoIdleBus(kwargs["name"]))
monkeypatch.setattr(runner_module, "discover_plugins", lambda: {})
monkeypatch.setattr(runner_module, "HookProcessService", _DummyService)
monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService)
monkeypatch.setattr(runner_module, "BinaryService", _DummyService)
monkeypatch.setattr(runner_module, "TagService", _DummyService)
monkeypatch.setattr(runner_module, "CrawlService", _DummyService)
monkeypatch.setattr(runner_module, "SnapshotService", _DummyService)
monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService)
monkeypatch.setattr(runner_module, "_emit_machine_config", lambda *args, **kwargs: asyncio.sleep(0))
monkeypatch.setattr(runner_module, "setup_abx_services", lambda *args, **kwargs: None)
@ -207,7 +203,57 @@ def test_run_snapshot_does_not_wait_for_crawl_background_daemons(monkeypatch):
monkeypatch.setattr(crawl_runner, "load_snapshot_payload", lambda snapshot_id: snapshot_payload)
monkeypatch.setattr(crawl_runner, "enqueue_discovered_snapshots_from_outputs", lambda snapshot: asyncio.sleep(0))
asyncio.run(crawl_runner.run_snapshot(str(snapshot.id)))
crawl_runner.bus.wait_until_idle = _NoIdleBus("unused").wait_until_idle
asyncio.run(crawl_runner.run_crawl(str(snapshot.id), [str(snapshot.id)]))
@pytest.mark.django_db(transaction=True)
def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus():
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.services.runner import CrawlRunner
from abx_dl.events import CrawlAbortEvent, CrawlEvent
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
)
snapshot = Snapshot.objects.create(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.STARTED,
)
runner = CrawlRunner(crawl)
async def run() -> CrawlAbortEvent | None:
abort_event_holder: dict[str, CrawlAbortEvent | None] = {"event": None}
async def on_CrawlEvent(event: CrawlEvent) -> None:
watcher = asyncio.create_task(runner.watch_for_cancelled_crawl(event, poll_interval=0.01))
await asyncio.sleep(0.02)
await sync_to_async(Crawl.objects.filter(id=crawl.id).update, thread_sensitive=True)(
status=Crawl.StatusChoices.SEALED,
retry_at=None,
)
abort_event = await runner.bus.find(CrawlAbortEvent, child_of=event, past=True, future=1.0)
abort_event_holder["event"] = abort_event if isinstance(abort_event, CrawlAbortEvent) else None
await watcher
runner.bus.on(CrawlEvent, on_CrawlEvent)
await runner.bus.emit(
CrawlEvent(
url=snapshot.url,
snapshot_id=str(snapshot.id),
output_dir=str(crawl.output_dir),
),
).now()
await runner.bus.wait_until_idle()
return abort_event_holder["event"]
abort_event = asyncio.run(run())
assert abort_event is not None
@pytest.mark.django_db(transaction=True)
@ -429,7 +475,6 @@ def test_runner_prepare_refreshes_network_interface_and_attaches_current_process
proc = _Proc()
monkeypatch.setattr(runner_module, "discover_plugins", lambda: {})
monkeypatch.setattr(runner_module, "create_bus", lambda **kwargs: _DummyBus(kwargs["name"]))
monkeypatch.setattr(runner_module, "HookProcessService", _DummyService)
monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService)
monkeypatch.setattr(runner_module, "BinaryService", _DummyService)
@ -439,12 +484,20 @@ def test_runner_prepare_refreshes_network_interface_and_attaches_current_process
monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService)
from archivebox.machine.models import NetworkInterface, Process
from archivebox.config import configset as configset_module
from archivebox.config import common as config_common
refresh_calls = []
monkeypatch.setattr(NetworkInterface, "current", classmethod(lambda cls, refresh=False: refresh_calls.append(refresh) or _Iface()))
monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc))
monkeypatch.setattr(configset_module, "get_config", lambda **kwargs: {"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60})
original_get_config = config_common.get_config
monkeypatch.setattr(
config_common,
"get_config",
lambda **kwargs: original_get_config(
overrides={"PLUGINS": "", "CHROME_BINARY": "", "CHROME_KEEPALIVE": False, "TIMEOUT": 60},
**kwargs,
),
)
crawl_runner = runner_module.CrawlRunner(crawl)
crawl_runner.load_run_state()
@ -458,7 +511,7 @@ def test_runner_prepare_refreshes_network_interface_and_attaches_current_process
def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
from archivebox.machine.models import Machine, NetworkInterface, Process
from archivebox.services import runner as runner_module
from archivebox.config import configset as configset_module
from archivebox.config import common as config_common
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
@ -496,7 +549,12 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
)
monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc))
monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine))
monkeypatch.setattr(configset_module, "get_config", lambda **kwargs: {"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60})
original_get_config = config_common.get_config
monkeypatch.setattr(
config_common,
"get_config",
lambda **kwargs: original_get_config(overrides={"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60}, **kwargs),
)
crawl_runner = runner_module.CrawlRunner(crawl)
crawl_runner.load_run_state()
@ -510,7 +568,7 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
def test_load_run_state_does_not_force_chrome_keepalive(monkeypatch):
from archivebox.machine.models import Machine, NetworkInterface, Process
from archivebox.services import runner as runner_module
from archivebox.config import configset as configset_module
from archivebox.config import common as config_common
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
@ -543,18 +601,23 @@ def test_load_run_state_does_not_force_chrome_keepalive(monkeypatch):
)
monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc))
monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine))
monkeypatch.setattr(configset_module, "get_config", lambda **kwargs: {"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60})
original_get_config = config_common.get_config
monkeypatch.setattr(
config_common,
"get_config",
lambda **kwargs: original_get_config(overrides={"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60}, **kwargs),
)
crawl_runner = runner_module.CrawlRunner(crawl)
crawl_runner.load_run_state()
assert "CHROME_KEEPALIVE" not in crawl_runner.base_config
assert crawl_runner.base_config["CHROME_KEEPALIVE"] is False
def test_load_run_state_uses_enabled_plugins_when_plugins_key_missing(monkeypatch):
from archivebox.machine.models import Machine, NetworkInterface, Process
from archivebox.services import runner as runner_module
from archivebox.config import configset as configset_module
from archivebox.config import common as config_common
from archivebox import hooks as hooks_module
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
@ -589,7 +652,12 @@ def test_load_run_state_uses_enabled_plugins_when_plugins_key_missing(monkeypatc
)
monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc))
monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine))
monkeypatch.setattr(configset_module, "get_config", lambda **kwargs: {"CHROME_BINARY": "", "TIMEOUT": 60})
original_get_config = config_common.get_config
monkeypatch.setattr(
config_common,
"get_config",
lambda **kwargs: original_get_config(overrides={"CHROME_BINARY": "", "TIMEOUT": 60}, **kwargs),
)
monkeypatch.setattr(
hooks_module,
"discover_hooks",
@ -610,6 +678,7 @@ def test_load_run_state_uses_enabled_plugins_when_plugins_key_missing(monkeypatc
assert len(snapshot_ids) == 1
@pytest.mark.django_db(transaction=True)
def test_run_snapshot_skips_descendant_when_max_size_already_reached(monkeypatch, tmp_path):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
@ -622,7 +691,6 @@ def test_run_snapshot_skips_descendant_when_max_size_already_reached(monkeypatch
)
monkeypatch.setattr(runner_module, "discover_plugins", lambda: {})
monkeypatch.setattr(runner_module, "create_bus", lambda **kwargs: _DummyBus(kwargs["name"]))
monkeypatch.setattr(runner_module, "HookProcessService", _DummyService)
monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService)
monkeypatch.setattr(runner_module, "BinaryService", _DummyService)
@ -660,7 +728,24 @@ def test_run_snapshot_skips_descendant_when_max_size_already_reached(monkeypatch
}
crawl_runner.seal_snapshot_due_to_limit = lambda snapshot_id: cancelled.append(snapshot_id)
asyncio.run(crawl_runner.run_snapshot("child-1"))
async def run_in_crawl_start_context() -> None:
from abx_dl.events import CrawlStartEvent
async def run_child_snapshot(event: CrawlStartEvent) -> None:
await crawl_runner.run_snapshot("child-1")
crawl_runner.bus.on(CrawlStartEvent, run_child_snapshot)
await crawl_runner.bus.emit(
CrawlStartEvent(
url="https://example.com",
snapshot_id="child-1",
output_dir=str(tmp_path),
event_timeout=0,
event_handler_timeout=0,
),
).now()
asyncio.run(run_in_crawl_start_context())
assert cancelled == ["child-1"]
@ -731,6 +816,30 @@ def test_seal_snapshot_cancels_queued_descendants_after_max_size():
assert child.retry_at is None
def test_sealed_crawl_does_not_create_discovered_snapshots():
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.SEALED,
retry_at=None,
max_depth=3,
)
root = Snapshot.objects.create(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.SEALED,
retry_at=None,
)
assert crawl.create_snapshots_from_urls() == []
assert crawl.create_discovered_snapshot(root, url="https://example.com/child", depth=1) is None
assert crawl.snapshot_set.count() == 1
def test_create_crawl_api_queues_crawl_without_spawning_runner(monkeypatch):
from django.contrib.auth import get_user_model
from archivebox.api.v1_crawls import CrawlCreateSchema, create_crawl
@ -792,10 +901,7 @@ def test_crawl_runner_does_not_seal_unfinished_crawl(monkeypatch):
},
)
monkeypatch.setattr(runner_module.CrawlRunner, "_create_live_ui", lambda self: None)
monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl_setup", lambda self, snapshot_id: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "enqueue_snapshot", lambda self, snapshot_id: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "wait_for_snapshot_tasks", lambda self: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl_cleanup", lambda self, snapshot_id: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl", lambda self, root_snapshot_id, snapshot_ids: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "finalize_run_state", lambda self: None)
asyncio.run(runner_module.CrawlRunner(crawl, snapshot_ids=[str(snapshot.id)]).run())
@ -845,10 +951,7 @@ def test_crawl_runner_calls_load_and_finalize_run_state(monkeypatch):
},
)
monkeypatch.setattr(runner_module.CrawlRunner, "_create_live_ui", lambda self: None)
monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl_setup", lambda self, snapshot_id: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "enqueue_snapshot", lambda self, snapshot_id: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "wait_for_snapshot_tasks", lambda self: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl_cleanup", lambda self, snapshot_id: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl", lambda self, root_snapshot_id, snapshot_ids: asyncio.sleep(0))
monkeypatch.setenv("DJANGO_ALLOW_ASYNC_UNSAFE", "true")
method_calls: list[str] = []
@ -916,7 +1019,7 @@ def test_wait_for_snapshot_tasks_returns_after_completed_tasks_are_pruned():
asyncio.run(run_test())
def test_crawl_runner_calls_crawl_cleanup_after_snapshot_phase(monkeypatch):
def test_crawl_runner_calls_crawl_lifecycle(monkeypatch):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
@ -947,20 +1050,18 @@ def test_crawl_runner_calls_crawl_cleanup_after_snapshot_phase(monkeypatch):
},
)
monkeypatch.setattr(runner_module.CrawlRunner, "_create_live_ui", lambda self: None)
monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl_setup", lambda self, snapshot_id: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "enqueue_snapshot", lambda self, snapshot_id: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "wait_for_snapshot_tasks", lambda self: asyncio.sleep(0))
monkeypatch.setattr(runner_module.CrawlRunner, "finalize_run_state", lambda self: None)
cleanup_calls = []
lifecycle_calls = []
monkeypatch.setattr(
runner_module.CrawlRunner,
"run_crawl_cleanup",
lambda self, snapshot_id: cleanup_calls.append("abx_cleanup") or asyncio.sleep(0),
"run_crawl",
lambda self, root_snapshot_id, snapshot_ids: lifecycle_calls.append((root_snapshot_id, snapshot_ids)) or asyncio.sleep(0),
)
asyncio.run(runner_module.CrawlRunner(crawl, snapshot_ids=[str(snapshot.id)]).run())
assert cleanup_calls == ["abx_cleanup"]
assert lifecycle_calls == [(str(snapshot.id), [str(snapshot.id)])]
def test_abx_process_service_background_process_finishes_after_process_exit(monkeypatch, tmp_path):
@ -1144,7 +1245,7 @@ def test_run_pending_crawls_prioritizes_queued_crawl_before_unrelated_binary_bac
@pytest.mark.django_db(transaction=True)
def test_crawl_completed_event_does_not_seal_active_snapshots():
def test_crawl_completed_event_requeues_active_snapshots():
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
@ -1186,7 +1287,53 @@ def test_crawl_completed_event_does_not_seal_active_snapshots():
crawl.refresh_from_db()
assert crawl.status == Crawl.StatusChoices.STARTED
assert crawl.retry_at is None
assert crawl.retry_at is not None
@pytest.mark.django_db(transaction=True)
def test_crawl_cleanup_event_requeues_unfinished_crawl():
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.services.crawl_service import CrawlService
from abx_dl.events import CrawlCleanupEvent
from abx_dl.orchestrator import create_bus
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.STARTED,
retry_at=None,
)
snapshot = Snapshot.objects.create(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.QUEUED,
retry_at=None,
)
bus = create_bus(name=f"test_crawl_cleanup_requeues_unfinished_{str(crawl.id).replace('-', '_')}")
CrawlService(bus, crawl_id=str(crawl.id))
try:
async def emit_cleanup() -> None:
event = CrawlCleanupEvent(
url="https://example.com",
snapshot_id=str(snapshot.id),
output_dir=str(crawl.output_dir),
)
emitted = bus.emit(event)
await emitted.now()
await emitted.event_results_list()
asyncio.run(emit_cleanup())
finally:
asyncio.run(bus.wait_until_idle())
asyncio.run(bus.destroy())
crawl.refresh_from_db()
assert crawl.status == Crawl.StatusChoices.STARTED
assert crawl.retry_at is not None
@pytest.mark.django_db(transaction=True)
@ -1307,7 +1454,8 @@ def test_snapshot_completed_event_bus_seals_finished_crawl():
)
bus = create_bus(name=f"test_snapshot_completed_bus_finished_crawl_{str(crawl.id).replace('-', '_')}")
SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: asyncio.sleep(0))
service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: asyncio.sleep(0))
assert service is not None
try:
async def emit_completed() -> None:
@ -1318,7 +1466,7 @@ def test_snapshot_completed_event_bus_seals_finished_crawl():
output_dir=str(snapshot.output_dir),
),
)
await emitted.now()
await emitted.wait()
await emitted.event_results_list()
asyncio.run(emit_completed())

4
archivebox/tests/test_schedule_e2e.py
View File

@ -519,8 +519,8 @@ def test_web_ui_add_depth_two_crawls_and_renders_real_outputs_over_running_serve
assert ("wget", "succeeded") in result_statuses
assert any(plugin.endswith("parse_html_urls") and status == "succeeded" for plugin, status in result_statuses)
assert len([status for _plugin, status, _files, _size in archive_results if status == "failed"]) <= 2
assert list((tmp_path / "users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl"))
assert list((tmp_path / "users/system/snapshots").rglob("wget/**/*.html"))
assert list((tmp_path / "archive/users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl"))
assert list((tmp_path / "archive/users/system/snapshots").rglob("wget/**/*.html"))
progress = requests.get(
f"http://127.0.0.1:{port}/admin/live-progress/",

2
archivebox/tests/test_snapshot.py
View File

@ -48,7 +48,7 @@ def test_snapshot_creates_snapshot_with_correct_url(tmp_path, process, disable_e
domain = urlparse(snapshot_url).hostname or "unknown"
# Verify crawl symlink exists and is relative
target_path = tmp_path / "users" / username / "snapshots" / snapshot_date_str / domain / snapshot_id
target_path = tmp_path / "archive" / "users" / username / "snapshots" / snapshot_date_str / domain / snapshot_id
symlinks = [p for p in tmp_path.rglob(str(snapshot_id)) if p.is_symlink()]
assert symlinks, "Snapshot symlink should exist under crawl dir"
link_path = symlinks[0]

3
archivebox/tests/test_urls.py
View File

@ -49,7 +49,8 @@ def _build_script(body: str) -> str:
from django.contrib.auth import get_user_model
from archivebox.core.models import Snapshot, ArchiveResult
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.common import get_config
SERVER_CONFIG = get_config()
from archivebox.core.host_utils import (
get_admin_host,
get_admin_base_url,

4
archivebox/workers/management/commands/runner_watch.py
View File

@ -23,7 +23,7 @@ class Command(BaseCommand):
import psutil
from archivebox.config.common import STORAGE_CONFIG
from archivebox.config.common import get_config
from archivebox.machine.models import Machine, Process
from archivebox.workers.supervisord_util import (
RUNNER_WORKER,
@ -35,7 +35,7 @@ class Command(BaseCommand):
pidfile = kwargs.get("pidfile") or os.environ.get("ARCHIVEBOX_RUNSERVER_PIDFILE")
if not pidfile:
pidfile = str(STORAGE_CONFIG.TMP_DIR / "runserver.pid")
pidfile = str(get_config().TMP_DIR / "runserver.pid")
interval = max(0.2, float(kwargs.get("interval", 1.0)))
last_pid = None

4
archivebox/workers/supervisord_util.py
View File

@ -577,12 +577,12 @@ def watch_worker(supervisor, daemon_name, interval=5):
def start_server_workers(host="0.0.0.0", port="8000", daemonize=False, debug=False, reload=False, nothreading=False):
from archivebox.config.common import STORAGE_CONFIG
from archivebox.config.common import get_config
supervisor = get_or_create_supervisord_process(daemonize=daemonize)
if debug:
pidfile = str(STORAGE_CONFIG.TMP_DIR / "runserver.pid") if reload else None
pidfile = str(get_config().TMP_DIR / "runserver.pid") if reload else None
server_worker = RUNSERVER_WORKER(host=host, port=port, reload=reload, pidfile=pidfile, nothreading=nothreading)
bg_workers: list[tuple[dict[str, str], bool]] = (
[(RUNNER_WORKER, True), (RUNNER_WATCH_WORKER(pidfile), False)] if reload else [(RUNNER_WORKER, False)]

6
pyproject.toml
View File

@ -78,10 +78,10 @@ dependencies = [
"w3lib>=2.2.1", # used for parsing content-type encoding from http response headers & html tags
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus>=2.5.0", # EventBus API
"abxbus>=2.5.4", # EventBus API
"abxpkg>=1.10.7", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.10.54", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.10.49", # shared ArchiveBox downloader package with blocking install preflight
"abx-plugins>=1.10.55", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.10.50", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]