From 0cc4251c8a2db0750a90689d4e528f53df68a7a8 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 16 May 2026 23:50:16 -0700 Subject: [PATCH] Update ArchiveBox for abx dependency releases --- .dockerignore | 3 + .gitignore | 3 + Dockerfile | 1 + archivebox/__init__.py | 2 +- archivebox/api/v1_cli.py | 7 +- archivebox/api/v1_core.py | 11 +- archivebox/api/v1_crawls.py | 11 +- archivebox/cli/__init__.py | 4 +- archivebox/cli/archivebox_add.py | 10 +- archivebox/cli/archivebox_archiveresult.py | 2 +- archivebox/cli/archivebox_config.py | 39 +- archivebox/cli/archivebox_help.py | 4 +- archivebox/cli/archivebox_init.py | 116 +--- archivebox/cli/archivebox_install.py | 6 +- archivebox/cli/archivebox_manage.py | 5 +- archivebox/cli/archivebox_persona.py | 4 +- archivebox/cli/archivebox_remove.py | 4 +- archivebox/cli/archivebox_run.py | 4 - archivebox/cli/archivebox_schedule.py | 7 +- archivebox/cli/archivebox_search.py | 9 +- archivebox/cli/archivebox_server.py | 16 +- archivebox/cli/archivebox_status.py | 28 +- archivebox/cli/archivebox_update.py | 207 +++--- archivebox/cli/archivebox_version.py | 21 +- archivebox/config/__init__.py | 98 +-- archivebox/config/collection.py | 257 +------ archivebox/config/common.py | 313 ++++++++- archivebox/config/configset.py | 368 ++-------- archivebox/config/constants.py | 18 +- archivebox/config/django.py | 16 +- archivebox/config/ldap.py | 4 - archivebox/config/paths.py | 95 ++- archivebox/config/version.py | 2 - archivebox/core/__init__.py | 20 - archivebox/core/admin_archiveresults.py | 4 +- archivebox/core/admin_snapshots.py | 8 +- archivebox/core/forms.py | 4 +- archivebox/core/host_utils.py | 165 ++--- archivebox/core/middleware.py | 33 +- .../core/migrations/0007_archiveresult.py | 7 +- .../core/migrations/0023_upgrade_to_0_9_0.py | 150 ++++- .../migrations/0024_assign_default_crawl.py | 11 +- ...options_alter_snapshot_options_and_more.py | 9 +- archivebox/core/models.py | 629 ++++++++++++------ archivebox/core/settings.py | 48 +- archivebox/core/templatetags/config_tags.py | 8 +- archivebox/core/urls.py | 18 +- archivebox/core/views.py | 120 ++-- .../migrations/0002_upgrade_from_0_8_6.py | 64 +- archivebox/crawls/models.py | 31 +- archivebox/hooks.py | 131 ++-- archivebox/ldap/auth.py | 4 +- .../0012_add_machine_config_if_missing.py | 22 + archivebox/machine/models.py | 56 +- archivebox/misc/checks.py | 57 +- archivebox/misc/folders.py | 16 +- archivebox/misc/legacy.py | 19 +- archivebox/misc/logging_util.py | 38 +- archivebox/misc/serve_static.py | 22 +- archivebox/misc/shell_welcome_message.py | 6 +- archivebox/misc/system.py | 20 +- archivebox/misc/util.py | 36 +- archivebox/personas/importers.py | 4 +- archivebox/search/__init__.py | 66 +- archivebox/services/archive_result_service.py | 37 +- archivebox/services/crawl_service.py | 11 +- archivebox/services/runner.py | 466 ++++++++----- archivebox/tests/conftest.py | 10 +- archivebox/tests/test_add_view.py | 35 +- archivebox/tests/test_admin_views.py | 12 +- archivebox/tests/test_auth_ldap.py | 31 +- archivebox/tests/test_cli_init.py | 25 +- archivebox/tests/test_cli_real_flows.py | 4 +- archivebox/tests/test_cli_run.py | 26 +- archivebox/tests/test_config.py | 24 + archivebox/tests/test_config_views.py | 2 - archivebox/tests/test_hooks.py | 2 +- archivebox/tests/test_machine_models.py | 20 + archivebox/tests/test_migrations_07_to_09.py | 69 +- archivebox/tests/test_migrations_08_to_09.py | 264 +++++++- archivebox/tests/test_persona_runtime.py | 2 +- archivebox/tests/test_recursive_crawl.py | 6 +- archivebox/tests/test_runner.py | 238 +++++-- archivebox/tests/test_schedule_e2e.py | 4 +- archivebox/tests/test_snapshot.py | 2 +- archivebox/tests/test_urls.py | 3 +- .../management/commands/runner_watch.py | 4 +- archivebox/workers/supervisord_util.py | 4 +- pyproject.toml | 6 +- 89 files changed, 2903 insertions(+), 1925 deletions(-) create mode 100644 archivebox/machine/migrations/0012_add_machine_config_if_missing.py diff --git a/.dockerignore b/.dockerignore index 6abd82c0..0892b9a5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -49,6 +49,9 @@ typings/ tmp/ data/ data*/ +- +personas/ +sources/ output/ index.sqlite3 index.sqlite3-wal diff --git a/.gitignore b/.gitignore index 8d4be0cc..c70fb712 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,9 @@ archivebox/tests/data/ archive/ output/ logs/ +/- +/personas/ +/sources/ index.sqlite3 queue.sqlite3 *.sqlite* diff --git a/Dockerfile b/Dockerfile index db20c670..b241ac6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -310,6 +310,7 @@ RUN --mount=type=cache,target=/home/archivebox/.npm_cache,sharing=locked,id=npm- "@postlight/parser@^2.2.3" \ "readability-extractor@github:ArchiveBox/readability-extractor" \ "single-file-cli@^1.1.54" \ + "abxbus@^2.5.4" \ "puppeteer-core@^23.5.0" \ "puppeteer@^23.5.0" \ "@puppeteer/browsers@^2.4.0" \ diff --git a/archivebox/__init__.py b/archivebox/__init__.py index bd464ac9..2986ddef 100755 --- a/archivebox/__init__.py +++ b/archivebox/__init__.py @@ -81,7 +81,7 @@ LOADED_PLUGINS = ALL_PLUGINS # Setup basic config, constants, paths, and version from .config.constants import CONSTANTS # noqa -from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR # noqa +from .config.paths import PACKAGE_DIR, DATA_DIR # noqa from .config.version import VERSION # noqa # Set MACHINE_ID env var so hook scripts can use it diff --git a/archivebox/api/v1_cli.py b/archivebox/api/v1_cli.py index 2c317ad4..17ac4985 100644 --- a/archivebox/api/v1_cli.py +++ b/archivebox/api/v1_cli.py @@ -8,9 +8,10 @@ from enum import Enum from django.http import HttpRequest from ninja import Router, Schema +from pydantic import Field from archivebox.misc.util import ansi_to_html -from archivebox.config.common import ARCHIVING_CONFIG +from archivebox.config.common import get_config # from .auth import API_AUTH_METHODS @@ -61,7 +62,7 @@ class AddCommandSchema(Schema): depth: int = 0 parser: str = "auto" plugins: str = "" - update: bool = not ARCHIVING_CONFIG.ONLY_NEW # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW + update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW) overwrite: bool = False index_only: bool = False @@ -87,7 +88,7 @@ class ScheduleCommandSchema(Schema): tag: str = "" depth: int = 0 overwrite: bool = False - update: bool = not ARCHIVING_CONFIG.ONLY_NEW + update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW) clear: bool = False diff --git a/archivebox/api/v1_core.py b/archivebox/api/v1_core.py index 4dea76ae..db33b5ca 100644 --- a/archivebox/api/v1_core.py +++ b/archivebox/api/v1_core.py @@ -8,7 +8,6 @@ from datetime import datetime from django.db.models import Model, Q, Sum from django.db.models.functions import Coalesce -from django.conf import settings from django.http import HttpRequest, HttpResponse from django.core.exceptions import ValidationError from django.contrib.auth import get_user_model @@ -22,7 +21,7 @@ from ninja.errors import HttpError from archivebox.core.models import Snapshot, ArchiveResult, Tag from archivebox.api.auth import auth_using_token -from archivebox.config.common import SERVER_CONFIG +from archivebox.config.common import get_config from archivebox.core.tag_utils import ( build_tag_cards, delete_tag as delete_tag_record, @@ -632,10 +631,10 @@ def search_tags( def _public_tag_listing_enabled() -> bool: - explicit = getattr(settings, "PUBLIC_SNAPSHOTS_LIST", None) - if explicit is not None: - return bool(explicit) - return bool(getattr(settings, "PUBLIC_INDEX", SERVER_CONFIG.PUBLIC_INDEX)) + config = get_config() + if config.PUBLIC_SNAPSHOTS_LIST is not None: + return config.PUBLIC_SNAPSHOTS_LIST + return config.PUBLIC_INDEX def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool: diff --git a/archivebox/api/v1_crawls.py b/archivebox/api/v1_crawls.py index a925ff18..bd77c480 100644 --- a/archivebox/api/v1_crawls.py +++ b/archivebox/api/v1_crawls.py @@ -173,17 +173,22 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema): crawl.retry_at = payload["retry_at"] update_fields.append("retry_at") - crawl.save(update_fields=update_fields) - if payload.get("status") == Crawl.StatusChoices.SEALED: + cancelled_at = timezone.now() + crawl.retry_at = None + if "retry_at" not in update_fields: + update_fields.append("retry_at") + crawl.save(update_fields=update_fields) Snapshot.objects.filter( crawl=crawl, status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED], ).update( status=Snapshot.StatusChoices.SEALED, retry_at=None, - modified_at=timezone.now(), + modified_at=cancelled_at, ) + else: + crawl.save(update_fields=update_fields) return crawl diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index 13a62c4f..f3ea2f1b 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -165,9 +165,9 @@ def cli(ctx, help=False): os.environ["ARCHIVEBOX_RUNSERVER"] = "1" if "--reload" in sys.argv: os.environ["ARCHIVEBOX_AUTORELOAD"] = "1" - from archivebox.config.common import STORAGE_CONFIG + from archivebox.config.common import get_config - os.environ["ARCHIVEBOX_RUNSERVER_PIDFILE"] = str(STORAGE_CONFIG.TMP_DIR / "runserver.pid") + os.environ["ARCHIVEBOX_RUNSERVER_PIDFILE"] = str(get_config().TMP_DIR / "runserver.pid") from archivebox.config.django import setup_django from archivebox.misc.checks import check_data_folder diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index ae41dae2..f3dd0254 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -16,8 +16,8 @@ from django.db.models import QuerySet from archivebox.misc.util import enforce_types, docstring from archivebox.misc.util import parse_filesize_to_bytes from archivebox import CONSTANTS -from archivebox.config.common import ARCHIVING_CONFIG, SERVER_CONFIG from archivebox.config.permissions import USER, HOSTNAME +from archivebox.config.common import get_config if TYPE_CHECKING: @@ -94,13 +94,13 @@ def add( from archivebox.personas.models import Persona from archivebox.misc.logging_util import printable_filesize from archivebox.misc.system import get_dir_size - from archivebox.config.configset import get_config from archivebox.services.runner import run_crawl + config = get_config() created_by_id = created_by_id or get_or_create_system_user_pk() started_at = timezone.now() if update is None: - update = not ARCHIVING_CONFIG.ONLY_NEW + update = not config.ONLY_NEW # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt" @@ -118,7 +118,7 @@ def add( # Read URLs directly into crawl urls_content = sources_file.read_text() persona_name = (persona or "Default").strip() or "Default" - plugins = plugins or str(get_config().get("PLUGINS") or "") + plugins = plugins or str(config.get("PLUGINS") or "") persona_obj, _ = Persona.objects.get_or_create(name=persona_name) persona_obj.ensure_dirs() @@ -221,7 +221,7 @@ def add( except Exception: rel_output_str = str(crawl.output_dir) - bind_addr = SERVER_CONFIG.BIND_ADDR or "127.0.0.1:8000" + bind_addr = config.BIND_ADDR or "127.0.0.1:8000" if bind_addr.startswith("http://") or bind_addr.startswith("https://"): base_url = bind_addr else: diff --git a/archivebox/cli/archivebox_archiveresult.py b/archivebox/cli/archivebox_archiveresult.py index 9c1eaf7b..545f7157 100644 --- a/archivebox/cli/archivebox_archiveresult.py +++ b/archivebox/cli/archivebox_archiveresult.py @@ -73,7 +73,7 @@ def create_archiveresults( 0: Success 1: Failure """ - from archivebox.config.configset import get_config + from archivebox.config.common import get_config from archivebox.hooks import discover_hooks from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT from archivebox.core.models import Snapshot diff --git a/archivebox/cli/archivebox_config.py b/archivebox/cli/archivebox_config.py index 41f5f29d..6f65d51a 100644 --- a/archivebox/cli/archivebox_config.py +++ b/archivebox/cli/archivebox_config.py @@ -28,13 +28,24 @@ def config( from archivebox.misc.checks import check_data_folder from archivebox.misc.logging_util import printable_config - from archivebox.config.collection import load_all_config, write_config_file, get_real_name - from archivebox.config.configset import get_flat_config, get_all_configs + from abx_plugins.plugins.base.utils import resolve_alias + from archivebox.config.collection import write_config_file + from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs + from archivebox.hooks import discover_plugin_configs check_data_folder() - FLAT_CONFIG = get_flat_config() + FLAT_CONFIG = get_config().as_dict() CONFIGS = get_all_configs() + plugin_schemas = { + plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict) + } + core_config_aliases = { + alias.upper(): field_name + for field_name, field in ArchiveBoxConfig.model_fields.items() + for alias in (field_name, str(field.alias or "")) + if alias + } config_options: list[str] = list(kwargs.pop("key=value", []) or keys or [f"{key}={val}" for key, val in kwargs.items()]) no_args = not (get or set or reset or config_options) @@ -42,7 +53,9 @@ def config( matching_config = {} if search: if config_options: - config_options = [get_real_name(key) for key in config_options] + config_options = [ + core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options + ] matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG} for config_section in CONFIGS.values(): aliases = getattr(config_section, "aliases", {}) @@ -63,7 +76,9 @@ def config( elif get or no_args: if config_options: - config_options = [get_real_name(key) for key in config_options] + config_options = [ + core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options + ] matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG} failed_config = [key for key in config_options if key not in FLAT_CONFIG] if failed_config: @@ -85,17 +100,11 @@ def config( print(_format_toml(kv_in_section)) print("[grey53]################################################################[/grey53]") - # Display plugin config section - from archivebox.hooks import discover_plugin_configs - - plugin_configs = discover_plugin_configs() plugin_keys = {} # Collect all plugin config keys - for plugin_name, schema in plugin_configs.items(): - if "properties" not in schema: - continue - for key in schema["properties"].keys(): + for schema in plugin_schemas.values(): + for key in schema.keys(): if key in matching_config: plugin_keys[key] = matching_config[key] @@ -120,7 +129,7 @@ def config( raw_key, val = line.split("=", 1) raw_key = raw_key.upper().strip() - key = get_real_name(raw_key) + key = core_config_aliases.get(raw_key) or resolve_alias(raw_key, plugin_schemas) if key != raw_key: print( f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]", @@ -134,7 +143,7 @@ def config( if new_config: before = FLAT_CONFIG matching_config = write_config_file(new_config) - after = {**load_all_config(), **get_flat_config()} + after = get_config().as_dict() print(printable_config(matching_config)) side_effect_changes = {} diff --git a/archivebox/cli/archivebox_help.py b/archivebox/cli/archivebox_help.py index 86d0be86..05be3050 100755 --- a/archivebox/cli/archivebox_help.py +++ b/archivebox/cli/archivebox_help.py @@ -15,6 +15,7 @@ def help() -> None: from archivebox.cli import ArchiveBoxGroup from archivebox.config import CONSTANTS + from archivebox.config.common import get_config from archivebox.config.permissions import IN_DOCKER from archivebox.misc.logging_util import log_cli_command @@ -67,7 +68,8 @@ def help() -> None: [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link] """) - if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir(): + config = get_config() + if os.access(config.ARCHIVE_DIR, os.R_OK) and config.ARCHIVE_DIR.is_dir(): pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~") EXAMPLE_USAGE = f""" [light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow] diff --git a/archivebox/cli/archivebox_init.py b/archivebox/cli/archivebox_init.py index 2376e1f3..3593ffa5 100755 --- a/archivebox/cli/archivebox_init.py +++ b/archivebox/cli/archivebox_init.py @@ -5,7 +5,6 @@ __package__ = "archivebox.cli" import os import sys from pathlib import Path -from collections.abc import Mapping from rich import print import rich_click as click @@ -13,17 +12,13 @@ import rich_click as click from archivebox.misc.util import docstring, enforce_types -def _normalize_snapshot_record(link_dict: Mapping[str, object]) -> tuple[str, dict[str, object]] | None: - url = link_dict.get("url") - if not isinstance(url, str) or not url: - return None - - record: dict[str, object] = {"url": url} - for key in ("timestamp", "title", "tags", "sources"): - value = link_dict.get(key) - if value is not None: - record[key] = value - return url, record +def _display_data_path(path: Path, data_dir: Path) -> str: + path = Path(path).resolve() + data_dir = Path(data_dir).resolve() + try: + return f"./{path.relative_to(data_dir)}" + except ValueError: + return str(path) @enforce_types @@ -31,11 +26,12 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non """Initialize a new ArchiveBox collection in the current directory""" from archivebox.config import CONSTANTS, VERSION, DATA_DIR - from archivebox.config.common import SERVER_CONFIG + from archivebox.config.common import get_config from archivebox.config.collection import write_config_file - from archivebox.misc.legacy import parse_json_main_index, parse_json_links_details from archivebox.misc.db import apply_migrations + config = get_config() + # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK): # print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr) # print("[red] You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr) @@ -68,14 +64,16 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non else: print("\n[green][+] Building archive folder structure...[/green]") - print( - f" + ./{CONSTANTS.ARCHIVE_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.SOURCES_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.LOGS_DIR.relative_to(DATA_DIR)}...", - ) + archive_path = _display_data_path(config.ARCHIVE_DIR, DATA_DIR) + sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, DATA_DIR) + logs_path = _display_data_path(CONSTANTS.LOGS_DIR, DATA_DIR) + print(f" + {archive_path}, {sources_path}, {logs_path}...") Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True) - Path(CONSTANTS.ARCHIVE_DIR).mkdir(exist_ok=True) + config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True) + config.USERS_DIR.mkdir(parents=True, exist_ok=True) Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True) - print(f" + ./{CONSTANTS.CONFIG_FILE.relative_to(DATA_DIR)}...") + print(f" + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...") # create the .archivebox_id file with a unique ID for this collection from archivebox.config.paths import _get_collection_id @@ -83,7 +81,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non _get_collection_id(DATA_DIR, force_create=True) # create the ArchiveBox.conf file - write_config_file({"SECRET_KEY": SERVER_CONFIG.SECRET_KEY}) + write_config_file({"SECRET_KEY": config.SECRET_KEY}) if os.access(CONSTANTS.DATABASE_FILE, os.F_OK): print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]") @@ -99,11 +97,9 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK) print() - print(f" √ ./{CONSTANTS.DATABASE_FILE.relative_to(DATA_DIR)}") + print(f" √ {_display_data_path(CONSTANTS.DATABASE_FILE, DATA_DIR)}") # from django.contrib.auth.models import User - # if SHELL_CONFIG.IS_TTY and not User.objects.filter(is_superuser=True).exclude(username='system').exists(): - # print('{green}[+] Creating admin user account...{reset}'.format(**SHELL_CONFIG.ANSI)) # call_command("createsuperuser", interactive=True) print() @@ -111,86 +107,44 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non from archivebox.core.models import Snapshot - all_links = Snapshot.objects.none() - pending_links: dict[str, dict[str, object]] = {} + snapshot_count = 0 if existing_index: - all_links = Snapshot.objects.all() - print(f" √ Loaded {all_links.count()} links from existing main index.") + snapshot_count = Snapshot.objects.count() + print(f" √ Loaded {snapshot_count} links from existing main index.") - if quick: - print(" > Skipping orphan snapshot import (quick mode)") - else: - try: - # Import orphaned links from legacy JSON indexes - orphaned_json_links: dict[str, dict[str, object]] = {} - for link_dict in parse_json_main_index(DATA_DIR): - normalized = _normalize_snapshot_record(link_dict) - if normalized is None: - continue - url, record = normalized - if not all_links.filter(url=url).exists(): - orphaned_json_links[url] = record - if orphaned_json_links: - pending_links.update(orphaned_json_links) - print(f" [yellow]√ Added {len(orphaned_json_links)} orphaned links from existing JSON index...[/yellow]") - - orphaned_data_dir_links: dict[str, dict[str, object]] = {} - for link_dict in parse_json_links_details(DATA_DIR): - normalized = _normalize_snapshot_record(link_dict) - if normalized is None: - continue - url, record = normalized - if not all_links.filter(url=url).exists(): - orphaned_data_dir_links[url] = record - if orphaned_data_dir_links: - pending_links.update(orphaned_data_dir_links) - print(f" [yellow]√ Added {len(orphaned_data_dir_links)} orphaned links from existing archive directories.[/yellow]") - - if pending_links: - for link_dict in pending_links.values(): - Snapshot.from_json(link_dict) - - # Hint for orphaned snapshot directories - print() - print(" [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:") - print(" archivebox update") - - except (KeyboardInterrupt, SystemExit): - print(file=sys.stderr) - print("[yellow]:stop_sign: Stopped checking archive directories due to Ctrl-C/SIGTERM[/yellow]", file=sys.stderr) - print(" Your archive data is safe, but you should re-run `archivebox init` to finish the process later.", file=sys.stderr) - print(file=sys.stderr) - print(" [violet]Hint:[/violet] In the future you can run a quick init without checking dirs like so:", file=sys.stderr) - print(" archivebox init --quick", file=sys.stderr) - raise SystemExit(1) + print(" > Skipping orphan snapshot import during init.") + print() + print(" [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:") + print(" archivebox update") print("\n[green]----------------------------------------------------------------------[/green]") from django.contrib.auth.models import User - if (SERVER_CONFIG.ADMIN_USERNAME and SERVER_CONFIG.ADMIN_PASSWORD) and not User.objects.filter( - username=SERVER_CONFIG.ADMIN_USERNAME, + config = get_config() + if (config.ADMIN_USERNAME and config.ADMIN_PASSWORD) and not User.objects.filter( + username=config.ADMIN_USERNAME, ).exists(): print("[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]") - User.objects.create_superuser(username=SERVER_CONFIG.ADMIN_USERNAME, password=SERVER_CONFIG.ADMIN_PASSWORD) + User.objects.create_superuser(username=config.ADMIN_USERNAME, password=config.ADMIN_PASSWORD) if existing_index: print("[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]") else: - print(f"[green][√] Done. A new ArchiveBox collection was initialized ({len(all_links) + len(pending_links)} links).[/green]") + print(f"[green][√] Done. A new ArchiveBox collection was initialized ({snapshot_count} links).[/green]") CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True) CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True) CONSTANTS.DEFAULT_LIB_DIR.mkdir(parents=True, exist_ok=True) (CONSTANTS.DEFAULT_LIB_DIR / "bin").mkdir(parents=True, exist_ok=True) - from archivebox.config.common import STORAGE_CONFIG from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir - STORAGE_CONFIG.TMP_DIR.mkdir(parents=True, exist_ok=True) - STORAGE_CONFIG.LIB_DIR.mkdir(parents=True, exist_ok=True) - (STORAGE_CONFIG.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True) + config = get_config() + config.TMP_DIR.mkdir(parents=True, exist_ok=True) + config.LIB_DIR.mkdir(parents=True, exist_ok=True) + (config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True) working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True) if working_tmp_dir: diff --git a/archivebox/cli/archivebox_install.py b/archivebox/cli/archivebox_install.py index a8f956cb..781c7024 100755 --- a/archivebox/cli/archivebox_install.py +++ b/archivebox/cli/archivebox_install.py @@ -22,11 +22,13 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo """ from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP - from archivebox.config.paths import ARCHIVE_DIR + from archivebox.config.common import get_config from archivebox.misc.logging import stderr from archivebox.cli.archivebox_init import init - if not (os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()): + config = get_config() + archive_dir = config.ARCHIVE_DIR + if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()): init() # must init full index because we need a db to store Binary entries in # Show what we're installing diff --git a/archivebox/cli/archivebox_manage.py b/archivebox/cli/archivebox_manage.py index 7105161c..491f23b9 100644 --- a/archivebox/cli/archivebox_manage.py +++ b/archivebox/cli/archivebox_manage.py @@ -10,10 +10,11 @@ from archivebox.misc.util import docstring, enforce_types def manage(args: list[str] | None = None) -> None: """Run an ArchiveBox Django management command""" - from archivebox.config.common import SHELL_CONFIG + from archivebox.config.common import get_config from archivebox.misc.logging import stderr - if (args and "createsuperuser" in args) and (SHELL_CONFIG.IN_DOCKER and not SHELL_CONFIG.IS_TTY): + config = get_config() + if (args and "createsuperuser" in args) and (config.IN_DOCKER and not config.IS_TTY): stderr("[!] Warning: you need to pass -it to use interactive commands in docker", color="lightyellow") stderr(" docker run -it archivebox manage {}".format(" ".join(args or ["..."])), color="lightyellow") stderr("") diff --git a/archivebox/cli/archivebox_persona.py b/archivebox/cli/archivebox_persona.py index 7f930665..d2cb9127 100644 --- a/archivebox/cli/archivebox_persona.py +++ b/archivebox/cli/archivebox_persona.py @@ -267,7 +267,7 @@ def extract_cookies_via_cdp( Returns True if successful, False otherwise. """ - from archivebox.config.common import STORAGE_CONFIG + from archivebox.config.common import get_config # Find the cookie extraction script chrome_plugin_dir = Path(__file__).parent.parent / "plugins" / "chrome" @@ -278,7 +278,7 @@ def extract_cookies_via_cdp( return False # Get node modules dir - node_modules_dir = STORAGE_CONFIG.LIB_DIR / "npm" / "node_modules" + node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules" # Set up environment env = os.environ.copy() diff --git a/archivebox/cli/archivebox_remove.py b/archivebox/cli/archivebox_remove.py index be3efcb4..c6e923e3 100644 --- a/archivebox/cli/archivebox_remove.py +++ b/archivebox/cli/archivebox_remove.py @@ -12,7 +12,7 @@ import rich_click as click from django.db.models import QuerySet from archivebox.config import DATA_DIR -from archivebox.config.constants import CONSTANTS +from archivebox.config.common import get_config from archivebox.config.django import setup_django from archivebox.misc.util import enforce_types, docstring from archivebox.misc.checks import check_data_folder @@ -70,7 +70,7 @@ def remove( for snapshot in snapshots: if delete: shutil.rmtree(snapshot.output_dir, ignore_errors=True) - legacy_path = CONSTANTS.ARCHIVE_DIR / snapshot.timestamp + legacy_path = get_config().ARCHIVE_DIR / snapshot.timestamp if legacy_path.is_symlink(): legacy_path.unlink(missing_ok=True) finally: diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py index cca1e671..d3ca7e40 100644 --- a/archivebox/cli/archivebox_run.py +++ b/archivebox/cli/archivebox_run.py @@ -333,10 +333,6 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str): sys.exit(1) if daemon: - if not sys.stdin.isatty(): - exit_code = process_stdin_records() - if exit_code != 0: - sys.exit(exit_code) sys.exit(run_runner(daemon=True)) if not sys.stdin.isatty(): diff --git a/archivebox/cli/archivebox_schedule.py b/archivebox/cli/archivebox_schedule.py index bb9c1dac..a43a3350 100644 --- a/archivebox/cli/archivebox_schedule.py +++ b/archivebox/cli/archivebox_schedule.py @@ -6,7 +6,7 @@ import rich_click as click from rich import print from archivebox.misc.util import enforce_types, docstring -from archivebox.config.common import ARCHIVING_CONFIG +from archivebox.config.common import get_config @enforce_types @@ -21,7 +21,7 @@ def schedule( tag: str = "", depth: int | str = 0, overwrite: bool = False, - update: bool = not ARCHIVING_CONFIG.ONLY_NEW, + update: bool | None = None, import_path: str | None = None, ): """Manage database-backed scheduled crawls processed by the crawl runner.""" @@ -33,6 +33,9 @@ def schedule( from archivebox.crawls.schedule_utils import validate_schedule from archivebox.services.runner import run_pending_crawls + if update is None: + update = not get_config().ONLY_NEW + depth = int(depth) result: dict[str, object] = { "created_schedule_ids": [], diff --git a/archivebox/cli/archivebox_search.py b/archivebox/cli/archivebox_search.py index 86af83bb..5047b2f3 100644 --- a/archivebox/cli/archivebox_search.py +++ b/archivebox/cli/archivebox_search.py @@ -13,6 +13,7 @@ import rich_click as click from django.db.models import Q, QuerySet from archivebox.config import DATA_DIR +from archivebox.config.common import get_config from archivebox.misc.logging import stderr from archivebox.misc.util import enforce_types, docstring @@ -59,14 +60,14 @@ def _snapshots_to_json( from datetime import datetime, timezone as tz from archivebox.config import VERSION - from archivebox.config.common import SERVER_CONFIG from archivebox.misc.util import to_json + config = get_config() main_index_header = ( { "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.", "schema": "archivebox.index.json", - "copyright_info": SERVER_CONFIG.FOOTER_INFO, + "copyright_info": config.FOOTER_INFO, "meta": { "project": "ArchiveBox", "version": VERSION, @@ -119,9 +120,9 @@ def _snapshots_to_html( from django.template.loader import render_to_string from archivebox.config import VERSION - from archivebox.config.common import SERVER_CONFIG from archivebox.config.version import get_COMMIT_HASH + config = get_config() template = "static_index.html" if with_headers else "minimal_index.html" snapshot_list = list(snapshots.iterator(chunk_size=500)) @@ -134,7 +135,7 @@ def _snapshots_to_html( "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"), "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"), "links": snapshot_list, - "FOOTER_INFO": SERVER_CONFIG.FOOTER_INFO, + "FOOTER_INFO": config.FOOTER_INFO, }, ) diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index 861ce775..d0a1743e 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -2,14 +2,15 @@ __package__ = "archivebox.cli" -from collections.abc import Iterable import sys +import os +from collections.abc import Iterable import rich_click as click from rich import print from archivebox.misc.util import docstring, enforce_types -from archivebox.config.common import SERVER_CONFIG +from archivebox.config.common import get_config def stop_existing_background_runner(*, machine, process_model, supervisor=None, stop_worker_fn=None, log=print) -> int: @@ -95,7 +96,7 @@ def stop_existing_server_workers(*, supervisor, stop_worker_fn, host: str, port: @enforce_types def server( - runserver_args: Iterable[str] = (SERVER_CONFIG.BIND_ADDR,), + runserver_args: Iterable[str] | None = None, reload: bool = False, init: bool = False, debug: bool = False, @@ -104,7 +105,8 @@ def server( ) -> None: """Run the ArchiveBox HTTP server""" - runserver_args = list(runserver_args) + config = get_config() + runserver_args = list(runserver_args or (config.BIND_ADDR,)) if init: from archivebox.cli.archivebox_init import init as archivebox_init @@ -116,11 +118,9 @@ def server( check_data_folder() - from archivebox.config.common import SHELL_CONFIG - - run_in_debug = SHELL_CONFIG.DEBUG or debug or reload + run_in_debug = config.DEBUG or debug or reload if debug or reload: - SHELL_CONFIG.DEBUG = True + os.environ["DEBUG"] = "True" from django.contrib.auth.models import User diff --git a/archivebox/cli/archivebox_status.py b/archivebox/cli/archivebox_status.py index b64ecddb..d0e5e2e5 100644 --- a/archivebox/cli/archivebox_status.py +++ b/archivebox/cli/archivebox_status.py @@ -8,8 +8,8 @@ import rich_click as click from rich import print from archivebox.misc.util import enforce_types, docstring -from archivebox.config import DATA_DIR, CONSTANTS, ARCHIVE_DIR -from archivebox.config.common import SHELL_CONFIG +from archivebox.config import DATA_DIR, CONSTANTS +from archivebox.config.common import get_config from archivebox.misc.legacy import parse_json_links_details from archivebox.misc.system import get_dir_size from archivebox.misc.logging_util import printable_filesize @@ -24,6 +24,7 @@ def status(out_dir: Path = DATA_DIR) -> None: from django.db.models.functions import Coalesce from archivebox.core.models import Snapshot + config = get_config() User = get_user_model() print("[green]\\[*] Scanning archive main index...[/green]") @@ -36,13 +37,14 @@ def status(out_dir: Path = DATA_DIR) -> None: links = list(Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0))) num_sql_links = len(links) num_link_details = sum(1 for link in parse_json_links_details(out_dir=out_dir)) + archive_dir = config.ARCHIVE_DIR print(f" > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {CONSTANTS.SQL_INDEX_FILENAME})") - print(f" > JSON Link Details: {num_link_details} links".ljust(36), f"(found in {ARCHIVE_DIR.name}/*/index.json)") + print(f" > JSON Link Details: {num_link_details} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)") print() print("[green]\\[*] Scanning archive data directories...[/green]") - users_dir = out_dir / "users" - scan_roots = [root for root in (ARCHIVE_DIR, users_dir) if root.exists()] - scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(ARCHIVE_DIR) + users_dir = config.USERS_DIR + scan_roots = [root for root in (archive_dir, users_dir) if root.exists()] + scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir) print(f"[yellow] {scan_roots_display}[/yellow]") num_bytes = num_dirs = num_files = 0 for root in scan_roots: @@ -65,11 +67,17 @@ def status(out_dir: Path = DATA_DIR) -> None: expected_snapshot_dirs = {str(Path(snapshot.output_dir).resolve()) for snapshot in links if Path(snapshot.output_dir).exists()} discovered_snapshot_dirs = set() - if ARCHIVE_DIR.exists(): - discovered_snapshot_dirs.update(str(entry.resolve()) for entry in ARCHIVE_DIR.iterdir() if entry.is_dir()) + if archive_dir.exists(): + discovered_snapshot_dirs.update( + str(entry.resolve()) + for entry in archive_dir.iterdir() + if entry.is_dir() and not entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry) + ) if users_dir.exists(): - discovered_snapshot_dirs.update(str(entry.resolve()) for entry in users_dir.glob("*/snapshots/*/*/*") if entry.is_dir()) + discovered_snapshot_dirs.update( + str(entry.resolve()) for entry in users_dir.glob(f"*/{CONSTANTS.SNAPSHOTS_DIR_NAME}/*/*/*") if entry.is_dir() + ) orphaned_dirs = sorted(discovered_snapshot_dirs - expected_snapshot_dirs) num_present = len(discovered_snapshot_dirs) @@ -123,7 +131,7 @@ def status(out_dir: Path = DATA_DIR) -> None: f"[{snapshot.num_outputs} {('X', '√')[snapshot.is_archived]} {printable_filesize(snapshot.archive_size)}] " f'"{snapshot.title}": {snapshot.url}' "[/grey53]" - )[: SHELL_CONFIG.TERM_WIDTH], + )[: config.TERM_WIDTH], ) print("[grey53] ...") diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index 659fcb97..f4da8cb0 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -260,34 +260,50 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 Only processes real directories (skips symlinks - those are already migrated). For each old dir found in archive/: 1. Load or create DB snapshot - 2. Trigger fs migration on save() to move to data/users/{user}/... + 2. Trigger fs migration on save() to move to data/archive/users/{user}/... 3. Leave symlink in archive/ pointing to new location After this drains, archive/ should only contain symlinks and we can trust 1:1 mapping between DB and filesystem. """ from archivebox.core.models import Snapshot - from archivebox.config import CONSTANTS + from archivebox.config.common import get_config + from archivebox.crawls.models import Crawl from django.db import transaction + from django.utils import timezone stats = {"processed": 0, "migrated": 0, "skipped": 0, "invalid": 0} + crawl_output_dirs: dict[str, Path] = {} + crawl_url_lines: dict[str, list[str]] = {} + crawl_url_sets: dict[str, set[str]] = {} + dirty_crawl_ids: set[str] = set() - archive_dir = CONSTANTS.ARCHIVE_DIR + runtime_config = get_config() + archive_dir = runtime_config.ARCHIVE_DIR if not archive_dir.exists(): return stats - print("[DEBUG Phase1] Scanning for old directories in archive/...") + for crawl in Crawl.objects.filter(label__startswith="[migration] orphaned").iterator(): + url_entries = crawl._iter_url_lines() + existing_urls = {url for _raw_line, url in url_entries if url} + lines = (crawl.urls or "").splitlines() + changed = False + for url in crawl.snapshot_set.order_by("timestamp").values_list("url", flat=True): + if url not in existing_urls: + lines.append(url) + existing_urls.add(url) + changed = True + if changed: + Crawl.objects.filter(pk=crawl.pk).update(urls="\n".join(lines), modified_at=timezone.now()) # Scan for real directories only (skip symlinks - they're already migrated) all_entries = list(os.scandir(archive_dir)) - print(f"[DEBUG Phase1] Total entries in archive/: {len(all_entries)}") entries = [ (e.stat().st_mtime, e.path) for e in all_entries - if e.is_dir(follow_symlinks=False) # Skip symlinks + if e.is_dir(follow_symlinks=False) and Snapshot.is_legacy_archive_dir(Path(e.path)) # Skip symlinks and 0.9.x roots ] entries.sort(reverse=True) # Newest first - print(f"[DEBUG Phase1] Real directories (not symlinks): {len(entries)}") print(f"[*] Found {len(entries)} old directories to drain") for mtime, entry_path in entries: @@ -313,7 +329,42 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 continue try: - snapshot.save() + Snapshot.objects.bulk_create([snapshot]) + snapshot.migrate_filesystem_to_current_version(source_dir=entry_path, config=runtime_config) + Snapshot.objects.filter(pk=snapshot.pk).update( + fs_version=snapshot.fs_version, + modified_at=timezone.now(), + ) + migration_cleanup = getattr(snapshot, "_pending_fs_migration_cleanup", None) + new_dir = None + if migration_cleanup: + old_dir, new_dir = migration_cleanup + transaction.on_commit( + lambda old_dir=old_dir, new_dir=new_dir, snapshot=snapshot: snapshot._cleanup_old_migration_dir(old_dir, new_dir), + ) + delattr(snapshot, "_pending_fs_migration_cleanup") + + crawl = _get_snapshot_crawl(snapshot) + crawl_dir = None + if crawl is not None: + crawl_cache_key = str(crawl.id) + crawl_dir = crawl_output_dirs.get(crawl_cache_key) + if crawl_dir is None: + crawl_dir = Path(crawl.output_dir) + crawl_output_dirs[crawl_cache_key] = crawl_dir + + existing_urls = crawl_url_sets.get(crawl_cache_key) + if existing_urls is None: + url_entries = crawl._iter_url_lines() + existing_urls = {url for _raw_line, url in url_entries if url} + crawl_url_sets[crawl_cache_key] = existing_urls + crawl_url_lines[crawl_cache_key] = (crawl.urls or "").splitlines() + if snapshot.url not in existing_urls: + crawl_url_lines[crawl_cache_key].append(snapshot.url) + existing_urls.add(snapshot.url) + dirty_crawl_ids.add(crawl_cache_key) + + snapshot.ensure_crawl_symlink(crawl_dir=crawl_dir, snapshot_dir=new_dir) stats["migrated"] += 1 print(f" [{stats['processed']}] Imported orphaned snapshot: {entry_path.name}") except Exception as e: @@ -326,8 +377,6 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 if not has_valid_crawl: # Create a new crawl (created_by will default to system user) - from archivebox.crawls.models import Crawl - crawl = Crawl.objects.create(urls=snapshot.url) # Use queryset update to avoid triggering save() hooks from archivebox.core.models import Snapshot as SnapshotModel @@ -335,59 +384,57 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 SnapshotModel.objects.filter(pk=snapshot.pk).update(crawl=crawl) # Refresh the instance snapshot.crawl = crawl - print(f"[DEBUG Phase1] Created missing crawl for snapshot {str(snapshot.id)[:8]}") # Check if needs migration (0.8.x → 0.9.x) - print( - f"[DEBUG Phase1] Snapshot {str(snapshot.id)[:8]}: fs_version={snapshot.fs_version}, needs_migration={snapshot.fs_migration_needed}", - ) - if snapshot.fs_migration_needed: - try: - # Calculate paths using actual directory (entry_path), not snapshot.timestamp - # because snapshot.timestamp might be truncated - old_dir = entry_path - new_dir = snapshot.get_storage_path_for_version("0.9.0") - print(f"[DEBUG Phase1] Migrating {old_dir.name} → {new_dir}") - - # Manually migrate files - if not new_dir.exists() and old_dir.exists(): - new_dir.mkdir(parents=True, exist_ok=True) - import shutil - - file_count = 0 - for old_file in old_dir.rglob("*"): - if old_file.is_file(): - rel_path = old_file.relative_to(old_dir) - new_file = new_dir / rel_path - if not new_file.exists(): - new_file.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(old_file, new_file) - file_count += 1 - print(f"[DEBUG Phase1] Copied {file_count} files") - - # Update only fs_version field using queryset update (bypasses validation) - from archivebox.core.models import Snapshot as SnapshotModel - - SnapshotModel.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0") - - # Commit the transaction - transaction.commit() - - # Cleanup: delete old dir and create symlink - if old_dir.exists() and old_dir != new_dir: - snapshot._cleanup_old_migration_dir(old_dir, new_dir) - + try: + old_version = snapshot.fs_version + snapshot.migrate_filesystem_to_current_version(source_dir=entry_path, config=runtime_config) + if snapshot.fs_version != old_version or getattr(snapshot, "_pending_fs_migration_cleanup", None): + Snapshot.objects.filter(pk=snapshot.pk).update( + fs_version=snapshot.fs_version, + modified_at=timezone.now(), + ) + migration_cleanup = getattr(snapshot, "_pending_fs_migration_cleanup", None) + new_dir = None + if migration_cleanup: + old_dir, new_dir = migration_cleanup + transaction.on_commit( + lambda old_dir=old_dir, new_dir=new_dir, snapshot=snapshot: snapshot._cleanup_old_migration_dir(old_dir, new_dir), + ) + delattr(snapshot, "_pending_fs_migration_cleanup") + crawl_dir = None + if snapshot.crawl_id: + crawl_cache_key = str(snapshot.crawl_id) + crawl_dir = crawl_output_dirs.get(crawl_cache_key) + if crawl_dir is None: + crawl = _get_snapshot_crawl(snapshot) + if crawl is not None: + crawl_dir = Path(crawl.output_dir) + crawl_output_dirs[crawl_cache_key] = crawl_dir + snapshot.ensure_crawl_symlink(crawl_dir=crawl_dir, snapshot_dir=new_dir) stats["migrated"] += 1 print(f" [{stats['processed']}] Migrated: {entry_path.name}") - except Exception as e: + else: stats["skipped"] += 1 - print(f" [{stats['processed']}] Skipped (error: {e}): {entry_path.name}") - else: + except Exception as e: stats["skipped"] += 1 + print(f" [{stats['processed']}] Skipped (error: {e}): {entry_path.name}") if stats["processed"] % batch_size == 0: + for crawl_id in tuple(dirty_crawl_ids): + Crawl.objects.filter(pk=crawl_id).update( + urls="\n".join(crawl_url_lines[crawl_id]), + modified_at=timezone.now(), + ) + dirty_crawl_ids.clear() transaction.commit() + for crawl_id in tuple(dirty_crawl_ids): + Crawl.objects.filter(pk=crawl_id).update( + urls="\n".join(crawl_url_lines[crawl_id]), + modified_at=timezone.now(), + ) + dirty_crawl_ids.clear() transaction.commit() return stats @@ -404,10 +451,12 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) - after Phase 1 has drained all old archive/ directories. """ from archivebox.core.models import Snapshot + from archivebox.config.common import get_config from django.db import transaction from django.utils import timezone stats = {"processed": 0, "reconciled": 0, "queued": 0} + runtime_config = get_config() queryset = Snapshot.objects.all() if resume: @@ -416,7 +465,7 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) - print(f"[*] Processing {total} snapshots from database (most recent first)...") # Process from most recent to least recent - for snapshot in queryset.select_related("crawl").order_by("-bookmarked_at").iterator(chunk_size=batch_size): + for snapshot in queryset.select_related("crawl__created_by").order_by("-bookmarked_at").iterator(chunk_size=batch_size): stats["processed"] += 1 # Skip snapshots with missing crawl references (orphaned by migration errors) @@ -424,41 +473,47 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) - continue try: - print( - f"[DEBUG Phase2] Snapshot {str(snapshot.id)[:8]}: fs_version={snapshot.fs_version}, needs_migration={snapshot.fs_migration_needed}", - ) - # Check if snapshot has a directory on disk from pathlib import Path - output_dir = Path(snapshot.output_dir) + output_dir = Path(snapshot.get_storage_path_for_version(snapshot.fs_version, config=runtime_config)) has_directory = output_dir.exists() and output_dir.is_dir() + current_fs_version = Snapshot._fs_current_version() + update_values = { + "status": Snapshot.StatusChoices.QUEUED, + "retry_at": timezone.now(), + "modified_at": timezone.now(), + } # Only reconcile if directory exists (don't create empty directories for orphans) if has_directory: - snapshot.reconcile_with_index_json() + json_path = output_dir / "index.json" + jsonl_path = output_dir / "index.jsonl" + if json_path.exists() or not jsonl_path.exists(): + old_title = snapshot.title + snapshot.reconcile_with_index_json(output_dir=output_dir) + if snapshot.title != old_title: + update_values["title"] = snapshot.title # Clean up invalid field values from old migrations if not isinstance(snapshot.current_step, int): - snapshot.current_step = 0 + update_values["current_step"] = 0 - # If still needs migration, it's an orphan (no directory on disk) - # Mark it as migrated to prevent save() from triggering filesystem migration if snapshot.fs_migration_needed: - if has_directory: - print(f"[DEBUG Phase2] WARNING: Snapshot {str(snapshot.id)[:8]} has directory but still needs migration") + legacy_dir = snapshot.get_storage_path_for_version("0.8.0", config=runtime_config) + current_dir = snapshot.get_storage_path_for_version(current_fs_version, config=runtime_config) + if legacy_dir.exists() or current_dir.exists(): + snapshot.migrate_filesystem_to_current_version(config=runtime_config) + snapshot.status = update_values["status"] + snapshot.retry_at = update_values["retry_at"] + if "current_step" in update_values: + snapshot.current_step = update_values["current_step"] + snapshot.save(update_fields=tuple([*update_values.keys(), "fs_version"])) else: - print(f"[DEBUG Phase2] Orphan snapshot {str(snapshot.id)[:8]} - marking as migrated without filesystem operation") - # Use queryset update to set fs_version without triggering save() hooks - from archivebox.core.models import Snapshot as SnapshotModel - - SnapshotModel.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0") - snapshot.fs_version = "0.9.0" - - # Queue for archiving (state machine will handle it) - snapshot.status = Snapshot.StatusChoices.QUEUED - snapshot.retry_at = timezone.now() - snapshot.save() + update_values["fs_version"] = current_fs_version + Snapshot.objects.filter(pk=snapshot.pk).update(**update_values) + else: + Snapshot.objects.filter(pk=snapshot.pk).update(**update_values) stats["reconciled"] += 1 if has_directory else 0 stats["queued"] += 1 diff --git a/archivebox/cli/archivebox_version.py b/archivebox/cli/archivebox_version.py index cd088079..ad3e7224 100755 --- a/archivebox/cli/archivebox_version.py +++ b/archivebox/cli/archivebox_version.py @@ -107,9 +107,8 @@ def version( from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER from archivebox.config.paths import get_data_locations, get_code_locations - from archivebox.config.common import SHELL_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG from archivebox.misc.logging_util import printable_folder_status - from archivebox.config.configset import get_config + from archivebox.config.common import get_config console = Console() prnt = console.print @@ -127,7 +126,7 @@ def version( ) prnt( f"IN_DOCKER={IN_DOCKER}", - f"IN_QEMU={SHELL_CONFIG.IN_QEMU}", + f"IN_QEMU={config.IN_QEMU}", f"ARCH={p.machine}", f"OS={p.system}", f"PLATFORM={platform.platform()}", @@ -144,8 +143,8 @@ def version( prnt( f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}", - f"FS_PERMS={STORAGE_CONFIG.OUTPUT_PERMISSIONS}", - f"FS_ATOMIC={STORAGE_CONFIG.ENFORCE_ATOMIC_WRITES}", + f"FS_PERMS={config.OUTPUT_PERMISSIONS}", + f"FS_ATOMIC={config.ENFORCE_ATOMIC_WRITES}", f"FS_REMOTE={OUTPUT_IS_REMOTE_FS}", ) except Exception: @@ -154,16 +153,16 @@ def version( ) prnt( - f"DEBUG={SHELL_CONFIG.DEBUG}", - f"IS_TTY={SHELL_CONFIG.IS_TTY}", + f"DEBUG={config.DEBUG}", + f"IS_TTY={config.IS_TTY}", f"SUDO={CONSTANTS.IS_ROOT}", f"ID={CONSTANTS.MACHINE_ID}:{CONSTANTS.COLLECTION_ID}", - f"SEARCH_BACKEND={SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}", + f"SEARCH_BACKEND={config.SEARCH_BACKEND_ENGINE}", f"LDAP={LDAP_ENABLED}", ) prnt() - if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)): + if not (os.access(config.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)): PANEL_TEXT = "\n".join( ( "", @@ -226,7 +225,7 @@ def version( _format_binary_abspath( installed.abspath, pwd=Path.cwd(), - lib_dir=STORAGE_CONFIG.LIB_DIR, + lib_dir=config.LIB_DIR, personas_dir=Path.home() / ".config" / "abx" / "personas", home=Path.home(), ) @@ -283,7 +282,7 @@ def version( prnt(f" [red]Error getting code locations: {e}[/red]") prnt() - if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK): + if os.access(config.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK): prnt("[bright_yellow][i] Data locations:[/bright_yellow]") try: for name, path in get_data_locations().items(): diff --git a/archivebox/config/__init__.py b/archivebox/config/__init__.py index 5f4f9032..356bf8a6 100644 --- a/archivebox/config/__init__.py +++ b/archivebox/config/__init__.py @@ -1,9 +1,4 @@ -""" -ArchiveBox config exports. - -This module provides backwards-compatible config exports for extractors -and other modules that expect to import config values directly. -""" +"""Minimal import-time config exports.""" __package__ = "archivebox.config" __order__ = 200 @@ -11,95 +6,6 @@ __order__ = 200 from .paths import ( PACKAGE_DIR, DATA_DIR, - ARCHIVE_DIR, ) -from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR # noqa +from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR # noqa from .version import VERSION # noqa - - -############################################################################### -# Config value exports for extractors -# These provide backwards compatibility with extractors that import from ..config -############################################################################### - - -def _get_config(): - """Lazy import to avoid circular imports.""" - from .common import ARCHIVING_CONFIG, STORAGE_CONFIG - - return ARCHIVING_CONFIG, STORAGE_CONFIG - - -# Direct exports (evaluated at import time for backwards compat) -# These are recalculated each time the module attribute is accessed - - -def __getattr__(name: str): - """ - Module-level __getattr__ for lazy config loading. - - Only provides backwards compatibility for GENERIC/SHARED config. - Plugin-specific config (binaries, args, toggles) should come from plugin config.json files. - """ - - # Generic timeout settings (used by multiple plugins) - if name == "TIMEOUT": - cfg, _ = _get_config() - return cfg.TIMEOUT - - # Generic SSL/Security settings (used by multiple plugins) - if name == "CHECK_SSL_VALIDITY": - cfg, _ = _get_config() - return cfg.CHECK_SSL_VALIDITY - - # Generic storage settings (used by multiple plugins) - if name == "RESTRICT_FILE_NAMES": - _, storage = _get_config() - return storage.RESTRICT_FILE_NAMES - - # Generic user agent / cookies (used by multiple plugins) - if name == "COOKIES_FILE": - cfg, _ = _get_config() - return cfg.COOKIES_FILE - if name == "USER_AGENT": - cfg, _ = _get_config() - return cfg.USER_AGENT - - # Generic resolution settings (used by multiple plugins) - if name == "RESOLUTION": - cfg, _ = _get_config() - return cfg.RESOLUTION - - # Allowlist/Denylist patterns (compiled regexes) - if name == "SAVE_ALLOWLIST_PTN": - cfg, _ = _get_config() - return cfg.SAVE_ALLOWLIST_PTNS - if name == "SAVE_DENYLIST_PTN": - cfg, _ = _get_config() - return cfg.SAVE_DENYLIST_PTNS - - raise AttributeError(f"module 'archivebox.config' has no attribute '{name}'") - - -# Re-export common config classes for direct imports -def get_CONFIG(): - """Get all config sections as a dict.""" - from .common import ( - SHELL_CONFIG, - STORAGE_CONFIG, - GENERAL_CONFIG, - SERVER_CONFIG, - ARCHIVING_CONFIG, - SEARCH_BACKEND_CONFIG, - ) - from .ldap import LDAP_CONFIG - - return { - "SHELL_CONFIG": SHELL_CONFIG, - "STORAGE_CONFIG": STORAGE_CONFIG, - "GENERAL_CONFIG": GENERAL_CONFIG, - "SERVER_CONFIG": SERVER_CONFIG, - "ARCHIVING_CONFIG": ARCHIVING_CONFIG, - "SEARCHBACKEND_CONFIG": SEARCH_BACKEND_CONFIG, - "LDAP_CONFIG": LDAP_CONFIG, - } diff --git a/archivebox/config/collection.py b/archivebox/config/collection.py index 215bafc4..b3474104 100644 --- a/archivebox/config/collection.py +++ b/archivebox/config/collection.py @@ -1,176 +1,18 @@ __package__ = "archivebox.config" import os -import json -from typing import Any - -from pathlib import Path -from configparser import ConfigParser from benedict import benedict - from archivebox.config.constants import CONSTANTS - -from archivebox.misc.logging import stderr - - -class CaseConfigParser(ConfigParser): - def optionxform(self, optionstr: str) -> str: - return optionstr - - -def get_real_name(key: str) -> str: - """get the up-to-date canonical name for a given old alias or current key""" - # Config aliases are no longer used with the simplified config system - # Just return the key as-is since we no longer have a complex alias mapping - return key - - -def load_config_val( - key: str, - default: Any = None, - type: type | None = None, - aliases: tuple[str, ...] | None = None, - config: benedict | None = None, - env_vars: os._Environ | None = None, - config_file_vars: dict[str, str] | None = None, -) -> Any: - """parse bool, int, and str key=value pairs from env""" - - assert isinstance(config, dict) - - is_read_only = type is None - if is_read_only: - if callable(default): - return default(config) - return default - - # get value from environment variables or config files - config_keys_to_check = (key, *(aliases or ())) - val = None - for key in config_keys_to_check: - if env_vars: - val = env_vars.get(key) - if val: - break - - if config_file_vars: - val = config_file_vars.get(key) - if val: - break - - is_unset = val is None - if is_unset: - if callable(default): - return default(config) - return default - - assert isinstance(val, str) - - # calculate value based on expected type - BOOL_TRUEIES = ("true", "yes", "1") - BOOL_FALSEIES = ("false", "no", "0") - - if type is bool: - if val.lower() in BOOL_TRUEIES: - return True - elif val.lower() in BOOL_FALSEIES: - return False - else: - raise ValueError(f"Invalid configuration option {key}={val} (expected a boolean: True/False)") - - elif type is str: - if val.lower() in (*BOOL_TRUEIES, *BOOL_FALSEIES): - raise ValueError(f"Invalid configuration option {key}={val} (expected a string, but value looks like a boolean)") - return val.strip() - - elif type is int: - if not val.strip().isdigit(): - raise ValueError(f"Invalid configuration option {key}={val} (expected an integer)") - return int(val.strip()) - - elif type is list or type is dict: - return json.loads(val) - - elif type is Path: - return Path(val) - - raise Exception("Config values can only be str, bool, int, or json") - - -def load_config_file() -> benedict | None: - """load the ini-formatted config file from DATA_DIR/Archivebox.conf""" - - config_path = CONSTANTS.CONFIG_FILE - if os.access(config_path, os.R_OK): - config_file = CaseConfigParser() - config_file.read(config_path) - # flatten into one namespace - config_file_vars = benedict({key.upper(): val for section, options in config_file.items() for key, val in options.items()}) - # print('[i] Loaded config file', os.path.abspath(config_path)) - # print(config_file_vars) - return config_file_vars - return None - - -class PluginConfigSection: - """Pseudo-section for all plugin config keys written to [PLUGINS] section in ArchiveBox.conf""" - - toml_section_header = "PLUGINS" - - def __init__(self, key: str): - self._key = key - - def __getattr__(self, name: str) -> Any: - # Allow hasattr checks to pass for the key - if name == self._key: - return None - raise AttributeError(f"PluginConfigSection has no attribute '{name}'") - - def update_in_place(self, warn: bool = True, persist: bool = False, **kwargs): - """No-op update since plugins read config dynamically via get_config().""" - pass - - -def section_for_key(key: str) -> Any: - """Find the config section containing a given key.""" - from archivebox.config.common import ( - SHELL_CONFIG, - STORAGE_CONFIG, - GENERAL_CONFIG, - SERVER_CONFIG, - ARCHIVING_CONFIG, - SEARCH_BACKEND_CONFIG, - ) - - # First check core config sections - for section in [ - SHELL_CONFIG, - STORAGE_CONFIG, - GENERAL_CONFIG, - SERVER_CONFIG, - ARCHIVING_CONFIG, - SEARCH_BACKEND_CONFIG, - ]: - if hasattr(section, key): - return section - - # Check if this is a plugin config key - from archivebox.hooks import discover_plugin_configs - - plugin_configs = discover_plugin_configs() - for plugin_name, schema in plugin_configs.items(): - if "properties" in schema and key in schema["properties"]: - # All plugin config goes to [PLUGINS] section - return PluginConfigSection(key) - - raise ValueError(f"No config section found for key: {key}") +from archivebox.config.configset import CaseConfigParser def write_config_file(config: dict[str, str]) -> benedict: """load the ini-formatted config file from DATA_DIR/Archivebox.conf""" + from archivebox.config.common import get_all_configs + from archivebox.hooks import discover_plugin_configs from archivebox.misc.system import atomic_write CONFIG_HEADER = """# This is the config file for your ArchiveBox collection. @@ -197,15 +39,25 @@ def write_config_file(config: dict[str, str]) -> benedict: with open(config_path, encoding="utf-8") as old: atomic_write(f"{config_path}.bak", old.read()) + config_sections = get_all_configs() + plugin_configs = discover_plugin_configs() + # Set up sections in empty config file for key, val in config.items(): - section = section_for_key(key) - assert section is not None + section_name = None + for section in config_sections.values(): + if key in type(section).model_fields: + section_name = section.toml_section_header + break - if not hasattr(section, "toml_section_header"): - raise ValueError(f"{key} is read-only (defined in {type(section).__module__}.{type(section).__name__}). Refusing to set.") + if section_name is None: + for schema in plugin_configs.values(): + if "properties" in schema and key in schema["properties"]: + section_name = "PLUGINS" + break - section_name = section.toml_section_header + if section_name is None: + raise ValueError(f"No config section found for key: {key}") if section_name in config_file: existing_config = dict(config_file[section_name]) @@ -213,7 +65,6 @@ def write_config_file(config: dict[str, str]) -> benedict: existing_config = {} config_file[section_name] = benedict({**existing_config, key: val}) - section.update_in_place(warn=False, persist=False, **{key: val}) with open(config_path, "w+", encoding="utf-8") as new: config_file.write(new) @@ -221,9 +72,9 @@ def write_config_file(config: dict[str, str]) -> benedict: updated_config = {} try: # validate the updated_config by attempting to re-parse it - from archivebox.config.configset import get_flat_config + from archivebox.config.common import get_config - updated_config = {**load_all_config(), **get_flat_config()} + updated_config = get_config().as_dict() except BaseException: # lgtm [py/catch-base-exception] # something went horribly wrong, revert to the previous version with open(f"{config_path}.bak", encoding="utf-8") as old: @@ -235,71 +86,3 @@ def write_config_file(config: dict[str, str]) -> benedict: os.remove(f"{config_path}.bak") return benedict({key.upper(): updated_config.get(key.upper()) for key in config.keys()}) - - -def load_config( - defaults: dict[str, Any], - config: benedict | None = None, - out_dir: str | None = None, - env_vars: os._Environ | None = None, - config_file_vars: dict[str, str] | None = None, -) -> benedict: - - env_vars = env_vars or os.environ - config_file_vars = config_file_vars or load_config_file() - - extended_config = benedict(config.copy() if config else {}) - for key, default in defaults.items(): - try: - # print('LOADING CONFIG KEY:', key, 'DEFAULT=', default) - extended_config[key] = load_config_val( - key, - default=default["default"], - type=default.get("type"), - aliases=default.get("aliases"), - config=extended_config, - env_vars=env_vars, - config_file_vars=config_file_vars, - ) - except KeyboardInterrupt: - raise SystemExit(0) - except Exception as e: - stderr() - stderr(f"[X] Error while loading configuration value: {key}", color="red", config=extended_config) - stderr(f" {e.__class__.__name__}: {e}") - stderr() - stderr(" Check your config for mistakes and try again (your archive data is unaffected).") - stderr() - stderr(" For config documentation and examples see:") - stderr(" https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration") - stderr() - # raise - # raise SystemExit(2) - - return benedict(extended_config) - - -def load_all_config(): - """Load all config sections and return as a flat dict.""" - from archivebox.config.common import ( - SHELL_CONFIG, - STORAGE_CONFIG, - GENERAL_CONFIG, - SERVER_CONFIG, - ARCHIVING_CONFIG, - SEARCH_BACKEND_CONFIG, - ) - - flat_config = benedict() - - for config_section in [ - SHELL_CONFIG, - STORAGE_CONFIG, - GENERAL_CONFIG, - SERVER_CONFIG, - ARCHIVING_CONFIG, - SEARCH_BACKEND_CONFIG, - ]: - flat_config.update(dict(config_section)) - - return flat_config diff --git a/archivebox/config/common.py b/archivebox/config/common.py index 2be64d9b..300c6bcb 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -1,25 +1,37 @@ __package__ = "archivebox.config" +import json import re import secrets import sys import shutil -from typing import ClassVar +from collections.abc import Mapping +from typing import Any, ClassVar, cast from pathlib import Path from rich.console import Console -from pydantic import Field, field_validator +from pydantic import BaseModel, Field, create_model, field_validator, model_validator +from pydantic_settings import SettingsConfigDict +from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs from archivebox.config.configset import BaseConfigSet +from archivebox.config.configset import COMPUTED_CONFIG_KEYS from .constants import CONSTANTS +from .ldap import LDAPConfig from .version import get_COMMIT_HASH, get_BUILD_TIME, VERSION from .permissions import IN_DOCKER +ConfigOverrides = Mapping[str, object] +ConfigPayload = dict[str, object] +PluginSchemaDocuments = dict[str, dict[str, Any]] + ###################### Config ########################## _STDOUT_CONSOLE = Console() _STDERR_CONSOLE = Console(stderr=True) +_WARNED_SERVER_SECURITY_MODES: set[str] = set() +_WARNED_ARCHIVING_CONFIGS: set[tuple[int, bool]] = set() def rprint(*args, file=None, **kwargs): @@ -58,12 +70,13 @@ class ShellConfig(BaseConfigSet): return get_BUILD_TIME() -SHELL_CONFIG = ShellConfig() - - class StorageConfig(BaseConfigSet): toml_section_header: str = "STORAGE_CONFIG" + # ARCHIVE_DIR / USERS_DIR are resolved dynamically via get_config(). + ARCHIVE_DIR: Path = Field(default=CONSTANTS.ARCHIVE_DIR) + USERS_DIR: Path = Field(default=CONSTANTS.USERS_DIR) + # TMP_DIR must be a local, fast, readable/writable dir by archivebox user, # must be a short path due to unix path length restrictions for socket files (<100 chars) # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets @@ -90,18 +103,12 @@ class StorageConfig(BaseConfigSet): DIR_OUTPUT_PERMISSIONS: str = Field(default="755") # computed from OUTPUT_PERMISSIONS -STORAGE_CONFIG = StorageConfig() - - class GeneralConfig(BaseConfigSet): toml_section_header: str = "GENERAL_CONFIG" TAG_SEPARATOR_PATTERN: str = Field(default=r"[,]") -GENERAL_CONFIG = GeneralConfig() - - class ServerConfig(BaseConfigSet): toml_section_header: str = "SERVER_CONFIG" @@ -130,6 +137,7 @@ class ServerConfig(BaseConfigSet): PUBLIC_INDEX: bool = Field(default=True) PUBLIC_SNAPSHOTS: bool = Field(default=True) + PUBLIC_SNAPSHOTS_LIST: bool | None = Field(default=None) PUBLIC_ADD_VIEW: bool = Field(default=False) ADMIN_USERNAME: str | None = Field(default=None) @@ -186,15 +194,14 @@ class ServerConfig(BaseConfigSet): ) -SERVER_CONFIG = ServerConfig() - - -def _print_server_security_mode_warning() -> None: - if not SERVER_CONFIG.IS_LOWER_SECURITY_MODE: +def _print_server_security_mode_warning(config: ServerConfig) -> None: + if not config.IS_LOWER_SECURITY_MODE: + return + if config.SERVER_SECURITY_MODE in _WARNED_SERVER_SECURITY_MODES: return rprint( - f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={SERVER_CONFIG.SERVER_SECURITY_MODE}[/yellow]", + f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={config.SERVER_SECURITY_MODE}[/yellow]", file=sys.stderr, ) rprint( @@ -217,19 +224,33 @@ def _print_server_security_mode_warning() -> None: "[yellow] 3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]", file=sys.stderr, ) - - -_print_server_security_mode_warning() + _WARNED_SERVER_SECURITY_MODES.add(config.SERVER_SECURITY_MODE) class ArchivingConfig(BaseConfigSet): toml_section_header: str = "ARCHIVING_CONFIG" + PLUGINS: str = Field( + default="", + description="Comma-separated plugin selection for this run. Empty means use enabled plugin defaults.", + ) + ENABLED_PLUGINS: str = Field( + default="", + description="Comma-separated plugin selection override used by the UI and API.", + ) + ENABLED_EXTRACTORS: str = Field( + default="", + description="Legacy comma-separated plugin selection override.", + ) + ONLY_NEW: bool = Field(default=True) OVERWRITE: bool = Field(default=False) TIMEOUT: int = Field(default=60) MAX_URL_ATTEMPTS: int = Field(default=50) + MAX_DEPTH: int = Field(default=0) + MAX_URLS: int = Field(default=0) + MAX_SIZE: int = Field(default=0) RESOLUTION: str = Field(default="1440,2000") CHECK_SSL_VALIDITY: bool = Field(default=True) @@ -298,10 +319,6 @@ class ArchivingConfig(BaseConfigSet): ) -ARCHIVING_CONFIG = ArchivingConfig() -ARCHIVING_CONFIG.warn_if_invalid() - - class SearchBackendConfig(BaseConfigSet): toml_section_header: str = "SEARCH_BACKEND_CONFIG" @@ -312,4 +329,250 @@ class SearchBackendConfig(BaseConfigSet): SEARCH_PROCESS_HTML: bool = Field(default=True) -SEARCH_BACKEND_CONFIG = SearchBackendConfig() +def _plugin_user_config_value(value: Any) -> str: + if isinstance(value, Path): + return str(value) + if isinstance(value, (dict, list, bool, int, float)) or value is None: + return json.dumps(value) + return str(value) + + +def _plugin_user_config(config: Mapping[str, object]) -> dict[str, str]: + return {key: _plugin_user_config_value(value) for key, value in config.items()} + + +def _discover_plugin_config_schemas() -> PluginSchemaDocuments: + from archivebox.hooks import discover_plugin_configs + + schemas: PluginSchemaDocuments = {} + if BASE_CONFIG_PATH.exists(): + schemas["base"] = { + "properties": json.loads(BASE_CONFIG_PATH.read_text()).get("properties", {}), + } + schemas.update(discover_plugin_configs()) + return schemas + + +def _plugin_config_properties(plugin_schemas: PluginSchemaDocuments) -> dict[str, dict[str, Any]]: + properties: dict[str, dict[str, Any]] = {} + for schema in plugin_schemas.values(): + schema_properties = schema.get("properties") or {} + if isinstance(schema_properties, dict): + properties.update(schema_properties) + return properties + + +def _plugin_config_model(plugin_schemas: PluginSchemaDocuments) -> type[BaseModel]: + return build_config_model("ArchiveBoxPluginConfig", _plugin_config_properties(plugin_schemas)) + + +def _archivebox_config_input_names() -> set[str]: + names = set(ArchiveBoxConfig.model_fields) + for field in ArchiveBoxConfig.model_fields.values(): + if isinstance(field.alias, str): + names.add(field.alias) + return names + + +class ArchiveBoxBaseConfig( + ShellConfig, + StorageConfig, + GeneralConfig, + ServerConfig, + ArchivingConfig, + SearchBackendConfig, + LDAPConfig, +): + """Merged, typed ArchiveBox config. + + Core ArchiveBox fields are declared above. Plugin-owned fields are added to + the concrete ArchiveBoxConfig model from plugin JSONSchema below, so + ArchiveBox does not hardcode any individual plugin config names. + """ + + model_config = SettingsConfigDict( + env_prefix="", + extra="ignore", + validate_default=True, + use_enum_values=True, + arbitrary_types_allowed=True, + populate_by_name=True, + ) + + DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR) + ABX_RUNTIME: str = Field(default="archivebox") + CRAWL_DIR: Path | None = Field(default=None) + CRAWL_OUTPUT_DIR: Path | None = Field(default=None) + SNAP_DIR: Path | None = Field(default=None) + computed_config_keys: ClassVar[tuple[str, ...]] = COMPUTED_CONFIG_KEYS + + @model_validator(mode="after") + def resolve_runtime_paths(self): + self.DATA_DIR = self.DATA_DIR.expanduser().resolve() + + archive_dir = self.ARCHIVE_DIR.expanduser() + if archive_dir == (CONSTANTS.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME) and self.DATA_DIR != CONSTANTS.DATA_DIR: + archive_dir = self.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME + if not archive_dir.is_absolute(): + archive_dir = self.DATA_DIR / archive_dir + self.ARCHIVE_DIR = archive_dir.resolve() + + users_dir = self.USERS_DIR.expanduser() + if users_dir == (CONSTANTS.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME): + users_dir = self.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME + if not users_dir.is_absolute(): + users_dir = self.ARCHIVE_DIR / users_dir + self.USERS_DIR = users_dir.resolve() + + return self + + +def _build_archivebox_config_model(plugin_schemas: PluginSchemaDocuments) -> type[ArchiveBoxBaseConfig]: + core_fields = set(ArchiveBoxBaseConfig.model_fields) + plugin_fields: dict[str, Any] = { + key: (field.annotation, field) for key, field in _plugin_config_model(plugin_schemas).model_fields.items() if key not in core_fields + } + return cast( + type[ArchiveBoxBaseConfig], + create_model( + "ArchiveBoxConfig", + __base__=ArchiveBoxBaseConfig, + __module__=__name__, + **plugin_fields, + ), + ) + + +PLUGIN_CONFIG_SCHEMAS = _discover_plugin_config_schemas() +ArchiveBoxConfig = _build_archivebox_config_model(PLUGIN_CONFIG_SCHEMAS) + + +def get_config( + defaults: ConfigOverrides | None = None, + overrides: ConfigOverrides | None = None, + persona: Any = None, + user: Any = None, + crawl: Any = None, + snapshot: Any = None, + archiveresult: Any = None, + machine: Any = None, +) -> ArchiveBoxBaseConfig: + """ + Get merged config from all sources. + + Priority (highest to lowest): + 1. Explicit overrides + 2. Per-snapshot config and output path + 3. Per-crawl config and output path + 4. Per-user config + 5. Per-persona derived config + 6. Current machine derived config + 7. Environment variables + 8. Config file (ArchiveBox.conf) + 9. Plugin schema defaults + 10. Core config defaults + """ + if snapshot is None and archiveresult is not None: + snapshot = archiveresult.snapshot + + if crawl is None and snapshot is not None: + crawl = snapshot.crawl + + if machine is None: + try: + from django.apps import apps + + if apps.ready: + from archivebox.machine.models import Machine + + machine = Machine.current() + except Exception: + machine = None + + if persona is None and crawl is not None: + from archivebox.personas.models import Persona + + persona_id = crawl.persona_id + if persona_id: + persona = Persona.objects.filter(id=persona_id).first() + if persona is None: + raise Persona.DoesNotExist(f"Crawl {crawl.id} references missing Persona {persona_id}") + + if persona is None: + crawl_config = crawl.config or {} + default_persona_name = str(crawl_config.get("DEFAULT_PERSONA") or "").strip() + if default_persona_name: + persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default") + persona.ensure_dirs() + + config_data: ConfigPayload = dict(defaults or {}) + config_data.update(ArchiveBoxConfig().model_dump(mode="json")) + + plugin_schemas = { + plugin_name: schema.get("properties", {}) for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict) + } + + scope_overrides: ConfigPayload = {} + + if machine is not None and machine.config: + from archivebox.machine.models import _sanitize_machine_config + + scope_overrides.update(_sanitize_machine_config(machine.config)) + + if persona is not None: + scope_overrides.update(persona.get_derived_config()) + + if user is not None and user.config: + scope_overrides.update(user.config) + + if crawl is not None and crawl.config: + scope_overrides.update(crawl.config) + + if crawl is not None: + scope_overrides["CRAWL_OUTPUT_DIR"] = crawl.output_dir + scope_overrides["CRAWL_DIR"] = crawl.output_dir + + if snapshot is not None and snapshot.config: + scope_overrides.update(snapshot.config) + + if snapshot is not None: + scope_overrides["SNAP_DIR"] = snapshot.output_dir + + if overrides: + scope_overrides.update(overrides) + + archivebox_scope_overrides = {key: value for key, value in scope_overrides.items() if key in _archivebox_config_input_names()} + config_data.update(archivebox_scope_overrides) + + plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()} + plugin_sections = resolve_plugin_configs( + plugin_schemas, + global_config=plugin_global_config, + user_config={**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **_plugin_user_config(scope_overrides)}, + ) + for plugin_config in plugin_sections.values(): + config_data.update(plugin_config) + config_data.update(archivebox_scope_overrides) + + config_data["ABX_RUNTIME"] = "archivebox" + + config = ArchiveBoxConfig.model_validate(config_data) + archiving_warning_key = (config.TIMEOUT, config.USE_COLOR) + if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS: + config.warn_if_invalid() + _WARNED_ARCHIVING_CONFIGS.add(archiving_warning_key) + _print_server_security_mode_warning(config) + return config + + +def get_all_configs() -> dict[str, BaseConfigSet]: + """Get all config section objects as a dictionary.""" + return { + "SHELL_CONFIG": ShellConfig(), + "STORAGE_CONFIG": StorageConfig(), + "GENERAL_CONFIG": GeneralConfig(), + "SERVER_CONFIG": ServerConfig(), + "ARCHIVING_CONFIG": ArchivingConfig(), + "SEARCH_BACKEND_CONFIG": SearchBackendConfig(), + "LDAP_CONFIG": LDAPConfig(), + } diff --git a/archivebox/config/configset.py b/archivebox/config/configset.py index da055632..b4c15efe 100644 --- a/archivebox/config/configset.py +++ b/archivebox/config/configset.py @@ -1,21 +1,31 @@ -""" -Simplified config system for ArchiveBox. - -This replaces the complex abx_spec_config/base_configset.py with a simpler -approach that still supports environment variables, config files, and -per-object overrides. -""" +"""Pydantic-backed config loading for ArchiveBox.""" __package__ = "archivebox.config" -import os -import json from pathlib import Path -from typing import Any +from typing import Any, ClassVar from configparser import ConfigParser from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict +COMPUTED_CONFIG_KEYS = ( + "TERM_WIDTH", + "COMMIT_HASH", + "BUILD_TIME", + "USES_SUBDOMAIN_ROUTING", + "ENABLES_FULL_JS_REPLAY", + "CONTROL_PLANE_ENABLED", + "BLOCK_UNSAFE_METHODS", + "SHOULD_NEUTER_RISKY_REPLAY", + "IS_UNSAFE_MODE", + "IS_DANGEROUS_MODE", + "IS_LOWER_SECURITY_MODE", + "URL_ALLOWLIST_PTN", + "URL_DENYLIST_PTN", + "SAVE_ALLOWLIST_PTNS", + "SAVE_DENYLIST_PTNS", +) + class CaseConfigParser(ConfigParser): def optionxform(self, optionstr: str) -> str: @@ -74,7 +84,9 @@ class BaseConfigSet(BaseSettings): env_prefix="", extra="ignore", validate_default=True, + populate_by_name=True, ) + computed_config_keys: ClassVar[tuple[str, ...]] = () @classmethod def settings_customise_sources( @@ -108,301 +120,57 @@ class BaseConfigSet(BaseSettings): # Flatten all sections into single namespace return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)} - def update_in_place(self, warn: bool = True, persist: bool = False, **kwargs) -> None: - """ - Update config values in place. + def __getitem__(self, key: str) -> Any: + if key in type(self).model_fields: + return getattr(self, key) + if self.__pydantic_extra__ and key in self.__pydantic_extra__: + return self.__pydantic_extra__[key] + if key in self.computed_config_keys: + return getattr(self, key) + raise KeyError(key) - This allows runtime updates to config without reloading. - """ - for key, value in kwargs.items(): - if hasattr(self, key): - # Use object.__setattr__ to bypass pydantic's frozen model - object.__setattr__(self, key, value) + def __setitem__(self, key: str, value: Any) -> None: + if key in type(self).model_fields: + object.__setattr__(self, key, value) + return + if key in self.computed_config_keys: + raise KeyError(f"{key} is computed and cannot be set") + if self.model_config.get("extra") != "allow": + raise KeyError(f"Unknown config key: {key}") + extra = self.__pydantic_extra__ + if extra is None: + extra = {} + object.__setattr__(self, "__pydantic_extra__", extra) + extra[key] = value + def update(self, *args, **kwargs) -> None: + values = dict(*args, **kwargs) + for key, value in values.items(): + if key in self.computed_config_keys: + continue + self[key] = value -def get_config( - defaults: dict | None = None, - persona: Any = None, - user: Any = None, - crawl: Any = None, - snapshot: Any = None, - archiveresult: Any = None, - machine: Any = None, -) -> dict[str, Any]: - """ - Get merged config from all sources. + def __contains__(self, key: str) -> bool: + return ( + key in type(self).model_fields + or bool(self.__pydantic_extra__ and key in self.__pydantic_extra__) + or key in self.computed_config_keys + ) - Priority (highest to lowest): - 1. Per-snapshot config (snapshot.config JSON field) - 2. Per-crawl config (crawl.config JSON field) - 3. Per-user config (user.config JSON field) - 4. Per-persona config (persona.get_derived_config() - includes CHROME_USER_DATA_DIR etc.) - 5. Environment variables - 6. Config file (ArchiveBox.conf) - 7. Plugin schema defaults (config.json) - 8. Core config defaults + def get(self, key: str, default: Any = None) -> Any: + return self[key] if key in self else default - Args: - defaults: Default values to start with - persona: Persona object (provides derived paths like CHROME_USER_DATA_DIR) - user: User object with config JSON field - crawl: Crawl object with config JSON field - snapshot: Snapshot object with config JSON field - archiveresult: ArchiveResult object (auto-fetches snapshot) - machine: Unused legacy argument kept for call compatibility + def as_dict(self) -> dict[str, Any]: + data = self.model_dump() + for key in self.computed_config_keys: + data[key] = getattr(self, key) + return data - Note: Objects are auto-fetched from relationships if not provided: - - snapshot auto-fetched from archiveresult.snapshot - - crawl auto-fetched from snapshot.crawl - - user auto-fetched from crawl.created_by + def items(self): + return self.as_dict().items() - Returns: - Merged config dict - """ - # Auto-fetch related objects from relationships - if snapshot is None and archiveresult and hasattr(archiveresult, "snapshot"): - snapshot = archiveresult.snapshot + def keys(self): + return self.as_dict().keys() - if crawl is None and snapshot and hasattr(snapshot, "crawl"): - crawl = snapshot.crawl - - if user is None and crawl and hasattr(crawl, "created_by"): - user = crawl.created_by - - if persona is None and crawl is not None: - from archivebox.personas.models import Persona - - persona_id = getattr(crawl, "persona_id", None) - if persona_id: - persona = Persona.objects.filter(id=persona_id).first() - if persona is None: - raise Persona.DoesNotExist(f"Crawl {getattr(crawl, 'id', None)} references missing Persona {persona_id}") - - if persona is None: - crawl_config = getattr(crawl, "config", None) or {} - default_persona_name = str(crawl_config.get("DEFAULT_PERSONA") or "").strip() - if default_persona_name: - persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default") - persona.ensure_dirs() - from archivebox.config.constants import CONSTANTS - from archivebox.config.common import ( - SHELL_CONFIG, - STORAGE_CONFIG, - GENERAL_CONFIG, - SERVER_CONFIG, - ARCHIVING_CONFIG, - SEARCH_BACKEND_CONFIG, - ) - - # Start with defaults - config = dict(defaults or {}) - - # Add plugin config defaults from JSONSchema config.json files - try: - from archivebox.hooks import get_config_defaults_from_plugins - - plugin_defaults = get_config_defaults_from_plugins() - config.update(plugin_defaults) - except ImportError: - pass # hooks not available yet during early startup - - # Add all core config sections - config.update(dict(SHELL_CONFIG)) - config.update(dict(STORAGE_CONFIG)) - config.update(dict(GENERAL_CONFIG)) - config.update(dict(SERVER_CONFIG)) - config.update(dict(ARCHIVING_CONFIG)) - config.update(dict(SEARCH_BACKEND_CONFIG)) - - # Load from archivebox.config.file - config_file = CONSTANTS.CONFIG_FILE - if config_file.exists(): - file_config = BaseConfigSet.load_from_file(config_file) - config.update(file_config) - - # Override with environment variables (for keys that exist in config) - for key in config: - env_val = os.environ.get(key) - if env_val is not None: - config[key] = _parse_env_value(env_val, config.get(key)) - - # Also add NEW environment variables (not yet in config) - # This is important for worker subprocesses that receive config via Process.env - for key, value in os.environ.items(): - if key.isupper() and key not in config: # Only uppercase keys (config convention) - config[key] = _parse_env_value(value, None) - - # Also check plugin config aliases in environment - try: - from archivebox.hooks import discover_plugin_configs - - plugin_configs = discover_plugin_configs() - for plugin_name, schema in plugin_configs.items(): - for key, prop_schema in schema.get("properties", {}).items(): - # Check x-aliases - for alias in prop_schema.get("x-aliases", []): - if alias in os.environ and key not in os.environ: - config[key] = _parse_env_value(os.environ[alias], config.get(key)) - break - # Check x-fallback - fallback = prop_schema.get("x-fallback") - if fallback and fallback in config and key not in config: - config[key] = config[fallback] - except ImportError: - pass - - # Apply persona config overrides (includes derived paths like CHROME_USER_DATA_DIR) - if persona and hasattr(persona, "get_derived_config"): - config.update(persona.get_derived_config()) - - # Apply user config overrides - if user and hasattr(user, "config") and user.config: - config.update(user.config) - - # Apply crawl config overrides - if crawl and hasattr(crawl, "config") and crawl.config: - config.update(crawl.config) - - # Add crawl path aliases for hooks that need shared crawl state. - if crawl and hasattr(crawl, "output_dir"): - config["CRAWL_OUTPUT_DIR"] = str(crawl.output_dir) - config["CRAWL_DIR"] = str(crawl.output_dir) - - # Apply snapshot config overrides (highest priority) - if snapshot and hasattr(snapshot, "config") and snapshot.config: - config.update(snapshot.config) - - if snapshot and hasattr(snapshot, "output_dir"): - config["SNAP_DIR"] = str(snapshot.output_dir) - - # Normalize all aliases to canonical names (after all sources merged) - # This handles aliases that came from user/crawl/snapshot configs, not just env - try: - from archivebox.hooks import discover_plugin_configs - - plugin_configs = discover_plugin_configs() - aliases_to_normalize = {} # {alias_key: canonical_key} - - # Build alias mapping from all plugin schemas - for plugin_name, schema in plugin_configs.items(): - for canonical_key, prop_schema in schema.get("properties", {}).items(): - for alias in prop_schema.get("x-aliases", []): - aliases_to_normalize[alias] = canonical_key - - # Normalize: copy alias values to canonical keys (aliases take precedence) - for alias_key, canonical_key in aliases_to_normalize.items(): - if alias_key in config: - # Alias exists - copy to canonical key (overwriting any default) - config[canonical_key] = config[alias_key] - # Remove alias from config to keep it clean - del config[alias_key] - except ImportError: - pass - - if not config.get("DATA_DIR"): - config["DATA_DIR"] = str(CONSTANTS.DATA_DIR) - config["ABX_RUNTIME"] = "archivebox" - - return config - - -def get_flat_config() -> dict[str, Any]: - """ - Get a flat dictionary of all config values. - - Replaces abx.pm.hook.get_FLAT_CONFIG() - """ - return get_config() - - -def get_all_configs() -> dict[str, BaseConfigSet]: - """ - Get all config section objects as a dictionary. - - Replaces abx.pm.hook.get_CONFIGS() - """ - from archivebox.config.common import ( - SHELL_CONFIG, - SERVER_CONFIG, - ARCHIVING_CONFIG, - SEARCH_BACKEND_CONFIG, - ) - - return { - "SHELL_CONFIG": SHELL_CONFIG, - "SERVER_CONFIG": SERVER_CONFIG, - "ARCHIVING_CONFIG": ARCHIVING_CONFIG, - "SEARCH_BACKEND_CONFIG": SEARCH_BACKEND_CONFIG, - } - - -def _parse_env_value(value: str, default: Any = None) -> Any: - """Parse an environment variable value based on expected type.""" - if default is None: - # Try to guess the type - if value.lower() in ("true", "false", "yes", "no", "1", "0"): - return value.lower() in ("true", "yes", "1") - try: - return int(value) - except ValueError: - pass - try: - return json.loads(value) - except (json.JSONDecodeError, ValueError): - pass - return value - - # Parse based on default's type - if isinstance(default, bool): - return value.lower() in ("true", "yes", "1") - elif isinstance(default, int): - return int(value) - elif isinstance(default, float): - return float(value) - elif isinstance(default, (list, dict)): - return json.loads(value) - elif isinstance(default, Path): - return Path(value) - else: - return value - - -# Default worker concurrency settings -DEFAULT_WORKER_CONCURRENCY = { - "crawl": 2, - "snapshot": 3, - "wget": 2, - "ytdlp": 2, - "screenshot": 3, - "singlefile": 2, - "title": 5, - "favicon": 5, - "headers": 5, - "archivedotorg": 2, - "readability": 3, - "mercury": 3, - "git": 2, - "pdf": 2, - "dom": 3, -} - - -def get_worker_concurrency() -> dict[str, int]: - """ - Get worker concurrency settings. - - Can be configured via WORKER_CONCURRENCY env var as JSON dict. - """ - config = get_config() - - # Start with defaults - concurrency = DEFAULT_WORKER_CONCURRENCY.copy() - - # Override with config - if "WORKER_CONCURRENCY" in config: - custom = config["WORKER_CONCURRENCY"] - if isinstance(custom, str): - custom = json.loads(custom) - concurrency.update(custom) - - return concurrency + def values(self): + return self.as_dict().values() diff --git a/archivebox/config/constants.py b/archivebox/config/constants.py index 40fc11d9..74f60844 100644 --- a/archivebox/config/constants.py +++ b/archivebox/config/constants.py @@ -25,6 +25,7 @@ from .paths import ( PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR, + USERS_DIR, get_collection_id, get_machine_id, get_machine_type, @@ -48,6 +49,7 @@ class ConstantsDict: PACKAGE_DIR: Path = PACKAGE_DIR DATA_DIR: Path = DATA_DIR ARCHIVE_DIR: Path = ARCHIVE_DIR + USERS_DIR: Path = USERS_DIR MACHINE_TYPE: str = get_machine_type() MACHINE_ID: str = get_machine_id() @@ -76,13 +78,17 @@ class ConstantsDict: # Data dirs ARCHIVE_DIR_NAME: str = "archive" + USERS_DIR_NAME: str = "users" + SNAPSHOTS_DIR_NAME: str = "snapshots" + CRAWLS_DIR_NAME: str = "crawls" SOURCES_DIR_NAME: str = "sources" PERSONAS_DIR_NAME: str = "personas" CACHE_DIR_NAME: str = "cache" LOGS_DIR_NAME: str = "logs" CUSTOM_PLUGINS_DIR_NAME: str = "custom_plugins" CUSTOM_TEMPLATES_DIR_NAME: str = "custom_templates" - ARCHIVE_DIR: Path = DATA_DIR / ARCHIVE_DIR_NAME + ARCHIVE_DIR: Path = ARCHIVE_DIR + USERS_DIR: Path = USERS_DIR SOURCES_DIR: Path = DATA_DIR / SOURCES_DIR_NAME PERSONAS_DIR: Path = DATA_DIR / PERSONAS_DIR_NAME LOGS_DIR: Path = DATA_DIR / LOGS_DIR_NAME @@ -110,6 +116,16 @@ class ConstantsDict: DEFAULT_LIB_DIR: Path = DATA_DIR / LIB_DIR_NAME / MACHINE_TYPE # ./data/lib/arm64-linux-docker DEFAULT_LIB_BIN_DIR: Path = DEFAULT_LIB_DIR / "bin" # ./data/lib/arm64-linux-docker/bin + RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset( + ( + USERS_DIR_NAME, + SNAPSHOTS_DIR_NAME, + CRAWLS_DIR_NAME, + "invalid", + ".DS_Store", + ), + ) + # Config constants TIMEZONE: str = "UTC" DEFAULT_CLI_COLORS: dict[str, str] = DEFAULT_CLI_COLORS diff --git a/archivebox/config/django.py b/archivebox/config/django.py index 7f954ad1..c8c1a68a 100644 --- a/archivebox/config/django.py +++ b/archivebox/config/django.py @@ -12,16 +12,16 @@ import django.db from archivebox.misc import logging -from . import CONSTANTS -from .common import SHELL_CONFIG +from .constants import CONSTANTS +from .common import get_config +CONFIG = get_config() -if not SHELL_CONFIG.USE_COLOR: +if not CONFIG.USE_COLOR: os.environ["NO_COLOR"] = "1" -if not SHELL_CONFIG.SHOW_PROGRESS: +if not CONFIG.SHOW_PROGRESS: os.environ["TERM"] = "dumb" -# recreate rich console obj based on new config values STDOUT = CONSOLE = Console() STDERR = Console(stderr=True) logging.CONSOLE = CONSOLE @@ -107,15 +107,15 @@ def setup_django(check_db=False, in_memory_db=False) -> None: traceback.print_exc() return - from django.conf import settings from archivebox.core.settings_logging import ERROR_LOG as DEFAULT_ERROR_LOG # log startup message to the error log - error_log = getattr(settings, "ERROR_LOG", DEFAULT_ERROR_LOG) + error_log = DEFAULT_ERROR_LOG with open(error_log, "a", encoding="utf-8") as f: command = " ".join(sys.argv) ts = datetime.now(timezone.utc).strftime("%Y-%m-%d__%H:%M:%S") - f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={SHELL_CONFIG.IN_DOCKER} IS_TTY={SHELL_CONFIG.IS_TTY}\n") + config = get_config() + f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={config.IN_DOCKER} IS_TTY={config.IS_TTY}\n") if check_db: # make sure the data dir is owned by a non-root user diff --git a/archivebox/config/ldap.py b/archivebox/config/ldap.py index a2eadee3..40218c5e 100644 --- a/archivebox/config/ldap.py +++ b/archivebox/config/ldap.py @@ -50,7 +50,3 @@ class LDAPConfig(BaseConfigSet): return False, f"LDAP_* config options must all be set if LDAP_ENABLED=True\nMissing: {', '.join(missing)}" return True, "" - - -# Singleton instance -LDAP_CONFIG = LDAPConfig() diff --git a/archivebox/config/paths.py b/archivebox/config/paths.py index 59885dcc..0023af0f 100644 --- a/archivebox/config/paths.py +++ b/archivebox/config/paths.py @@ -8,16 +8,30 @@ import platform from pathlib import Path from functools import cache from datetime import datetime +from typing import TYPE_CHECKING from benedict import benedict from .permissions import SudoPermission, IS_ROOT, ARCHIVEBOX_USER +if TYPE_CHECKING: + from archivebox.config.common import ArchiveBoxConfig + ############################################################################################# PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent # archivebox source code dir DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve() # archivebox user data dir -ARCHIVE_DIR: Path = DATA_DIR / "archive" # archivebox snapshot data dir + + +def _env_path(key: str, default: Path) -> Path: + path = Path(os.environ.get(key) or default).expanduser() + if not path.is_absolute(): + path = DATA_DIR / path + return path.resolve() + + +ARCHIVE_DIR: Path = _env_path("ARCHIVE_DIR", DATA_DIR / "archive") # archivebox snapshot data dir +USERS_DIR: Path = _env_path("USERS_DIR", ARCHIVE_DIR / "users") # archivebox user-scoped crawl/snapshot data dir IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes") @@ -154,15 +168,15 @@ def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool: return len(f"file://{socket_file}") <= 96 -@cache -def get_or_create_working_tmp_dir(autofix=True, quiet=True): - from archivebox import CONSTANTS - from archivebox.config.common import STORAGE_CONFIG +def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs): + from archivebox.config.constants import CONSTANTS + from archivebox.config.common import get_config from archivebox.misc.checks import check_tmp_dir + config = config or get_config(**config_kwargs) # try a few potential directories in order of preference CANDIDATES = [ - STORAGE_CONFIG.TMP_DIR, # + config.TMP_DIR, # CONSTANTS.DEFAULT_TMP_DIR, # ./data/tmp/ Path("/var/run/archivebox") / get_collection_id(), # /var/run/archivebox/abc5d8512 Path("/tmp") / "archivebox" / get_collection_id(), # /tmp/archivebox/abc5d8512 @@ -182,8 +196,8 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True): except Exception: pass if check_tmp_dir(candidate, throw=False, quiet=True, must_exist=True): - if autofix and STORAGE_CONFIG.TMP_DIR != candidate: - STORAGE_CONFIG.update_in_place(TMP_DIR=candidate) + if autofix and config.TMP_DIR != candidate: + os.environ["TMP_DIR"] = str(candidate) return candidate try: if ( @@ -200,23 +214,23 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True): # Fall back to the shortest writable path so read-only CLI commands can still run, # and let later permission checks surface the missing socket support if needed. if fallback_candidate: - if autofix and STORAGE_CONFIG.TMP_DIR != fallback_candidate: - STORAGE_CONFIG.update_in_place(TMP_DIR=fallback_candidate) + if autofix and config.TMP_DIR != fallback_candidate: + os.environ["TMP_DIR"] = str(fallback_candidate) return fallback_candidate if not quiet: raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!") -@cache -def get_or_create_working_lib_dir(autofix=True, quiet=False): - from archivebox import CONSTANTS - from archivebox.config.common import STORAGE_CONFIG +def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs): + from archivebox.config.constants import CONSTANTS + from archivebox.config.common import get_config from archivebox.misc.checks import check_lib_dir + config = config or get_config(**config_kwargs) # try a few potential directories in order of preference CANDIDATES = [ - STORAGE_CONFIG.LIB_DIR, # + config.LIB_DIR, # CONSTANTS.DEFAULT_LIB_DIR, # ./data/lib/arm64-linux-docker Path("/usr/local/share/archivebox") / get_collection_id(), # /usr/local/share/archivebox/abc5 *( @@ -231,23 +245,23 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False): except Exception: pass if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True): - if autofix and STORAGE_CONFIG.LIB_DIR != candidate: - STORAGE_CONFIG.update_in_place(LIB_DIR=candidate) + if autofix and config.LIB_DIR != candidate: + os.environ["LIB_DIR"] = str(candidate) return candidate if not quiet: raise OSError(f"ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!") -@cache -def get_data_locations(): - from archivebox.config import CONSTANTS - from archivebox.config.common import STORAGE_CONFIG +def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs): + from archivebox.config.constants import CONSTANTS + from archivebox.config.common import get_config + config = config or get_config(**config_kwargs) try: - tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True) or STORAGE_CONFIG.TMP_DIR + tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True, config=config) or config.TMP_DIR except Exception: - tmp_dir = STORAGE_CONFIG.TMP_DIR + tmp_dir = config.TMP_DIR return benedict( { @@ -271,10 +285,20 @@ def get_data_locations(): "is_mount": os.path.ismount(DATABASE_FILE.resolve()), }, "ARCHIVE_DIR": { - "path": ARCHIVE_DIR.resolve(), + "path": config.ARCHIVE_DIR.resolve(), "enabled": True, - "is_valid": os.path.isdir(ARCHIVE_DIR) and os.access(ARCHIVE_DIR, os.R_OK) and os.access(ARCHIVE_DIR, os.W_OK), - "is_mount": os.path.ismount(ARCHIVE_DIR.resolve()), + "is_valid": os.path.isdir(config.ARCHIVE_DIR) + and os.access(config.ARCHIVE_DIR, os.R_OK) + and os.access(config.ARCHIVE_DIR, os.W_OK), + "is_mount": os.path.ismount(config.ARCHIVE_DIR.resolve()), + }, + "USERS_DIR": { + "path": config.USERS_DIR.resolve(), + "enabled": os.path.isdir(config.USERS_DIR), + "is_valid": os.path.isdir(config.USERS_DIR) + and os.access(config.USERS_DIR, os.R_OK) + and os.access(config.USERS_DIR, os.W_OK), + "is_mount": os.path.ismount(config.USERS_DIR.resolve()), }, "SOURCES_DIR": { "path": CONSTANTS.SOURCES_DIR.resolve(), @@ -311,15 +335,15 @@ def get_data_locations(): ) -@cache -def get_code_locations(): - from archivebox.config import CONSTANTS - from archivebox.config.common import STORAGE_CONFIG +def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs): + from archivebox.config.constants import CONSTANTS + from archivebox.config.common import get_config + config = config or get_config(**config_kwargs) try: - lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True) or STORAGE_CONFIG.LIB_DIR + lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR except Exception: - lib_dir = STORAGE_CONFIG.LIB_DIR + lib_dir = config.LIB_DIR lib_bin_dir = lib_dir / "bin" @@ -336,10 +360,9 @@ def get_code_locations(): "is_valid": os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK), # read + list }, "CUSTOM_TEMPLATES_DIR": { - "path": STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR.resolve(), - "enabled": os.path.isdir(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR), - "is_valid": os.path.isdir(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR) - and os.access(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK), # read + "path": config.CUSTOM_TEMPLATES_DIR.resolve(), + "enabled": os.path.isdir(config.CUSTOM_TEMPLATES_DIR), + "is_valid": os.path.isdir(config.CUSTOM_TEMPLATES_DIR) and os.access(config.CUSTOM_TEMPLATES_DIR, os.R_OK), # read }, "USER_PLUGINS_DIR": { "path": CONSTANTS.USER_PLUGINS_DIR.resolve(), diff --git a/archivebox/config/version.py b/archivebox/config/version.py index fde5533d..38eda872 100644 --- a/archivebox/config/version.py +++ b/archivebox/config/version.py @@ -12,8 +12,6 @@ from datetime import datetime IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes") PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent # archivebox source code dir -DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve() # archivebox user data dir -ARCHIVE_DIR: Path = DATA_DIR / "archive" # archivebox snapshot data dir ############################################################################################# diff --git a/archivebox/core/__init__.py b/archivebox/core/__init__.py index f50f21bf..013c1431 100644 --- a/archivebox/core/__init__.py +++ b/archivebox/core/__init__.py @@ -7,23 +7,3 @@ def register_admin(admin_site): from archivebox.core.admin import register_admin as do_register do_register(admin_site) - - -def get_CONFIG(): - from archivebox.config.common import ( - SHELL_CONFIG, - STORAGE_CONFIG, - GENERAL_CONFIG, - SERVER_CONFIG, - ARCHIVING_CONFIG, - SEARCH_BACKEND_CONFIG, - ) - - return { - "SHELL_CONFIG": SHELL_CONFIG, - "STORAGE_CONFIG": STORAGE_CONFIG, - "GENERAL_CONFIG": GENERAL_CONFIG, - "SERVER_CONFIG": SERVER_CONFIG, - "ARCHIVING_CONFIG": ARCHIVING_CONFIG, - "SEARCHBACKEND_CONFIG": SEARCH_BACKEND_CONFIG, - } diff --git a/archivebox/core/admin_archiveresults.py b/archivebox/core/admin_archiveresults.py index ba8affbb..62cdab4c 100644 --- a/archivebox/core/admin_archiveresults.py +++ b/archivebox/core/admin_archiveresults.py @@ -20,7 +20,7 @@ from django.utils import timezone from django.utils.text import smart_split from archivebox.config import DATA_DIR -from archivebox.config.common import SERVER_CONFIG +from archivebox.config.common import get_config from archivebox.misc.paginators import AcceleratedPaginator from archivebox.base_models.admin import BaseModelAdmin from archivebox.hooks import get_plugin_icon @@ -469,7 +469,7 @@ class ArchiveResultAdmin(BaseModelAdmin): list_filter = ("status", "plugin", "start_ts") ordering = ["-start_ts"] - list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE + list_per_page = get_config().SNAPSHOTS_PER_PAGE paginator = AcceleratedPaginator save_on_top = True diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index 266ed974..b8764ea5 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -15,7 +15,7 @@ from django.template import Template, RequestContext from django.contrib.admin.helpers import ActionForm from archivebox.config import DATA_DIR -from archivebox.config.common import SERVER_CONFIG +from archivebox.config.common import get_config from archivebox.misc.util import htmldecode, urldecode from archivebox.misc.paginators import AcceleratedPaginator from archivebox.misc.logging_util import printable_filesize @@ -248,7 +248,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): ordering = ["-created_at"] actions = ["add_tags", "remove_tags", "resnapshot_snapshot", "update_snapshots", "overwrite_snapshots", "delete_snapshots"] inlines = [] # Removed TagInline, using TagEditorWidget instead - list_per_page = min(max(5, SERVER_CONFIG.SNAPSHOTS_PER_PAGE), 5000) + list_per_page = min(max(5, get_config().SNAPSHOTS_PER_PAGE), 5000) action_form = SnapshotActionForm paginator = AcceleratedPaginator @@ -897,7 +897,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): return None def _get_expected_hook_total(self, obj) -> int: - from archivebox.config.configset import get_config + from archivebox.config.common import get_config try: config = get_config(crawl=obj.crawl, snapshot=obj) @@ -976,7 +976,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): # Monkey patch here plus core_tags.py admin_cls.change_list_template = "private_index_grid.html" - admin_cls.list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE + admin_cls.list_per_page = get_config().SNAPSHOTS_PER_PAGE admin_cls.list_max_show_all = admin_cls.list_per_page # Call monkey patched view diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py index edc0403d..6731262d 100644 --- a/archivebox/core/forms.py +++ b/archivebox/core/forms.py @@ -7,7 +7,7 @@ from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_byt from taggit.utils import edit_string_for_tags, parse_tags from archivebox.base_models.admin import KeyValueWidget from archivebox.crawls.schedule_utils import validate_schedule -from archivebox.config.common import SEARCH_BACKEND_CONFIG +from archivebox.config.common import get_config from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget from archivebox.hooks import get_plugins, discover_plugin_configs, get_plugin_icon from archivebox.personas.models import Persona @@ -259,7 +259,7 @@ class AddLinkForm(forms.Form): (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in extensions ] - required_search_plugin = f"search_backend_{SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}".strip() + required_search_plugin = f"search_backend_{get_config().SEARCH_BACKEND_ENGINE}".strip() search_choices = [choice[0] for choice in get_choice_field(self, "search_plugins").choices] if required_search_plugin in search_choices: get_choice_field(self, "search_plugins").initial = [required_search_plugin] diff --git a/archivebox/core/host_utils.py b/archivebox/core/host_utils.py index 21479788..2b6fd618 100644 --- a/archivebox/core/host_utils.py +++ b/archivebox/core/host_utils.py @@ -1,9 +1,10 @@ from __future__ import annotations import re +from typing import Any from urllib.parse import urlparse -from archivebox.config.common import SERVER_CONFIG +from archivebox.config.common import get_config _SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$") @@ -35,16 +36,19 @@ def normalize_base_url(value: str | None) -> str: return _normalize_base_url(value) -def get_listen_host() -> str: - return (SERVER_CONFIG.LISTEN_HOST or "").strip() +def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + return (config.LISTEN_HOST or "").strip() -def get_listen_parts() -> tuple[str, str | None]: - return split_host_port(get_listen_host()) +def get_listen_parts(config: dict[str, Any] | None = None, **config_kwargs: Any) -> tuple[str, str | None]: + config = config or get_config(**config_kwargs) + return split_host_port(get_listen_host(config=config)) -def _build_listen_host(subdomain: str | None) -> str: - host, port = get_listen_parts() +def _build_listen_host(subdomain: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + host, port = get_listen_parts(config=config) if not host: return "" full_host = f"{subdomain}.{host}" if subdomain else host @@ -53,34 +57,38 @@ def _build_listen_host(subdomain: str | None) -> str: return full_host -def get_admin_host() -> str: - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: - return get_listen_host().lower() - override = _normalize_base_url(SERVER_CONFIG.ADMIN_BASE_URL) +def get_admin_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + if not config.USES_SUBDOMAIN_ROUTING: + return get_listen_host(config=config).lower() + override = _normalize_base_url(config.ADMIN_BASE_URL) if override: return urlparse(override).netloc.lower() - return _build_listen_host("admin") + return _build_listen_host("admin", config=config) -def get_web_host() -> str: - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: - return get_listen_host().lower() - override = _normalize_base_url(SERVER_CONFIG.ARCHIVE_BASE_URL) +def get_web_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + if not config.USES_SUBDOMAIN_ROUTING: + return get_listen_host(config=config).lower() + override = _normalize_base_url(config.ARCHIVE_BASE_URL) if override: return urlparse(override).netloc.lower() - return _build_listen_host("web") + return _build_listen_host("web", config=config) -def get_api_host() -> str: - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: - return get_listen_host().lower() - return _build_listen_host("api") +def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + if not config.USES_SUBDOMAIN_ROUTING: + return get_listen_host(config=config).lower() + return _build_listen_host("api", config=config) -def get_public_host() -> str: - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: - return get_listen_host().lower() - return _build_listen_host("public") +def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + if not config.USES_SUBDOMAIN_ROUTING: + return get_listen_host(config=config).lower() + return _build_listen_host("public", config=config) def get_snapshot_subdomain(snapshot_id: str) -> str: @@ -89,16 +97,18 @@ def get_snapshot_subdomain(snapshot_id: str) -> str: return f"snap-{suffix}" -def get_snapshot_host(snapshot_id: str) -> str: - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: - return get_listen_host().lower() - return _build_listen_host(get_snapshot_subdomain(snapshot_id)) +def get_snapshot_host(snapshot_id: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + if not config.USES_SUBDOMAIN_ROUTING: + return get_listen_host(config=config).lower() + return _build_listen_host(get_snapshot_subdomain(snapshot_id), config=config) -def get_original_host(domain: str) -> str: - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: - return get_listen_host().lower() - return _build_listen_host(domain) +def get_original_host(domain: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + if not config.USES_SUBDOMAIN_ROUTING: + return get_listen_host(config=config).lower() + return _build_listen_host(domain, config=config) def is_snapshot_subdomain(subdomain: str) -> bool: @@ -114,11 +124,12 @@ def get_snapshot_lookup_key(snapshot_ref: str) -> str: return value -def get_listen_subdomain(request_host: str) -> str: - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: +def get_listen_subdomain(request_host: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + if not config.USES_SUBDOMAIN_ROUTING: return "" req_host, req_port = split_host_port(request_host) - listen_host, listen_port = get_listen_parts() + listen_host, listen_port = get_listen_parts(config=config) if not listen_host: return "" if listen_port and req_port and listen_port != req_port: @@ -156,73 +167,79 @@ def _build_base_url_for_host(host: str, request=None) -> str: return f"{scheme}://{host}" -def get_admin_base_url(request=None) -> str: - override = _normalize_base_url(SERVER_CONFIG.ADMIN_BASE_URL) +def get_admin_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + override = _normalize_base_url(config.ADMIN_BASE_URL) if override: return override - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: - return _build_base_url_for_host(get_listen_host(), request=request) - return _build_base_url_for_host(get_admin_host(), request=request) + if not config.USES_SUBDOMAIN_ROUTING: + return _build_base_url_for_host(get_listen_host(config=config), request=request) + return _build_base_url_for_host(get_admin_host(config=config), request=request) -def get_web_base_url(request=None) -> str: - override = _normalize_base_url(SERVER_CONFIG.ARCHIVE_BASE_URL) +def get_web_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + override = _normalize_base_url(config.ARCHIVE_BASE_URL) if override: return override - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: - return _build_base_url_for_host(get_listen_host(), request=request) - return _build_base_url_for_host(get_web_host(), request=request) + if not config.USES_SUBDOMAIN_ROUTING: + return _build_base_url_for_host(get_listen_host(config=config), request=request) + return _build_base_url_for_host(get_web_host(config=config), request=request) -def get_api_base_url(request=None) -> str: - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: - return _build_base_url_for_host(get_listen_host(), request=request) - return _build_base_url_for_host(get_api_host(), request=request) +def get_api_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + if not config.USES_SUBDOMAIN_ROUTING: + return _build_base_url_for_host(get_listen_host(config=config), request=request) + return _build_base_url_for_host(get_api_host(config=config), request=request) -def get_public_base_url(request=None) -> str: - return _build_base_url_for_host(get_public_host(), request=request) +def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + return _build_base_url_for_host(get_public_host(config=config), request=request) # Backwards-compat aliases (archive == web) -def get_archive_base_url(request=None) -> str: - return get_web_base_url(request=request) +def get_archive_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return get_web_base_url(request=request, config=config, **config_kwargs) -def get_snapshot_base_url(snapshot_id: str, request=None) -> str: - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: - return _build_url(get_web_base_url(request=request), f"/snapshot/{snapshot_id}") - return _build_base_url_for_host(get_snapshot_host(snapshot_id), request=request) +def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + if not config.USES_SUBDOMAIN_ROUTING: + return _build_url(get_web_base_url(request=request, config=config), f"/snapshot/{snapshot_id}") + return _build_base_url_for_host(get_snapshot_host(snapshot_id, config=config), request=request) -def get_original_base_url(domain: str, request=None) -> str: - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: - return _build_url(get_web_base_url(request=request), f"/original/{domain}") - return _build_base_url_for_host(get_original_host(domain), request=request) +def get_original_base_url(domain: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + if not config.USES_SUBDOMAIN_ROUTING: + return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}") + return _build_base_url_for_host(get_original_host(domain, config=config), request=request) -def build_admin_url(path: str = "", request=None) -> str: - return _build_url(get_admin_base_url(request), path) +def build_admin_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return _build_url(get_admin_base_url(request, config=config, **config_kwargs), path) -def build_web_url(path: str = "", request=None) -> str: - return _build_url(get_web_base_url(request), path) +def build_web_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return _build_url(get_web_base_url(request, config=config, **config_kwargs), path) -def build_api_url(path: str = "", request=None) -> str: - return _build_url(get_api_base_url(request), path) +def build_api_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return _build_url(get_api_base_url(request, config=config, **config_kwargs), path) -def build_archive_url(path: str = "", request=None) -> str: - return _build_url(get_archive_base_url(request), path) +def build_archive_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return _build_url(get_archive_base_url(request, config=config, **config_kwargs), path) -def build_snapshot_url(snapshot_id: str, path: str = "", request=None) -> str: - return _build_url(get_snapshot_base_url(snapshot_id, request=request), path) +def build_snapshot_url(snapshot_id: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return _build_url(get_snapshot_base_url(snapshot_id, request=request, config=config, **config_kwargs), path) -def build_original_url(domain: str, path: str = "", request=None) -> str: - return _build_url(get_original_base_url(domain, request=request), path) +def build_original_url(domain: str, path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return _build_url(get_original_base_url(domain, request=request, config=config, **config_kwargs), path) def _build_url(base_url: str, path: str) -> str: diff --git a/archivebox/core/middleware.py b/archivebox/core/middleware.py index 3dddcbc6..c40dae08 100644 --- a/archivebox/core/middleware.py +++ b/archivebox/core/middleware.py @@ -12,7 +12,7 @@ from django.contrib.staticfiles import finders from django.utils.http import http_date from django.http import HttpResponseForbidden, HttpResponseNotModified -from archivebox.config.common import SERVER_CONFIG +from archivebox.config.common import get_config from archivebox.config import VERSION from archivebox.config.version import get_COMMIT_HASH from archivebox.core.host_utils import ( @@ -85,7 +85,7 @@ def CacheControlMiddleware(get_response): if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path): if not response.get("Cache-Control"): - policy = "public" if SERVER_CONFIG.PUBLIC_SNAPSHOTS else "private" + policy = "public" if get_config().PUBLIC_SNAPSHOTS else "private" response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300" # print('Set Cache-Control header to', response['Cache-Control']) return response @@ -98,7 +98,7 @@ def ServerSecurityModeMiddleware(get_response): allowed_methods = {"GET", "HEAD", "OPTIONS"} def middleware(request): - if SERVER_CONFIG.CONTROL_PLANE_ENABLED: + if get_config().CONTROL_PLANE_ENABLED: return get_response(request) request.user = AnonymousUser() @@ -123,19 +123,20 @@ def HostRoutingMiddleware(get_response): def middleware(request): request_host = (request.get_host() or "").lower() - admin_host = get_admin_host() - web_host = get_web_host() - api_host = get_api_host() - public_host = get_public_host() - listen_host = get_listen_host() - subdomain = get_listen_subdomain(request_host) + config = get_config() + admin_host = get_admin_host(config=config) + web_host = get_web_host(config=config) + api_host = get_api_host(config=config) + public_host = get_public_host(config=config) + listen_host = get_listen_host(config=config) + subdomain = get_listen_subdomain(request_host, config=config) # Framework-owned assets must bypass snapshot/original-domain replay routing. # Otherwise pages on snapshot subdomains can receive HTML for JS/CSS requests. if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}: return get_response(request) - if SERVER_CONFIG.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host): + if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host): if ( request.path == "/admin" or request.path.startswith("/admin/") @@ -147,7 +148,7 @@ def HostRoutingMiddleware(get_response): target = f"{target}?{request.META['QUERY_STRING']}" return redirect(target) - if not SERVER_CONFIG.USES_SUBDOMAIN_ROUTING: + if not config.USES_SUBDOMAIN_ROUTING: if host_matches(request_host, listen_host): return get_response(request) @@ -164,7 +165,7 @@ def HostRoutingMiddleware(get_response): if host_matches(request_host, admin_host): snapshot_match = snapshot_path_re.match(request.path) - if SERVER_CONFIG.USES_SUBDOMAIN_ROUTING and snapshot_match: + if config.USES_SUBDOMAIN_ROUTING and snapshot_match: snapshot_id = snapshot_match.group("snapshot_id") replay_path = (snapshot_match.group("path") or "").strip("/") if replay_path == "index.html": @@ -224,17 +225,19 @@ def HostRoutingMiddleware(get_response): class ReverseProxyAuthMiddleware(RemoteUserMiddleware): - header = "HTTP_{normalized}".format(normalized=SERVER_CONFIG.REVERSE_PROXY_USER_HEADER.replace("-", "_").upper()) + header = "HTTP_REMOTE_USER" def process_request(self, request): - if SERVER_CONFIG.REVERSE_PROXY_WHITELIST == "": + config = get_config() + self.header = "HTTP_{normalized}".format(normalized=config.REVERSE_PROXY_USER_HEADER.replace("-", "_").upper()) + if config.REVERSE_PROXY_WHITELIST == "": return ip = request.META.get("REMOTE_ADDR") if not isinstance(ip, str): return - for cidr in SERVER_CONFIG.REVERSE_PROXY_WHITELIST.split(","): + for cidr in config.REVERSE_PROXY_WHITELIST.split(","): try: network = ipaddress.ip_network(cidr) except ValueError: diff --git a/archivebox/core/migrations/0007_archiveresult.py b/archivebox/core/migrations/0007_archiveresult.py index 3b31b15c..fcdfcd1c 100644 --- a/archivebox/core/migrations/0007_archiveresult.py +++ b/archivebox/core/migrations/0007_archiveresult.py @@ -12,12 +12,7 @@ try: ARCHIVE_DIR = CONSTANTS.ARCHIVE_DIR except ImportError: - try: - from archivebox.config import CONFIG - - ARCHIVE_DIR = Path(CONFIG.get("ARCHIVE_DIR", "./archive")) - except ImportError: - ARCHIVE_DIR = Path("./archive") + ARCHIVE_DIR = Path("./archive") try: from archivebox.misc.util import to_json diff --git a/archivebox/core/migrations/0023_upgrade_to_0_9_0.py b/archivebox/core/migrations/0023_upgrade_to_0_9_0.py index ea7bcb44..8ae9844a 100644 --- a/archivebox/core/migrations/0023_upgrade_to_0_9_0.py +++ b/archivebox/core/migrations/0023_upgrade_to_0_9_0.py @@ -4,6 +4,9 @@ from django.db import migrations, models, connection import django.utils.timezone +from uuid import UUID +from datetime import datetime +import json def get_table_columns(table_name): @@ -13,6 +16,18 @@ def get_table_columns(table_name): return {row[1] for row in cursor.fetchall()} +def normalize_cmd(cmd): + if not cmd: + return "[]" + try: + parsed = json.loads(cmd) + if isinstance(parsed, list): + return json.dumps([str(part) for part in parsed]) + except (TypeError, json.JSONDecodeError): + pass + return json.dumps(str(cmd).split()) + + def upgrade_core_tables(apps, schema_editor): """Upgrade core tables from v0.7.2 or v0.8.6rc0 to v0.9.0.""" from archivebox.uuid_compat import uuid7 @@ -64,29 +79,72 @@ def upgrade_core_tables(apps, schema_editor): if has_uuid and not has_abid: # Migrating from v0.7.2+ (has uuid column) print("Migrating ArchiveResult from v0.7.2+ schema (with uuid)...") - cursor.execute(""" - INSERT OR IGNORE INTO core_archiveresult_new ( - id, uuid, snapshot_id, cmd, pwd, cmd_version, - start_ts, end_ts, status, extractor, output + cursor.execute( + "SELECT id, uuid, snapshot_id, cmd, pwd, cmd_version, start_ts, end_ts, status, extractor, output FROM core_archiveresult", + ) + old_records = cursor.fetchall() + for record in old_records: + try: + new_uuid = UUID(str(record[1])).hex + except (TypeError, ValueError): + new_uuid = uuid7().hex + start_ts = record[6] or datetime.now().isoformat() + end_ts = record[7] or start_ts + cursor.execute( + """ + INSERT OR IGNORE INTO core_archiveresult_new ( + id, uuid, snapshot_id, cmd, pwd, cmd_version, + start_ts, end_ts, status, extractor, output + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + record[0], + new_uuid, + record[2], + normalize_cmd(record[3]), + record[4] or "", + record[5] or "", + start_ts, + end_ts, + "succeeded" if record[8] == "success" else (record[8] or "queued"), + record[9] or "", + record[10] or "", + ), ) - SELECT - id, uuid, snapshot_id, cmd, pwd, cmd_version, - start_ts, end_ts, status, extractor, output - FROM core_archiveresult; - """) elif has_abid and not has_uuid: # Migrating from v0.8.6rc0 (has abid instead of uuid) print("Migrating ArchiveResult from v0.8.6rc0 schema...") - cursor.execute(""" - INSERT OR IGNORE INTO core_archiveresult_new ( - id, uuid, snapshot_id, cmd, pwd, cmd_version, - start_ts, end_ts, status, extractor, output + cursor.execute( + "SELECT id, snapshot_id, cmd, pwd, cmd_version, start_ts, end_ts, status, extractor, output FROM core_archiveresult", + ) + old_records = cursor.fetchall() + for record in old_records: + try: + new_uuid = UUID(str(record[0])).hex + except (TypeError, ValueError): + new_uuid = uuid7().hex + start_ts = record[5] or datetime.now().isoformat() + end_ts = record[6] or start_ts + cursor.execute( + """ + INSERT OR IGNORE INTO core_archiveresult_new ( + uuid, snapshot_id, cmd, pwd, cmd_version, + start_ts, end_ts, status, extractor, output + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + new_uuid, + record[1], + normalize_cmd(record[2]), + record[3] or "", + record[4] or "", + start_ts, + end_ts, + "succeeded" if record[7] == "success" else (record[7] or "queued"), + record[8] or "", + record[9] or "", + ), ) - SELECT - id, abid as uuid, snapshot_id, cmd, pwd, cmd_version, - start_ts, end_ts, status, extractor, output - FROM core_archiveresult; - """) else: # Migrating from v0.7.2 (no uuid or abid column - generate fresh UUIDs) print("Migrating ArchiveResult from v0.7.2 schema (no uuid - generating UUIDs)...") @@ -96,6 +154,8 @@ def upgrade_core_tables(apps, schema_editor): old_records = cursor.fetchall() for record in old_records: new_uuid = uuid7().hex + start_ts = record[5] or datetime.now().isoformat() + end_ts = record[6] or start_ts cursor.execute( """ INSERT OR IGNORE INTO core_archiveresult_new ( @@ -107,14 +167,14 @@ def upgrade_core_tables(apps, schema_editor): record[0], new_uuid, record[1], - record[2], - record[3], - record[4], - record[5], - record[6], - record[7], - record[8], - record[9], + normalize_cmd(record[2]), + record[3] or "", + record[4] or "", + start_ts, + end_ts, + "succeeded" if record[7] == "success" else (record[7] or "queued"), + record[8] or "", + record[9] or "", ), ) @@ -174,18 +234,27 @@ def upgrade_core_tables(apps, schema_editor): if has_added and not has_bookmarked_at: # Migrating from v0.7.2 (has added/updated fields) print("Migrating Snapshot from v0.7.2 schema...") - # Transform added→bookmarked_at/created_at and updated→modified_at + # timestamp is the legacy bookmark/import timestamp and archive/{timestamp} identity. + # added is the DB row creation/import time, and updated was renamed to downloaded_at in 0.8.x. cursor.execute(""" INSERT OR IGNORE INTO core_snapshot_new ( id, url, timestamp, title, - bookmarked_at, created_at, modified_at, + bookmarked_at, created_at, modified_at, downloaded_at, status ) SELECT id, url, timestamp, title, - COALESCE(added, CURRENT_TIMESTAMP) as bookmarked_at, + COALESCE( + CASE + WHEN CAST(timestamp AS REAL) BETWEEN 788918400 AND 2082758400 + THEN datetime(CAST(timestamp AS REAL), 'unixepoch') + END, + added, + CURRENT_TIMESTAMP + ) as bookmarked_at, COALESCE(added, CURRENT_TIMESTAMP) as created_at, COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at, + updated as downloaded_at, 'queued' as status FROM core_snapshot; """) @@ -198,17 +267,21 @@ def upgrade_core_tables(apps, schema_editor): has_crawl_id = "crawl_id" in snapshot_cols # Build column list based on what exists - cols = ["id", "url", "timestamp", "title", "bookmarked_at", "created_at", "modified_at", "downloaded_at"] + insert_cols = ["id", "url", "timestamp", "title", "bookmarked_at", "created_at", "modified_at", "downloaded_at"] + select_cols = ["id", "url", "timestamp", "title", "bookmarked_at", "created_at", "modified_at", "downloaded_at"] if has_crawl_id: - cols.append("crawl_id") + insert_cols.append("crawl_id") + select_cols.append("REPLACE(crawl_id, '-', '')") if has_status: - cols.append("status") + insert_cols.append("status") + select_cols.append("status") if has_retry_at: - cols.append("retry_at") + insert_cols.append("retry_at") + select_cols.append("retry_at") cursor.execute(f""" - INSERT OR IGNORE INTO core_snapshot_new ({", ".join(cols)}) - SELECT {", ".join(cols)} + INSERT OR IGNORE INTO core_snapshot_new ({", ".join(insert_cols)}) + SELECT {", ".join(select_cols)} FROM core_snapshot; """) else: @@ -324,7 +397,7 @@ def upgrade_core_tables(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ ("core", "0022_auto_20231023_2008"), - ("crawls", "0001_initial"), + ("crawls", "0002_upgrade_from_0_8_6"), ("auth", "0012_alter_user_first_name_max_length"), ] @@ -359,6 +432,11 @@ class Migration(migrations.Migration): name="modified_at", field=models.DateTimeField(auto_now=True), ), + migrations.AddField( + model_name="snapshot", + name="downloaded_at", + field=models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True), + ), # Declare fs_version (already created in database with DEFAULT '0.8.0') migrations.AddField( model_name="snapshot", diff --git a/archivebox/core/migrations/0024_assign_default_crawl.py b/archivebox/core/migrations/0024_assign_default_crawl.py index e5dd70d8..bea3c850 100644 --- a/archivebox/core/migrations/0024_assign_default_crawl.py +++ b/archivebox/core/migrations/0024_assign_default_crawl.py @@ -23,6 +23,9 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor): print("✓ Fresh install or all snapshots already have crawls") return + cursor.execute("SELECT url FROM core_snapshot WHERE crawl_id IS NULL ORDER BY bookmarked_at, timestamp") + crawl_urls = "\n".join(url for (url,) in cursor.fetchall() if url) + # Get or create system user (pk=1) cursor.execute("SELECT id FROM auth_user WHERE id = 1") if not cursor.fetchone(): @@ -36,7 +39,7 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor): # Create a default crawl for migrated snapshots # At this point crawls_crawl is guaranteed to have v0.9.0 schema (crawls/0002 ran first) - crawl_id = str(uuid_lib.uuid4()) + crawl_id = uuid_lib.uuid4().hex now = datetime.now().isoformat() cursor.execute( @@ -45,11 +48,11 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor): id, created_at, modified_at, num_uses_succeeded, num_uses_failed, urls, max_depth, tags_str, label, notes, output_dir, status, retry_at, created_by_id, schedule_id, config, persona_id - ) VALUES (?, ?, ?, 0, 0, '', 0, '', 'Migrated from v0.7.2/v0.8.6', + ) VALUES (?, ?, ?, 0, 0, ?, 0, '', 'Migrated from v0.7.2/v0.8.6', 'Auto-created crawl for migrated snapshots', '', 'sealed', ?, 1, NULL, '{}', NULL) """, - [crawl_id, now, now, now], + [crawl_id, now, now, crawl_urls, now], ) # Assign all snapshots without a crawl to the default crawl @@ -118,7 +121,7 @@ class Migration(migrations.Migration): SELECT id, url, timestamp, title, bookmarked_at, created_at, modified_at, - crawl_id, parent_snapshot_id, + REPLACE(crawl_id, '-', ''), REPLACE(parent_snapshot_id, '-', ''), downloaded_at, depth, fs_version, COALESCE(config, '{}'), COALESCE(notes, ''), num_uses_succeeded, num_uses_failed, diff --git a/archivebox/core/migrations/0025_alter_archiveresult_options_alter_snapshot_options_and_more.py b/archivebox/core/migrations/0025_alter_archiveresult_options_alter_snapshot_options_and_more.py index 9d29d8c6..46e91e91 100644 --- a/archivebox/core/migrations/0025_alter_archiveresult_options_alter_snapshot_options_and_more.py +++ b/archivebox/core/migrations/0025_alter_archiveresult_options_alter_snapshot_options_and_more.py @@ -140,7 +140,7 @@ class Migration(migrations.Migration): name="retry_at", field=models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True), ), - # NOTE: bookmarked_at and created_at already added by migration 0023 + # NOTE: bookmarked_at, created_at, and downloaded_at already added by migration 0023 migrations.AddField( model_name="snapshot", name="config", @@ -160,11 +160,8 @@ class Migration(migrations.Migration): name="depth", field=models.PositiveSmallIntegerField(db_index=True, default=0), ), - migrations.AddField( - model_name="snapshot", - name="downloaded_at", - field=models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True), - ), + # NOTE: downloaded_at already added by migration 0023 so it can preserve + # v0.7.x updated / v0.8.x downloaded_at values without a duplicate table rebuild. # NOTE: fs_version already added by migration 0023 with default='0.8.0' # NOTE: modified_at already added by migration 0023 migrations.AddField( diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 7af1176e..51c003cc 100755 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -1,10 +1,10 @@ __package__ = "archivebox.core" -from typing import Optional, Any, cast +from typing import TYPE_CHECKING, Optional, Any, cast from collections.abc import Iterable, Sequence import uuid from archivebox.uuid_compat import uuid7 -from datetime import datetime, timedelta +from datetime import datetime import os import json @@ -21,9 +21,11 @@ from django.core.cache import cache from django.urls import reverse_lazy from django.contrib import admin from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist from django.utils.safestring import mark_safe from archivebox.config import CONSTANTS +from archivebox.config.common import get_config from archivebox.misc.system import get_dir_size, atomic_write from archivebox.misc.util import parse_date, domain as url_domain, to_json, ts_to_date_str, urlencode, htmlencode, urldecode from archivebox.hooks import ( @@ -44,6 +46,9 @@ from archivebox.workers.tasks import bg_archive_snapshot from archivebox.crawls.models import Crawl from archivebox.machine.models import Binary +if TYPE_CHECKING: + from archivebox.config.common import ArchiveBoxBaseConfig + class Tag(ModelWithUUID): # Keep AutoField for compatibility with main branch migrations @@ -165,11 +170,10 @@ class SnapshotQuerySet(models.QuerySet): def search(self, patterns: list[str]) -> "SnapshotQuerySet": """Search snapshots using the configured search backend""" - from archivebox.config.common import SEARCH_BACKEND_CONFIG from archivebox.search import query_search_index from archivebox.misc.logging import stderr - if not SEARCH_BACKEND_CONFIG.USE_SEARCHING_BACKEND: + if not get_config().USE_SEARCHING_BACKEND: stderr() stderr("[X] The search backend is not enabled, set config.USE_SEARCHING_BACKEND = True", color="red") raise SystemExit(2) @@ -191,13 +195,14 @@ class SnapshotQuerySet(models.QuerySet): import sys from datetime import datetime, timezone as tz from archivebox.config import VERSION - from archivebox.config.common import SERVER_CONFIG + + config = get_config() MAIN_INDEX_HEADER = ( { "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.", "schema": "archivebox.index.json", - "copyright_info": SERVER_CONFIG.FOOTER_INFO, + "copyright_info": config.FOOTER_INFO, "meta": { "project": "ArchiveBox", "version": VERSION, @@ -239,9 +244,10 @@ class SnapshotQuerySet(models.QuerySet): from datetime import datetime, timezone as tz from django.template.loader import render_to_string from archivebox.config import VERSION - from archivebox.config.common import SERVER_CONFIG from archivebox.config.version import get_COMMIT_HASH + config = get_config() + template = "static_index.html" if with_headers else "minimal_index.html" snapshot_list = list(self.iterator(chunk_size=500)) @@ -254,7 +260,7 @@ class SnapshotQuerySet(models.QuerySet): "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"), "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"), "links": snapshot_list, - "FOOTER_INFO": SERVER_CONFIG.FOOTER_INFO, + "FOOTER_INFO": config.FOOTER_INFO, }, ) @@ -392,29 +398,28 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea # Migrate filesystem if needed (happens automatically on save) if self.pk and self.fs_migration_needed: - print( - f"[DEBUG save()] Triggering filesystem migration for {str(self.id)[:8]}: {self.fs_version} → {self._fs_current_version()}", - ) - # Walk through migration chain automatically - current = self.fs_version - target = self._fs_current_version() - - while current != target: - next_ver = self._fs_next_version(current) - method = f"_fs_migrate_from_{current.replace('.', '_')}_to_{next_ver.replace('.', '_')}" - - # Only run if method exists (most are no-ops) - if hasattr(self, method): - print(f"[DEBUG save()] Running {method}()") - getattr(self, method)() - - current = next_ver - - # Update version - self.fs_version = target + self.migrate_filesystem_to_current_version() + update_fields = kwargs.get("update_fields") + if update_fields is not None: + kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "fs_version", "modified_at"])) + elif self.pk: + legacy_dir = get_config().ARCHIVE_DIR / self.timestamp + current_dir = self.get_storage_path_for_version(self._fs_current_version()) + if legacy_dir.exists() and not legacy_dir.is_symlink() and current_dir.exists() and legacy_dir != current_dir: + self.migrate_filesystem_to_current_version(source_dir=legacy_dir) super().save(*args, **kwargs) + + migration_cleanup = getattr(self, "_pending_fs_migration_cleanup", None) + if migration_cleanup: + from django.db import transaction + + old_dir, new_dir = migration_cleanup + transaction.on_commit(lambda: self._cleanup_old_migration_dir(old_dir, new_dir)) + delattr(self, "_pending_fs_migration_cleanup") + self.ensure_legacy_archive_symlink() + self.ensure_crawl_symlink() existing_urls = {url for _raw_line, url in self.crawl._iter_url_lines() if url} if self.crawl.url_passes_filters(self.url, snapshot=self) and self.url not in existing_urls: self.crawl.urls += f"\n{self.url}" @@ -465,45 +470,109 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea return "0.9.0" return self._fs_current_version() - def _fs_migrate_from_0_8_0_to_0_9_0(self): + @staticmethod + def is_legacy_archive_dir(path: Path) -> bool: + """Return True for old-style archive/{timestamp} snapshot directories.""" + if path.name in CONSTANTS.RESERVED_ARCHIVE_DIR_NAMES or path.name.startswith("."): + return False + try: + ts_int = int(float(path.name)) + except (TypeError, ValueError, OverflowError): + return False + return 788918400 <= ts_int <= 2082758400 + + def migrate_filesystem_to_current_version(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None) -> None: + """ + Copy legacy snapshot output into the current layout and defer old-dir cleanup. + + The ordering is intentionally crash-safe: + 1. Copy from the legacy directory into the new directory idempotently. + 2. Verify the new directory has every old file. + 3. Convert metadata in the new directory. + 4. Update fs_version in memory for the caller to save. + 5. Cleanup is scheduled only after the DB commit succeeds. + """ + current = self.fs_version + target = self._fs_current_version() + cleanup: tuple[Path, Path] | None = None + runtime_config = config or get_config() + + if source_dir and current == target: + current_dir = self.get_storage_path_for_version(target, config=runtime_config) + cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, target_dir=current_dir) + if cleanup: + self._pending_fs_migration_cleanup = cleanup + return + + while current != target: + next_ver = self._fs_next_version(current) + migrations = { + ("0.7.0", "0.9.0"): self._fs_migrate_from_0_7_0_to_0_9_0, + ("0.8.0", "0.9.0"): self._fs_migrate_from_0_8_0_to_0_9_0, + } + + migration = migrations.get((current, next_ver)) + if migration is None: + raise ValueError(f"No filesystem migration path from {current} to {next_ver}") + cleanup = migration(source_dir=source_dir, config=runtime_config) + + current = next_ver + source_dir = None + + self.fs_version = target + if cleanup: + self._pending_fs_migration_cleanup = cleanup + + def _fs_migrate_from_0_7_0_to_0_9_0(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None): + return self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, config=config) + + def _fs_migrate_from_0_8_0_to_0_9_0(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None): + return self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, config=config) + + def _fs_migrate_legacy_to_0_9_0( + self, + source_dir: Path | None = None, + target_dir: Path | None = None, + config: "ArchiveBoxBaseConfig | None" = None, + ): """ Migrate from flat to nested structure. 0.8.x: archive/{timestamp}/ - 0.9.x: users/{user}/snapshots/YYYYMMDD/{domain}/{uuid}/ - - Transaction handling: - 1. Copy files INSIDE transaction - 2. Convert index.json to index.jsonl INSIDE transaction - 3. Create symlink INSIDE transaction - 4. Update fs_version INSIDE transaction (done by save()) - 5. Exit transaction (DB commit) - 6. Delete old files OUTSIDE transaction (after commit) + 0.9.x: archive/users/{user}/snapshots/YYYYMMDD/{domain}/{uuid}/ """ + import filecmp import shutil - from django.db import transaction - old_dir = self.get_storage_path_for_version("0.8.0") - new_dir = self.get_storage_path_for_version("0.9.0") + old_dir = Path(source_dir) if source_dir else self.get_storage_path_for_version("0.8.0", config=config) + new_dir = Path(target_dir) if target_dir else self.get_storage_path_for_version("0.9.0", config=config) - print( - f"[DEBUG _fs_migrate] {self.timestamp}: old_exists={old_dir.exists()}, same={old_dir == new_dir}, new_exists={new_dir.exists()}", - ) - - if not old_dir.exists() or old_dir == new_dir: - # No migration needed - print("[DEBUG _fs_migrate] Returning None (early return)") + if old_dir == new_dir: return None - if new_dir.exists(): - # New directory already exists (files already copied), but we still need cleanup - # Return cleanup info so old directory can be cleaned up - print("[DEBUG _fs_migrate] Returning cleanup info (new_dir exists)") - return (old_dir, new_dir) + if old_dir.is_symlink(): + return None + + if not old_dir.exists(): + if new_dir.exists(): + self.convert_index_json_to_jsonl(output_dir=new_dir) + return None + return None + + if not new_dir.exists(): + new_dir.parent.mkdir(parents=True, exist_ok=True) + try: + old_dir.rename(new_dir) + self.convert_index_json_to_jsonl(output_dir=new_dir) + return (old_dir, new_dir) + except OSError: + pass new_dir.mkdir(parents=True, exist_ok=True) - # Copy all files (idempotent), skipping index.json (will be converted to jsonl) + # Copy all files idempotently. If a previous attempt already converted + # index.json to index.jsonl, recopying index.json is harmless; conversion + # below removes it again after ensuring index.jsonl exists. for old_file in old_dir.rglob("*"): if not old_file.is_file(): continue @@ -512,8 +581,9 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea new_file = new_dir / rel_path # Skip if already copied - if new_file.exists() and new_file.stat().st_size == old_file.stat().st_size: - continue + if new_file.exists(): + if new_file.stat().st_size == old_file.stat().st_size and filecmp.cmp(old_file, new_file, shallow=False): + continue new_file.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(old_file, new_file) @@ -524,16 +594,21 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea if old_files.keys() != new_files.keys(): missing = old_files.keys() - new_files.keys() - raise Exception(f"Migration incomplete: missing {missing}") + missing.discard(Path(CONSTANTS.JSON_INDEX_FILENAME)) + if missing: + raise Exception(f"Migration incomplete: missing {missing}") - # Convert index.json to index.jsonl in the new directory - self.convert_index_json_to_jsonl() + for rel_path, old_size in old_files.items(): + if rel_path == Path(CONSTANTS.JSON_INDEX_FILENAME): + continue + if new_files.get(rel_path) != old_size: + raise Exception(f"Migration incomplete: size mismatch for {rel_path}") + if not filecmp.cmp(old_dir / rel_path, new_dir / rel_path, shallow=False): + raise Exception(f"Migration incomplete: content mismatch for {rel_path}") - # Schedule cleanup AFTER transaction commits successfully - # This ensures DB changes are committed before we delete old files - transaction.on_commit(lambda: self._cleanup_old_migration_dir(old_dir, new_dir)) + # Convert index.json to index.jsonl in the new directory. + self.convert_index_json_to_jsonl(output_dir=new_dir) - # Return cleanup info for manual cleanup if needed (when called directly) return (old_dir, new_dir) def _cleanup_old_migration_dir(self, old_dir: Path, new_dir: Path): @@ -600,34 +675,31 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea except Exception: return "unknown" - def get_storage_path_for_version(self, version: str) -> Path: + def get_storage_path_for_version(self, version: str, config: "ArchiveBoxBaseConfig | None" = None) -> Path: """ Calculate storage path for specific filesystem version. Centralizes path logic so it's reusable. 0.7.x/0.8.x: archive/{timestamp} - 0.9.x: users/{username}/snapshots/YYYYMMDD/{domain}/{uuid}/ + 0.9.x: archive/users/{username}/snapshots/YYYYMMDD/{domain}/{uuid}/ """ - from datetime import datetime + runtime_config = config or get_config() if version in ("0.7.0", "0.8.0"): - return CONSTANTS.ARCHIVE_DIR / self.timestamp + return runtime_config.ARCHIVE_DIR / self.timestamp elif version in ("0.9.0", "1.0.0"): username = self.created_by.username - # Use created_at for date grouping (fallback to timestamp) - if self.created_at: - date_str = self.created_at.strftime("%Y%m%d") - else: - date_str = datetime.fromtimestamp(float(self.timestamp)).strftime("%Y%m%d") + date_base = self.bookmarked_at or self.created_at + date_str = date_base.strftime("%Y%m%d") if date_base else "unknown" domain = self.extract_domain_from_url(self.url) - return CONSTANTS.DATA_DIR / "users" / username / "snapshots" / date_str / domain / str(self.id) + return runtime_config.USERS_DIR / username / CONSTANTS.SNAPSHOTS_DIR_NAME / date_str / domain / str(self.id) else: # Unknown version - use current - return self.get_storage_path_for_version(self._fs_current_version()) + return self.get_storage_path_for_version(self._fs_current_version(), config=runtime_config) # ========================================================================= # Loading and Creation from Filesystem (Used by archivebox update ONLY) @@ -668,46 +740,70 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea pass if not data: - return None + timestamp = cls._select_best_timestamp( + index_timestamp=None, + folder_name=snapshot_dir.name, + ) + if not timestamp: + return None + try: + return cls.objects.select_related("crawl__created_by").get(timestamp=timestamp) + except cls.DoesNotExist: + return None + except cls.MultipleObjectsReturned: + return cls.objects.select_related("crawl__created_by").filter(timestamp=timestamp).first() url = data.get("url") if not url: - return None + timestamp = cls._select_best_timestamp( + index_timestamp=data.get("timestamp"), + folder_name=snapshot_dir.name, + ) + if not timestamp: + return None + try: + return cls.objects.select_related("crawl__created_by").get(timestamp=timestamp) + except cls.DoesNotExist: + return None + except cls.MultipleObjectsReturned: + return cls.objects.select_related("crawl__created_by").filter(timestamp=timestamp).first() # Get timestamp - prefer index file, fallback to folder name timestamp = cls._select_best_timestamp( index_timestamp=data.get("timestamp"), folder_name=snapshot_dir.name, ) + folder_timestamp = cls._select_best_timestamp( + index_timestamp=None, + folder_name=snapshot_dir.name, + ) if not timestamp: return None # Look up existing (try exact match first, then fuzzy match for truncated timestamps) try: - snapshot = cls.objects.get(url=url, timestamp=timestamp) - print(f"[DEBUG load_from_directory] Found existing snapshot for {url} @ {timestamp}: {str(snapshot.id)[:8]}") + snapshot = cls.objects.select_related("crawl__created_by").get(url=url, timestamp=timestamp) return snapshot except cls.DoesNotExist: - print(f"[DEBUG load_from_directory] NOT FOUND (exact): {url} @ {timestamp}") # Try fuzzy match - index.json may have truncated timestamp # e.g., index has "1767000340" but DB has "1767000340.624737" - candidates = cls.objects.filter(url=url, timestamp__startswith=timestamp) - if candidates.count() == 1: - snapshot = candidates.first() - if snapshot is None: - return None - print(f"[DEBUG load_from_directory] Found via fuzzy match: {snapshot.timestamp}") - return snapshot - elif candidates.count() > 1: - print("[DEBUG load_from_directory] Multiple fuzzy matches, using first") - return candidates.first() - print(f"[DEBUG load_from_directory] NOT FOUND (fuzzy): {url} @ {timestamp}") + # Do not fuzzy-match when the legacy folder name itself is a valid + # timestamp; distinct dirs like 1508259732 and 1508259732.0 must + # remain distinct snapshots. + if not folder_timestamp or timestamp != folder_timestamp: + candidates = cls.objects.select_related("crawl__created_by").filter(url=url, timestamp__startswith=timestamp) + if candidates.count() == 1: + snapshot = candidates.first() + if snapshot is None: + return None + return snapshot + elif candidates.count() > 1: + return candidates.first() return None except cls.MultipleObjectsReturned: # Should not happen with unique constraint - print(f"[DEBUG load_from_directory] Multiple snapshots found for {url} @ {timestamp}") - return cls.objects.filter(url=url, timestamp=timestamp).first() + return cls.objects.select_related("crawl__created_by").filter(url=url, timestamp=timestamp).first() @classmethod def create_from_directory(cls, snapshot_dir: Path) -> Optional["Snapshot"]: @@ -742,6 +838,29 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea except (json.JSONDecodeError, OSError): pass + if not data or not data.get("url"): + archive_org_path = snapshot_dir / "archive.org.txt" + try: + archived_url = archive_org_path.read_text(encoding="utf-8", errors="replace").strip().splitlines()[0].strip() + except (IndexError, OSError): + archived_url = "" + + if archived_url.startswith(("http://", "https://")): + if "://web.archive.org/web/" in archived_url and "/web/" in archived_url: + archive_target = archived_url.split("/web/", 1)[1].split("/", 1) + if len(archive_target) == 2: + candidate = archive_target[1] + if not candidate.startswith(("http://", "https://")) and "/" in candidate: + candidate = candidate.split("/", 1)[1] + if candidate.startswith(("http://", "https://")): + archived_url = candidate + + data = { + "url": archived_url, + "timestamp": snapshot_dir.name, + "title": "", + } + if not data: return None @@ -764,9 +883,6 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea # Detect version fs_version = cls._detect_fs_version_from_index(data) - # Get or create catchall crawl for orphaned snapshots - from archivebox.crawls.models import Crawl - system_user_id = get_or_create_system_user_pk() catchall_crawl, _ = Crawl.objects.get_or_create( label="[migration] orphaned snapshots", @@ -776,13 +892,36 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea "created_by_id": system_user_id, }, ) + if cls.objects.filter(crawl=catchall_crawl, url=url).exists(): + catchall_crawl = Crawl.objects.create( + label=f"[migration] orphaned snapshot {timestamp}", + urls=url, + max_depth=0, + created_by_id=system_user_id, + ) + + snapshot_kwargs = { + "url": url, + "timestamp": timestamp, + "title": data.get("title", ""), + "fs_version": fs_version, + "crawl": catchall_crawl, + } + try: + bookmarked_at = parse_date(data.get("bookmarked_at") or timestamp) + except (TypeError, ValueError, OSError): + bookmarked_at = None + try: + created_at = parse_date(data.get("created_at")) + except (TypeError, ValueError, OSError): + created_at = None + if bookmarked_at: + snapshot_kwargs["bookmarked_at"] = bookmarked_at + if created_at: + snapshot_kwargs["created_at"] = created_at return cls( - url=url, - timestamp=timestamp, - title=data.get("title", ""), - fs_version=fs_version, - crawl=catchall_crawl, + **snapshot_kwargs, ) @staticmethod @@ -790,8 +929,9 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea """ Select best timestamp from index.json vs folder name. - Validates range (1995-2035). - Prefers index.json if valid. + Validates range (1995-2035). When a valid legacy folder name is + available it is the stable filesystem identity, so preserve it over + normalized variants like "1508259732.0" found in old index files. """ def is_valid_timestamp(ts: object | None) -> bool: @@ -807,24 +947,25 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea index_valid = is_valid_timestamp(index_timestamp) if index_timestamp else False folder_valid = is_valid_timestamp(folder_name) - if index_valid and index_timestamp is not None: - return str(int(float(str(index_timestamp)))) if folder_valid: - return str(int(float(str(folder_name)))) + return str(folder_name).strip() + if index_valid and index_timestamp is not None: + return str(index_timestamp).strip() return None @classmethod def _ensure_unique_timestamp(cls, url: str, timestamp: str) -> str: """ Ensure timestamp is globally unique. - If collision with different URL, increment by 1 until unique. - - NOTE: Logic already exists in create_or_update_from_dict (line 266-267) - This is just an extracted, reusable version. + If there is a collision, add a tiny fractional suffix until unique. """ - while cls.objects.filter(timestamp=timestamp).exclude(url=url).exists(): - timestamp = str(int(float(timestamp)) + 1) - return timestamp + candidate = str(timestamp) + base = float(timestamp) + suffix = 0 + while cls.objects.filter(timestamp=candidate).exists(): + suffix += 1 + candidate = f"{base + (suffix / 1_000_000):.6f}".rstrip("0").rstrip(".") + return candidate @staticmethod def _detect_fs_version_from_index(data: dict) -> str: @@ -848,7 +989,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea # Index.json Reconciliation # ========================================================================= - def reconcile_with_index(self): + def reconcile_with_index(self, output_dir: Path | None = None): """ Merge index.json/index.jsonl with DB. DB is source of truth. @@ -863,17 +1004,18 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea import json # Try to convert index.json to index.jsonl first - self.convert_index_json_to_jsonl() + output_dir = Path(output_dir) if output_dir is not None else Path(self.output_dir) + self.convert_index_json_to_jsonl(output_dir=output_dir) # Check for index.jsonl (preferred) or index.json (legacy) - jsonl_path = Path(self.output_dir) / CONSTANTS.JSONL_INDEX_FILENAME - json_path = Path(self.output_dir) / CONSTANTS.JSON_INDEX_FILENAME + jsonl_path = output_dir / CONSTANTS.JSONL_INDEX_FILENAME + json_path = output_dir / CONSTANTS.JSON_INDEX_FILENAME index_data = {} if jsonl_path.exists(): # Read from JSONL format - jsonl_data = self.read_index_jsonl() + jsonl_data = self.read_index_jsonl(output_dir=output_dir) if jsonl_data["snapshot"]: index_data = jsonl_data["snapshot"] # Convert archive_results list to expected format @@ -896,11 +1038,11 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea self._merge_archive_results_from_index(index_data) # Write back in JSONL format - self.write_index_jsonl() + self.write_index_jsonl(output_dir=output_dir) - def reconcile_with_index_json(self): + def reconcile_with_index_json(self, output_dir: Path | None = None): """Deprecated: use reconcile_with_index() instead.""" - return self.reconcile_with_index() + return self.reconcile_with_index(output_dir=output_dir) def _merge_title_from_index(self, index_data: dict): """Merge title - prefer longest non-URL title.""" @@ -1022,7 +1164,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea with open(index_path, "w") as f: json.dump(data, f, indent=2, sort_keys=True) - def write_index_jsonl(self): + def write_index_jsonl(self, output_dir: Path | None = None): """ Write index.jsonl in flat JSONL format. @@ -1034,7 +1176,8 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea """ import json - index_path = Path(self.output_dir) / CONSTANTS.JSONL_INDEX_FILENAME + output_dir = Path(output_dir) if output_dir is not None else Path(self.output_dir) + index_path = output_dir / CONSTANTS.JSONL_INDEX_FILENAME index_path.parent.mkdir(parents=True, exist_ok=True) # Track unique binaries and processes to avoid duplicates @@ -1061,7 +1204,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea # Write ArchiveResult record f.write(json.dumps(ar.to_json()) + "\n") - def read_index_jsonl(self) -> dict: + def read_index_jsonl(self, output_dir: Path | None = None) -> dict: """ Read index.jsonl and return parsed records grouped by type. @@ -1076,7 +1219,8 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea TYPE_PROCESS, ) - index_path = Path(self.output_dir) / CONSTANTS.JSONL_INDEX_FILENAME + output_dir = Path(output_dir) if output_dir is not None else Path(self.output_dir) + index_path = output_dir / CONSTANTS.JSONL_INDEX_FILENAME result: dict[str, Any] = { "snapshot": None, "archive_results": [], @@ -1101,7 +1245,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea return result - def convert_index_json_to_jsonl(self) -> bool: + def convert_index_json_to_jsonl(self, output_dir: Path | None = None) -> bool: """ Convert index.json to index.jsonl format. @@ -1110,11 +1254,15 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea """ import json - json_path = Path(self.output_dir) / CONSTANTS.JSON_INDEX_FILENAME - jsonl_path = Path(self.output_dir) / CONSTANTS.JSONL_INDEX_FILENAME + output_dir = Path(output_dir) if output_dir is not None else Path(self.output_dir) + json_path = output_dir / CONSTANTS.JSON_INDEX_FILENAME + jsonl_path = output_dir / CONSTANTS.JSONL_INDEX_FILENAME # Skip if already converted or no json file exists - if jsonl_path.exists() or not json_path.exists(): + if jsonl_path.exists(): + json_path.unlink(missing_ok=True) + return False + if not json_path.exists(): return False try: @@ -1126,56 +1274,60 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea # Detect format version and extract records fs_version = data.get("fs_version", "0.7.0") - jsonl_path.parent.mkdir(parents=True, exist_ok=True) - with open(jsonl_path, "w") as f: - # Write Snapshot record - snapshot_record = { - "type": "Snapshot", - "id": str(self.id), - "crawl_id": str(self.crawl_id) if self.crawl_id else None, - "url": data.get("url", self.url), - "timestamp": data.get("timestamp", self.timestamp), - "title": data.get("title", self.title or ""), - "tags": data.get("tags", ""), - "fs_version": fs_version, - "bookmarked_at": data.get("bookmarked_at"), - "created_at": data.get("created_at"), + records = [] + snapshot_record = { + "type": "Snapshot", + "id": str(self.id), + "crawl_id": str(self.crawl_id) if self.crawl_id else None, + "url": data.get("url", self.url), + "timestamp": data.get("timestamp", self.timestamp), + "title": data.get("title", self.title or ""), + "tags": data.get("tags", ""), + "fs_version": fs_version, + "bookmarked_at": data.get("bookmarked_at"), + "created_at": data.get("created_at"), + } + records.append(snapshot_record) + + # Handle 0.8.x/0.9.x format (archive_results list) + for result_data in data.get("archive_results", []): + ar_record = { + "type": "ArchiveResult", + "snapshot_id": str(self.id), + "plugin": result_data.get("plugin", ""), + "status": result_data.get("status", ""), + "output_str": result_data.get("output", ""), + "start_ts": result_data.get("start_ts"), + "end_ts": result_data.get("end_ts"), } - f.write(json.dumps(snapshot_record) + "\n") + if result_data.get("cmd"): + ar_record["cmd"] = result_data["cmd"] + records.append(ar_record) - # Handle 0.8.x/0.9.x format (archive_results list) - for result_data in data.get("archive_results", []): - ar_record = { - "type": "ArchiveResult", - "snapshot_id": str(self.id), - "plugin": result_data.get("plugin", ""), - "status": result_data.get("status", ""), - "output_str": result_data.get("output", ""), - "start_ts": result_data.get("start_ts"), - "end_ts": result_data.get("end_ts"), - } - if result_data.get("cmd"): - ar_record["cmd"] = result_data["cmd"] - f.write(json.dumps(ar_record) + "\n") + # Handle 0.7.x format (history dict) + if "history" in data and isinstance(data["history"], dict): + for plugin, result_list in data["history"].items(): + if not isinstance(result_list, list): + continue + for result_data in result_list: + ar_record = { + "type": "ArchiveResult", + "snapshot_id": str(self.id), + "plugin": result_data.get("plugin") or result_data.get("extractor") or plugin, + "status": result_data.get("status", ""), + "output_str": result_data.get("output", ""), + "start_ts": result_data.get("start_ts"), + "end_ts": result_data.get("end_ts"), + } + if result_data.get("cmd"): + ar_record["cmd"] = result_data["cmd"] + records.append(ar_record) - # Handle 0.7.x format (history dict) - if "history" in data and isinstance(data["history"], dict): - for plugin, result_list in data["history"].items(): - if not isinstance(result_list, list): - continue - for result_data in result_list: - ar_record = { - "type": "ArchiveResult", - "snapshot_id": str(self.id), - "plugin": result_data.get("plugin") or result_data.get("extractor") or plugin, - "status": result_data.get("status", ""), - "output_str": result_data.get("output", ""), - "start_ts": result_data.get("start_ts"), - "end_ts": result_data.get("end_ts"), - } - if result_data.get("cmd"): - ar_record["cmd"] = result_data["cmd"] - f.write(json.dumps(ar_record) + "\n") + jsonl_path.parent.mkdir(parents=True, exist_ok=True) + tmp_jsonl_path = jsonl_path.with_name(f".{jsonl_path.name}.tmp") + with open(tmp_jsonl_path, "w", encoding="utf-8") as f: + f.write("".join(json.dumps(record) + "\n" for record in records)) + os.replace(tmp_jsonl_path, jsonl_path) # Remove old index.json after successful conversion try: @@ -1477,7 +1629,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea return current_path # Check for backwards-compat symlink - old_path = CONSTANTS.ARCHIVE_DIR / self.timestamp + old_path = get_config().ARCHIVE_DIR / self.timestamp if old_path.is_symlink(): link_target = Path(os.readlink(old_path)) return (old_path.parent / link_target).resolve() if not link_target.is_absolute() else link_target.resolve() @@ -1490,7 +1642,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea """Ensure the legacy archive/ path resolves to this snapshot.""" import os - legacy_path = CONSTANTS.ARCHIVE_DIR / self.timestamp + legacy_path = get_config().ARCHIVE_DIR / self.timestamp target = Path(self.get_storage_path_for_version(self._fs_current_version())) if target == legacy_path: @@ -1515,31 +1667,31 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea except OSError: return - def ensure_crawl_symlink(self) -> None: + def ensure_crawl_symlink(self, *, crawl_dir: Path | None = None, snapshot_dir: Path | None = None) -> None: """Ensure snapshot is symlinked under its crawl output directory.""" import os from pathlib import Path - from django.utils import timezone - from archivebox import DATA_DIR - from archivebox.crawls.models import Crawl - if not self.crawl_id: - return - crawl = Crawl.objects.filter(id=self.crawl_id).select_related("created_by").first() - if not crawl: - return + if crawl_dir is None: + if not self.crawl_id: + return + try: + crawl = self.crawl + except ObjectDoesNotExist: + crawl = None + if crawl is None: + crawl = Crawl.objects.filter(id=self.crawl_id).select_related("created_by").first() + if not crawl: + return + crawl_dir = Path(crawl.output_dir) - date_base = crawl.created_at or self.created_at or timezone.now() - date_str = date_base.strftime("%Y%m%d") domain = self.extract_domain_from_url(self.url) - username = crawl.created_by.username if getattr(crawl, "created_by_id", None) else "system" - crawl_dir = DATA_DIR / "users" / username / "crawls" / date_str / domain / str(crawl.id) - link_path = crawl_dir / "snapshots" / domain / str(self.id) + link_path = Path(crawl_dir) / CONSTANTS.SNAPSHOTS_DIR_NAME / domain / str(self.id) link_parent = link_path.parent link_parent.mkdir(parents=True, exist_ok=True) - target = Path(self.output_dir) + target = Path(snapshot_dir) if snapshot_dir is not None else Path(self.output_dir) if link_path.exists() or link_path.is_symlink(): if link_path.is_symlink(): if link_path.resolve() == target.resolve(): @@ -1572,14 +1724,11 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea if username == "system": username = "web" - date_base = self.created_at or self.bookmarked_at + date_base = self.bookmarked_at or self.created_at if date_base: date_str = date_base.strftime("%Y%m%d") else: - try: - date_str = datetime.fromtimestamp(float(self.timestamp)).strftime("%Y%m%d") - except (TypeError, ValueError, OSError): - return self.legacy_archive_path + return self.legacy_archive_path domain = self.extract_domain_from_url(self.url) return f"{username}/{date_str}/{domain}/{self.id}" @@ -1589,13 +1738,46 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea @cached_property def url_path(self) -> str: """URL path matching the current snapshot output_dir layout.""" + output_dir = Path(self.output_dir).resolve() try: - rel_path = Path(self.output_dir).resolve().relative_to(CONSTANTS.DATA_DIR) + rel_users_path = output_dir.relative_to(get_config().USERS_DIR) + except Exception: + rel_users_path = None + + if rel_users_path: + parts = rel_users_path.parts + # Configured users root: /snapshots//// + if len(parts) >= 5 and parts[1] == CONSTANTS.SNAPSHOTS_DIR_NAME: + username = parts[0] + if username == "system": + username = "web" + date_str = parts[2] + domain = parts[3] + snapshot_id = parts[4] + return f"{username}/{date_str}/{domain}/{snapshot_id}" + + try: + rel_path = output_dir.relative_to(CONSTANTS.DATA_DIR) except Exception: return self.legacy_archive_path parts = rel_path.parts - # New layout: users//snapshots//// + # New layout: archive/users//snapshots//// + if ( + len(parts) >= 7 + and parts[0] == CONSTANTS.ARCHIVE_DIR_NAME + and parts[1] == CONSTANTS.USERS_DIR_NAME + and parts[3] == CONSTANTS.SNAPSHOTS_DIR_NAME + ): + username = parts[2] + if username == "system": + username = "web" + date_str = parts[4] + domain = parts[5] + snapshot_id = parts[6] + return f"{username}/{date_str}/{domain}/{snapshot_id}" + + # Previous dev layout: users//snapshots//// if len(parts) >= 6 and parts[0] == "users" and parts[2] == "snapshots": username = parts[1] if username == "system": @@ -1729,7 +1911,8 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea import re from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.config.common import GENERAL_CONFIG + + config = get_config() overrides = overrides or {} @@ -1781,7 +1964,6 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea parent_snapshot.created_by.pk if parent_snapshot else get_or_create_system_user_pk() ) - # DEBUG: Check if crawl_id in record matches overrides crawl import sys record_crawl_id = record.get("crawl_id") @@ -1821,7 +2003,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea tag_list = list(dict.fromkeys(tag.strip() for tag in tags_raw if tag.strip())) elif tags_raw: tag_list = list( - dict.fromkeys(tag.strip() for tag in re.split(GENERAL_CONFIG.TAG_SEPARATOR_PATTERN, tags_raw) if tag.strip()), + dict.fromkeys(tag.strip() for tag in re.split(config.TAG_SEPARATOR_PATTERN, tags_raw) if tag.strip()), ) # Check for existing snapshot with same URL in same crawl @@ -1830,6 +2012,15 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea title = record.get("title") timestamp = record.get("timestamp") + timestamp_for_bookmark = Snapshot._select_best_timestamp(index_timestamp=timestamp, folder_name="") + try: + bookmarked_at = parse_date(record.get("bookmarked_at") or timestamp_for_bookmark) + except (TypeError, ValueError, OSError): + bookmarked_at = None + try: + created_at = parse_date(record.get("created_at")) + except (TypeError, ValueError, OSError): + created_at = None if snapshot: # Update existing snapshot @@ -1842,12 +2033,17 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea while Snapshot.objects.filter(timestamp=timestamp).exists(): timestamp = str(float(timestamp) + 1.0) - snapshot = Snapshot.objects.create( - url=url, - timestamp=timestamp, - title=title, - crawl=crawl, - ) + create_kwargs = { + "url": url, + "timestamp": timestamp, + "title": title, + "crawl": crawl, + } + if bookmarked_at: + create_kwargs["bookmarked_at"] = bookmarked_at + if created_at: + create_kwargs["created_at"] = created_at + snapshot = Snapshot.objects.create(**create_kwargs) # Update tags if tag_list: @@ -1864,8 +2060,10 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea update_fields.extend(["status", "retry_at"]) # Update additional fields if provided - for field_name in ("depth", "parent_snapshot_id", "crawl_id", "bookmarked_at"): + for field_name in ("depth", "parent_snapshot_id", "crawl_id", "bookmarked_at", "created_at", "downloaded_at"): value = record.get(field_name) + if field_name in ("bookmarked_at", "created_at", "downloaded_at") and value and isinstance(value, str): + value = parse_date(value) if value is not None and getattr(snapshot, field_name) != value: setattr(snapshot, field_name, value) update_fields.append(field_name) @@ -1889,7 +2087,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea This enables step-based execution where all hooks in a step can run in parallel. """ from archivebox.hooks import discover_hooks - from archivebox.config.configset import get_config + from archivebox.config.common import get_config # Get merged config with crawl-specific PLUGINS filter config = get_config(crawl=self.crawl, snapshot=self) @@ -2074,10 +2272,9 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea @cached_property def bookmarked_date(self) -> str | None: - max_ts = (timezone.now() + timedelta(days=30)).timestamp() - if self.timestamp and self.timestamp.replace(".", "").isdigit(): - if 0 < float(self.timestamp) < max_ts: - return self._ts_to_date_str(datetime.fromtimestamp(float(self.timestamp))) + if self.bookmarked_at: + return self._ts_to_date_str(self.bookmarked_at) + if self.timestamp: return str(self.timestamp) return None @@ -2360,8 +2557,6 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea def write_html_details(self, out_dir: Path | str | None = None) -> None: """Write HTML detail page for this snapshot to its output directory""" from django.template.loader import render_to_string - from archivebox.config.common import SERVER_CONFIG - from archivebox.config.configset import get_config from archivebox.core.widgets import TagEditorWidget from archivebox.misc.logging_util import printable_filesize @@ -2411,7 +2606,7 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea "status_color": "success" if is_archived else "danger", "oldest_archive_date": ts_to_date_str(self.oldest_archive_date), "SAVE_ARCHIVE_DOT_ORG": SAVE_ARCHIVE_DOT_ORG, - "PREVIEW_ORIGINALS": SERVER_CONFIG.PREVIEW_ORIGINALS, + "PREVIEW_ORIGINALS": config.PREVIEW_ORIGINALS, "best_preview_path": best_preview_path, "best_result": best_result, "archiveresults": outputs, diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 802036b6..067b6b23 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -13,8 +13,8 @@ from django.utils.crypto import get_random_string import archivebox -from archivebox.config import DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS # noqa -from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG, STORAGE_CONFIG # noqa +from archivebox.config.constants import CONSTANTS +from archivebox.config.common import get_config from archivebox.core.host_utils import normalize_base_url, get_admin_base_url, get_api_base_url from .settings_logging import SETTINGS_LOGGING @@ -23,6 +23,8 @@ IS_MIGRATING = "makemigrations" in sys.argv[:3] or "migrate" in sys.argv[:3] IS_TESTING = "test" in sys.argv[:3] or "PYTEST_CURRENT_TEST" in os.environ IS_SHELL = "shell" in sys.argv[:3] or "shell_plus" in sys.argv[:3] IS_GETTING_VERSION_OR_HELP = "version" in sys.argv or "help" in sys.argv or "--version" in sys.argv or "--help" in sys.argv +CONFIG = get_config() +PACKAGE_DIR = CONSTANTS.PACKAGE_DIR ################################################################################ ### ArchiveBox Plugin Settings @@ -45,7 +47,7 @@ LOGOUT_REDIRECT_URL = os.environ.get("LOGOUT_REDIRECT_URL", "/") PASSWORD_RESET_URL = "/accounts/password_reset/" APPEND_SLASH = True -DEBUG = SHELL_CONFIG.DEBUG or ("--debug" in sys.argv) +DEBUG = CONFIG.DEBUG or ("--debug" in sys.argv) INSTALLED_APPS = [ @@ -110,11 +112,9 @@ AUTHENTICATION_BACKENDS = [ # LDAP Authentication Configuration # Conditionally loaded if LDAP_ENABLED=True and django-auth-ldap is installed try: - from archivebox.config.ldap import LDAP_CONFIG - - if LDAP_CONFIG.LDAP_ENABLED: + if CONFIG.LDAP_ENABLED: # Validate LDAP configuration - is_valid, error_msg = LDAP_CONFIG.validate_ldap_config() + is_valid, error_msg = CONFIG.validate_ldap_config() if not is_valid: from rich import print @@ -127,23 +127,23 @@ try: ldap = importlib.import_module("ldap") # Configure LDAP authentication - AUTH_LDAP_SERVER_URI = LDAP_CONFIG.LDAP_SERVER_URI - AUTH_LDAP_BIND_DN = LDAP_CONFIG.LDAP_BIND_DN - AUTH_LDAP_BIND_PASSWORD = LDAP_CONFIG.LDAP_BIND_PASSWORD + AUTH_LDAP_SERVER_URI = CONFIG.LDAP_SERVER_URI + AUTH_LDAP_BIND_DN = CONFIG.LDAP_BIND_DN + AUTH_LDAP_BIND_PASSWORD = CONFIG.LDAP_BIND_PASSWORD # Configure user search AUTH_LDAP_USER_SEARCH = LDAPSearch( - LDAP_CONFIG.LDAP_USER_BASE, + CONFIG.LDAP_USER_BASE, getattr(ldap, "SCOPE_SUBTREE", 2), - LDAP_CONFIG.LDAP_USER_FILTER, + CONFIG.LDAP_USER_FILTER, ) # Map LDAP attributes to Django user model fields AUTH_LDAP_USER_ATTR_MAP = { - "username": LDAP_CONFIG.LDAP_USERNAME_ATTR, - "first_name": LDAP_CONFIG.LDAP_FIRSTNAME_ATTR, - "last_name": LDAP_CONFIG.LDAP_LASTNAME_ATTR, - "email": LDAP_CONFIG.LDAP_EMAIL_ATTR, + "username": CONFIG.LDAP_USERNAME_ATTR, + "first_name": CONFIG.LDAP_FIRSTNAME_ATTR, + "last_name": CONFIG.LDAP_LASTNAME_ATTR, + "email": CONFIG.LDAP_EMAIL_ATTR, } # Use custom LDAP backend that supports LDAP_CREATE_SUPERUSER @@ -175,9 +175,9 @@ except ImportError: STATIC_URL = "/static/" TEMPLATES_DIR_NAME = "templates" -CUSTOM_TEMPLATES_ENABLED = os.path.isdir(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR) and os.access(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK) +CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONFIG.CUSTOM_TEMPLATES_DIR) and os.access(CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK) STATICFILES_DIRS = [ - *([str(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []), + *([str(CONFIG.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []), # *[ # str(plugin_dir / 'static') # for plugin_dir in PLUGIN_DIRS.values() @@ -188,7 +188,7 @@ STATICFILES_DIRS = [ ] TEMPLATE_DIRS = [ - *([str(STORAGE_CONFIG.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []), + *([str(CONFIG.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []), # *[ # str(plugin_dir / 'templates') # for plugin_dir in PLUGIN_DIRS.values() @@ -328,14 +328,14 @@ STORAGES = { "BACKEND": "django.core.files.storage.FileSystemStorage", "OPTIONS": { "base_url": "/archive/", - "location": ARCHIVE_DIR, + "location": CONFIG.ARCHIVE_DIR, }, }, # "snapshots": { # "BACKEND": "django.core.files.storage.FileSystemStorage", # "OPTIONS": { # "base_url": "/snapshots/", - # "location": CONSTANTS.SNAPSHOTS_DIR, + # "location": CONSTANTS.USERS_DIR, # }, # }, # "personas": { @@ -353,10 +353,10 @@ CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"} ### Security Settings ################################################################################ -SECRET_KEY = SERVER_CONFIG.SECRET_KEY or get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789_") +SECRET_KEY = CONFIG.SECRET_KEY or get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789_") -ALLOWED_HOSTS = SERVER_CONFIG.ALLOWED_HOSTS.split(",") -CSRF_TRUSTED_ORIGINS = list(set(SERVER_CONFIG.CSRF_TRUSTED_ORIGINS.split(","))) +ALLOWED_HOSTS = CONFIG.ALLOWED_HOSTS.split(",") +CSRF_TRUSTED_ORIGINS = list(set(CONFIG.CSRF_TRUSTED_ORIGINS.split(","))) admin_base_url = normalize_base_url(get_admin_base_url()) if admin_base_url and admin_base_url not in CSRF_TRUSTED_ORIGINS: diff --git a/archivebox/core/templatetags/config_tags.py b/archivebox/core/templatetags/config_tags.py index 35e3d45c..8c305c57 100644 --- a/archivebox/core/templatetags/config_tags.py +++ b/archivebox/core/templatetags/config_tags.py @@ -4,19 +4,19 @@ from typing import Any from django import template -from archivebox.config.configset import get_config as _get_config +from archivebox.config.common import get_config register = template.Library() -@register.simple_tag -def get_config(key: str) -> Any: +@register.simple_tag(name="get_config") +def get_config_tag(key: str) -> Any: """ Get a config value by key. Usage: {% get_config "ARCHIVEDOTORG_ENABLED" as enabled %} """ try: - return _get_config().get(key) + return get_config().get(key) except (KeyError, AttributeError): return None diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py index 85a5bb85..09f66bee 100644 --- a/archivebox/core/urls.py +++ b/archivebox/core/urls.py @@ -1,11 +1,15 @@ __package__ = "archivebox.core" +import sys +from importlib.util import find_spec + from django.urls import path, re_path, include from django.views import static -from django.conf import settings from django.views.generic.base import RedirectView from django.http import HttpRequest +from archivebox.config.constants import CONSTANTS +from archivebox.config.common import get_config from archivebox.misc.serve_static import serve_static from archivebox.core.admin_site import archivebox_admin @@ -28,13 +32,13 @@ from archivebox.core.views import ( # GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE} -# print('DEBUG', settings.DEBUG) +CONFIG = get_config() +DEBUG = CONFIG.DEBUG or ("--debug" in sys.argv) urlpatterns = [ re_path(r"^static/(?P.*)$", serve_static), - # re_path(r"^media/(?P.*)$", static.serve, {"document_root": settings.MEDIA_ROOT}), - path("robots.txt", static.serve, {"document_root": settings.STATICFILES_DIRS[0], "path": "robots.txt"}), - path("favicon.ico", static.serve, {"document_root": settings.STATICFILES_DIRS[0], "path": "favicon.ico"}), + path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}), + path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}), path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"), path("public/", PublicIndexView.as_view(), name="public-index"), path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"), @@ -79,10 +83,10 @@ def _raise_test_error(_request: HttpRequest): raise ZeroDivisionError("Intentional test error route") -if settings.DEBUG_TOOLBAR: +if DEBUG and ("--nothreading" in sys.argv) and ("--reload" not in sys.argv) and find_spec("debug_toolbar"): urlpatterns += [path("__debug__/", include("debug_toolbar.urls"))] -if settings.DEBUG_REQUESTS_TRACKER: +if DEBUG and find_spec("requests_tracker"): urlpatterns += [path("__requests_tracker__/", include("requests_tracker.urls"))] diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 999ea5d6..3070d18c 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -6,7 +6,7 @@ import posixpath from glob import glob, escape from django.utils import timezone import inspect -from typing import cast, get_type_hints +from typing import cast from collections.abc import Callable from pathlib import Path from urllib.parse import quote, urlparse @@ -27,9 +27,9 @@ from django.utils.decorators import method_decorator from admin_data_views.typing import TableContext, ItemContext, SectionData from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink -from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, DATA_DIR, VERSION -from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG, SEARCH_BACKEND_CONFIG -from archivebox.config.configset import get_flat_config, get_config, get_all_configs +from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, VERSION +from archivebox.config.common import get_config, get_all_configs +from archivebox.config.configset import BaseConfigSet from archivebox.misc.util import base_url, htmlencode, ts_to_date_str, urldecode, without_fragment from archivebox.misc.serve_static import serve_static_with_byterange_support from archivebox.misc.logging_util import printable_filesize @@ -94,17 +94,17 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None: def _admin_login_redirect_or_forbidden(request: HttpRequest): - if SERVER_CONFIG.CONTROL_PLANE_ENABLED: + if get_config().CONTROL_PLANE_ENABLED: return redirect(f"/admin/login/?next={request.path}") return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.") class HomepageView(View): def get(self, request): - if request.user.is_authenticated and SERVER_CONFIG.CONTROL_PLANE_ENABLED: + if request.user.is_authenticated and get_config().CONTROL_PLANE_ENABLED: return redirect("/admin/core/snapshot/") - if SERVER_CONFIG.PUBLIC_INDEX: + if get_config().PUBLIC_INDEX: return redirect("/public") return _admin_login_redirect_or_forbidden(request) @@ -251,7 +251,7 @@ class SnapshotView(View): "num_failures": snapshot.num_failures, "oldest_archive_date": ts_to_date_str(snapshot.oldest_archive_date), "warc_path": warc_path, - "PREVIEW_ORIGINALS": SERVER_CONFIG.PREVIEW_ORIGINALS, + "PREVIEW_ORIGINALS": get_config().PREVIEW_ORIGINALS, "archiveresults": [*non_compact_outputs, *compact_outputs], "best_result": best_result, "snapshot": snapshot, # Pass the snapshot object for template tags @@ -264,7 +264,7 @@ class SnapshotView(View): return render(template_name="core/snapshot.html", request=request, context=context) def get(self, request, path): - if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS: + if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS: return _admin_login_redirect_or_forbidden(request) snapshot = None @@ -466,7 +466,7 @@ class SnapshotPathView(View): path: str = "", url: str | None = None, ): - if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS: + if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS: return _admin_login_redirect_or_forbidden(request) if username == "system": @@ -501,20 +501,20 @@ class SnapshotPathView(View): if date: try: if len(date) == 4: - qs = qs.filter(created_at__year=int(date)) + qs = qs.filter(bookmarked_at__year=int(date)) elif len(date) == 6: - qs = qs.filter(created_at__year=int(date[:4]), created_at__month=int(date[4:6])) + qs = qs.filter(bookmarked_at__year=int(date[:4]), bookmarked_at__month=int(date[4:6])) elif len(date) == 8: qs = qs.filter( - created_at__year=int(date[:4]), - created_at__month=int(date[4:6]), - created_at__day=int(date[6:8]), + bookmarked_at__year=int(date[:4]), + bookmarked_at__month=int(date[4:6]), + bookmarked_at__day=int(date[6:8]), ) except ValueError: pass if requested_url: - snapshot = qs.order_by("-created_at", "-bookmarked_at", "-timestamp").first() + snapshot = qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first() else: requested_domain = domain or "" if requested_domain.startswith(("http://", "https://")): @@ -524,9 +524,9 @@ class SnapshotPathView(View): # Prefer exact domain matches matches = [ - s for s in qs.order_by("-created_at", "-bookmarked_at") if Snapshot.extract_domain_from_url(s.url) == requested_domain + s for s in qs.order_by("-bookmarked_at", "-created_at") if Snapshot.extract_domain_from_url(s.url) == requested_domain ] - snapshot = matches[0] if matches else qs.order_by("-created_at", "-bookmarked_at", "-timestamp").first() + snapshot = matches[0] if matches else qs.order_by("-bookmarked_at", "-created_at", "-timestamp").first() if not snapshot: return HttpResponse( @@ -633,7 +633,7 @@ def _latest_response_match(domain: str, rel_path: str) -> tuple[Path, Path] | No return None domain = domain.split(":", 1)[0].lower() # TODO: optimize by querying output_files in DB instead of globbing filesystem - data_root = DATA_DIR / "users" + data_root = get_config().USERS_DIR escaped_domain = escape(domain) escaped_path = escape(rel_path) pattern = str(data_root / "*" / "snapshots" / "*" / escaped_domain / "*" / "responses" / escaped_domain / escaped_path) @@ -658,7 +658,7 @@ def _latest_responses_root(domain: str) -> Path | None: if not domain: return None domain = domain.split(":", 1)[0].lower() - data_root = DATA_DIR / "users" + data_root = get_config().USERS_DIR escaped_domain = escape(domain) pattern = str(data_root / "*" / "snapshots" / "*" / escaped_domain / "*" / "responses" / escaped_domain) matches = glob(pattern) @@ -675,7 +675,7 @@ def _latest_snapshot_for_domain(domain: str) -> Snapshot | None: return None requested_domain = domain.split(":", 1)[0].lower() - snapshots = SnapshotView.find_snapshots_for_url(f"https://{requested_domain}").order_by("-created_at", "-bookmarked_at", "-timestamp") + snapshots = SnapshotView.find_snapshots_for_url(f"https://{requested_domain}").order_by("-bookmarked_at", "-created_at", "-timestamp") for snapshot in snapshots: if Snapshot.extract_domain_from_url(snapshot.url).lower() == requested_domain: return snapshot @@ -734,7 +734,7 @@ def _serve_responses_path(request, responses_root: Path, rel_path: str, show_ind def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""): rel_path = path or "" is_directory_request = bool(path) and path.endswith("/") - show_indexes = bool(request.GET.get("files")) or (SERVER_CONFIG.USES_SUBDOMAIN_ROUTING and is_directory_request) + show_indexes = bool(request.GET.get("files")) or (get_config().USES_SUBDOMAIN_ROUTING and is_directory_request) if not show_indexes and (not rel_path or rel_path == "index.html"): return SnapshotView.render_live_index(request, snapshot) @@ -804,7 +804,7 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = if snapshot: return SnapshotView.render_live_index(request, snapshot) - if SERVER_CONFIG.PUBLIC_ADD_VIEW or request.user.is_authenticated: + if get_config().PUBLIC_ADD_VIEW or request.user.is_authenticated: target_url = _original_request_url(domain, path, request.META.get("QUERY_STRING", "")) return redirect(build_web_url(f"/web/{quote(target_url, safe=':/')}")) @@ -815,7 +815,7 @@ class SnapshotHostView(View): """Serve snapshot directory contents on ./.""" def get(self, request, snapshot_id: str, path: str = ""): - if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS: + if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS: return _admin_login_redirect_or_forbidden(request) snapshot = _find_snapshot_by_ref(snapshot_id) @@ -836,7 +836,7 @@ class SnapshotReplayView(View): """Serve snapshot directory contents on a one-domain replay path.""" def get(self, request, snapshot_id: str, path: str = ""): - if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS: + if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS: return _admin_login_redirect_or_forbidden(request) snapshot = _find_snapshot_by_ref(snapshot_id) @@ -850,7 +850,7 @@ class OriginalDomainHostView(View): """Serve responses from the most recent snapshot when using ./.""" def get(self, request, domain: str, path: str = ""): - if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS: + if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS: return _admin_login_redirect_or_forbidden(request) return _serve_original_domain_replay(request, domain, path) @@ -859,7 +859,7 @@ class OriginalDomainReplayView(View): """Serve original-domain replay content on a one-domain replay path.""" def get(self, request, domain: str, path: str = ""): - if not request.user.is_authenticated and not SERVER_CONFIG.PUBLIC_SNAPSHOTS: + if not request.user.is_authenticated and not get_config().PUBLIC_SNAPSHOTS: return _admin_login_redirect_or_forbidden(request) return _serve_original_domain_replay(request, domain, path) @@ -867,15 +867,15 @@ class OriginalDomainReplayView(View): class PublicIndexView(ListView): template_name = "public_index.html" model = Snapshot - paginate_by = SERVER_CONFIG.SNAPSHOTS_PER_PAGE + paginate_by = get_config().SNAPSHOTS_PER_PAGE ordering = ["-bookmarked_at", "-created_at"] def get_context_data(self, **kwargs): return { **super().get_context_data(**kwargs), "VERSION": VERSION, - "COMMIT_HASH": SHELL_CONFIG.COMMIT_HASH, - "FOOTER_INFO": SERVER_CONFIG.FOOTER_INFO, + "COMMIT_HASH": get_config().COMMIT_HASH, + "FOOTER_INFO": get_config().FOOTER_INFO, "search_mode": get_search_mode(self.request.GET.get("search_mode")), } @@ -935,7 +935,7 @@ class PublicIndexView(ListView): def get(self, *args, **kwargs): if self.request.user.is_authenticated: return redirect("/admin/core/snapshot/") - if SERVER_CONFIG.PUBLIC_INDEX: + if get_config().PUBLIC_INDEX: response = super().get(*args, **kwargs) return response else: @@ -957,7 +957,7 @@ class AddView(UserPassesTestMixin, FormView): return super().get_initial() def test_func(self): - return SERVER_CONFIG.PUBLIC_ADD_VIEW or self.request.user.is_authenticated + return get_config().PUBLIC_ADD_VIEW or self.request.user.is_authenticated def _can_override_crawl_config(self) -> bool: user = self.request.user @@ -975,7 +975,7 @@ class AddView(UserPassesTestMixin, FormView): return custom_config def get_context_data(self, **kwargs): - required_search_plugin = f"search_backend_{SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}".strip() + required_search_plugin = f"search_backend_{get_config().SEARCH_BACKEND_ENGINE}".strip() plugin_configs = discover_plugin_configs() plugin_dependency_map = { plugin_name: [ @@ -990,7 +990,7 @@ class AddView(UserPassesTestMixin, FormView): # We can't just call request.build_absolute_uri in the template, because it would include query parameters "absolute_add_path": self.request.build_absolute_uri(self.request.path), "VERSION": VERSION, - "FOOTER_INFO": SERVER_CONFIG.FOOTER_INFO, + "FOOTER_INFO": get_config().FOOTER_INFO, "required_search_plugin": required_search_plugin, "plugin_dependency_map_json": json.dumps(plugin_dependency_map, sort_keys=True), "stdout": "", @@ -1112,7 +1112,7 @@ class AddView(UserPassesTestMixin, FormView): class WebAddView(AddView): def _latest_snapshot_for_url(self, requested_url: str): - return SnapshotView.find_snapshots_for_url(requested_url).order_by("-created_at", "-bookmarked_at", "-timestamp").first() + return SnapshotView.find_snapshots_for_url(requested_url).order_by("-bookmarked_at", "-created_at", "-timestamp").first() def _normalize_add_url(self, requested_url: str) -> str: if requested_url.startswith(("http://", "https://")): @@ -1671,8 +1671,7 @@ def find_config_default(key: str) -> str: for config in CONFIGS.values(): if key in dict(config): - default_field = getattr(config, "model_fields", dict(config))[key] - default_val = default_field.default if hasattr(default_field, "default") else default_field + default_val = type(config).model_fields[key].default break if isinstance(default_val, Callable): @@ -1686,41 +1685,12 @@ def find_config_default(key: str) -> str: def find_config_type(key: str) -> str: - from typing import ClassVar - CONFIGS = get_all_configs() for config in CONFIGS.values(): - if hasattr(config, key): - # Try to get from pydantic model_fields first (more reliable) - if hasattr(config, "model_fields") and key in config.model_fields: - field = config.model_fields[key] - if hasattr(field, "annotation") and field.annotation is not None: - try: - return str(field.annotation.__name__) - except AttributeError: - return str(field.annotation) - - # Fallback to get_type_hints with proper namespace - try: - import typing - - namespace = { - "ClassVar": ClassVar, - "Optional": typing.Optional, - "Union": typing.Union, - "List": list, - "Dict": dict, - "Path": Path, - } - type_hints = get_type_hints(config, globalns=namespace, localns=namespace) - try: - return str(type_hints[key].__name__) - except AttributeError: - return str(type_hints[key]) - except Exception: - # If all else fails, return str - pass + if key in type(config).model_fields: + annotation = type(config).model_fields[key].annotation + return getattr(annotation, "__name__", str(annotation)) return "str" @@ -1748,8 +1718,6 @@ def find_config_source(key: str, merged_config: dict) -> str: pass # Check if it's from archivebox.config.file - from archivebox.config.configset import BaseConfigSet - file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE) if key in file_config: return "Config File" @@ -1869,10 +1837,8 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext: @render_with_item_view def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: from archivebox.machine.models import Machine - from archivebox.config.configset import BaseConfigSet CONFIGS = get_all_configs() - FLAT_CONFIG = get_flat_config() assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings." @@ -1909,7 +1875,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont sources_info.append(("Default", default_val, "gray")) # Final computed value - final_value = merged_config.get(key, FLAT_CONFIG.get(key, CONFIGS.get(key, None))) + final_value = merged_config.get(key, CONFIGS.get(key, None)) if not key_is_safe(key): final_value = "********" @@ -1923,7 +1889,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont section_header = mark_safe( f'[CONSTANTS]   {key}   (read-only, hardcoded by ArchiveBox)', ) - elif key in FLAT_CONFIG: + elif key in merged_config: section_header = mark_safe( f'data / ArchiveBox.conf   [{find_config_section(key)}]   {key}', ) @@ -1967,13 +1933,13 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont Configuration Sources (highest priority first):

{sources_html}

-

+

To change this value, edit data/ArchiveBox.conf or run:

archivebox config --set {key}="{ val.strip("'") if (val := find_config_default(key)) - else (str(FLAT_CONFIG[key] if key_is_safe(key) else "********")).strip("'") + else (str(final_value if key_is_safe(key) else "********")).strip("'") }"

'''), diff --git a/archivebox/crawls/migrations/0002_upgrade_from_0_8_6.py b/archivebox/crawls/migrations/0002_upgrade_from_0_8_6.py index 1665d62f..eee69b1a 100644 --- a/archivebox/crawls/migrations/0002_upgrade_from_0_8_6.py +++ b/archivebox/crawls/migrations/0002_upgrade_from_0_8_6.py @@ -19,6 +19,64 @@ def upgrade_crawl_table_from_v086(apps, schema_editor): has_seed_id = "seed_id" in crawl_cols has_urls = "urls" in crawl_cols + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='crawls_crawlschedule'") + if cursor.fetchone(): + cursor.execute("PRAGMA table_info(crawls_crawlschedule)") + schedule_cols = {row[1] for row in cursor.fetchall()} + schedule_sets = [] + if "id" in schedule_cols: + schedule_sets.append("id = REPLACE(id, '-', '')") + if "template_id" in schedule_cols: + schedule_sets.append( + "template_id = CASE " + "WHEN template_id IS NOT NULL " + "AND LENGTH(REPLACE(template_id, '-', '')) = 32 " + "AND REPLACE(template_id, '-', '') NOT GLOB '*[^0-9A-Fa-f]*' " + "THEN REPLACE(template_id, '-', '') ELSE NULL END", + ) + if schedule_sets: + cursor.execute(f"UPDATE crawls_crawlschedule SET {', '.join(schedule_sets)}") + + crawl_sets = [] + if "id" in crawl_cols: + crawl_sets.append("id = REPLACE(id, '-', '')") + if "persona_id" in crawl_cols: + crawl_sets.append( + "persona_id = CASE " + "WHEN persona_id IS NOT NULL " + "AND LENGTH(REPLACE(persona_id, '-', '')) = 32 " + "AND REPLACE(persona_id, '-', '') NOT GLOB '*[^0-9A-Fa-f]*' " + "THEN REPLACE(persona_id, '-', '') ELSE NULL END", + ) + if "schedule_id" in crawl_cols: + crawl_sets.append( + "schedule_id = CASE " + "WHEN schedule_id IS NOT NULL " + "AND LENGTH(REPLACE(schedule_id, '-', '')) = 32 " + "AND REPLACE(schedule_id, '-', '') NOT GLOB '*[^0-9A-Fa-f]*' " + "THEN REPLACE(schedule_id, '-', '') ELSE NULL END", + ) + if crawl_sets: + cursor.execute(f"UPDATE crawls_crawl SET {', '.join(crawl_sets)}") + + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_snapshot'") + if cursor.fetchone(): + cursor.execute("PRAGMA table_info(core_snapshot)") + snapshot_cols = {row[1] for row in cursor.fetchall()} + if "crawl_id" in snapshot_cols: + cursor.execute( + """ + UPDATE core_snapshot + SET crawl_id = CASE + WHEN crawl_id IS NOT NULL + AND LENGTH(REPLACE(crawl_id, '-', '')) = 32 + AND REPLACE(crawl_id, '-', '') NOT GLOB '*[^0-9A-Fa-f]*' + THEN REPLACE(crawl_id, '-', '') + ELSE NULL + END + """, + ) + # Only upgrade if we have v0.8.6rc0 schema if not (has_seed_id and not has_urls): return @@ -66,9 +124,9 @@ def upgrade_crawl_table_from_v086(apps, schema_editor): status, retry_at, created_by_id, schedule_id ) SELECT - id, created_at, modified_at, num_uses_succeeded, num_uses_failed, - '', config, max_depth, tags_str, NULL, '', '', '', - status, retry_at, created_by_id, schedule_id + REPLACE(id, '-', ''), created_at, modified_at, num_uses_succeeded, num_uses_failed, + '', config, max_depth, tags_str, REPLACE(persona_id, '-', ''), '', '', '', + status, retry_at, created_by_id, REPLACE(schedule_id, '-', '') FROM crawls_crawl; """) diff --git a/archivebox/crawls/models.py b/archivebox/crawls/models.py index e4327549..26596dfe 100755 --- a/archivebox/crawls/models.py +++ b/archivebox/crawls/models.py @@ -1,6 +1,7 @@ __package__ = "archivebox.crawls" from typing import TYPE_CHECKING +from io import StringIO import uuid import json import re @@ -272,17 +273,23 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith @property def output_dir(self) -> Path: """ - Construct output directory: users/{username}/crawls/{YYYYMMDD}/{domain}/{crawl-id} + Construct output directory: archive/users/{username}/crawls/{YYYYMMDD}/{domain}/{crawl-id} Domain is extracted from the first URL in the crawl. """ - from archivebox import DATA_DIR + from archivebox.config import CONSTANTS + from archivebox.config.common import get_config from archivebox.core.models import Snapshot date_str = self.created_at.strftime("%Y%m%d") - urls = self.get_urls_list() - domain = Snapshot.extract_domain_from_url(urls[0]) if urls else "unknown" + first_url = "" + for raw_line in StringIO(self.urls or ""): + candidate = raw_line.strip() + if candidate and not candidate.startswith("#"): + first_url = candidate + break + domain = Snapshot.extract_domain_from_url(first_url) if first_url else "unknown" - return DATA_DIR / "users" / self.created_by.username / "crawls" / date_str / domain / str(self.id) + return get_config().USERS_DIR / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id) def get_urls_list(self) -> list[str]: """Get list of URLs from urls field, filtering out comments and empty lines.""" @@ -359,7 +366,7 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith def get_url_allowlist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]: if use_effective_config: - from archivebox.config.configset import get_config + from archivebox.config.common import get_config config = get_config(crawl=self, snapshot=snapshot) else: @@ -368,7 +375,7 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith def get_url_denylist(self, *, use_effective_config: bool = False, snapshot=None) -> list[str]: if use_effective_config: - from archivebox.config.configset import get_config + from archivebox.config.common import get_config config = get_config(crawl=self, snapshot=snapshot) else: @@ -616,6 +623,9 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith from archivebox.core.models import Snapshot from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url + if self.status == self.StatusChoices.SEALED: + return [] + created_snapshots = [] for line in self.urls.splitlines(): @@ -692,6 +702,9 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith from archivebox.core.models import Snapshot from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url + if self.status == self.StatusChoices.SEALED: + return None + url = sanitize_extracted_url(fix_url_from_markdown(str(url or "").strip())) if not url: return None @@ -825,7 +838,7 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith import time from pathlib import Path from archivebox.hooks import run_hook, discover_hooks, process_hook_records, is_finite_background_hook - from archivebox.config.configset import get_config + from archivebox.config.common import get_config from archivebox.machine.models import Binary, Machine # Debug logging to file (since stdout/stderr redirected to /dev/null in progress mode) @@ -1050,7 +1063,7 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith persona.cleanup_runtime_for_crawl(self) # Run on_CrawlEnd hooks - from archivebox.config.configset import get_config + from archivebox.config.common import get_config config = get_config(crawl=self) diff --git a/archivebox/hooks.py b/archivebox/hooks.py index 832bc631..bf9c1eb7 100644 --- a/archivebox/hooks.py +++ b/archivebox/hooks.py @@ -45,12 +45,12 @@ __package__ = "archivebox" import os import json +from collections.abc import Iterable, Mapping from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, TypedDict +from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, TypedDict from abx_plugins import get_plugins_dir -from django.conf import settings from django.utils.safestring import mark_safe from archivebox.config.constants import CONSTANTS from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url @@ -59,10 +59,38 @@ if TYPE_CHECKING: from archivebox.machine.models import Process +class ConfigLookup(Protocol): + def get(self, key: str, default: Any = None) -> Any: ... + + def items(self) -> Iterable[tuple[str, Any]]: ... + + +class PluginSpecialConfig(TypedDict): + enabled: bool + timeout: int + binary: str + + +class ConfigDump(Protocol): + def as_dict(self) -> dict[str, Any]: ... + + +def _has_config_dump(config: object) -> TypeGuard[ConfigDump]: + return callable(getattr(config, "as_dict", None)) + + +def _config_to_overrides(config: ConfigLookup | Mapping[str, Any] | None) -> dict[str, Any]: + if config is None: + return {} + if _has_config_dump(config): + return dict(config.as_dict()) + return dict(config.items()) + + # Plugin directories BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve() USER_PLUGINS_DIR = Path( - os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or getattr(settings, "USER_PLUGINS_DIR", "") or str(CONSTANTS.USER_PLUGINS_DIR), + os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or str(CONSTANTS.USER_PLUGINS_DIR), ).expanduser() @@ -147,10 +175,30 @@ class HookResult(TypedDict, total=False): records: list[dict[str, Any]] # Parsed JSONL records with 'type' field +def _model_output_dir_from_child_path(path: Path, marker: str) -> Path | None: + """ + Infer the model output dir from a model dir or one of its plugin subdirs. + + Current ArchiveBox snapshot/crawl dirs are: + .../{snapshots,crawls}/YYYYMMDD/domain/uuid[/plugin] + """ + parts = path.resolve().parts + try: + marker_index = parts.index(marker) + except ValueError: + return None + + model_end_index = marker_index + 4 + if len(parts) < model_end_index: + return None + return Path(*parts[:model_end_index]) + + def discover_hooks( event_name: str, filter_disabled: bool = True, - config: dict[str, Any] | None = None, + config: ConfigLookup | None = None, + **config_kwargs: Any, ) -> list[Path]: """ Find all hook scripts for an event family. @@ -170,15 +218,15 @@ def discover_hooks( Event names are normalized by stripping a trailing `Event`. If no matching `on_{EventFamily}__*` scripts exist, returns []. filter_disabled: If True, skip hooks from disabled plugins (default: True) - config: Optional config dict from get_config() (merges file, env, machine, crawl, snapshot) - If None, will call get_config() with global scope + config: Optional pre-merged config dict from get_config(). + **config_kwargs: Scope/override args forwarded to get_config() when config is not supplied. Returns: Sorted list of hook script paths from enabled plugins only. Examples: # With proper config context (recommended): - from archivebox.config.configset import get_config + from archivebox.config.common import get_config config = get_config(crawl=my_crawl, snapshot=my_snapshot) discover_hooks('Snapshot', config=config) # Returns: [Path('.../on_Snapshot__10_title.py'), ...] (wget excluded if SAVE_WGET=False) @@ -217,9 +265,9 @@ def discover_hooks( if filter_disabled and hook_event_name != "BinaryRequest": # Get merged config if not provided (lazy import to avoid circular dependency) if config is None: - from archivebox.config.configset import get_config + from archivebox.config.common import get_config - config = get_config() + config = get_config(**config_kwargs) enabled_hooks = [] @@ -250,7 +298,7 @@ def discover_hooks( def run_hook( script: Path, output_dir: Path, - config: dict[str, Any], + config: ConfigLookup | Mapping[str, Any] | None = None, timeout: int | None = None, parent: Optional["Process"] = None, **kwargs: Any, @@ -267,7 +315,8 @@ def run_hook( Args: script: Path to the hook script (.sh, .py, or .js) output_dir: Working directory for the script (where output files go) - config: Merged config dict from get_config(crawl=..., snapshot=...) - REQUIRED + config: Optional pre-merged config dict from get_config(crawl=..., snapshot=...). + If omitted, pass scope/override args using kwargs prefixed with config_. timeout: Maximum execution time in seconds If None, auto-detects from PLUGINNAME_TIMEOUT config (fallback to TIMEOUT, default 300) parent: Optional parent Process (for tracking worker->hook hierarchy) @@ -277,20 +326,24 @@ def run_hook( Process model instance (use process.exit_code, process.stdout, process.get_records()) Example: - from archivebox.config.configset import get_config + from archivebox.config.common import get_config config = get_config(crawl=my_crawl, snapshot=my_snapshot) process = run_hook(hook_path, output_dir, config=config, url=url, snapshot_id=id) if process.status == 'exited': records = process.get_records() # Get parsed JSONL output """ from archivebox.machine.models import Process, Machine, NetworkInterface + from archivebox.config.common import get_config from archivebox.config.constants import CONSTANTS import sys + config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")} + resolved_config = get_config(overrides=_config_to_overrides(config), **config_scope) + # Auto-detect timeout from plugin config if not explicitly provided if timeout is None: plugin_name = script.parent.name - plugin_config = get_plugin_special_config(plugin_name, config) + plugin_config = get_plugin_special_config(plugin_name, resolved_config) timeout = plugin_config["timeout"] if timeout: timeout = min(int(timeout), int(CONSTANTS.MAX_HOOK_RUNTIME_SECONDS)) @@ -360,17 +413,18 @@ def run_hook( # Set up environment with base paths env = os.environ.copy() - env["DATA_DIR"] = str(config.get("DATA_DIR") or getattr(settings, "DATA_DIR", Path.cwd())) - env["ARCHIVE_DIR"] = str(config.get("ARCHIVE_DIR") or getattr(settings, "ARCHIVE_DIR", Path.cwd() / "archive")) + env["DATA_DIR"] = str(resolved_config.DATA_DIR) + env["ARCHIVE_DIR"] = str(resolved_config.ARCHIVE_DIR) env["ABX_RUNTIME"] = "archivebox" - env.setdefault("MACHINE_ID", getattr(settings, "MACHINE_ID", "") or os.environ.get("MACHINE_ID", "")) + env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID)) resolved_output_dir = output_dir.resolve() - output_parts = set(resolved_output_dir.parts) - if "snapshots" in output_parts: - env["SNAP_DIR"] = str(resolved_output_dir.parent) - if "crawls" in output_parts: - env["CRAWL_DIR"] = str(resolved_output_dir.parent) + snap_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME) + crawl_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.CRAWLS_DIR_NAME) + if snap_dir: + env["SNAP_DIR"] = str(snap_dir) + if crawl_dir: + env["CRAWL_DIR"] = str(crawl_dir) crawl_id = kwargs.get("_crawl_id") or kwargs.get("crawl_id") if crawl_id: @@ -384,8 +438,8 @@ def run_hook( pass # Get LIB_DIR and LIB_BIN_DIR from config - lib_dir = config.get("LIB_DIR", getattr(settings, "LIB_DIR", None)) - lib_bin_dir = config.get("LIB_BIN_DIR", getattr(settings, "LIB_BIN_DIR", None)) + lib_dir = resolved_config.LIB_DIR + lib_bin_dir = resolved_config.LIB_BIN_DIR if lib_dir: env["LIB_DIR"] = str(lib_dir) if not lib_bin_dir and lib_dir: @@ -394,11 +448,11 @@ def run_hook( # Set Node.js module resolution paths. # NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory. - node_modules_dir = config.get("NODE_MODULES_DIR") + node_modules_dir = resolved_config.get("NODE_MODULES_DIR") if not node_modules_dir and lib_dir: node_modules_dir = Path(lib_dir) / "npm" / "node_modules" - node_path_parts = [part for part in str(config.get("NODE_PATH") or "").split(os.pathsep) if part] + node_path_parts = [part for part in str(resolved_config.get("NODE_PATH") or "").split(os.pathsep) if part] if node_modules_dir: node_modules_dir = Path(node_modules_dir) node_modules_dir.mkdir(parents=True, exist_ok=True) @@ -425,7 +479,7 @@ def run_hook( "SNAP_DIR", "CRAWL_DIR", } - for key, value in config.items(): + for key, value in resolved_config.items(): if key in SKIP_KEYS: continue # Already handled specially above, don't overwrite if value is None: @@ -632,28 +686,29 @@ def get_plugin_name(plugin: str) -> str: return plugin -def get_enabled_plugins(config: dict[str, Any] | None = None) -> list[str]: +def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any) -> list[str]: """ Get the list of enabled plugins based on config and available hooks. Filters plugins by USE_/SAVE_ flags. Only returns plugins that are enabled. Args: - config: Merged config dict from get_config() - if None, uses global config + config: Optional pre-merged config dict from get_config(). + **config_kwargs: Scope/override args forwarded to get_config() when config is not supplied. Returns: Plugin names sorted alphabetically (numeric prefix controls order). Example: - from archivebox.config.configset import get_config + from archivebox.config.common import get_config config = get_config(crawl=my_crawl, snapshot=my_snapshot) enabled = get_enabled_plugins(config) # ['wget', 'media', 'chrome', ...] """ # Get merged config if not provided if config is None: - from archivebox.config.configset import get_config + from archivebox.config.common import get_config - config = get_config() + config = get_config(**config_kwargs) def normalize_enabled_plugins(value: Any) -> list[str]: if value is None: @@ -675,10 +730,12 @@ def get_enabled_plugins(config: dict[str, Any] | None = None) -> list[str]: return [str(value).strip()] if str(value).strip() else [] # Support explicit ENABLED_PLUGINS override (legacy) - if "ENABLED_PLUGINS" in config: - return normalize_enabled_plugins(config["ENABLED_PLUGINS"]) - if "ENABLED_EXTRACTORS" in config: - return normalize_enabled_plugins(config["ENABLED_EXTRACTORS"]) + enabled_plugins = config.get("ENABLED_PLUGINS") + if enabled_plugins: + return normalize_enabled_plugins(enabled_plugins) + enabled_extractors = config.get("ENABLED_EXTRACTORS") + if enabled_extractors: + return normalize_enabled_plugins(enabled_extractors) # Filter all plugins by enabled status all_plugins = get_plugins() @@ -870,7 +927,7 @@ def get_config_defaults_from_plugins() -> dict[str, Any]: return defaults -def get_plugin_special_config(plugin_name: str, config: dict[str, Any], _visited: set[str] | None = None) -> dict[str, Any]: +def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited: set[str] | None = None) -> PluginSpecialConfig: """ Extract special config keys for a plugin following naming conventions. @@ -897,7 +954,7 @@ def get_plugin_special_config(plugin_name: str, config: dict[str, Any], _visited } Examples: - >>> from archivebox.config.configset import get_config + >>> from archivebox.config.common import get_config >>> config = get_config(crawl=my_crawl, snapshot=my_snapshot) >>> get_plugin_special_config('wget', config) {'enabled': True, 'timeout': 120, 'binary': '/usr/bin/wget'} diff --git a/archivebox/ldap/auth.py b/archivebox/ldap/auth.py index dd1ac462..82f5a522 100644 --- a/archivebox/ldap/auth.py +++ b/archivebox/ldap/auth.py @@ -32,7 +32,7 @@ class ArchiveBoxLDAPBackend(BaseLDAPBackend): This method is called by django-auth-ldap after successful LDAP authentication. """ - from archivebox.config.ldap import LDAP_CONFIG + from archivebox.config.common import get_config base_authenticate = getattr(super(), "authenticate_ldap_user", None) if base_authenticate is None: @@ -40,7 +40,7 @@ class ArchiveBoxLDAPBackend(BaseLDAPBackend): user = base_authenticate(ldap_user, password) - if user and LDAP_CONFIG.LDAP_CREATE_SUPERUSER: + if user and get_config().LDAP_CREATE_SUPERUSER: # Grant superuser privileges to all LDAP-authenticated users if not user.is_superuser: user.is_superuser = True diff --git a/archivebox/machine/migrations/0012_add_machine_config_if_missing.py b/archivebox/machine/migrations/0012_add_machine_config_if_missing.py new file mode 100644 index 00000000..d462b8ce --- /dev/null +++ b/archivebox/machine/migrations/0012_add_machine_config_if_missing.py @@ -0,0 +1,22 @@ +from django.db import migrations + + +def add_machine_config_if_missing(apps, schema_editor): + cursor = schema_editor.connection.cursor() + cursor.execute("PRAGMA table_info(machine_machine)") + columns = {row[1] for row in cursor.fetchall()} + if "config" not in columns: + cursor.execute("ALTER TABLE machine_machine ADD COLUMN config TEXT") + + +class Migration(migrations.Migration): + dependencies = [ + ("machine", "0011_remove_binary_output_dir"), + ] + + operations = [ + migrations.RunPython( + add_machine_config_if_missing, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py index 52ad1d0f..d732e4cf 100755 --- a/archivebox/machine/models.py +++ b/archivebox/machine/models.py @@ -179,10 +179,44 @@ class Machine(ModelWithHealthStats): return cls._sanitize_config(_CURRENT_MACHINE) else: _CURRENT_MACHINE = None - _CURRENT_MACHINE, _ = cls.objects.update_or_create( - guid=get_host_guid(), - defaults={"hostname": socket.gethostname(), **get_os_info(), **get_vm_info(), "stats": get_host_stats()}, - ) + + host_guid = get_host_guid() + try: + _CURRENT_MACHINE = cls.objects.get(guid=host_guid) + except cls.DoesNotExist: + _CURRENT_MACHINE = cls.objects.create( + guid=host_guid, + hostname=socket.gethostname(), + **get_os_info(), + **get_vm_info(), + stats=get_host_stats(), + ) + else: + if timezone.now() >= _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL): + for key, value in { + "hostname": socket.gethostname(), + **get_os_info(), + **get_vm_info(), + "stats": get_host_stats(), + }.items(): + setattr(_CURRENT_MACHINE, key, value) + _CURRENT_MACHINE.save( + update_fields=[ + "hostname", + "hw_in_docker", + "hw_in_vm", + "hw_manufacturer", + "hw_product", + "hw_uuid", + "os_arch", + "os_family", + "os_platform", + "os_release", + "os_kernel", + "stats", + "modified_at", + ], + ) return cls._sanitize_config(_CURRENT_MACHINE) @classmethod @@ -427,9 +461,9 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine): Get output directory for this binary's hook logs. Path: data/machines/{machine_uuid}/binaries/{binary_name}/{binary_uuid} """ - from django.conf import settings + from archivebox.config.common import get_config - return Path(settings.DATA_DIR) / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id) + return get_config().DATA_DIR / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id) def to_json(self) -> dict: """ @@ -582,7 +616,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine): """ import json from archivebox.hooks import discover_hooks, run_hook - from archivebox.config.configset import get_config + from archivebox.config.common import get_config # Get merged config (Binary doesn't have crawl/snapshot context). config = get_config() @@ -658,9 +692,9 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine): self.save() # Symlink binary into LIB_BIN_DIR if configured - from django.conf import settings + from archivebox.config.common import get_config - lib_bin_dir = getattr(settings, "LIB_BIN_DIR", None) + lib_bin_dir = get_config().LIB_BIN_DIR if lib_bin_dir: self.symlink_to_lib_bin(lib_bin_dir) @@ -2303,7 +2337,7 @@ class Process(models.Model): """ import subprocess from pathlib import Path - from django.conf import settings + from archivebox.config.common import get_config chrome_utils = Path(__file__).parent.parent / "plugins" / "chrome" / "chrome_utils.js" if not chrome_utils.exists(): @@ -2311,7 +2345,7 @@ class Process(models.Model): try: result = subprocess.run( - ["node", str(chrome_utils), "killZombieChrome", str(settings.DATA_DIR)], + ["node", str(chrome_utils), "killZombieChrome", str(get_config().DATA_DIR)], capture_output=True, timeout=30, text=True, diff --git a/archivebox/misc/checks.py b/archivebox/misc/checks.py index 46444662..a23119ca 100644 --- a/archivebox/misc/checks.py +++ b/archivebox/misc/checks.py @@ -19,12 +19,15 @@ from rich.panel import Panel # that the check is called after django.setup() has been called -def check_data_folder() -> None: - from archivebox import DATA_DIR, ARCHIVE_DIR +def check_data_folder(config=None, **config_kwargs) -> None: + from archivebox import DATA_DIR from archivebox.config import CONSTANTS + from archivebox.config.common import get_config from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir - archive_dir_exists = os.path.isdir(ARCHIVE_DIR) + config = config or get_config(**config_kwargs) + archive_dir = config.ARCHIVE_DIR + archive_dir_exists = os.path.isdir(archive_dir) if not archive_dir_exists: print("[red][X] No archivebox index found in the current directory.[/red]", file=sys.stderr) print(f" {DATA_DIR}", file=sys.stderr) @@ -39,16 +42,17 @@ def check_data_folder() -> None: # Create data dir subdirs create_and_chown_dir(CONSTANTS.SOURCES_DIR) + create_and_chown_dir(config.USERS_DIR) create_and_chown_dir(CONSTANTS.PERSONAS_DIR / "Default") create_and_chown_dir(CONSTANTS.LOGS_DIR) # create_and_chown_dir(CONSTANTS.CACHE_DIR) # Create /tmp and /lib dirs if they don't exist - get_or_create_working_tmp_dir(autofix=True, quiet=False) - get_or_create_working_lib_dir(autofix=True, quiet=False) + get_or_create_working_tmp_dir(autofix=True, quiet=False, config=config) + get_or_create_working_lib_dir(autofix=True, quiet=False, config=config) # Check data dir permissions, /tmp, and /lib permissions - check_data_dir_permissions() + check_data_dir_permissions(config=config) def check_migrations(): @@ -141,7 +145,7 @@ def check_not_inside_source_dir(): raise SystemExit("[!] Cannot run from source dir, set DATA_DIR or cd to a data folder first") -def check_data_dir_permissions(): +def check_data_dir_permissions(config=None, **config_kwargs): from archivebox import DATA_DIR from archivebox.misc.logging import STDERR from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, DEFAULT_PUID, DEFAULT_PGID, IS_ROOT, USER @@ -183,35 +187,37 @@ def check_data_dir_permissions(): " [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]", ) - from archivebox.config.common import STORAGE_CONFIG + from archivebox.config.common import get_config + + config = config or get_config(**config_kwargs) + try: + tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True, config=config) or config.TMP_DIR + except Exception: + tmp_dir = config.TMP_DIR try: - tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True) or STORAGE_CONFIG.TMP_DIR + lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR except Exception: - tmp_dir = STORAGE_CONFIG.TMP_DIR - - try: - lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True) or STORAGE_CONFIG.LIB_DIR - except Exception: - lib_dir = STORAGE_CONFIG.LIB_DIR + lib_dir = config.LIB_DIR # Check /tmp dir permissions - check_tmp_dir(tmp_dir, throw=False, must_exist=True) + check_tmp_dir(tmp_dir, throw=False, must_exist=True, config=config) # Check /lib dir permissions - check_lib_dir(lib_dir, throw=False, must_exist=True) + check_lib_dir(lib_dir, throw=False, must_exist=True, config=config) - os.umask(0o777 - int(STORAGE_CONFIG.DIR_OUTPUT_PERMISSIONS, base=8)) + os.umask(0o777 - int(config.DIR_OUTPUT_PERMISSIONS, base=8)) -def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True): +def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs): from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir from archivebox.misc.logging import STDERR from archivebox.misc.logging_util import pretty_path from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP - from archivebox.config.common import STORAGE_CONFIG + from archivebox.config.common import get_config - tmp_dir = tmp_dir or STORAGE_CONFIG.TMP_DIR + config = config or get_config(**config_kwargs) + tmp_dir = tmp_dir or config.TMP_DIR socket_file = tmp_dir.absolute().resolve() / "supervisord.sock" if not must_exist and not os.path.isdir(tmp_dir): @@ -264,16 +270,15 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True): return False -def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_exist=True): +def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs): from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP from archivebox.misc.logging import STDERR from archivebox.misc.logging_util import pretty_path from archivebox.config.paths import dir_is_writable, get_or_create_working_lib_dir - from archivebox.config.common import STORAGE_CONFIG + from archivebox.config.common import get_config - lib_dir = lib_dir or STORAGE_CONFIG.LIB_DIR - - # assert lib_dir == STORAGE_CONFIG.LIB_DIR, "lib_dir is not the same as the one in the flat config" + config = config or get_config(**config_kwargs) + lib_dir = lib_dir or config.LIB_DIR if not must_exist and not os.path.isdir(lib_dir): return True diff --git a/archivebox/misc/folders.py b/archivebox/misc/folders.py index 2d9a55a5..38d4b2ed 100644 --- a/archivebox/misc/folders.py +++ b/archivebox/misc/folders.py @@ -13,11 +13,12 @@ import shutil from pathlib import Path from archivebox.config import DATA_DIR, CONSTANTS +from archivebox.config.common import get_config from archivebox.misc.util import enforce_types @enforce_types -def fix_invalid_folder_locations(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str]]: +def fix_invalid_folder_locations(out_dir: Path = DATA_DIR, config=None, **config_kwargs) -> tuple[list[str], list[str]]: """ Legacy cleanup: Move folders to their correct timestamp-named locations based on index.json. @@ -26,9 +27,16 @@ def fix_invalid_folder_locations(out_dir: Path = DATA_DIR) -> tuple[list[str], l """ fixed = [] cant_fix = [] - for entry in os.scandir(out_dir / CONSTANTS.ARCHIVE_DIR_NAME): + config = config or get_config(**config_kwargs) + archive_dir = config.ARCHIVE_DIR if Path(out_dir).resolve() == DATA_DIR.resolve() else out_dir / CONSTANTS.ARCHIVE_DIR_NAME + if not archive_dir.exists(): + return fixed, cant_fix + for entry in os.scandir(archive_dir): + entry_path = Path(entry.path) + if entry_path.name in CONSTANTS.RESERVED_ARCHIVE_DIR_NAMES or entry_path.name.startswith("."): + continue if entry.is_dir(follow_symlinks=True): - index_path = Path(entry.path) / "index.json" + index_path = entry_path / "index.json" if index_path.exists(): try: with open(index_path) as f: @@ -41,7 +49,7 @@ def fix_invalid_folder_locations(out_dir: Path = DATA_DIR) -> tuple[list[str], l continue if not entry.path.endswith(f"/{timestamp}"): - dest = out_dir / CONSTANTS.ARCHIVE_DIR_NAME / timestamp + dest = archive_dir / timestamp if dest.exists(): cant_fix.append(entry.path) else: diff --git a/archivebox/misc/legacy.py b/archivebox/misc/legacy.py index d4a62b05..477c27d6 100644 --- a/archivebox/misc/legacy.py +++ b/archivebox/misc/legacy.py @@ -58,7 +58,7 @@ def parse_json_main_index(out_dir: Path) -> Iterator[SnapshotDict]: return -def parse_json_links_details(out_dir: Path) -> Iterator[SnapshotDict]: +def parse_json_links_details(out_dir: Path, config=None, **config_kwargs) -> Iterator[SnapshotDict]: """ Parse links from individual snapshot index.jsonl/index.json files in archive directories. @@ -66,18 +66,29 @@ def parse_json_links_details(out_dir: Path) -> Iterator[SnapshotDict]: Prefers index.jsonl (new format) over index.json (legacy format). """ from archivebox.config import CONSTANTS + from archivebox.config.common import get_config - archive_dir = out_dir / CONSTANTS.ARCHIVE_DIR_NAME + config = config or get_config(**config_kwargs) + archive_dir = config.ARCHIVE_DIR if Path(out_dir).resolve() == CONSTANTS.DATA_DIR.resolve() else out_dir / CONSTANTS.ARCHIVE_DIR_NAME if not archive_dir.exists(): return for entry in os.scandir(archive_dir): if not entry.is_dir(): continue + entry_path = Path(entry.path) + if entry_path.name in CONSTANTS.RESERVED_ARCHIVE_DIR_NAMES or entry_path.name.startswith("."): + continue + try: + ts_int = int(float(entry_path.name)) + except (TypeError, ValueError, OverflowError): + continue + if not 788918400 <= ts_int <= 2082758400: + continue # Try index.jsonl first (new format) - jsonl_file = Path(entry.path) / CONSTANTS.JSONL_INDEX_FILENAME - json_file = Path(entry.path) / CONSTANTS.JSON_INDEX_FILENAME + jsonl_file = entry_path / CONSTANTS.JSONL_INDEX_FILENAME + json_file = entry_path / CONSTANTS.JSON_INDEX_FILENAME link = None diff --git a/archivebox/misc/logging_util.py b/archivebox/misc/logging_util.py index c5458eeb..fd6ca909 100644 --- a/archivebox/misc/logging_util.py +++ b/archivebox/misc/logging_util.py @@ -24,7 +24,7 @@ from rich import print from rich.panel import Panel from archivebox.config import CONSTANTS, DATA_DIR, VERSION -from archivebox.config.common import SHELL_CONFIG +from archivebox.config.common import get_config from archivebox.misc.system import get_dir_size from archivebox.misc.util import enforce_types from archivebox.misc.logging import ANSI @@ -55,10 +55,12 @@ _LAST_RUN_STATS = RuntimeStats() class TimedProgress: """Show a progress bar and measure elapsed time until .end() is called""" - def __init__(self, seconds, prefix=""): + def __init__(self, seconds, prefix="", config=None, **config_kwargs): - self.SHOW_PROGRESS = SHELL_CONFIG.SHOW_PROGRESS - self.ANSI = SHELL_CONFIG.ANSI + config = config or get_config(**config_kwargs) + self.SHOW_PROGRESS = config.SHOW_PROGRESS + self.ANSI = config.ANSI + self.TERM_WIDTH = config.TERM_WIDTH if self.SHOW_PROGRESS: self.p = Process(target=progress_bar, args=(seconds, prefix, self.ANSI)) @@ -94,7 +96,7 @@ class TimedProgress: # clear whole terminal line try: - sys.stdout.write("\r{}{}\r".format((" " * SHELL_CONFIG.TERM_WIDTH), self.ANSI["reset"])) + sys.stdout.write("\r{}{}\r".format((" " * self.TERM_WIDTH), self.ANSI["reset"])) except (OSError, BrokenPipeError): # ignore when the parent proc has stopped listening to our stdout pass @@ -103,15 +105,16 @@ class TimedProgress: @enforce_types -def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI) -> None: +def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI, config=None, **config_kwargs) -> None: """show timer in the form of progress bar, with percentage and seconds remaining""" output_buf = sys.stdout or sys.__stdout__ or sys.stderr or sys.__stderr__ chunk = "█" if output_buf and output_buf.encoding.upper() == "UTF-8" else "#" - last_width = SHELL_CONFIG.TERM_WIDTH + config = config or get_config(**config_kwargs) + last_width = config.TERM_WIDTH chunks = last_width - len(prefix) - 20 # number of progress chunks to show (aka max bar width) try: for s in range(seconds * chunks): - max_width = SHELL_CONFIG.TERM_WIDTH + max_width = config.TERM_WIDTH if max_width < last_width: # when the terminal size is shrunk, we have to write a newline # otherwise the progress bar will keep wrapping incorrectly @@ -153,7 +156,7 @@ def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI) -> sys.stdout.flush() # uncomment to have it disappear when it hits 100% instead of staying full red: # time.sleep(0.5) - # sys.stdout.write('\r{}{}\r'.format((' ' * SHELL_CONFIG.TERM_WIDTH), ANSI['reset'])) + # sys.stdout.write('\r{}{}\r'.format((' ' * get_config().TERM_WIDTH), ANSI['reset'])) # sys.stdout.flush() except (KeyboardInterrupt, BrokenPipeError): print() @@ -226,13 +229,22 @@ def log_indexing_process_finished(): _LAST_RUN_STATS.index_end_ts = end_ts -def log_indexing_started(out_path: str): - if SHELL_CONFIG.IS_TTY: - sys.stdout.write(f" > ./{Path(out_path).relative_to(DATA_DIR)}") +def _display_data_path(out_path: str) -> str: + path = Path(out_path).resolve() + try: + return f"./{path.relative_to(DATA_DIR)}" + except ValueError: + return str(path) + + +def log_indexing_started(out_path: str, config=None, **config_kwargs): + config = config or get_config(**config_kwargs) + if config.IS_TTY: + sys.stdout.write(f" > {_display_data_path(out_path)}") def log_indexing_finished(out_path: str): - print(f"\r √ ./{Path(out_path).relative_to(DATA_DIR)}") + print(f"\r √ {_display_data_path(out_path)}") ### Archiving Stage diff --git a/archivebox/misc/serve_static.py b/archivebox/misc/serve_static.py index 84da7764..6fa7ce2a 100644 --- a/archivebox/misc/serve_static.py +++ b/archivebox/misc/serve_static.py @@ -23,7 +23,7 @@ from django.http import StreamingHttpResponse, Http404, HttpResponse, HttpRespon from django.utils._os import safe_join from django.utils.http import http_date from django.utils.translation import gettext as _ -from archivebox.config.common import SERVER_CONFIG +from archivebox.config.common import get_config from archivebox.misc.logging_util import printable_filesize @@ -60,8 +60,9 @@ def _hash_for_path(document_root: Path, rel_path: str) -> str | None: return file_map.get(rel_path) -def _cache_policy() -> str: - return "public" if SERVER_CONFIG.PUBLIC_SNAPSHOTS else "private" +def _cache_policy(config=None, **config_kwargs) -> str: + config = config or get_config(**config_kwargs) + return "public" if config.PUBLIC_SNAPSHOTS else "private" def _format_direntry_timestamp(stat_result: os.stat_result) -> str: @@ -604,14 +605,23 @@ def _is_risky_replay_document(fullpath: Path, content_type: str) -> bool: return any(marker in head for marker in RISKY_REPLAY_MARKERS) -def _apply_archive_replay_headers(response: HttpResponse, *, fullpath: Path, content_type: str, is_archive_replay: bool) -> HttpResponse: +def _apply_archive_replay_headers( + response: HttpResponse, + *, + fullpath: Path, + content_type: str, + is_archive_replay: bool, + config=None, + **config_kwargs, +) -> HttpResponse: if not is_archive_replay: return response response.headers.setdefault("X-Content-Type-Options", "nosniff") - response.headers.setdefault("X-ArchiveBox-Security-Mode", SERVER_CONFIG.SERVER_SECURITY_MODE) + config = config or get_config(**config_kwargs) + response.headers.setdefault("X-ArchiveBox-Security-Mode", config.SERVER_SECURITY_MODE) - if SERVER_CONFIG.SHOULD_NEUTER_RISKY_REPLAY and _is_risky_replay_document(fullpath, content_type): + if config.SHOULD_NEUTER_RISKY_REPLAY and _is_risky_replay_document(fullpath, content_type): response.headers["Content-Security-Policy"] = ( "sandbox; " "default-src 'self' data: blob:; " diff --git a/archivebox/misc/shell_welcome_message.py b/archivebox/misc/shell_welcome_message.py index d33f9307..da9d5b6c 100644 --- a/archivebox/misc/shell_welcome_message.py +++ b/archivebox/misc/shell_welcome_message.py @@ -16,15 +16,15 @@ import archivebox from benedict import benedict # noqa from django.utils import timezone # noqa from datetime import datetime, timedelta # noqa -from django.conf import settings # noqa from archivebox import CONSTANTS # noqa from archivebox.cli import * # noqa -from archivebox.config.configset import get_config +from archivebox.config.common import get_config -CONFIG = get_config() if __name__ == "__main__": + CONFIG = get_config() + # load the rich extension for ipython for pretty printing # https://rich.readthedocs.io/en/stable/introduction.html#ipython-extension get_ipython().run_line_magic("load_ext", "rich") # type: ignore # noqa diff --git a/archivebox/misc/system.py b/archivebox/misc/system.py index 4d3e7b12..c1fdb071 100644 --- a/archivebox/misc/system.py +++ b/archivebox/misc/system.py @@ -12,7 +12,7 @@ from subprocess import PIPE, Popen, CalledProcessError, CompletedProcess, Timeou from atomicwrites import atomic_write as lib_atomic_write -from archivebox.config.common import STORAGE_CONFIG +from archivebox.config.common import get_config from archivebox.misc.util import enforce_types, ExtendedEncoder IS_WINDOWS = os.name == "nt" @@ -87,7 +87,7 @@ def run(cmd, *args, input=None, capture_output=True, timeout=None, check=False, @enforce_types -def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool = True) -> None: +def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool = True, config=None, **config_kwargs) -> None: """Safe atomic write to filesystem by writing to temp file + atomic rename""" mode = "wb+" if isinstance(contents, bytes) else "w" @@ -101,7 +101,8 @@ def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool elif isinstance(contents, (bytes, str)): f.write(contents) except OSError as e: - if STORAGE_CONFIG.ENFORCE_ATOMIC_WRITES: + config = config or get_config(**config_kwargs) + if config.ENFORCE_ATOMIC_WRITES: print(f"[X] OSError: Failed to write {path} with fcntl.F_FULLFSYNC. ({e})") print( " You can store the archive/ subfolder on a hard drive or network share that doesn't support support synchronous writes,", @@ -119,11 +120,12 @@ def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool f.write(contents) # set file permissions - os.chmod(path, int(STORAGE_CONFIG.OUTPUT_PERMISSIONS, base=8)) + config = config or get_config(**config_kwargs) + os.chmod(path, int(config.OUTPUT_PERMISSIONS, base=8)) @enforce_types -def chmod_file(path: str, cwd: str = "") -> None: +def chmod_file(path: str, cwd: str = "", config=None, **config_kwargs) -> None: """chmod -R /""" root = Path(cwd or os.getcwd()) / path @@ -132,14 +134,16 @@ def chmod_file(path: str, cwd: str = "") -> None: if not root.is_dir(): # path is just a plain file - os.chmod(root, int(STORAGE_CONFIG.OUTPUT_PERMISSIONS, base=8)) + config = config or get_config(**config_kwargs) + os.chmod(root, int(config.OUTPUT_PERMISSIONS, base=8)) else: + config = config or get_config(**config_kwargs) for subpath in Path(path).glob("**/*"): if subpath.is_dir(): # directories need execute permissions to be able to list contents - os.chmod(subpath, int(STORAGE_CONFIG.DIR_OUTPUT_PERMISSIONS, base=8)) + os.chmod(subpath, int(config.DIR_OUTPUT_PERMISSIONS, base=8)) else: - os.chmod(subpath, int(STORAGE_CONFIG.OUTPUT_PERMISSIONS, base=8)) + os.chmod(subpath, int(config.OUTPUT_PERMISSIONS, base=8)) @enforce_types diff --git a/archivebox/misc/util.py b/archivebox/misc/util.py index aebb172b..103abd03 100644 --- a/archivebox/misc/util.py +++ b/archivebox/misc/util.py @@ -404,16 +404,17 @@ def parse_date(date: Any) -> datetime | None: @enforce_types -def download_url(url: str, timeout: int | None = None) -> str: +def download_url(url: str, timeout: int | None = None, config=None, **config_kwargs) -> str: """Download the contents of a remote url and return the text""" - from archivebox.config.common import ARCHIVING_CONFIG + from archivebox.config.common import get_config - timeout = timeout or ARCHIVING_CONFIG.TIMEOUT + config = config or get_config(**config_kwargs) + timeout = timeout or config.TIMEOUT session = requests.Session() - if ARCHIVING_CONFIG.COOKIES_FILE and Path(ARCHIVING_CONFIG.COOKIES_FILE).is_file(): - cookie_jar = http.cookiejar.MozillaCookieJar(ARCHIVING_CONFIG.COOKIES_FILE) + if config.COOKIES_FILE and Path(config.COOKIES_FILE).is_file(): + cookie_jar = http.cookiejar.MozillaCookieJar(config.COOKIES_FILE) cookie_jar.load(ignore_discard=True, ignore_expires=True) for cookie in cookie_jar: if cookie.value is not None: @@ -421,8 +422,8 @@ def download_url(url: str, timeout: int | None = None) -> str: response = session.get( url, - headers={"User-Agent": ARCHIVING_CONFIG.USER_AGENT}, - verify=ARCHIVING_CONFIG.CHECK_SSL_VALIDITY, + headers={"User-Agent": config.USER_AGENT}, + verify=config.CHECK_SSL_VALIDITY, timeout=timeout, ) @@ -440,19 +441,20 @@ def download_url(url: str, timeout: int | None = None) -> str: @enforce_types -def get_headers(url: str, timeout: int | None = None) -> str: +def get_headers(url: str, timeout: int | None = None, config=None, **config_kwargs) -> str: """Download the contents of a remote url and return the headers""" # TODO: get rid of this and use an abx pluggy hook instead - from archivebox.config.common import ARCHIVING_CONFIG + from archivebox.config.common import get_config - timeout = timeout or ARCHIVING_CONFIG.TIMEOUT + config = config or get_config(**config_kwargs) + timeout = timeout or config.TIMEOUT try: response = requests.head( url, - headers={"User-Agent": ARCHIVING_CONFIG.USER_AGENT}, - verify=ARCHIVING_CONFIG.CHECK_SSL_VALIDITY, + headers={"User-Agent": config.USER_AGENT}, + verify=config.CHECK_SSL_VALIDITY, timeout=timeout, allow_redirects=True, ) @@ -463,8 +465,8 @@ def get_headers(url: str, timeout: int | None = None) -> str: except RequestException: response = requests.get( url, - headers={"User-Agent": ARCHIVING_CONFIG.USER_AGENT}, - verify=ARCHIVING_CONFIG.CHECK_SSL_VALIDITY, + headers={"User-Agent": config.USER_AGENT}, + verify=config.CHECK_SSL_VALIDITY, timeout=timeout, stream=True, ) @@ -692,7 +694,7 @@ for url_str, num_urls in _test_url_strs.items(): ### Chrome Helpers -def chrome_cleanup(): +def chrome_cleanup(config=None, **config_kwargs): """ Cleans up any state or runtime files that Chrome leaves behind when killed by a timeout or other error. Handles: @@ -713,9 +715,9 @@ def chrome_cleanup(): # Also clean up the active persona's explicit CHROME_USER_DATA_DIR if set # (in case it's a custom path not under PERSONAS_DIR) - from archivebox.config.configset import get_config + from archivebox.config.common import get_config - config = get_config() + config = config or get_config(**config_kwargs) chrome_user_data_dir = config.get("CHROME_USER_DATA_DIR") if chrome_user_data_dir: singleton_lock = Path(chrome_user_data_dir) / "SingletonLock" diff --git a/archivebox/personas/importers.py b/archivebox/personas/importers.py index d82d20ec..bd09796e 100644 --- a/archivebox/personas/importers.py +++ b/archivebox/personas/importers.py @@ -603,13 +603,13 @@ def export_browser_state( return False, None, "Missing browser source." from abx_plugins import get_plugins_dir - from archivebox.config.common import STORAGE_CONFIG + from archivebox.config.common import get_config state_script = Path(__file__).with_name("export_browser_state.js") if not state_script.exists(): return False, None, f"Browser state export script not found at {state_script}" - node_modules_dir = STORAGE_CONFIG.LIB_DIR / "npm" / "node_modules" + node_modules_dir = get_config().LIB_DIR / "npm" / "node_modules" chrome_plugin_dir = Path(get_plugins_dir()).resolve() env = os.environ.copy() diff --git a/archivebox/search/__init__.py b/archivebox/search/__init__.py index 525c4dbe..a30f0b16 100644 --- a/archivebox/search/__init__.py +++ b/archivebox/search/__init__.py @@ -14,13 +14,15 @@ Search backends must provide a search.py module with: __package__ = "archivebox.search" +import os +from contextlib import contextmanager from typing import Any from django.db.models import Case, IntegerField, QuerySet, Value, When from archivebox.misc.util import enforce_types from archivebox.misc.logging import stderr -from archivebox.config.common import SEARCH_BACKEND_CONFIG +from archivebox.config.common import get_config # Cache discovered backends to avoid repeated filesystem scans @@ -28,13 +30,34 @@ _search_backends_cache: dict | None = None SEARCH_MODES = ("meta", "contents", "deep") -def get_default_search_mode() -> str: - return "meta" if SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE == "ripgrep" else "contents" +@contextmanager +def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any): + """Expose ArchiveBox collection roots to in-process search backends.""" + config = config or get_config(**config_kwargs) + updates = { + "DATA_DIR": str(config.DATA_DIR), + "SNAP_DIR": str(config.USERS_DIR), + } + previous = {key: os.environ.get(key) for key in updates} + os.environ.update(updates) + try: + yield + finally: + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value -def get_search_mode(search_mode: str | None) -> str: +def get_default_search_mode(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + return "meta" if config.SEARCH_BACKEND_ENGINE == "ripgrep" else "contents" + + +def get_search_mode(search_mode: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: normalized = (search_mode or "").strip().lower() - return normalized if normalized in SEARCH_MODES else get_default_search_mode() + return normalized if normalized in SEARCH_MODES else get_default_search_mode(config=config, **config_kwargs) def prioritize_metadata_matches( @@ -90,7 +113,7 @@ def get_available_backends() -> dict: return _search_backends_cache -def get_backend() -> Any: +def get_backend(config: dict[str, Any] | None = None, **config_kwargs: Any) -> Any: """ Get the configured search backend module. @@ -99,7 +122,8 @@ def get_backend() -> Any: Falls back to 'ripgrep' if configured backend is not found. """ - backend_name = SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE + config = config or get_config(**config_kwargs) + backend_name = config.SEARCH_BACKEND_ENGINE backends = get_available_backends() if backend_name in backends: @@ -117,7 +141,7 @@ def get_backend() -> Any: @enforce_types -def query_search_index(query: str, search_mode: str | None = None) -> QuerySet: +def query_search_index(query: str, search_mode: str | None = None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> QuerySet: """ Search for snapshots matching the query. @@ -125,16 +149,17 @@ def query_search_index(query: str, search_mode: str | None = None) -> QuerySet: """ from archivebox.core.models import Snapshot - if not SEARCH_BACKEND_CONFIG.USE_SEARCHING_BACKEND: + config = config or get_config(**config_kwargs) + if not config.USE_SEARCHING_BACKEND: return Snapshot.objects.none() - search_mode = "contents" if search_mode is None else get_search_mode(search_mode) + search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config) if search_mode == "meta": return Snapshot.objects.none() backends = get_available_backends() backend_names: list[str] = [] - configured_backend = SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE + configured_backend = config.SEARCH_BACKEND_ENGINE if search_mode == "deep": if "ripgrep" in backends: backend_names.append("ripgrep") @@ -154,10 +179,11 @@ def query_search_index(query: str, search_mode: str | None = None) -> QuerySet: for backend_name in backend_names: backend = backends[backend_name] try: - if backend_name == "ripgrep": - snapshot_pks.extend(backend.search(query, search_mode=search_mode)) - else: - snapshot_pks.extend(backend.search(query)) + with search_backend_env(config=config): + if backend_name == "ripgrep": + snapshot_pks.extend(backend.search(query, search_mode=search_mode)) + else: + snapshot_pks.extend(backend.search(query)) successful_backends += 1 except Exception as err: errors.append(err) @@ -177,18 +203,20 @@ def query_search_index(query: str, search_mode: str | None = None) -> QuerySet: @enforce_types -def flush_search_index(snapshots: QuerySet) -> None: +def flush_search_index(snapshots: QuerySet, config: dict[str, Any] | None = None, **config_kwargs: Any) -> None: """ Remove snapshots from the search index. """ - if not SEARCH_BACKEND_CONFIG.USE_INDEXING_BACKEND or not snapshots: + config = config or get_config(**config_kwargs) + if not config.USE_INDEXING_BACKEND or not snapshots: return - backend = get_backend() + backend = get_backend(config=config) snapshot_pks = [str(pk) for pk in snapshots.values_list("pk", flat=True)] try: - backend.flush(snapshot_pks) + with search_backend_env(config=config): + backend.flush(snapshot_pks) except Exception as err: stderr() stderr( diff --git a/archivebox/services/archive_result_service.py b/archivebox/services/archive_result_service.py index 4bc17896..b383bec9 100644 --- a/archivebox/services/archive_result_service.py +++ b/archivebox/services/archive_result_service.py @@ -242,29 +242,30 @@ class ArchiveResultService(BaseService): process_query = process_query.filter(pid=process_started.pid) process = await process_query.order_by("-modified_at").afirst() - result, _created = await ArchiveResult.objects.aget_or_create( + start_ts = parse_event_datetime(event.start_ts) + end_ts = parse_event_datetime(event.end_ts) or timezone.now() + defaults = { + "status": _normalize_status(event.status), + "output_str": event.output_str, + "output_json": event.output_json, + "output_files": output_files, + "output_size": output_size, + "output_mimetypes": output_mimetypes, + "start_ts": start_ts or timezone.now(), + "end_ts": end_ts, + } + if process is not None: + defaults["process"] = process + if event.error: + defaults["notes"] = event.error + + result, _created = await ArchiveResult.objects.aupdate_or_create( snapshot=snapshot, plugin=event.plugin, hook_name=event.hook_name, - defaults={ - "status": ArchiveResult.StatusChoices.STARTED, - "process": process, - }, + defaults=defaults, ) - result.process = process or result.process - result.status = _normalize_status(event.status) - result.output_str = event.output_str - result.output_json = event.output_json - result.output_files = output_files - result.output_size = output_size - result.output_mimetypes = output_mimetypes - result.start_ts = parse_event_datetime(event.start_ts) or result.start_ts or timezone.now() - result.end_ts = parse_event_datetime(event.end_ts) or timezone.now() - if event.error: - result.notes = event.error - await result.asave() - if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS): next_title = _extract_snapshot_title(str(snapshot.output_dir), event.plugin, result.output_str, snapshot_url=snapshot.url) if next_title and _should_update_snapshot_title(snapshot.title or "", next_title, snapshot_url=snapshot.url): diff --git a/archivebox/services/crawl_service.py b/archivebox/services/crawl_service.py index 5e1b1c3b..d168f287 100644 --- a/archivebox/services/crawl_service.py +++ b/archivebox/services/crawl_service.py @@ -1,6 +1,8 @@ from __future__ import annotations from asgiref.sync import sync_to_async +from django.utils import timezone + from abx_dl.events import CrawlCleanupEvent, CrawlCompletedEvent, CrawlSetupEvent, CrawlStartEvent from abx_dl.services.base import BaseService @@ -42,9 +44,12 @@ class CrawlService(BaseService): is_finished = await sync_to_async(crawl.is_finished, thread_sensitive=True)() if is_finished: crawl.status = Crawl.StatusChoices.SEALED + crawl.retry_at = None elif crawl.status != Crawl.StatusChoices.SEALED: crawl.status = Crawl.StatusChoices.STARTED - crawl.retry_at = None + crawl.retry_at = timezone.now() + else: + crawl.retry_at = None await crawl.asave(update_fields=["status", "retry_at", "modified_at"]) async def on_CrawlCompletedEvent__save_to_db(self, event: CrawlCompletedEvent) -> None: @@ -55,7 +60,9 @@ class CrawlService(BaseService): if not is_finished: if crawl.status != Crawl.StatusChoices.SEALED: crawl.status = Crawl.StatusChoices.STARTED - crawl.retry_at = None + crawl.retry_at = timezone.now() + else: + crawl.retry_at = None await crawl.asave(update_fields=["status", "retry_at", "modified_at"]) return diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 8ff0efad..087d2b1b 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -8,6 +8,7 @@ import shutil import subprocess import sys import time +from collections.abc import Mapping from contextlib import nullcontext from pathlib import Path from tempfile import TemporaryDirectory @@ -19,8 +20,11 @@ from rich.console import Console from abx_dl.events import ( BinaryRequestEvent, + CrawlAbortEvent, CrawlCleanupEvent, + CrawlCompletedEvent, CrawlEvent, + CrawlSetupEvent, CrawlStartEvent, InstallEvent, MachineEvent, @@ -44,7 +48,11 @@ from abx_dl.orchestrator import ( from abx_dl.services.process_service import ProcessService as HookProcessService from abx_dl.services.binary_service import BinaryService as HookBinaryService from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService -from abxbus.event_bus import EventBus +from abxbus import BaseEvent +from abxbus.event_bus import EventBus, get_current_event, in_handler_context +from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError + +from archivebox.config.configset import BaseConfigSet from .archive_result_service import ArchiveResultService from .binary_service import BinaryService @@ -66,7 +74,15 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name) -def _normalize_runtime_config(config: dict[str, Any]) -> dict[str, Any]: +def _normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | None) -> dict[str, Any]: + if config is None: + return {} + if isinstance(config, BaseConfigSet): + config = config.model_dump(mode="json") + elif isinstance(config, str): + config = json.loads(config) + else: + config = dict(config) return {key: value for key, value in json.loads(json.dumps(config, default=str)).items() if value is not None} @@ -78,22 +94,28 @@ def _runner_task_context() -> contextvars.Context: return context +def _is_external_task_cancelled(error: asyncio.CancelledError) -> bool: + return not isinstance(error, (EventHandlerAbortedError, EventHandlerCancelledError)) + + async def _emit_machine_config( bus, *, config: dict[str, Any], derived_config: dict[str, Any], + parent_event=None, ) -> None: user_config = _normalize_runtime_config(config) derived_machine_config = _normalize_runtime_config(derived_config) - await bus.emit( + emitter = parent_event.emit if parent_event is not None else bus.emit + await emitter( MachineEvent( config=user_config, config_type="user", ), ).now() if derived_machine_config: - await bus.emit( + await emitter( MachineEvent( config=derived_machine_config, config_type="derived", @@ -185,6 +207,22 @@ class CrawlRunner: self.crawl_output_dir = "" self._live_stream = None self.root_crawl_event_id: str | None = None + self.root_crawl_start_event_id: str | None = None + self._skip_wait_until_idle = False + + async def crawl_is_cancelled(self) -> bool: + from archivebox.crawls.models import Crawl + + return await Crawl.objects.filter(id=self.crawl.id, status=Crawl.StatusChoices.SEALED).aexists() + + async def watch_for_cancelled_crawl(self, parent_event: BaseEvent, *, poll_interval: float = 1.0) -> None: + while True: + await asyncio.sleep(poll_interval) + if not await self.crawl_is_cancelled(): + continue + abort_event = parent_event.emit(CrawlAbortEvent()) + await _run_event_now(abort_event, abort_event.event_timeout) + return def runtime_plugins(self) -> dict[str, Plugin]: return filter_plugins(self.plugins, self.selected_plugins, include_providers=True) if self.selected_plugins else self.plugins @@ -195,6 +233,7 @@ class CrawlRunner: runtime="archivebox", crawl_id=str(self.crawl.id), ) + root_snapshot_id: str | None = None try: snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)() live_ui = self._create_live_ui() @@ -210,14 +249,11 @@ class CrawlRunner: ) if snapshot_ids: root_snapshot_id = snapshot_ids[0] - await self.run_crawl_setup(root_snapshot_id) - for snapshot_id in snapshot_ids: - await self.enqueue_snapshot(snapshot_id) - await self.wait_for_snapshot_tasks() - await self.run_crawl_cleanup(root_snapshot_id) + await self.run_crawl(root_snapshot_id, snapshot_ids) finally: await heartbeat.stop() - await self.bus.wait_until_idle() + if not self._skip_wait_until_idle: + await self.bus.wait_until_idle() if self._live_stream is not None: try: self._live_stream.close() @@ -230,27 +266,82 @@ class CrawlRunner: task = self.snapshot_tasks.get(snapshot_id) if task is not None and not task.done(): return - task = asyncio.create_task(self.run_snapshot(snapshot_id), context=_runner_task_context()) + current_event = get_current_event() + if isinstance(current_event, CrawlStartEvent): + task = asyncio.create_task(self.run_snapshot(snapshot_id)) + elif in_handler_context(): + return + else: + task = asyncio.create_task(self.run_snapshot(snapshot_id), context=_runner_task_context()) self.snapshot_tasks[snapshot_id] = task async def wait_for_snapshot_tasks(self) -> None: + task_errors: list[Exception] = [] while True: pending_tasks: list[asyncio.Task[None]] = [] for snapshot_id, task in list(self.snapshot_tasks.items()): if task.done(): if self.snapshot_tasks.get(snapshot_id) is task: self.snapshot_tasks.pop(snapshot_id, None) - task.result() + try: + task.result() + except asyncio.CancelledError as err: + if _is_external_task_cancelled(err): + raise + await sync_to_async(recover_orphaned_snapshots, thread_sensitive=True)() + await sync_to_async(recover_orphaned_crawls, thread_sensitive=True)() + except Exception as err: + task_errors.append(err) continue pending_tasks.append(task) if not pending_tasks: - return + await self.enqueue_pending_snapshots_from_projection() + if not self.snapshot_tasks: + if task_errors: + if len(task_errors) == 1: + raise task_errors[0] + raise ExceptionGroup("One or more snapshot tasks failed", task_errors) + return + continue done, _pending = await asyncio.wait(pending_tasks, return_when=asyncio.FIRST_COMPLETED) for task in done: - task.result() + for snapshot_id, tracked_task in list(self.snapshot_tasks.items()): + if tracked_task is task: + self.snapshot_tasks.pop(snapshot_id, None) + break + try: + task.result() + except asyncio.CancelledError as err: + if _is_external_task_cancelled(err): + raise + await sync_to_async(recover_orphaned_snapshots, thread_sensitive=True)() + await sync_to_async(recover_orphaned_crawls, thread_sensitive=True)() + except Exception as err: + task_errors.append(err) + await self.enqueue_pending_snapshots_from_projection() + + async def enqueue_pending_snapshots_from_projection(self) -> None: + from archivebox.core.models import Snapshot + + if not isinstance(get_current_event(), CrawlStartEvent): + return + + pending_snapshot_ids = await sync_to_async( + lambda: [ + str(snapshot_id) + for snapshot_id in self.crawl.snapshot_set.exclude(status=Snapshot.StatusChoices.SEALED) + .filter(retry_at__lte=timezone.now()) + .order_by("depth", "created_at") + .values_list("id", flat=True) + ], + thread_sensitive=True, + )() + for snapshot_id in pending_snapshot_ids: + if snapshot_id not in self.snapshot_tasks: + await self.enqueue_snapshot(snapshot_id) def load_run_state(self) -> list[str]: - from archivebox.config.configset import get_config + from archivebox.config.common import get_config from archivebox.hooks import discover_hooks from archivebox.machine.models import Machine, NetworkInterface, Process, _sanitize_machine_config @@ -361,7 +452,7 @@ class CrawlRunner: def load_snapshot_payload(self, snapshot_id: str) -> dict[str, Any]: from archivebox.core.models import Snapshot - from archivebox.config.configset import get_config + from archivebox.config.common import get_config snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id) config = get_config(crawl=self.crawl, snapshot=snapshot) @@ -388,7 +479,7 @@ class CrawlRunner: "depth": snapshot.depth, "status": snapshot.status, "output_dir": str(snapshot.output_dir), - "config": config, + "config": _normalize_runtime_config(config), "_snapshot": snapshot, } @@ -431,11 +522,11 @@ class CrawlRunner: if has_capacity: continue break - if self.process_discovered_snapshots_inline: + if self.process_discovered_snapshots_inline and isinstance(get_current_event(), CrawlStartEvent): await self.enqueue_snapshot(str(child_snapshot.id)) - async def run_crawl_setup(self, snapshot_id: str) -> None: - snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(snapshot_id) + async def run_crawl(self, root_snapshot_id: str, snapshot_ids: list[str]) -> None: + snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(root_snapshot_id) config = _normalize_runtime_config(snapshot["config"]) derived_config = _normalize_runtime_config(self.derived_config) output_dir = Path(self.crawl_output_dir) @@ -449,32 +540,19 @@ class CrawlRunner: setup_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("CrawlSetup")] crawl_setup_phase_timeout = compute_phase_timeout(setup_hooks, config) install_phase_timeout = compute_install_phase_timeout(get_install_plugins(plugins), config) - await _emit_machine_config(self.bus, config=config, derived_config=derived_config) - setup_abx_services( - self.bus, - plugins=plugins, - url=snapshot["url"], - snapshot=abx_snapshot, - output_dir=output_dir, - install_enabled=False, - crawl_setup_enabled=True, - crawl_start_enabled=False, - snapshot_cleanup_enabled=False, - crawl_cleanup_enabled=False, - crawl_setup_phase_timeout=crawl_setup_phase_timeout, - snapshot_phase_timeout=0.0, - snapshot_cleanup_phase_timeout=0.0, - crawl_cleanup_phase_timeout=crawl_setup_phase_timeout, - persist_derived=False, - auto_install=True, - emit_jsonl=False, - MachineService=None, - BinaryService=HookBinaryService, - ProcessService=None, - ArchiveResultService=None, - TagService=None, - SnapshotService=None, + snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")] + max_snapshot_count = max(1, int(self.crawl.max_urls or len(snapshot_ids) or 1)) + snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config) * max_snapshot_count + crawl_cleanup_phase_timeout = crawl_setup_phase_timeout + crawl_lifecycle_timeout = ( + crawl_setup_phase_timeout + + snapshot_phase_timeout + + crawl_cleanup_phase_timeout + + CrawlCompletedEvent.model_fields["event_timeout"].default + + 30.0 ) + await _emit_machine_config(self.bus, config=config, derived_config=derived_config) + install_cancel_watcher: asyncio.Task[None] | None = None install_event = self.bus.emit( InstallEvent( url=snapshot["url"], @@ -484,16 +562,135 @@ class CrawlRunner: event_handler_slow_timeout=slow_warning_timeout(install_phase_timeout), ), ) - await _run_event_now(install_event, install_phase_timeout) + + async def on_archivebox_InstallEvent(event: InstallEvent) -> None: + nonlocal install_cancel_watcher + if event.event_id != install_event.event_id: + return + install_cancel_watcher = asyncio.create_task(self.watch_for_cancelled_crawl(event)) + + on_archivebox_InstallEvent.__name__ = "on_archivebox_InstallEvent__cancel_watcher" + self.bus.on(InstallEvent, on_archivebox_InstallEvent) + setup_abx_services( + self.bus, + plugins=plugins, + url=snapshot["url"], + snapshot=abx_snapshot, + output_dir=output_dir, + install_enabled=False, + crawl_setup_enabled=True, + crawl_event_enabled=False, + crawl_start_enabled=False, + snapshot_cleanup_enabled=False, + crawl_cleanup_enabled=True, + crawl_completed_enabled=False, + crawl_setup_phase_timeout=crawl_setup_phase_timeout, + snapshot_phase_timeout=0.0, + snapshot_cleanup_phase_timeout=0.0, + crawl_cleanup_phase_timeout=crawl_setup_phase_timeout, + persist_derived=False, + auto_install=True, + emit_jsonl=False, + abort_requested=self.crawl_is_cancelled, + MachineService=None, + BinaryService=HookBinaryService, + ProcessService=None, + ArchiveResultService=None, + TagService=None, + SnapshotService=None, + ) + try: + await _run_event_now(install_event, install_phase_timeout) + finally: + if install_cancel_watcher is not None: + install_cancel_watcher.cancel() + await asyncio.gather(install_cancel_watcher, return_exceptions=True) + + async def on_archivebox_CrawlStartEvent(event: CrawlStartEvent) -> None: + if event.event_id != self.root_crawl_start_event_id: + return + for snapshot_id in snapshot_ids: + await self.enqueue_snapshot(snapshot_id) + await self.wait_for_snapshot_tasks() + + async def on_archivebox_CrawlEvent(event: CrawlEvent) -> None: + if event.event_id != self.root_crawl_event_id: + return + cancel_watcher = asyncio.create_task(self.watch_for_cancelled_crawl(event)) + try: + try: + if not await self.crawl_is_cancelled(): + await _run_event_now( + event.emit( + CrawlSetupEvent( + url=snapshot["url"], + snapshot_id=snapshot["id"], + output_dir=str(output_dir), + event_timeout=crawl_setup_phase_timeout, + event_handler_slow_timeout=slow_warning_timeout(crawl_setup_phase_timeout), + ), + ), + crawl_setup_phase_timeout, + ) + if not await self.crawl_is_cancelled(): + crawl_start_event = event.emit( + CrawlStartEvent( + url=snapshot["url"], + snapshot_id=snapshot["id"], + output_dir=str(output_dir), + event_timeout=0, + event_handler_timeout=0, + event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout), + ), + ) + self.root_crawl_start_event_id = crawl_start_event.event_id + await _run_event_now(crawl_start_event, None) + finally: + await _run_event_now( + event.emit( + CrawlCleanupEvent( + url=snapshot["url"], + snapshot_id=snapshot["id"], + output_dir=str(output_dir), + event_timeout=crawl_setup_phase_timeout, + event_handler_slow_timeout=slow_warning_timeout(crawl_setup_phase_timeout), + ), + ), + crawl_setup_phase_timeout, + ) + finally: + cancel_watcher.cancel() + await asyncio.gather(cancel_watcher, return_exceptions=True) + await _run_event_now( + event.emit( + CrawlCompletedEvent( + url=snapshot["url"], + snapshot_id=snapshot["id"], + output_dir=str(output_dir), + ), + ), + CrawlCompletedEvent.model_fields["event_timeout"].default, + ) + + on_archivebox_CrawlStartEvent.__name__ = "on_archivebox_CrawlStartEvent__run_snapshots" + on_archivebox_CrawlEvent.__name__ = "on_archivebox_CrawlEvent__run_recursive_crawl" + self.bus.on(CrawlStartEvent, on_archivebox_CrawlStartEvent) + self.bus.on(CrawlEvent, on_archivebox_CrawlEvent) + crawl_event = CrawlEvent( url=snapshot["url"], snapshot_id=snapshot["id"], output_dir=str(output_dir), - event_timeout=crawl_setup_phase_timeout, - event_handler_slow_timeout=slow_warning_timeout(crawl_setup_phase_timeout), + event_timeout=0, + event_handler_timeout=0, + event_handler_slow_timeout=slow_warning_timeout(crawl_lifecycle_timeout), ) self.root_crawl_event_id = crawl_event.event_id - await _run_event_now(self.bus.emit(crawl_event), crawl_setup_phase_timeout) + emitted_crawl_event = self.bus.emit(crawl_event) + await _run_event_now(emitted_crawl_event, None) + if await self.crawl_is_cancelled(): + self._skip_wait_until_idle = True + return for plugin, hook in setup_hooks: if hook.is_background: continue @@ -523,96 +720,62 @@ class CrawlRunner: if completed_process.status == "failed": raise RuntimeError(f"Crawl setup hook {plugin.name}:{hook.name} failed") - async def run_crawl_cleanup(self, snapshot_id: str) -> None: - snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(snapshot_id) - if self.root_crawl_event_id is None: - return - config = _normalize_runtime_config(snapshot["config"]) - output_dir = Path(self.crawl_output_dir) - plugins = self.runtime_plugins() - setup_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("CrawlSetup")] - crawl_cleanup_phase_timeout = compute_phase_timeout(setup_hooks, config) - await _run_event_now( - self.bus.emit( - CrawlCleanupEvent( - url=snapshot["url"], - snapshot_id=snapshot["id"], - output_dir=str(output_dir), - event_parent_id=self.root_crawl_event_id, - event_timeout=crawl_cleanup_phase_timeout, - event_handler_slow_timeout=slow_warning_timeout(crawl_cleanup_phase_timeout), - ), - ), - crawl_cleanup_phase_timeout, - ) - async def run_snapshot(self, snapshot_id: str) -> None: async with self.snapshot_semaphore: + crawl_start_event = get_current_event() + if not isinstance(crawl_start_event, CrawlStartEvent): + raise RuntimeError("Snapshot events must be emitted from a CrawlStartEvent handler") snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(snapshot_id) if snapshot["status"] == "sealed": return if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() == "max_size": await sync_to_async(self.seal_snapshot_due_to_limit, thread_sensitive=True)(snapshot_id) return - try: - config = _normalize_runtime_config(snapshot["config"]) - derived_config = _normalize_runtime_config(self.derived_config) - output_dir = Path(snapshot["output_dir"]) - plugins = self.runtime_plugins() - abx_snapshot = AbxSnapshot( - id=snapshot["id"], - url=snapshot["url"], - depth=int(snapshot["depth"]), - crawl_id=str(self.crawl.id), - ) - snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")] - snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config) - await _emit_machine_config(self.bus, config=config, derived_config=derived_config) - HookSnapshotService( - self.bus, - url=snapshot["url"], - snapshot=abx_snapshot, - output_dir=output_dir, - plugins=plugins, - snapshot_phase_timeout=snapshot_phase_timeout, - snapshot_cleanup_enabled=True, - snapshot_cleanup_phase_timeout=snapshot_phase_timeout, - ) - crawl_start_event = CrawlStartEvent( - url=snapshot["url"], - snapshot_id=snapshot["id"], - output_dir=str(output_dir), - event_timeout=snapshot_phase_timeout, - event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout), - ) - await _run_event_now(self.bus.emit(crawl_start_event), snapshot_phase_timeout) - snapshot_event = SnapshotEvent( - url=snapshot["url"], - snapshot_id=snapshot["id"], - output_dir=str(output_dir), - depth=int(snapshot["depth"]), - event_parent_id=crawl_start_event.event_id, - event_timeout=snapshot_phase_timeout, - event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout), - ) - emitted_snapshot_event = self.bus.emit(snapshot_event) - await _run_event_now(emitted_snapshot_event, snapshot_phase_timeout) - completed_snapshot = await self.bus.find( - SnapshotCompletedEvent, - child_of=emitted_snapshot_event, - past=True, - future=snapshot_phase_timeout, - ) - if completed_snapshot is None: - raise RuntimeError(f"Snapshot {snapshot_id} did not complete") - await completed_snapshot.now(timeout=snapshot_phase_timeout) - await completed_snapshot.wait(timeout=snapshot_phase_timeout) - await completed_snapshot.event_results_list() - await self.enqueue_discovered_snapshots_from_outputs(snapshot) - finally: - current_task = asyncio.current_task() - if current_task is not None and self.snapshot_tasks.get(snapshot_id) is current_task: - self.snapshot_tasks.pop(snapshot_id, None) + config = _normalize_runtime_config(snapshot["config"]) + derived_config = _normalize_runtime_config(self.derived_config) + output_dir = Path(snapshot["output_dir"]) + plugins = self.runtime_plugins() + abx_snapshot = AbxSnapshot( + id=snapshot["id"], + url=snapshot["url"], + depth=int(snapshot["depth"]), + crawl_id=str(self.crawl.id), + ) + snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")] + snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config) + await _emit_machine_config(self.bus, config=config, derived_config=derived_config, parent_event=crawl_start_event) + HookSnapshotService( + self.bus, + url=snapshot["url"], + snapshot=abx_snapshot, + output_dir=output_dir, + plugins=plugins, + snapshot_phase_timeout=snapshot_phase_timeout, + snapshot_cleanup_enabled=True, + snapshot_cleanup_phase_timeout=snapshot_phase_timeout, + abort_requested=self.crawl_is_cancelled, + ) + snapshot_event = SnapshotEvent( + url=snapshot["url"], + snapshot_id=snapshot["id"], + output_dir=str(output_dir), + depth=int(snapshot["depth"]), + event_timeout=snapshot_phase_timeout, + event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout), + ) + emitted_snapshot_event = crawl_start_event.emit(snapshot_event) + await _run_event_now(emitted_snapshot_event, snapshot_phase_timeout) + completed_snapshot = await self.bus.find( + SnapshotCompletedEvent, + child_of=emitted_snapshot_event, + past=True, + future=snapshot_phase_timeout, + ) + if completed_snapshot is None: + raise RuntimeError(f"Snapshot {snapshot_id} did not complete") + await completed_snapshot.wait(timeout=snapshot_phase_timeout) + await completed_snapshot.event_results_list() + await self.enqueue_discovered_snapshots_from_outputs(snapshot) def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None: from archivebox.core.models import Snapshot @@ -646,7 +809,7 @@ def run_crawl( async def _run_binary(binary_id: str) -> None: - from archivebox.config.configset import get_config + from archivebox.config.common import get_config from archivebox.machine.models import Binary, Machine binary = await Binary.objects.aget(id=binary_id) @@ -698,7 +861,7 @@ def run_binary(binary_id: str) -> None: async def _run_install(plugin_names: list[str] | None = None) -> None: - from archivebox.config.configset import get_config + from archivebox.config.common import get_config from archivebox.machine.models import Machine plugins = discover_plugins() @@ -948,10 +1111,26 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> run_crawl(str(queued_crawl.id), process_discovered_snapshots_inline=True) continue + pending = Crawl.objects.filter( + retry_at__lte=timezone.now(), + status=Crawl.StatusChoices.STARTED, + ) + if crawl_id: + pending = pending.filter(id=crawl_id) + pending = pending.order_by("retry_at", "created_at") + + crawl = pending.first() + if crawl is not None: + if not crawl.claim_processing_lock(lock_seconds=60): + continue + run_crawl(str(crawl.id), process_discovered_snapshots_inline=True) + continue + if crawl_id is None: snapshot = ( Snapshot.objects.filter(retry_at__lte=timezone.now()) .exclude(status=Snapshot.StatusChoices.SEALED) + .exclude(crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED]) .select_related("crawl") .order_by("retry_at", "created_at") .first() @@ -981,22 +1160,7 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> run_binary(str(binary.id)) continue - pending = Crawl.objects.filter( - retry_at__lte=timezone.now(), - status=Crawl.StatusChoices.STARTED, - ) - if crawl_id: - pending = pending.filter(id=crawl_id) - pending = pending.order_by("retry_at", "created_at") - - crawl = pending.first() - if crawl is None: - if daemon: - time.sleep(2.0) - continue - return 0 - - if not crawl.claim_processing_lock(lock_seconds=60): + if daemon: + time.sleep(2.0) continue - - run_crawl(str(crawl.id), process_discovered_snapshots_inline=True) + return 0 diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py index 843ef2cb..d89abecd 100644 --- a/archivebox/tests/conftest.py +++ b/archivebox/tests/conftest.py @@ -20,6 +20,8 @@ PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve() SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve() # Force ArchiveBox imports to see a temp DATA_DIR during test collection. os.environ["DATA_DIR"] = str(SESSION_DATA_DIR) +os.environ.pop("ARCHIVE_DIR", None) +os.environ.pop("USERS_DIR", None) os.environ.pop("CRAWL_DIR", None) os.environ.pop("SNAP_DIR", None) @@ -40,7 +42,7 @@ def _assert_safe_runtime_paths(*, cwd: Path | None = None, env: dict[str, str] | if cwd is not None: _assert_not_repo_path(cwd, label="cwd") - for key in ("DATA_DIR", "CRAWL_DIR", "SNAP_DIR"): + for key in ("DATA_DIR", "ARCHIVE_DIR", "USERS_DIR", "CRAWL_DIR", "SNAP_DIR"): value = (env or {}).get(key) if value: _assert_not_repo_path(Path(value), label=key) @@ -139,6 +141,8 @@ def isolate_test_runtime(tmp_path, monkeypatch): original_popen = subprocess.Popen os.chdir(tmp_path) os.environ.pop("DATA_DIR", None) + os.environ.pop("ARCHIVE_DIR", None) + os.environ.pop("USERS_DIR", None) os.environ.pop("CRAWL_DIR", None) os.environ.pop("SNAP_DIR", None) @@ -218,6 +222,8 @@ def run_archivebox_cmd_cwd( _assert_not_repo_path(cwd, label="cwd") base_env = os.environ.copy() base_env.pop("DATA_DIR", None) + base_env.pop("ARCHIVE_DIR", None) + base_env.pop("USERS_DIR", None) base_env.pop("CRAWL_DIR", None) base_env.pop("SNAP_DIR", None) base_env["USE_COLOR"] = "False" @@ -258,6 +264,8 @@ def run_python_cwd( _assert_not_repo_path(cwd, label="cwd") base_env = os.environ.copy() base_env.pop("DATA_DIR", None) + base_env.pop("ARCHIVE_DIR", None) + base_env.pop("USERS_DIR", None) base_env.pop("CRAWL_DIR", None) base_env.pop("SNAP_DIR", None) _assert_safe_runtime_paths(cwd=cwd, env=base_env) diff --git a/archivebox/tests/test_add_view.py b/archivebox/tests/test_add_view.py index 2047c947..839044a0 100644 --- a/archivebox/tests/test_add_view.py +++ b/archivebox/tests/test_add_view.py @@ -4,7 +4,6 @@ import pytest from django.contrib.auth import get_user_model from django.urls import reverse -from archivebox.config.common import SERVER_CONFIG, SEARCH_BACKEND_CONFIG from archivebox.core.models import Tag from archivebox.crawls.models import Crawl @@ -26,7 +25,7 @@ def admin_user(db): def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, monkeypatch): - monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True) + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") response = client.get(reverse("add"), HTTP_HOST=WEB_HOST) body = response.content.decode() @@ -52,8 +51,8 @@ def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, m def test_add_view_checks_configured_search_backend_by_default(client, monkeypatch): - monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True) - monkeypatch.setattr(SEARCH_BACKEND_CONFIG, "SEARCH_BACKEND_ENGINE", "sqlite") + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") + monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "sqlite") response = client.get(reverse("add"), HTTP_HOST=WEB_HOST) body = response.content.decode() @@ -67,7 +66,7 @@ def test_add_view_checks_configured_search_backend_by_default(client, monkeypatc def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_user, monkeypatch): - monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True) + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") client.force_login(admin_user) response = client.post( @@ -107,7 +106,7 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_ def test_add_view_starts_background_runner_after_creating_crawl(client, admin_user, monkeypatch): - monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True) + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") client.force_login(admin_user) runner_calls = [] @@ -137,7 +136,7 @@ def test_add_view_starts_background_runner_after_creating_crawl(client, admin_us def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkeypatch): - monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True) + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") client.force_login(admin_user) response = client.post( @@ -185,7 +184,7 @@ def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkey def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_user, monkeypatch): - monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True) + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") client.force_login(admin_user) response = client.post( @@ -225,7 +224,7 @@ def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_us def test_add_view_exposes_api_token_for_tag_widget_autocomplete(client, admin_user, monkeypatch): - monkeypatch.setattr(SERVER_CONFIG, "PUBLIC_ADD_VIEW", True) + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") client.force_login(admin_user) response = client.get(reverse("add"), HTTP_HOST=WEB_HOST) @@ -234,9 +233,9 @@ def test_add_view_exposes_api_token_for_tag_widget_autocomplete(client, admin_us assert b"window.ARCHIVEBOX_API_KEY" in response.content -def test_tags_autocomplete_requires_auth_when_public_snapshots_list_disabled(client, settings): - settings.PUBLIC_SNAPSHOTS_LIST = False - settings.PUBLIC_INDEX = False +def test_tags_autocomplete_requires_auth_when_public_snapshots_list_disabled(client, monkeypatch): + monkeypatch.setenv("PUBLIC_SNAPSHOTS_LIST", "false") + monkeypatch.setenv("PUBLIC_INDEX", "false") Tag.objects.create(name="archive") response = client.get( @@ -248,9 +247,9 @@ def test_tags_autocomplete_requires_auth_when_public_snapshots_list_disabled(cli assert response.status_code == 401 -def test_tags_autocomplete_allows_public_access_when_public_snapshots_list_enabled(client, settings): - settings.PUBLIC_SNAPSHOTS_LIST = True - settings.PUBLIC_INDEX = False +def test_tags_autocomplete_allows_public_access_when_public_snapshots_list_enabled(client, monkeypatch): + monkeypatch.setenv("PUBLIC_SNAPSHOTS_LIST", "true") + monkeypatch.setenv("PUBLIC_INDEX", "false") Tag.objects.create(name="archive") response = client.get( @@ -263,9 +262,9 @@ def test_tags_autocomplete_allows_public_access_when_public_snapshots_list_enabl assert response.json()["tags"][0]["name"] == "archive" -def test_tags_autocomplete_allows_authenticated_user_when_public_snapshots_list_disabled(client, admin_user, settings): - settings.PUBLIC_SNAPSHOTS_LIST = False - settings.PUBLIC_INDEX = False +def test_tags_autocomplete_allows_authenticated_user_when_public_snapshots_list_disabled(client, admin_user, monkeypatch): + monkeypatch.setenv("PUBLIC_SNAPSHOTS_LIST", "false") + monkeypatch.setenv("PUBLIC_INDEX", "false") Tag.objects.create(name="archive") client.force_login(admin_user) diff --git a/archivebox/tests/test_admin_views.py b/archivebox/tests/test_admin_views.py index 0bfc0158..2b0cfd9c 100644 --- a/archivebox/tests/test_admin_views.py +++ b/archivebox/tests/test_admin_views.py @@ -22,7 +22,6 @@ from django.contrib.auth import get_user_model from django.contrib.auth.models import UserManager from django.utils import timezone -from archivebox.config.common import SEARCH_BACKEND_CONFIG pytestmark = pytest.mark.django_db @@ -924,16 +923,15 @@ class TestAdminSnapshotListView: assert f"/admin/core/snapshot/{snapshot.pk}/redo-failed/".encode() in response.content def test_snapshot_view_url_uses_canonical_replay_url_for_mode(self, snapshot, monkeypatch): - from archivebox.config.common import SERVER_CONFIG from archivebox.core.admin_site import archivebox_admin from archivebox.core.admin_snapshots import SnapshotAdmin admin = SnapshotAdmin(snapshot.__class__, archivebox_admin) - monkeypatch.setattr(SERVER_CONFIG, "SERVER_SECURITY_MODE", "safe-subdomains-fullreplay") + monkeypatch.setenv("SERVER_SECURITY_MODE", "safe-subdomains-fullreplay") assert admin.get_snapshot_view_url(snapshot) == f"http://snap-{str(snapshot.pk).replace('-', '')[-12:]}.archivebox.localhost:8000" - monkeypatch.setattr(SERVER_CONFIG, "SERVER_SECURITY_MODE", "safe-onedomain-nojsreplay") + monkeypatch.setenv("SERVER_SECURITY_MODE", "safe-onedomain-nojsreplay") assert admin.get_snapshot_view_url(snapshot) == f"http://archivebox.localhost:8000/snapshot/{snapshot.pk}" def test_find_snapshots_for_url_matches_fragment_suffixed_variants(self, crawl, db): @@ -1381,7 +1379,7 @@ class TestAdminSnapshotSearch: """Tests for admin snapshot search functionality.""" def test_admin_search_mode_selector_defaults_to_meta_for_ripgrep(self, client, admin_user, monkeypatch): - monkeypatch.setattr(SEARCH_BACKEND_CONFIG, "SEARCH_BACKEND_ENGINE", "ripgrep") + monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep") client.login(username="testadmin", password="testpassword") response = client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_HOST) @@ -1392,7 +1390,7 @@ class TestAdminSnapshotSearch: assert b'name="search_mode" value="deep"' in response.content def test_admin_search_mode_selector_defaults_to_contents_for_non_ripgrep(self, client, admin_user, monkeypatch): - monkeypatch.setattr(SEARCH_BACKEND_CONFIG, "SEARCH_BACKEND_ENGINE", "sqlite") + monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "sqlite") client.login(username="testadmin", password="testpassword") response = client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_HOST) @@ -1632,7 +1630,7 @@ class TestPublicIndexSearch: @override_settings(PUBLIC_INDEX=True) def test_public_search_mode_selector_defaults_to_meta_for_ripgrep(self, client, monkeypatch): - monkeypatch.setattr(SEARCH_BACKEND_CONFIG, "SEARCH_BACKEND_ENGINE", "ripgrep") + monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep") response = client.get("/public/", HTTP_HOST=PUBLIC_HOST) diff --git a/archivebox/tests/test_auth_ldap.py b/archivebox/tests/test_auth_ldap.py index 2364b16b..5a7df5ea 100644 --- a/archivebox/tests/test_auth_ldap.py +++ b/archivebox/tests/test_auth_ldap.py @@ -17,20 +17,21 @@ class TestLDAPConfig(unittest.TestCase): def test_ldap_config_defaults(self): """Test that LDAP config loads with correct defaults.""" - from archivebox.config.ldap import LDAP_CONFIG + from archivebox.config.common import get_config # Check default values - self.assertFalse(LDAP_CONFIG.LDAP_ENABLED) - self.assertIsNone(LDAP_CONFIG.LDAP_SERVER_URI) - self.assertIsNone(LDAP_CONFIG.LDAP_BIND_DN) - self.assertIsNone(LDAP_CONFIG.LDAP_BIND_PASSWORD) - self.assertIsNone(LDAP_CONFIG.LDAP_USER_BASE) - self.assertEqual(LDAP_CONFIG.LDAP_USER_FILTER, "(uid=%(user)s)") - self.assertEqual(LDAP_CONFIG.LDAP_USERNAME_ATTR, "username") - self.assertEqual(LDAP_CONFIG.LDAP_FIRSTNAME_ATTR, "givenName") - self.assertEqual(LDAP_CONFIG.LDAP_LASTNAME_ATTR, "sn") - self.assertEqual(LDAP_CONFIG.LDAP_EMAIL_ATTR, "mail") - self.assertFalse(LDAP_CONFIG.LDAP_CREATE_SUPERUSER) + config = get_config() + self.assertFalse(config.LDAP_ENABLED) + self.assertIsNone(config.LDAP_SERVER_URI) + self.assertIsNone(config.LDAP_BIND_DN) + self.assertIsNone(config.LDAP_BIND_PASSWORD) + self.assertIsNone(config.LDAP_USER_BASE) + self.assertEqual(config.LDAP_USER_FILTER, "(uid=%(user)s)") + self.assertEqual(config.LDAP_USERNAME_ATTR, "username") + self.assertEqual(config.LDAP_FIRSTNAME_ATTR, "givenName") + self.assertEqual(config.LDAP_LASTNAME_ATTR, "sn") + self.assertEqual(config.LDAP_EMAIL_ATTR, "mail") + self.assertFalse(config.LDAP_CREATE_SUPERUSER) def test_ldap_config_validation_disabled(self): """Test that validation passes when LDAP is disabled.""" @@ -74,10 +75,10 @@ class TestLDAPConfig(unittest.TestCase): self.assertEqual(error_msg, "") def test_ldap_config_in_get_config(self): - """Test that LDAP_CONFIG is included in get_CONFIG().""" - from archivebox.config import get_CONFIG + """Test that LDAP_CONFIG is included in the typed config sections.""" + from archivebox.config.common import get_all_configs - all_config = get_CONFIG() + all_config = get_all_configs() self.assertIn("LDAP_CONFIG", all_config) self.assertEqual(all_config["LDAP_CONFIG"].__class__.__name__, "LDAPConfig") diff --git a/archivebox/tests/test_cli_init.py b/archivebox/tests/test_cli_init.py index 3510a086..eb24d477 100644 --- a/archivebox/tests/test_cli_init.py +++ b/archivebox/tests/test_cli_init.py @@ -8,10 +8,10 @@ import os import sqlite3 import subprocess -from archivebox.config.common import STORAGE_CONFIG +from archivebox.config.common import get_config -DIR_PERMISSIONS = STORAGE_CONFIG.OUTPUT_PERMISSIONS.replace("6", "7").replace("4", "5") +DIR_PERMISSIONS = get_config().OUTPUT_PERMISSIONS.replace("6", "7").replace("4", "5") def test_init_creates_database_file(tmp_path): @@ -35,6 +35,23 @@ def test_init_creates_archive_directory(tmp_path): assert archive_dir.is_dir() +def test_init_respects_configured_archive_and_users_dirs(tmp_path): + """Test that init creates configured archive/users storage roots.""" + os.chdir(tmp_path) + archive_dir = tmp_path / "mounted_archive" + users_dir = archive_dir / "custom_users" + env = os.environ.copy() + env["ARCHIVE_DIR"] = str(archive_dir) + env["USERS_DIR"] = str(users_dir) + + result = subprocess.run(["archivebox", "init"], env=env, capture_output=True) + + assert result.returncode == 0 + assert archive_dir.is_dir() + assert users_dir.is_dir() + assert not (tmp_path / "archive").exists() + + def test_init_creates_sources_directory(tmp_path): """Test that init creates sources directory.""" os.chdir(tmp_path) @@ -145,11 +162,11 @@ def test_init_sets_correct_file_permissions(tmp_path): # Check database permissions db_path = tmp_path / "index.sqlite3" - assert oct(db_path.stat().st_mode)[-3:] in (STORAGE_CONFIG.OUTPUT_PERMISSIONS, DIR_PERMISSIONS) + assert oct(db_path.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS) # Check directory permissions archive_dir = tmp_path / "archive" - assert oct(archive_dir.stat().st_mode)[-3:] in (STORAGE_CONFIG.OUTPUT_PERMISSIONS, DIR_PERMISSIONS) + assert oct(archive_dir.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS) def test_init_is_idempotent(tmp_path): diff --git a/archivebox/tests/test_cli_real_flows.py b/archivebox/tests/test_cli_real_flows.py index 132dfe3d..68f2e69f 100644 --- a/archivebox/tests/test_cli_real_flows.py +++ b/archivebox/tests/test_cli_real_flows.py @@ -158,7 +158,7 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(tmp_path, pro failed_results = [(url, plugin, output) for url, plugin, status, _files, _size, output in archive_results if status == "failed"] assert len(failed_results) <= 2, failed_results - snapshot_root = tmp_path / "users/system/snapshots" + snapshot_root = tmp_path / "archive/users/system/snapshots" html_outputs = [path for path in snapshot_root.rglob("wget/**/*.html") if path.is_file()] header_outputs = [path for path in snapshot_root.rglob("headers/**/headers.json") if path.is_file() and path.stat().st_size > 0] title_outputs = [path for path in snapshot_root.rglob("title/title.txt") if path.is_file() and path.stat().st_size > 0] @@ -257,6 +257,6 @@ def test_cli_recursive_crawl_processes_discovered_html_urls(tmp_path, process): assert by_url_plugin[("https://example.com", "parse_html_urls")] == "succeeded" assert by_url_plugin[("https://iana.org/domains/example", "wget")] == "succeeded" - urls_outputs = list((tmp_path / "users/system/snapshots").rglob("parse_html_urls/urls.jsonl")) + urls_outputs = list((tmp_path / "archive/users/system/snapshots").rglob("parse_html_urls/urls.jsonl")) assert urls_outputs assert any("https://iana.org/domains/example" in path.read_text() for path in urls_outputs) diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index 8fa0c887..1d539ea1 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -274,7 +274,7 @@ class TestRunEmpty: class TestRunDaemonMode: - def test_run_daemon_processes_stdin_before_runner(self, monkeypatch): + def test_run_daemon_starts_runner_without_reading_stdin(self, monkeypatch): from archivebox.cli import archivebox_run class FakeStdin: @@ -286,7 +286,7 @@ class TestRunDaemonMode: monkeypatch.setattr( archivebox_run, "process_stdin_records", - lambda: calls.append("stdin") or 0, + lambda: (_ for _ in ()).throw(AssertionError("daemon mode must not block on stdin")), ) monkeypatch.setattr( archivebox_run, @@ -298,27 +298,7 @@ class TestRunDaemonMode: archivebox_run.main.callback(daemon=True, crawl_id=None, snapshot_id=None, binary_id=None) assert exit_info.value.code == 0 - assert calls == ["stdin", "runner:True"] - - def test_run_daemon_skips_runner_if_stdin_processing_fails(self, monkeypatch): - from archivebox.cli import archivebox_run - - class FakeStdin: - def isatty(self): - return False - - monkeypatch.setattr(sys, "stdin", FakeStdin()) - monkeypatch.setattr(archivebox_run, "process_stdin_records", lambda: 1) - monkeypatch.setattr( - archivebox_run, - "run_runner", - lambda daemon=False: (_ for _ in ()).throw(AssertionError("runner should not start after stdin failure")), - ) - - with pytest.raises(SystemExit) as exit_info: - archivebox_run.main.callback(daemon=True, crawl_id=None, snapshot_id=None, binary_id=None) - - assert exit_info.value.code == 1 + assert calls == ["runner:True"] @pytest.mark.django_db diff --git a/archivebox/tests/test_config.py b/archivebox/tests/test_config.py index ab9a557e..58add203 100644 --- a/archivebox/tests/test_config.py +++ b/archivebox/tests/test_config.py @@ -106,6 +106,30 @@ def test_config_invalid_key_fails(tmp_path, process): assert result.returncode != 0 or "failed" in result.stdout.lower() +def test_config_ignores_legacy_unknown_keys(tmp_path, process): + """Old ArchiveBox.conf keys should not prevent startup during upgrades.""" + os.chdir(tmp_path) + (tmp_path / "ArchiveBox.conf").write_text( + """ +[ARCHIVING_CONFIG] +MAX_MEDIA_SIZE = "750m" + +[SEARCH_BACKEND_CONFIG] +SEARCH_BACKEND_HOST_NAME = "sonic" +SEARCH_BACKEND_PASSWORD = "SecretPassword" +""", + ) + + result = subprocess.run( + ["archivebox", "version"], + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "Extra inputs are not permitted" not in result.stderr + + def test_config_set_requires_equals_sign(tmp_path, process): """Test that --set requires KEY=VALUE format.""" os.chdir(tmp_path) diff --git a/archivebox/tests/test_config_views.py b/archivebox/tests/test_config_views.py index 632258b0..7aeec1d5 100644 --- a/archivebox/tests/test_config_views.py +++ b/archivebox/tests/test_config_views.py @@ -252,7 +252,6 @@ def test_live_config_value_view_renames_source_field_and_uses_plugin_definition_ request.user = SimpleNamespace(is_superuser=True) monkeypatch.setattr(core_views, "get_all_configs", lambda: {}) - monkeypatch.setattr(core_views, "get_flat_config", lambda: {}) monkeypatch.setattr(core_views, "get_config", lambda: {"PARSE_DOM_OUTLINKS_ENABLED": True}) monkeypatch.setattr(core_views, "find_config_default", lambda key: "True") monkeypatch.setattr(core_views, "find_config_type", lambda key: "bool") @@ -308,7 +307,6 @@ def test_live_config_value_view_priority_text_matches_runtime_precedence(monkeyp request.user = SimpleNamespace(is_superuser=True) monkeypatch.setattr(core_views, "get_all_configs", lambda: {}) - monkeypatch.setattr(core_views, "get_flat_config", lambda: {"CHECK_SSL_VALIDITY": True}) monkeypatch.setattr(core_views, "get_config", lambda: {"CHECK_SSL_VALIDITY": False}) monkeypatch.setattr(core_views, "find_config_default", lambda key: "True") monkeypatch.setattr(core_views, "find_config_type", lambda key: "bool") diff --git a/archivebox/tests/test_hooks.py b/archivebox/tests/test_hooks.py index b8bd0f1a..27636b3c 100755 --- a/archivebox/tests/test_hooks.py +++ b/archivebox/tests/test_hooks.py @@ -697,7 +697,7 @@ print(json.dumps({ ) hook_path.chmod(0o755) - output_dir = tmp_path / "users" / "system" / "snapshots" / "20260513" / "example.com" / "test" / "envprobe" + output_dir = tmp_path / "archive" / "users" / "system" / "snapshots" / "20260513" / "example.com" / "test" / "envprobe" process = run_hook( hook_path, output_dir, diff --git a/archivebox/tests/test_machine_models.py b/archivebox/tests/test_machine_models.py index c67f8135..a91f3e82 100644 --- a/archivebox/tests/test_machine_models.py +++ b/archivebox/tests/test_machine_models.py @@ -154,6 +154,26 @@ class TestMachineModel(TestCase): self.assertNotIn("CHROME_USER_DATA_DIR", refreshed.config) self.assertNotIn("CHROMIUM_VERSION", refreshed.config) + def test_get_config_auto_applies_current_machine_config(self): + """get_config() should include sanitized Machine.current() config by default.""" + import archivebox.machine.models as models + from archivebox.config.common import get_config + + machine = Machine.current() + machine.config = { + "CHROME_BINARY": "/tmp/chromium", + "ABX_INSTALL_CACHE": {"chrome": "2026-03-24T00:00:00+00:00"}, + "CHROME_ISOLATION": "snapshot", + } + machine.save(update_fields=["config"]) + models._CURRENT_MACHINE = machine + + config = get_config() + + self.assertEqual(config.CHROME_BINARY, "/tmp/chromium") + self.assertEqual(config["ABX_INSTALL_CACHE"], {"chrome": "2026-03-24T00:00:00+00:00"}) + self.assertEqual(config.CHROME_ISOLATION, "crawl") + def test_machine_manager_current(self): """Machine.objects.current() should return current machine.""" machine = Machine.current() diff --git a/archivebox/tests/test_migrations_07_to_09.py b/archivebox/tests/test_migrations_07_to_09.py index cfb04b72..418cb257 100644 --- a/archivebox/tests/test_migrations_07_to_09.py +++ b/archivebox/tests/test_migrations_07_to_09.py @@ -93,8 +93,12 @@ class TestMigrationFrom07x(unittest.TestCase): self.assertTrue(ok, msg) def test_migration_preserves_archiveresults(self): - """Migration should preserve all archive results.""" + """Migration should preserve ArchiveResult rows and link each one to a Process.""" expected_count = len(self.original_data["archiveresults"]) + expected_counts = {} + for result in self.original_data["archiveresults"]: + key = (result["extractor"], result["status"]) + expected_counts[key] = expected_counts.get(key, 0) + 1 result = run_archivebox(self.work_dir, ["init"], timeout=45) self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") @@ -102,6 +106,20 @@ class TestMigrationFrom07x(unittest.TestCase): ok, msg = verify_archiveresult_count(self.db_path, expected_count) self.assertTrue(ok, msg) + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute("SELECT plugin, status, COUNT(*) FROM core_archiveresult GROUP BY plugin, status") + migrated_counts = {(plugin, status): count for plugin, status, count in cursor.fetchall()} + cursor.execute("SELECT COUNT(*) FROM core_archiveresult WHERE process_id IS NULL") + missing_process_count = cursor.fetchone()[0] + cursor.execute("SELECT COUNT(*) FROM machine_process") + process_count = cursor.fetchone()[0] + conn.close() + + self.assertEqual(migrated_counts, expected_counts) + self.assertEqual(missing_process_count, 0) + self.assertEqual(process_count, expected_count) + def test_migration_preserves_foreign_keys(self): """Migration should maintain foreign key relationships.""" result = run_archivebox(self.work_dir, ["init"], timeout=45) @@ -110,6 +128,55 @@ class TestMigrationFrom07x(unittest.TestCase): ok, msg = verify_foreign_keys(self.db_path) self.assertTrue(ok, msg) + def test_migration_preserves_legacy_timestamp_meanings(self): + """0.7.x timestamp is bookmark identity; added is row creation; updated is downloaded.""" + snapshot = self.original_data["snapshots"][0] + legacy_bookmark_ts = "1609459200.123456" + legacy_added = "2024-08-28 09:40:00" + legacy_updated = "2024-08-29 10:41:00" + + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute( + """ + UPDATE core_snapshot + SET timestamp = ?, added = ?, updated = ? + WHERE id = ? + """, + (legacy_bookmark_ts, legacy_added, legacy_updated, snapshot["id"]), + ) + conn.commit() + conn.close() + + result = run_archivebox(self.work_dir, ["init"], timeout=45) + self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") + + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute( + "SELECT timestamp, bookmarked_at, created_at, modified_at, downloaded_at FROM core_snapshot WHERE id = ?", + (snapshot["id"],), + ) + timestamp, bookmarked_at, created_at, modified_at, downloaded_at = cursor.fetchone() + conn.close() + + self.assertEqual(timestamp, legacy_bookmark_ts) + self.assertTrue(bookmarked_at.startswith("2021-01-01"), bookmarked_at) + self.assertTrue(created_at.startswith("2024-08-28"), created_at) + self.assertTrue(modified_at.startswith("2024-08-29"), modified_at) + self.assertTrue(downloaded_at.startswith("2024-08-29"), downloaded_at) + + def test_update_saves_migrated_snapshots_without_foreign_key_errors(self): + """Migrated 0.7.x snapshots should be writable through the current ORM.""" + result = run_archivebox(self.work_dir, ["init"], timeout=45) + self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") + + result = run_archivebox(self.work_dir, ["update"], timeout=60) + output = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, f"Update failed after migration: {result.stderr}") + self.assertNotIn("FOREIGN KEY constraint failed", output) + self.assertNotIn("Skipping snapshot", output) + def test_status_works_after_migration(self): """Status command should work after migration.""" result = run_archivebox(self.work_dir, ["init"], timeout=45) diff --git a/archivebox/tests/test_migrations_08_to_09.py b/archivebox/tests/test_migrations_08_to_09.py index 7264a06d..db1114c0 100644 --- a/archivebox/tests/test_migrations_08_to_09.py +++ b/archivebox/tests/test_migrations_08_to_09.py @@ -15,6 +15,7 @@ import sqlite3 import tempfile import unittest import json +import uuid from pathlib import Path from .migrations_helpers import ( @@ -134,8 +135,13 @@ class TestMigrationFrom08x(unittest.TestCase): self.assertTrue(ok, msg) def test_migration_preserves_archiveresults(self): - """Migration should preserve all archive results.""" + """Migration should preserve ArchiveResult rows and link each one to a Process.""" expected_count = len(self.original_data["archiveresults"]) + expected_counts = {} + for result in self.original_data["archiveresults"]: + status = "succeeded" if result["status"] == "success" else result["status"] + key = (result["extractor"], status) + expected_counts[key] = expected_counts.get(key, 0) + 1 result = run_archivebox(self.work_dir, ["init"], timeout=45) self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") @@ -143,6 +149,20 @@ class TestMigrationFrom08x(unittest.TestCase): ok, msg = verify_archiveresult_count(self.db_path, expected_count) self.assertTrue(ok, msg) + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute("SELECT plugin, status, COUNT(*) FROM core_archiveresult GROUP BY plugin, status") + migrated_counts = {(plugin, status): count for plugin, status, count in cursor.fetchall()} + cursor.execute("SELECT COUNT(*) FROM core_archiveresult WHERE process_id IS NULL") + missing_process_count = cursor.fetchone()[0] + cursor.execute("SELECT COUNT(*) FROM machine_process") + process_count = cursor.fetchone()[0] + conn.close() + + self.assertEqual(migrated_counts, expected_counts) + self.assertEqual(missing_process_count, 0) + self.assertEqual(process_count, expected_count) + def test_migration_preserves_archiveresult_status(self): """Migration should preserve archive result status values.""" result = run_archivebox(self.work_dir, ["init"], timeout=45) @@ -213,6 +233,77 @@ class TestMigrationFrom08x(unittest.TestCase): ok, msg = verify_foreign_keys(self.db_path) self.assertTrue(ok, msg) + def test_migration_preserves_08_timestamp_meanings(self): + """0.8.x already has separated timestamp/bookmarked_at/created_at/downloaded_at fields.""" + snapshot = self.original_data["snapshots"][0] + legacy_timestamp = "1609459200.123456" + bookmarked_at = "2021-01-01 00:00:00" + created_at = "2024-08-28 09:40:00" + modified_at = "2024-08-29 10:41:00" + downloaded_at = "2024-08-30 11:42:00" + + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute( + """ + UPDATE core_snapshot + SET timestamp = ?, bookmarked_at = ?, created_at = ?, modified_at = ?, downloaded_at = ? + WHERE id = ? + """, + (legacy_timestamp, bookmarked_at, created_at, modified_at, downloaded_at, snapshot["id"]), + ) + conn.commit() + conn.close() + + result = run_archivebox(self.work_dir, ["init"], timeout=45) + self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") + + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute( + "SELECT timestamp, bookmarked_at, created_at, modified_at, downloaded_at FROM core_snapshot WHERE id = ?", + (snapshot["id"],), + ) + migrated = cursor.fetchone() + conn.close() + + self.assertEqual(migrated[0], legacy_timestamp) + self.assertTrue(migrated[1].startswith("2021-01-01"), migrated[1]) + self.assertTrue(migrated[2].startswith("2024-08-28"), migrated[2]) + self.assertTrue(migrated[3].startswith("2024-08-29"), migrated[3]) + self.assertTrue(migrated[4].startswith("2024-08-30"), migrated[4]) + + def test_hyphenated_crawl_ids_are_normalized_before_snapshot_saves(self): + """0.8.x crawl UUIDs with dashes should migrate to Django's SQLite UUID format.""" + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + for crawl in self.original_data["crawls"]: + hyphenated = str(uuid.UUID(hex=crawl["id"])) + cursor.execute("UPDATE crawls_crawl SET id = ? WHERE id = ?", (hyphenated, crawl["id"])) + cursor.execute("UPDATE core_snapshot SET crawl_id = ? WHERE crawl_id = ?", (hyphenated, crawl["id"])) + crawl["id"] = hyphenated + conn.commit() + conn.close() + + result = run_archivebox(self.work_dir, ["init"], timeout=45) + self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") + + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute("SELECT COUNT(*) FROM crawls_crawl WHERE id LIKE '%-%'") + hyphenated_crawls = cursor.fetchone()[0] + cursor.execute("SELECT COUNT(*) FROM core_snapshot WHERE crawl_id LIKE '%-%'") + hyphenated_snapshot_refs = cursor.fetchone()[0] + conn.close() + + self.assertEqual(hyphenated_crawls, 0) + self.assertEqual(hyphenated_snapshot_refs, 0) + + result = run_archivebox(self.work_dir, ["update"], timeout=60) + output = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, f"Update failed after migration: {result.stderr}") + self.assertNotIn("FOREIGN KEY constraint failed", output) + def test_migration_removes_seed_id_column(self): """Migration should remove seed_id column from archivebox.crawls.crawl.""" result = run_archivebox(self.work_dir, ["init"], timeout=45) @@ -517,6 +608,140 @@ class TestFilesystemMigration08to09(unittest.TestCase): """Clean up temporary directory.""" shutil.rmtree(self.work_dir, ignore_errors=True) + def test_update_migrates_db_snapshot_when_legacy_index_missing(self): + """A legacy folder with no index file should still migrate if its timestamp exists in DB.""" + create_data_dir_structure(self.work_dir) + conn = sqlite3.connect(str(self.db_path)) + conn.executescript(SCHEMA_0_7) + conn.close() + original_data = seed_0_7_data(self.db_path) + snapshot = original_data["snapshots"][0] + + snapshot_dir = self.work_dir / "archive" / snapshot["timestamp"] + snapshot_dir.mkdir(parents=True, exist_ok=True) + (snapshot_dir / "screenshot.png").write_text("existing-db-snapshot") + + result = run_archivebox(self.work_dir, ["init"], timeout=60) + self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") + result = run_archivebox(self.work_dir, ["update"], timeout=120) + self.assertEqual(result.returncode, 0, f"Update failed: {result.stderr}") + + migrated_files = list((self.work_dir / "archive" / "users").glob("*/snapshots/*/*/*/screenshot.png")) + self.assertEqual(len(migrated_files), 1) + self.assertEqual(migrated_files[0].read_text(), "existing-db-snapshot") + self.assertFalse((self.work_dir / "invalid").exists()) + + def test_update_recovers_orphan_with_corrupt_index_from_archive_org_url(self): + """A corrupt legacy index can be imported when archive.org.txt has the original URL.""" + create_data_dir_structure(self.work_dir) + conn = sqlite3.connect(str(self.db_path)) + conn.executescript(SCHEMA_0_7) + conn.close() + seed_0_7_data(self.db_path) + + timestamp = "1339747993" + original_url = "http://www.wired.com/wiredenterprise/2012/01/seamicro-and-google/all/1" + snapshot_dir = self.work_dir / "archive" / timestamp + snapshot_dir.mkdir(parents=True, exist_ok=True) + (snapshot_dir / "index.json").write_text("") + (snapshot_dir / "archive.org.txt").write_text(f"https://web.archive.org/web/20170531210128/{original_url}\n") + (snapshot_dir / "output.pdf").write_text("orphan-output") + + result = run_archivebox(self.work_dir, ["init"], timeout=60) + self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") + result = run_archivebox(self.work_dir, ["update"], timeout=120) + self.assertEqual(result.returncode, 0, f"Update failed: {result.stderr}") + + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute("SELECT url, timestamp FROM core_snapshot WHERE timestamp = ?", (timestamp,)) + row = cursor.fetchone() + conn.close() + + self.assertEqual(row, (original_url, timestamp)) + migrated_files = list((self.work_dir / "archive" / "users").glob("*/snapshots/*/*/*/output.pdf")) + self.assertEqual(len(migrated_files), 1) + self.assertEqual(migrated_files[0].read_text(), "orphan-output") + self.assertFalse((self.work_dir / "invalid").exists()) + + def test_update_preserves_legacy_folder_timestamp_over_index_float_variant(self): + """Legacy folder timestamp is the on-disk identity even if index.json has a .0 variant.""" + create_data_dir_structure(self.work_dir) + conn = sqlite3.connect(str(self.db_path)) + conn.executescript(SCHEMA_0_7) + conn.close() + seed_0_7_data(self.db_path) + + timestamp = "1508259732" + url = "https://example.com/folder-timestamp" + snapshot_dir = self.work_dir / "archive" / timestamp + snapshot_dir.mkdir(parents=True, exist_ok=True) + (snapshot_dir / "index.json").write_text( + json.dumps( + { + "url": url, + "timestamp": "1508259732.0", + "title": "Folder Timestamp", + }, + ), + ) + (snapshot_dir / "output.html").write_text("folder timestamp output") + + result = run_archivebox(self.work_dir, ["init"], timeout=60) + self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") + result = run_archivebox(self.work_dir, ["update"], timeout=120) + self.assertEqual(result.returncode, 0, f"Update failed: {result.stderr}") + + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute("SELECT timestamp FROM core_snapshot WHERE url = ?", (url,)) + row = cursor.fetchone() + conn.close() + + self.assertEqual(row, (timestamp,)) + self.assertTrue((self.work_dir / "archive" / timestamp).is_symlink()) + self.assertFalse((self.work_dir / "archive" / f"{timestamp}.0").exists()) + self.assertFalse((self.work_dir / "invalid").exists()) + + def test_update_preserves_distinct_legacy_dirs_with_integer_and_float_timestamps(self): + """Sibling legacy dirs like 1508259732 and 1508259732.0 must not fuzzy-merge.""" + create_data_dir_structure(self.work_dir) + conn = sqlite3.connect(str(self.db_path)) + conn.executescript(SCHEMA_0_7) + conn.close() + seed_0_7_data(self.db_path) + + url = "https://example.com/duplicate-timestamp" + for timestamp, payload in [("1508259732.0", "float-dir"), ("1508259732", "int-dir")]: + snapshot_dir = self.work_dir / "archive" / timestamp + snapshot_dir.mkdir(parents=True, exist_ok=True) + (snapshot_dir / "index.json").write_text( + json.dumps( + { + "url": url, + "timestamp": timestamp, + "title": payload, + }, + ), + ) + (snapshot_dir / f"{payload}.txt").write_text(payload) + + result = run_archivebox(self.work_dir, ["init"], timeout=60) + self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}") + result = run_archivebox(self.work_dir, ["update"], timeout=120) + self.assertEqual(result.returncode, 0, f"Update failed: {result.stderr}") + + conn = sqlite3.connect(str(self.db_path)) + cursor = conn.cursor() + cursor.execute("SELECT timestamp FROM core_snapshot WHERE url = ? ORDER BY timestamp", (url,)) + rows = cursor.fetchall() + conn.close() + + self.assertEqual(rows, [("1508259732",), ("1508259732.0",)]) + self.assertTrue((self.work_dir / "archive" / "1508259732").is_symlink()) + self.assertTrue((self.work_dir / "archive" / "1508259732.0").is_symlink()) + self.assertFalse((self.work_dir / "invalid").exists()) + def test_archiveresult_files_preserved_after_migration(self): """ Test that ArchiveResult output files are reorganized into new structure. @@ -524,7 +749,7 @@ class TestFilesystemMigration08to09(unittest.TestCase): This test verifies that: 1. Migration preserves ArchiveResult data in Process/Binary records 2. Running `archivebox update` reorganizes files into new structure - 3. New structure: users/username/snapshots/YYYYMMDD/example.com/snap-uuid-here/output.ext + 3. New structure: archive/users/username/snapshots/YYYYMMDD/example.com/snap-uuid-here/output.ext 4. All files are moved (no data loss) 5. Old archive/timestamp/ directories are cleaned up """ @@ -536,7 +761,7 @@ class TestFilesystemMigration08to09(unittest.TestCase): conn = sqlite3.connect(str(self.db_path)) cursor = conn.cursor() for i, snapshot in enumerate(original_data["snapshots"]): - legacy_timestamp = str(1704110400 + (i * 86400)) + legacy_timestamp = "1609459200.123456" if i == 0 else str(1704110400 + (i * 86400)) cursor.execute( "UPDATE core_snapshot SET timestamp = ? WHERE id = ?", (legacy_timestamp, snapshot["id"]), @@ -572,7 +797,7 @@ class TestFilesystemMigration08to09(unittest.TestCase): # Count archive directories and files BEFORE migration archive_dir = self.work_dir / "archive" - dirs_before = list(archive_dir.glob("*")) if archive_dir.exists() else [] + dirs_before = [d for d in archive_dir.glob("*") if d.name.replace(".", "").isdigit()] if archive_dir.exists() else [] dirs_before_count = len([d for d in dirs_before if d.is_dir()]) # Count total files in all archive directories @@ -600,7 +825,7 @@ class TestFilesystemMigration08to09(unittest.TestCase): self.assertEqual(result.returncode, 0, f"Init (migration) failed: {result.stderr}") # Count archive directories and files AFTER migration - dirs_after = list(archive_dir.glob("*")) if archive_dir.exists() else [] + dirs_after = [d for d in archive_dir.glob("*") if d.name.replace(".", "").isdigit()] if archive_dir.exists() else [] dirs_after_count = len([d for d in dirs_after if d.is_dir()]) files_after = [] @@ -640,8 +865,8 @@ class TestFilesystemMigration08to09(unittest.TestCase): self.assertEqual(result.returncode, 0, f"Update failed: {result.stderr}") # Check new filesystem structure - # New structure: users/username/snapshots/YYYYMMDD/example.com/snap-uuid-here/output.ext - users_dir = self.work_dir / "users" + # New structure: archive/users/username/snapshots/YYYYMMDD/example.com/snap-uuid-here/output.ext + users_dir = self.work_dir / "archive" / "users" snapshots_base = None if users_dir.exists(): @@ -656,7 +881,7 @@ class TestFilesystemMigration08to09(unittest.TestCase): print(f"[*] New structure base: {snapshots_base}") # Count files in new structure - # Structure: users/{username}/snapshots/YYYYMMDD/{domain}/{uuid}/files... + # Structure: archive/users/{username}/snapshots/YYYYMMDD/{domain}/{uuid}/files... files_new_structure = [] new_sample_files = {} @@ -679,6 +904,18 @@ class TestFilesystemMigration08to09(unittest.TestCase): print(f"[*] Files in new structure: {files_new_count}") print(f"[*] Sample files in new structure: {len(new_sample_files)}") + migrated_2021_files = list(users_dir.glob("*/snapshots/20210101/*/*/favicon.ico")) + self.assertGreater( + len(migrated_2021_files), + 0, + "Legacy snapshot should be bucketed by normalized bookmarked_at, not created_at/import time", + ) + + crawl_snapshot_links = list(users_dir.glob("*/crawls/*/*/*/snapshots/*/*")) + crawl_snapshot_symlinks = [path for path in crawl_snapshot_links if path.is_symlink()] + crawl_dirs = list(users_dir.glob("*/crawls/*/*/*")) + print(f"[*] Crawl snapshot symlinks: {len(crawl_snapshot_symlinks)}") + # Check old structure (should be gone or empty) old_archive_dir = self.work_dir / "archive" old_files_remaining = [] @@ -705,6 +942,17 @@ class TestFilesystemMigration08to09(unittest.TestCase): "No files found in new structure after update", ) + self.assertGreater( + len(crawl_snapshot_symlinks), + 0, + "No crawl snapshot symlinks created for migrated snapshots", + ) + + self.assertFalse( + any((crawl_dir / "index.jsonl").exists() for crawl_dir in crawl_dirs), + "Migrated crawl dirs should match normal 0.9 crawl dirs and not add crawl index.jsonl files", + ) + # CRITICAL: Verify old structure is cleaned up self.assertEqual( old_files_count, diff --git a/archivebox/tests/test_persona_runtime.py b/archivebox/tests/test_persona_runtime.py index 1ce0ac9a..d1a4e3d3 100644 --- a/archivebox/tests/test_persona_runtime.py +++ b/archivebox/tests/test_persona_runtime.py @@ -196,7 +196,7 @@ def test_get_config_raises_for_missing_persona_id(initialized_archive): import django django.setup() - from archivebox.config.configset import get_config + from archivebox.config.common import get_config from archivebox.crawls.models import Crawl from archivebox.personas.models import Persona diff --git a/archivebox/tests/test_recursive_crawl.py b/archivebox/tests/test_recursive_crawl.py index 5fcfdf65..d26d27e4 100644 --- a/archivebox/tests/test_recursive_crawl.py +++ b/archivebox/tests/test_recursive_crawl.py @@ -185,7 +185,7 @@ def test_parser_extractors_emit_snapshot_jsonl(tmp_path, process, recursive_test if status == "succeeded" and output: assert "parsed" in output.lower(), "Parser summary should report parsed URLs" - urls_jsonl_files = list(Path("users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl")) + urls_jsonl_files = list(Path("archive/users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl")) assert urls_jsonl_files, "parse_html_urls should write urls.jsonl output" records = [] @@ -485,7 +485,7 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p assert len([row for row in parser_results if row[3] == "failed"]) <= 2 assert len([row for row in wget_results if row[2] == "failed"]) <= 2 - urls_jsonl_files = list(Path("users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl")) + urls_jsonl_files = list(Path("archive/users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl")) assert urls_jsonl_files, "parse_html_urls should write urls.jsonl files" parsed_urls = set() for path in urls_jsonl_files: @@ -495,7 +495,7 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p assert set(recursive_test_site["child_urls"]).issubset(parsed_urls) assert set(recursive_test_site["deep_urls"]).issubset(parsed_urls) - snapshot_dirs = [path.parent for path in Path("users/system/snapshots").rglob("index.jsonl")] + snapshot_dirs = [path.parent for path in Path("archive/users/system/snapshots").rglob("index.jsonl")] assert snapshot_dirs for snapshot_dir in snapshot_dirs: assert (snapshot_dir / "index.jsonl").exists() diff --git a/archivebox/tests/test_runner.py b/archivebox/tests/test_runner.py index fbc87e76..60238388 100644 --- a/archivebox/tests/test_runner.py +++ b/archivebox/tests/test_runner.py @@ -6,6 +6,7 @@ from pathlib import Path from types import SimpleNamespace import pytest +from asgiref.sync import sync_to_async from django.test import RequestFactory @@ -84,6 +85,7 @@ class _DummyService: pass +@pytest.mark.django_db(transaction=True) def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl @@ -107,8 +109,10 @@ def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch): created_buses: list[_DummyBus] = [] + original_create_bus = runner_module.create_bus + def fake_create_bus(*, name, total_timeout=3600.0, **kwargs): - bus = _DummyBus(name) + bus = original_create_bus(name=name, total_timeout=total_timeout, **kwargs) created_buses.append(bus) return bus @@ -119,7 +123,6 @@ def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch): monkeypatch.setattr(runner_module, "BinaryService", _DummyService) monkeypatch.setattr(runner_module, "TagService", _DummyService) monkeypatch.setattr(runner_module, "CrawlService", _DummyService) - monkeypatch.setattr(runner_module, "SnapshotService", _DummyService) monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService) monkeypatch.setattr(runner_module, "_emit_machine_config", lambda *args, **kwargs: asyncio.sleep(0)) monkeypatch.setattr(runner_module, "setup_abx_services", lambda *args, **kwargs: None) @@ -156,17 +159,11 @@ def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch): monkeypatch.setattr(crawl_runner, "load_snapshot_payload", lambda snapshot_id: snapshot_data[snapshot_id]) monkeypatch.setattr(crawl_runner, "enqueue_discovered_snapshots_from_outputs", lambda snapshot: asyncio.sleep(0)) - async def run_both(): - await asyncio.gather( - crawl_runner.run_snapshot(str(snapshot_a.id)), - crawl_runner.run_snapshot(str(snapshot_b.id)), - ) - - asyncio.run(run_both()) + asyncio.run(crawl_runner.run_crawl(str(snapshot_a.id), [str(snapshot_a.id), str(snapshot_b.id)])) from abx_dl.events import SnapshotEvent - snapshot_events = [event for event in crawl_runner.bus.emitted if isinstance(event, SnapshotEvent)] + snapshot_events = asyncio.run(crawl_runner.bus.filter(SnapshotEvent, past=True)) assert len(snapshot_events) == 2 assert {event.snapshot_id for event in snapshot_events} == {str(snapshot_a.id), str(snapshot_b.id)} assert {event.url for event in snapshot_events} == {snapshot_a.url, snapshot_b.url} @@ -174,6 +171,7 @@ def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch): assert len(created_buses) == 1 +@pytest.mark.django_db(transaction=True) def test_run_snapshot_does_not_wait_for_crawl_background_daemons(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl @@ -190,14 +188,12 @@ def test_run_snapshot_does_not_wait_for_crawl_background_daemons(monkeypatch): status=Snapshot.StatusChoices.QUEUED, ) - monkeypatch.setattr(runner_module, "create_bus", lambda **kwargs: _NoIdleBus(kwargs["name"])) monkeypatch.setattr(runner_module, "discover_plugins", lambda: {}) monkeypatch.setattr(runner_module, "HookProcessService", _DummyService) monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService) monkeypatch.setattr(runner_module, "BinaryService", _DummyService) monkeypatch.setattr(runner_module, "TagService", _DummyService) monkeypatch.setattr(runner_module, "CrawlService", _DummyService) - monkeypatch.setattr(runner_module, "SnapshotService", _DummyService) monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService) monkeypatch.setattr(runner_module, "_emit_machine_config", lambda *args, **kwargs: asyncio.sleep(0)) monkeypatch.setattr(runner_module, "setup_abx_services", lambda *args, **kwargs: None) @@ -207,7 +203,57 @@ def test_run_snapshot_does_not_wait_for_crawl_background_daemons(monkeypatch): monkeypatch.setattr(crawl_runner, "load_snapshot_payload", lambda snapshot_id: snapshot_payload) monkeypatch.setattr(crawl_runner, "enqueue_discovered_snapshots_from_outputs", lambda snapshot: asyncio.sleep(0)) - asyncio.run(crawl_runner.run_snapshot(str(snapshot.id))) + crawl_runner.bus.wait_until_idle = _NoIdleBus("unused").wait_until_idle + asyncio.run(crawl_runner.run_crawl(str(snapshot.id), [str(snapshot.id)])) + + +@pytest.mark.django_db(transaction=True) +def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus(): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import CrawlRunner + from abx_dl.events import CrawlAbortEvent, CrawlEvent + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.STARTED, + ) + runner = CrawlRunner(crawl) + + async def run() -> CrawlAbortEvent | None: + abort_event_holder: dict[str, CrawlAbortEvent | None] = {"event": None} + + async def on_CrawlEvent(event: CrawlEvent) -> None: + watcher = asyncio.create_task(runner.watch_for_cancelled_crawl(event, poll_interval=0.01)) + await asyncio.sleep(0.02) + await sync_to_async(Crawl.objects.filter(id=crawl.id).update, thread_sensitive=True)( + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + abort_event = await runner.bus.find(CrawlAbortEvent, child_of=event, past=True, future=1.0) + abort_event_holder["event"] = abort_event if isinstance(abort_event, CrawlAbortEvent) else None + await watcher + + runner.bus.on(CrawlEvent, on_CrawlEvent) + await runner.bus.emit( + CrawlEvent( + url=snapshot.url, + snapshot_id=str(snapshot.id), + output_dir=str(crawl.output_dir), + ), + ).now() + await runner.bus.wait_until_idle() + return abort_event_holder["event"] + + abort_event = asyncio.run(run()) + + assert abort_event is not None @pytest.mark.django_db(transaction=True) @@ -429,7 +475,6 @@ def test_runner_prepare_refreshes_network_interface_and_attaches_current_process proc = _Proc() monkeypatch.setattr(runner_module, "discover_plugins", lambda: {}) - monkeypatch.setattr(runner_module, "create_bus", lambda **kwargs: _DummyBus(kwargs["name"])) monkeypatch.setattr(runner_module, "HookProcessService", _DummyService) monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService) monkeypatch.setattr(runner_module, "BinaryService", _DummyService) @@ -439,12 +484,20 @@ def test_runner_prepare_refreshes_network_interface_and_attaches_current_process monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService) from archivebox.machine.models import NetworkInterface, Process - from archivebox.config import configset as configset_module + from archivebox.config import common as config_common refresh_calls = [] monkeypatch.setattr(NetworkInterface, "current", classmethod(lambda cls, refresh=False: refresh_calls.append(refresh) or _Iface())) monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc)) - monkeypatch.setattr(configset_module, "get_config", lambda **kwargs: {"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60}) + original_get_config = config_common.get_config + monkeypatch.setattr( + config_common, + "get_config", + lambda **kwargs: original_get_config( + overrides={"PLUGINS": "", "CHROME_BINARY": "", "CHROME_KEEPALIVE": False, "TIMEOUT": 60}, + **kwargs, + ), + ) crawl_runner = runner_module.CrawlRunner(crawl) crawl_runner.load_run_state() @@ -458,7 +511,7 @@ def test_runner_prepare_refreshes_network_interface_and_attaches_current_process def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch): from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.services import runner as runner_module - from archivebox.config import configset as configset_module + from archivebox.config import common as config_common from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl @@ -496,7 +549,12 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch): ) monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc)) monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine)) - monkeypatch.setattr(configset_module, "get_config", lambda **kwargs: {"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60}) + original_get_config = config_common.get_config + monkeypatch.setattr( + config_common, + "get_config", + lambda **kwargs: original_get_config(overrides={"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60}, **kwargs), + ) crawl_runner = runner_module.CrawlRunner(crawl) crawl_runner.load_run_state() @@ -510,7 +568,7 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch): def test_load_run_state_does_not_force_chrome_keepalive(monkeypatch): from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.services import runner as runner_module - from archivebox.config import configset as configset_module + from archivebox.config import common as config_common from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl @@ -543,18 +601,23 @@ def test_load_run_state_does_not_force_chrome_keepalive(monkeypatch): ) monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc)) monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine)) - monkeypatch.setattr(configset_module, "get_config", lambda **kwargs: {"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60}) + original_get_config = config_common.get_config + monkeypatch.setattr( + config_common, + "get_config", + lambda **kwargs: original_get_config(overrides={"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60}, **kwargs), + ) crawl_runner = runner_module.CrawlRunner(crawl) crawl_runner.load_run_state() - assert "CHROME_KEEPALIVE" not in crawl_runner.base_config + assert crawl_runner.base_config["CHROME_KEEPALIVE"] is False def test_load_run_state_uses_enabled_plugins_when_plugins_key_missing(monkeypatch): from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.services import runner as runner_module - from archivebox.config import configset as configset_module + from archivebox.config import common as config_common from archivebox import hooks as hooks_module from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl @@ -589,7 +652,12 @@ def test_load_run_state_uses_enabled_plugins_when_plugins_key_missing(monkeypatc ) monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc)) monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine)) - monkeypatch.setattr(configset_module, "get_config", lambda **kwargs: {"CHROME_BINARY": "", "TIMEOUT": 60}) + original_get_config = config_common.get_config + monkeypatch.setattr( + config_common, + "get_config", + lambda **kwargs: original_get_config(overrides={"CHROME_BINARY": "", "TIMEOUT": 60}, **kwargs), + ) monkeypatch.setattr( hooks_module, "discover_hooks", @@ -610,6 +678,7 @@ def test_load_run_state_uses_enabled_plugins_when_plugins_key_missing(monkeypatc assert len(snapshot_ids) == 1 +@pytest.mark.django_db(transaction=True) def test_run_snapshot_skips_descendant_when_max_size_already_reached(monkeypatch, tmp_path): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl @@ -622,7 +691,6 @@ def test_run_snapshot_skips_descendant_when_max_size_already_reached(monkeypatch ) monkeypatch.setattr(runner_module, "discover_plugins", lambda: {}) - monkeypatch.setattr(runner_module, "create_bus", lambda **kwargs: _DummyBus(kwargs["name"])) monkeypatch.setattr(runner_module, "HookProcessService", _DummyService) monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService) monkeypatch.setattr(runner_module, "BinaryService", _DummyService) @@ -660,7 +728,24 @@ def test_run_snapshot_skips_descendant_when_max_size_already_reached(monkeypatch } crawl_runner.seal_snapshot_due_to_limit = lambda snapshot_id: cancelled.append(snapshot_id) - asyncio.run(crawl_runner.run_snapshot("child-1")) + async def run_in_crawl_start_context() -> None: + from abx_dl.events import CrawlStartEvent + + async def run_child_snapshot(event: CrawlStartEvent) -> None: + await crawl_runner.run_snapshot("child-1") + + crawl_runner.bus.on(CrawlStartEvent, run_child_snapshot) + await crawl_runner.bus.emit( + CrawlStartEvent( + url="https://example.com", + snapshot_id="child-1", + output_dir=str(tmp_path), + event_timeout=0, + event_handler_timeout=0, + ), + ).now() + + asyncio.run(run_in_crawl_start_context()) assert cancelled == ["child-1"] @@ -731,6 +816,30 @@ def test_seal_snapshot_cancels_queued_descendants_after_max_size(): assert child.retry_at is None +def test_sealed_crawl_does_not_create_discovered_snapshots(): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.SEALED, + retry_at=None, + max_depth=3, + ) + root = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + ) + + assert crawl.create_snapshots_from_urls() == [] + assert crawl.create_discovered_snapshot(root, url="https://example.com/child", depth=1) is None + assert crawl.snapshot_set.count() == 1 + + def test_create_crawl_api_queues_crawl_without_spawning_runner(monkeypatch): from django.contrib.auth import get_user_model from archivebox.api.v1_crawls import CrawlCreateSchema, create_crawl @@ -792,10 +901,7 @@ def test_crawl_runner_does_not_seal_unfinished_crawl(monkeypatch): }, ) monkeypatch.setattr(runner_module.CrawlRunner, "_create_live_ui", lambda self: None) - monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl_setup", lambda self, snapshot_id: asyncio.sleep(0)) - monkeypatch.setattr(runner_module.CrawlRunner, "enqueue_snapshot", lambda self, snapshot_id: asyncio.sleep(0)) - monkeypatch.setattr(runner_module.CrawlRunner, "wait_for_snapshot_tasks", lambda self: asyncio.sleep(0)) - monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl_cleanup", lambda self, snapshot_id: asyncio.sleep(0)) + monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl", lambda self, root_snapshot_id, snapshot_ids: asyncio.sleep(0)) monkeypatch.setattr(runner_module.CrawlRunner, "finalize_run_state", lambda self: None) asyncio.run(runner_module.CrawlRunner(crawl, snapshot_ids=[str(snapshot.id)]).run()) @@ -845,10 +951,7 @@ def test_crawl_runner_calls_load_and_finalize_run_state(monkeypatch): }, ) monkeypatch.setattr(runner_module.CrawlRunner, "_create_live_ui", lambda self: None) - monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl_setup", lambda self, snapshot_id: asyncio.sleep(0)) - monkeypatch.setattr(runner_module.CrawlRunner, "enqueue_snapshot", lambda self, snapshot_id: asyncio.sleep(0)) - monkeypatch.setattr(runner_module.CrawlRunner, "wait_for_snapshot_tasks", lambda self: asyncio.sleep(0)) - monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl_cleanup", lambda self, snapshot_id: asyncio.sleep(0)) + monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl", lambda self, root_snapshot_id, snapshot_ids: asyncio.sleep(0)) monkeypatch.setenv("DJANGO_ALLOW_ASYNC_UNSAFE", "true") method_calls: list[str] = [] @@ -916,7 +1019,7 @@ def test_wait_for_snapshot_tasks_returns_after_completed_tasks_are_pruned(): asyncio.run(run_test()) -def test_crawl_runner_calls_crawl_cleanup_after_snapshot_phase(monkeypatch): +def test_crawl_runner_calls_crawl_lifecycle(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot @@ -947,20 +1050,18 @@ def test_crawl_runner_calls_crawl_cleanup_after_snapshot_phase(monkeypatch): }, ) monkeypatch.setattr(runner_module.CrawlRunner, "_create_live_ui", lambda self: None) - monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl_setup", lambda self, snapshot_id: asyncio.sleep(0)) - monkeypatch.setattr(runner_module.CrawlRunner, "enqueue_snapshot", lambda self, snapshot_id: asyncio.sleep(0)) - monkeypatch.setattr(runner_module.CrawlRunner, "wait_for_snapshot_tasks", lambda self: asyncio.sleep(0)) + monkeypatch.setattr(runner_module.CrawlRunner, "finalize_run_state", lambda self: None) - cleanup_calls = [] + lifecycle_calls = [] monkeypatch.setattr( runner_module.CrawlRunner, - "run_crawl_cleanup", - lambda self, snapshot_id: cleanup_calls.append("abx_cleanup") or asyncio.sleep(0), + "run_crawl", + lambda self, root_snapshot_id, snapshot_ids: lifecycle_calls.append((root_snapshot_id, snapshot_ids)) or asyncio.sleep(0), ) asyncio.run(runner_module.CrawlRunner(crawl, snapshot_ids=[str(snapshot.id)]).run()) - assert cleanup_calls == ["abx_cleanup"] + assert lifecycle_calls == [(str(snapshot.id), [str(snapshot.id)])] def test_abx_process_service_background_process_finishes_after_process_exit(monkeypatch, tmp_path): @@ -1144,7 +1245,7 @@ def test_run_pending_crawls_prioritizes_queued_crawl_before_unrelated_binary_bac @pytest.mark.django_db(transaction=True) -def test_crawl_completed_event_does_not_seal_active_snapshots(): +def test_crawl_completed_event_requeues_active_snapshots(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot @@ -1186,7 +1287,53 @@ def test_crawl_completed_event_does_not_seal_active_snapshots(): crawl.refresh_from_db() assert crawl.status == Crawl.StatusChoices.STARTED - assert crawl.retry_at is None + assert crawl.retry_at is not None + + +@pytest.mark.django_db(transaction=True) +def test_crawl_cleanup_event_requeues_unfinished_crawl(): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.crawl_service import CrawlService + from abx_dl.events import CrawlCleanupEvent + from abx_dl.orchestrator import create_bus + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.STARTED, + retry_at=None, + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.QUEUED, + retry_at=None, + ) + + bus = create_bus(name=f"test_crawl_cleanup_requeues_unfinished_{str(crawl.id).replace('-', '_')}") + CrawlService(bus, crawl_id=str(crawl.id)) + try: + + async def emit_cleanup() -> None: + event = CrawlCleanupEvent( + url="https://example.com", + snapshot_id=str(snapshot.id), + output_dir=str(crawl.output_dir), + ) + emitted = bus.emit(event) + await emitted.now() + await emitted.event_results_list() + + asyncio.run(emit_cleanup()) + finally: + asyncio.run(bus.wait_until_idle()) + asyncio.run(bus.destroy()) + + crawl.refresh_from_db() + assert crawl.status == Crawl.StatusChoices.STARTED + assert crawl.retry_at is not None @pytest.mark.django_db(transaction=True) @@ -1307,7 +1454,8 @@ def test_snapshot_completed_event_bus_seals_finished_crawl(): ) bus = create_bus(name=f"test_snapshot_completed_bus_finished_crawl_{str(crawl.id).replace('-', '_')}") - SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: asyncio.sleep(0)) + service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: asyncio.sleep(0)) + assert service is not None try: async def emit_completed() -> None: @@ -1318,7 +1466,7 @@ def test_snapshot_completed_event_bus_seals_finished_crawl(): output_dir=str(snapshot.output_dir), ), ) - await emitted.now() + await emitted.wait() await emitted.event_results_list() asyncio.run(emit_completed()) diff --git a/archivebox/tests/test_schedule_e2e.py b/archivebox/tests/test_schedule_e2e.py index 9f85a915..973cc607 100644 --- a/archivebox/tests/test_schedule_e2e.py +++ b/archivebox/tests/test_schedule_e2e.py @@ -519,8 +519,8 @@ def test_web_ui_add_depth_two_crawls_and_renders_real_outputs_over_running_serve assert ("wget", "succeeded") in result_statuses assert any(plugin.endswith("parse_html_urls") and status == "succeeded" for plugin, status in result_statuses) assert len([status for _plugin, status, _files, _size in archive_results if status == "failed"]) <= 2 - assert list((tmp_path / "users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl")) - assert list((tmp_path / "users/system/snapshots").rglob("wget/**/*.html")) + assert list((tmp_path / "archive/users/system/snapshots").rglob("parse_html_urls/**/urls.jsonl")) + assert list((tmp_path / "archive/users/system/snapshots").rglob("wget/**/*.html")) progress = requests.get( f"http://127.0.0.1:{port}/admin/live-progress/", diff --git a/archivebox/tests/test_snapshot.py b/archivebox/tests/test_snapshot.py index ac8609d5..4147ec51 100644 --- a/archivebox/tests/test_snapshot.py +++ b/archivebox/tests/test_snapshot.py @@ -48,7 +48,7 @@ def test_snapshot_creates_snapshot_with_correct_url(tmp_path, process, disable_e domain = urlparse(snapshot_url).hostname or "unknown" # Verify crawl symlink exists and is relative - target_path = tmp_path / "users" / username / "snapshots" / snapshot_date_str / domain / snapshot_id + target_path = tmp_path / "archive" / "users" / username / "snapshots" / snapshot_date_str / domain / snapshot_id symlinks = [p for p in tmp_path.rglob(str(snapshot_id)) if p.is_symlink()] assert symlinks, "Snapshot symlink should exist under crawl dir" link_path = symlinks[0] diff --git a/archivebox/tests/test_urls.py b/archivebox/tests/test_urls.py index 37bcbb08..a1e4d9df 100644 --- a/archivebox/tests/test_urls.py +++ b/archivebox/tests/test_urls.py @@ -49,7 +49,8 @@ def _build_script(body: str) -> str: from django.contrib.auth import get_user_model from archivebox.core.models import Snapshot, ArchiveResult - from archivebox.config.common import SERVER_CONFIG + from archivebox.config.common import get_config + SERVER_CONFIG = get_config() from archivebox.core.host_utils import ( get_admin_host, get_admin_base_url, diff --git a/archivebox/workers/management/commands/runner_watch.py b/archivebox/workers/management/commands/runner_watch.py index 9cb82152..671ce2dc 100644 --- a/archivebox/workers/management/commands/runner_watch.py +++ b/archivebox/workers/management/commands/runner_watch.py @@ -23,7 +23,7 @@ class Command(BaseCommand): import psutil - from archivebox.config.common import STORAGE_CONFIG + from archivebox.config.common import get_config from archivebox.machine.models import Machine, Process from archivebox.workers.supervisord_util import ( RUNNER_WORKER, @@ -35,7 +35,7 @@ class Command(BaseCommand): pidfile = kwargs.get("pidfile") or os.environ.get("ARCHIVEBOX_RUNSERVER_PIDFILE") if not pidfile: - pidfile = str(STORAGE_CONFIG.TMP_DIR / "runserver.pid") + pidfile = str(get_config().TMP_DIR / "runserver.pid") interval = max(0.2, float(kwargs.get("interval", 1.0))) last_pid = None diff --git a/archivebox/workers/supervisord_util.py b/archivebox/workers/supervisord_util.py index 9304a6a4..9dc836df 100644 --- a/archivebox/workers/supervisord_util.py +++ b/archivebox/workers/supervisord_util.py @@ -577,12 +577,12 @@ def watch_worker(supervisor, daemon_name, interval=5): def start_server_workers(host="0.0.0.0", port="8000", daemonize=False, debug=False, reload=False, nothreading=False): - from archivebox.config.common import STORAGE_CONFIG + from archivebox.config.common import get_config supervisor = get_or_create_supervisord_process(daemonize=daemonize) if debug: - pidfile = str(STORAGE_CONFIG.TMP_DIR / "runserver.pid") if reload else None + pidfile = str(get_config().TMP_DIR / "runserver.pid") if reload else None server_worker = RUNSERVER_WORKER(host=host, port=port, reload=reload, pidfile=pidfile, nothreading=nothreading) bg_workers: list[tuple[dict[str, str], bool]] = ( [(RUNNER_WORKER, True), (RUNNER_WATCH_WORKER(pidfile), False)] if reload else [(RUNNER_WORKER, False)] diff --git a/pyproject.toml b/pyproject.toml index 18df0053..5585b995 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,10 +78,10 @@ dependencies = [ "w3lib>=2.2.1", # used for parsing content-type encoding from http response headers & html tags ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management - "abxbus>=2.5.0", # EventBus API + "abxbus>=2.5.4", # EventBus API "abxpkg>=1.10.7", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.10.54", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.10.49", # shared ArchiveBox downloader package with blocking install preflight + "abx-plugins>=1.10.55", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.10.50", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]