diff --git a/archivebox/api/v1_cli.py b/archivebox/api/v1_cli.py index 67a298ed..2ad48699 100644 --- a/archivebox/api/v1_cli.py +++ b/archivebox/api/v1_cli.py @@ -8,10 +8,8 @@ from enum import Enum from django.http import HttpRequest from ninja import Router, Schema -from pydantic import Field from archivebox.misc.util import ansi_to_html -from archivebox.config.common import get_config # from .auth import API_AUTH_METHODS @@ -67,8 +65,7 @@ class AddCommandSchema(Schema): snapshot_max_size: int = 0 parser: str = "auto" plugins: str = "" - update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW) - overwrite: bool = False + only_new: bool | None = None index_only: bool = False @@ -92,8 +89,7 @@ class ScheduleCommandSchema(Schema): every: str | None = None tag: str = "" depth: int = 0 - overwrite: bool = False - update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW) + only_new: bool | None = None clear: bool = False @@ -121,6 +117,9 @@ class RemoveCommandSchema(Schema): def cli_add(request: HttpRequest, args: AddCommandSchema): from archivebox.cli.archivebox_add import add + config_overrides: dict[str, object] = {} + if args.only_new is not None: + config_overrides["ONLY_NEW"] = bool(args.only_new) crawl, snapshots = add( urls=args.urls, snapshot_ids=args.snapshot_ids, @@ -130,13 +129,12 @@ def cli_add(request: HttpRequest, args: AddCommandSchema): crawl_max_size=args.crawl_max_size, crawl_timeout=args.crawl_timeout, snapshot_max_size=args.snapshot_max_size, - update=args.update, index_only=args.index_only, - overwrite=args.overwrite, plugins=args.plugins, parser=args.parser, bg=True, # Always run in background for API calls created_by_id=request.user.pk, + config=config_overrides or None, ) snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)] @@ -188,6 +186,9 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema): def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema): from archivebox.cli.archivebox_schedule import schedule + config_overrides: dict[str, object] = {} + if args.only_new is not None: + config_overrides["ONLY_NEW"] = bool(args.only_new) result = schedule( import_path=args.import_path, add=args.add, @@ -199,8 +200,7 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema): every=args.every, tag=args.tag, depth=args.depth, - overwrite=args.overwrite, - update=args.update, + config=config_overrides or None, ) stdout = getattr(request, "stdout", None) diff --git a/archivebox/api/v1_core.py b/archivebox/api/v1_core.py index 68c56c93..8e490564 100644 --- a/archivebox/api/v1_core.py +++ b/archivebox/api/v1_core.py @@ -32,7 +32,7 @@ from archivebox.core.permissions import public_snapshots_queryset from archivebox.api.auth import auth_using_token from archivebox.config.common import get_config from archivebox.core.host_utils import build_web_url -from archivebox.misc.util import validate_url_length +from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length from archivebox.core.tag_utils import ( add_snapshot_counts, build_tag_cards, @@ -786,7 +786,8 @@ def _filter_snapshots_for_rss( ) crawl_id = crawl_id.strip() if crawl_id: - queryset = queryset.filter(crawl__id__icontains=crawl_id) + matching_crawl_pks = list(filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).values_list("pk", flat=True)[:100]) + queryset = queryset.filter(crawl_id__in=matching_crawl_pks) created_by = created_by.strip() if created_by: @@ -845,7 +846,7 @@ def _snapshots_rss_response( class SnapshotFilterSchema(FilterSchema): - id: Annotated[str | None, FilterLookup(["id__icontains", "timestamp__startswith"])] = None + id: Annotated[str | None, FilterLookup(["id__istartswith", "id__iendswith", "timestamp__startswith"])] = None created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None @@ -856,7 +857,9 @@ class SnapshotFilterSchema(FilterSchema): modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None search: Annotated[ str | None, - FilterLookup(["url__icontains", "title__icontains", "tags__name__icontains", "id__icontains", "timestamp__startswith"]), + FilterLookup( + ["url__icontains", "title__icontains", "tags__name__icontains", "id__istartswith", "id__iendswith", "timestamp__startswith"], + ), ] = None url: Annotated[str | None, FilterLookup("url")] = None tag: Annotated[str | None, FilterLookup("tags__name")] = None @@ -919,7 +922,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema): raise HttpError(400, "depth must be between 0 and 4") if data.crawl_id: - crawl = Crawl.objects.get(id__icontains=data.crawl_id) + crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), data.crawl_id).get() crawl_tags = normalize_tag_list(crawl.tags_str.split(",")) tags = tags or crawl_tags else: @@ -976,7 +979,7 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS try: snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id)) except Snapshot.DoesNotExist: - snapshot = Snapshot.objects.get(Q(id__icontains=snapshot_id)) + snapshot = filter_queryset_by_uuid_substring(Snapshot.objects.all(), snapshot_id).get() payload = data.dict(exclude_unset=True) update_fields = ["modified_at"] @@ -1252,6 +1255,7 @@ def search_tags( "tags": build_tag_cards( query=q, request=request, + preview_limit=0, sort=normalized_sort, created_by=normalized_created_by, year=normalized_year, diff --git a/archivebox/api/v1_crawls.py b/archivebox/api/v1_crawls.py index 58113258..11fef6d7 100644 --- a/archivebox/api/v1_crawls.py +++ b/archivebox/api/v1_crawls.py @@ -14,8 +14,15 @@ from ninja import Router, Schema from ninja.errors import HttpError from archivebox.core.models import Snapshot +from archivebox.core.permissions import ( + PERMISSIONS_PUBLIC, + PERMISSIONS_UNLISTED, + is_admin_user, + normalize_permissions, +) from archivebox.config.common import get_config from archivebox.crawls.models import Crawl +from archivebox.misc.util import filter_queryset_by_uuid_substring from .auth import API_AUTH_METHODS, auth_using_token @@ -127,7 +134,7 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn """Get a specific Crawl by id.""" setattr(request, "with_snapshots", with_snapshots) setattr(request, "with_archiveresults", with_archiveresults) - crawl = Crawl.objects.get(id__icontains=crawl_id) + crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get() if crawl and as_rss: query = request.GET.copy() @@ -139,21 +146,38 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn def crawl_file(request: HttpRequest, crawl_id: str, path: str): + # Try to resolve the crawl first; if it doesn't exist, return 404. + try: + crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get() + except Crawl.DoesNotExist: + raise HttpError(404, "Crawl not found") + + # Determine the effective viewer: session user takes precedence, otherwise + # fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or + # Authorization: Bearer ... (so that programmatic clients still work). user = getattr(request, "user", None) - is_superuser = bool( - getattr(user, "is_authenticated", False) and getattr(user, "is_active", False) and getattr(user, "is_superuser", False), - ) - if not is_superuser: + is_authenticated = bool(getattr(user, "is_authenticated", False) and getattr(user, "is_active", False)) + if not is_authenticated: token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") auth_header = request.headers.get("Authorization", "") if not token and auth_header.lower().startswith("bearer "): token = auth_header.split(None, 1)[1].strip() token_user = auth_using_token(token=token, request=request) if token else None - is_superuser = bool(token_user and token_user.is_active and token_user.is_superuser) - if not is_superuser: - raise HttpError(403, "Permission denied") + if token_user and token_user.is_active: + user = token_user + is_authenticated = True + # Re-bind so is_admin_user() / ownership checks below see the token user. + setattr(request, "user", token_user) + + # Gate access using the same model as SnapshotView/can_view_snapshot: + # admins always pass; owners can see their own crawls; otherwise the crawl + # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls. + if not is_admin_user(request): + permissions = normalize_permissions(crawl.permissions) + is_owner = bool(is_authenticated and getattr(crawl, "created_by_id", None) == getattr(user, "id", None)) + if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}: + raise HttpError(404, "Crawl not found") - crawl = Crawl.objects.get(id__icontains=crawl_id) crawl_root = Path(crawl.output_dir).resolve() file_path = (crawl_root / path).resolve() if not file_path.is_file() or crawl_root not in file_path.parents: @@ -185,7 +209,7 @@ def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfol @router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl") def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema): """Update a crawl (e.g., set status=sealed to cancel queued work).""" - crawl = Crawl.objects.get(id__icontains=crawl_id) + crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get() payload = data.dict(exclude_unset=True) update_fields = ["modified_at"] @@ -227,7 +251,7 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema): @router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl") def delete_crawl(request: HttpRequest, crawl_id: str): - crawl = Crawl.objects.get(id__icontains=crawl_id) + crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get() crawl_id_str = str(crawl.id) snapshot_count = crawl.snapshot_set.count() deleted_count, _ = crawl.delete() diff --git a/archivebox/base_models/admin.py b/archivebox/base_models/admin.py index f3f697a8..606dd7b1 100644 --- a/archivebox/base_models/admin.py +++ b/archivebox/base_models/admin.py @@ -48,7 +48,7 @@ class KeyValueWidget(forms.Widget): from archivebox.hooks import discover_plugin_configs options: dict[str, ConfigOption] = {} - skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "CRAWL_OUTPUT_DIR", "SNAP_DIR"} + skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "SNAP_DIR"} for key, field in ArchiveBoxConfig.model_fields.items(): if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys: continue @@ -227,9 +227,6 @@ class KeyValueWidget(forms.Widget): return 'Example: ["value"]'; }} if (types.includes('object')) {{ - if (key === 'SAVE_ALLOWLIST' || key === 'SAVE_DENYLIST') {{ - return 'Example: {{"^https://example\\\\.com": ["wget"]}}'; - }} return 'Example: {{"key": "value"}}'; }} return ''; @@ -238,8 +235,6 @@ class KeyValueWidget(forms.Widget): function isRegexConfigKey_{widget_id}(key) {{ return key === 'URL_ALLOWLIST' || key === 'URL_DENYLIST' || - key === 'SAVE_ALLOWLIST' || - key === 'SAVE_DENYLIST' || key.endsWith('_PATTERN') || key.includes('REGEX'); }} @@ -633,17 +628,74 @@ class KeyValueWidget(forms.Widget): window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id}; + function focusConfigKeyFromHash_{widget_id}() {{ + // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly + // to (or creates) the matching row in this editor. Used by + // the in-banner "pin via admin" link and the + // ``→ Edit in Machine.config`` shortcut on the live + // config detail page. + var hash = (window.location.hash || '').replace(/^#/, '').trim(); + if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{ + return; + }} + var container = document.getElementById('{widget_id}_rows'); + if (!container) {{ + return; + }} + var match = null; + container.querySelectorAll('.key-value-row').forEach(function(row) {{ + if (match) {{ return; }} + var keyInput = row.querySelector('.kv-key'); + if (keyInput && keyInput.value.trim() === hash) {{ + match = row; + }} + }}); + if (!match) {{ + // No existing row for this key — prepopulate one with the + // key filled in but value left blank so the operator just + // types/pastes the value and hits save. + window.addKeyValueRow_{widget_id}(); + var rows = container.querySelectorAll('.key-value-row'); + match = rows[rows.length - 1]; + var keyInput = match.querySelector('.kv-key'); + if (keyInput) {{ + keyInput.value = hash; + keyInput.dispatchEvent(new Event('input', {{ bubbles: true }})); + }} + }} + if (!match) {{ + return; + }} + match.scrollIntoView({{ behavior: 'smooth', block: 'center' }}); + var prevOutline = match.style.outline; + match.style.outline = '2px solid #f59e0b'; + match.style.outlineOffset = '2px'; + match.style.transition = 'outline 1.2s ease-out'; + setTimeout(function() {{ + match.style.outline = prevOutline || 'none'; + }}, 1400); + var valueInput = match.querySelector('.kv-value'); + if (valueInput) {{ + valueInput.focus(); + try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}} + }} + }} + // Initialize on load document.addEventListener('DOMContentLoaded', function() {{ initializeRows_{widget_id}(); updateHiddenField_{widget_id}(); + focusConfigKeyFromHash_{widget_id}(); }}); // Also run immediately in case DOM is already ready if (document.readyState !== 'loading') {{ initializeRows_{widget_id}(); updateHiddenField_{widget_id}(); + focusConfigKeyFromHash_{widget_id}(); }} + window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id}); + // Update on any input change var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows'); diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index 036c049f..dfba0284 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -16,6 +16,14 @@ if "--debug" in sys.argv: os.environ["DEBUG"] = "True" sys.argv.remove("--debug") +# Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`, +# `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before +# the subcommand executes. Strip it from argv here so each subcommand's own click parser +# never sees it. Ignored for `help` and `init` themselves. +if "--init" in sys.argv: + sys.argv = [arg for arg in sys.argv if arg != "--init"] + os.environ["ARCHIVEBOX_WANTS_INIT"] = "1" + class ArchiveBoxGroup(click.Group): """lazy loading click group for archivebox commands""" @@ -172,6 +180,16 @@ def cli(ctx, help=False): from archivebox.misc.checks import check_data_folder, check_migrations setup_django() + if os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" and subcommand not in ("init", "help"): + # Universal `--init` was passed: build/upgrade the data folder before + # the regular preflight runs, so it succeeds on a fresh dir and an + # out-of-date schema both. Drop the env var afterwards so spawned + # subprocesses (supervisord workers, daphne, runner, etc.) inherit + # a clean env and don't re-trigger init in every child. + from archivebox.cli.archivebox_init import init as archivebox_init + + archivebox_init(quick=True) + os.environ.pop("ARCHIVEBOX_WANTS_INIT", None) check_data_folder() if subcommand != "update": check_migrations(auto_apply=True) diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 4aace1b6..280e0e43 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -9,7 +9,7 @@ import json import os from pathlib import Path -from typing import TYPE_CHECKING +from typing import Any, TYPE_CHECKING import rich_click as click @@ -58,11 +58,10 @@ def add( parser: str = "auto", plugins: str = "", persona: str = "Default", - overwrite: bool = False, - update: bool | None = None, index_only: bool = False, bg: bool = False, created_by_id: int | None = None, + config: dict[str, Any] | None = None, ) -> tuple[Crawl, QuerySet[Snapshot]]: """Add a new URL or list of URLs to your archive. @@ -87,10 +86,11 @@ def add( from archivebox.config.permissions import USER, HOSTNAME from archivebox.config.common import get_config - config = get_config() + config_overrides = dict(config or {}) + runtime_config = get_config() crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None if crawl_max_concurrent_snapshots is None: - crawl_max_concurrent_snapshots = config.CRAWL_MAX_CONCURRENT_SNAPSHOTS + crawl_max_concurrent_snapshots = runtime_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots) if depth not in (0, 1, 2, 3, 4): @@ -119,8 +119,6 @@ def add( created_by_id = created_by_id or get_or_create_system_user_pk() created_by = get_user_model().objects.filter(pk=created_by_id).first() started_at = timezone.now() - if update is None: - update = not config.ONLY_NEW if isinstance(urls, str): url_list = [line.strip() for line in urls.splitlines() if line.strip()] @@ -157,9 +155,7 @@ def add( crawl_config = { "PERMISSIONS": str(effective_persona_config.PERMISSIONS), - **({"ONLY_NEW": not update} if bool(not update) != bool(effective_persona_config.ONLY_NEW) else {}), **({"INDEX_ONLY": True} if index_only else {}), - **({"OVERWRITE": True} if overwrite else {}), **({"PLUGINS": plugins} if plugins else {}), **( {"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots} @@ -175,6 +171,11 @@ def add( **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}), **({"URL_DENYLIST": url_denylist} if url_denylist else {}), } + # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest + # priority — they win over persona/plugin/env defaults and get stamped + # directly onto crawl.config so the runtime resolution and admin UI both + # reflect them faithfully. + crawl_config.update(config_overrides) crawl = Crawl.objects.create( urls=urls_content, @@ -314,8 +315,13 @@ def add( @click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)") @click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...") @click.option("--persona", default="Default", help="Authentication profile to use when archiving") -@click.option("--overwrite", "-F", is_flag=True, help="Overwrite existing data if URLs have been archived previously") -@click.option("--update", is_flag=True, default=None, help="Retry any previously skipped/failed URLs when re-adding them") +@click.option( + "--only-new/--no-only-new", + "only_new", + default=None, + help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). " + "Pass --no-only-new to force re-archive of URLs that already exist.", +) @click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now") @click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)") @click.argument("urls", nargs=-1, type=click.Path()) @@ -345,6 +351,12 @@ def main(**kwargs): if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1: raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots") + # Translate --only-new/--no-only-new into a crawl config override. + # add() takes config overrides as a dict; no per-flag kwargs. + only_new = kwargs.pop("only_new", None) + if only_new is not None: + kwargs["config"] = {"ONLY_NEW": bool(only_new)} + add(urls=urls, **kwargs) diff --git a/archivebox/cli/archivebox_remove.py b/archivebox/cli/archivebox_remove.py index bff95c66..67b9d3d8 100644 --- a/archivebox/cli/archivebox_remove.py +++ b/archivebox/cli/archivebox_remove.py @@ -3,11 +3,13 @@ __package__ = "archivebox.cli" __command__ = "archivebox remove" +import time from pathlib import Path from collections.abc import Iterable import rich_click as click +from django.db import OperationalError from django.db.models import QuerySet from archivebox.config import DATA_DIR @@ -62,13 +64,44 @@ def remove( log_list_finished(snapshots) log_removal_started(snapshots, yes=yes) - to_remove = snapshots.count() - - from archivebox.search import flush_search_index from archivebox.core.models import Snapshot + from archivebox.search import flush_search_index + + # Freeze the target set up-front so a concurrent daemon writing new + # snapshots can't extend the deletion under us, and so the cursor isn't + # held open across the per-row deletes below. + snapshot_pks = list(snapshots.values_list("pk", flat=True)) + to_remove = len(snapshot_pks) + + # Search-index flush touches a separate backend (FTS / sonic), not the + # main index.sqlite3 writer lock, so it's safe to do once up front. + flush_search_index(snapshots=Snapshot.objects.filter(pk__in=snapshot_pks)) + + # Delete one snapshot at a time. Each ``.delete()`` is its own short + # Django-atomic block, so the writer lock is released between rows and + # an in-flight daemon transaction can interleave instead of deadlocking. + # Filesystem cleanup for each row is scheduled via ``transaction.on_commit`` + # in ``base_models/models.py`` and runs AFTER its row's tx commits — so + # rmtree doesn't hold the lock either. + # + # The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock + # errors when called inside an atomic block (because it can't safely + # release+reacquire a transaction), so we wrap each row's delete in our + # own retry loop at this outer (non-atomic) level. Each attempt is a + # fresh atomic; an exception cleanly rolls it back before we sleep. + retry_interval = 1.0 + retry_timeout = 60.0 + for pk in snapshot_pks: + deadline = time.monotonic() + retry_timeout + while True: + try: + Snapshot.objects.filter(pk=pk).delete() + break + except OperationalError as err: + if "database is locked" not in str(err) or time.monotonic() >= deadline: + raise + time.sleep(retry_interval) - flush_search_index(snapshots=snapshots) - snapshots.delete() all_snapshots = Snapshot.objects.all() log_removal_finished(all_snapshots.count(), to_remove) diff --git a/archivebox/cli/archivebox_schedule.py b/archivebox/cli/archivebox_schedule.py index a43a3350..f5854f86 100644 --- a/archivebox/cli/archivebox_schedule.py +++ b/archivebox/cli/archivebox_schedule.py @@ -6,7 +6,6 @@ import rich_click as click from rich import print from archivebox.misc.util import enforce_types, docstring -from archivebox.config.common import get_config @enforce_types @@ -20,9 +19,8 @@ def schedule( every: str | None = None, tag: str = "", depth: int | str = 0, - overwrite: bool = False, - update: bool | None = None, import_path: str | None = None, + config: dict[str, object] | None = None, ): """Manage database-backed scheduled crawls processed by the crawl runner.""" @@ -33,9 +31,7 @@ def schedule( from archivebox.crawls.schedule_utils import validate_schedule from archivebox.services.runner import run_pending_crawls - if update is None: - update = not get_config().ONLY_NEW - + config_overrides = dict(config or {}) depth = int(depth) result: dict[str, object] = { "created_schedule_ids": [], @@ -79,10 +75,12 @@ def schedule( status=Crawl.StatusChoices.SEALED, retry_at=None, config={ - "ONLY_NEW": not update, - "OVERWRITE": overwrite, "DEPTH": 0 if is_update_schedule else depth, "SCHEDULE_KIND": "update" if is_update_schedule else "crawl", + # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the + # template defaults. Anything left unset falls through to the + # standard config stack at crawl-resolution time. + **config_overrides, }, ) crawl_schedule = CrawlSchedule.objects.create( @@ -163,8 +161,13 @@ def schedule( default="0", help="Recursively archive linked pages up to N hops away", ) -@click.option("--overwrite", is_flag=True, help="Overwrite existing data if URLs have been archived previously") -@click.option("--update", is_flag=True, help="Retry previously failed/skipped URLs when scheduled crawls run") +@click.option( + "--only-new/--no-only-new", + "only_new", + default=None, + help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). " + "Pass --no-only-new to force re-archive on each scheduled run.", +) @click.option("--clear", is_flag=True, help="Disable all currently enabled schedules") @click.option("--show", is_flag=True, help="Print all currently enabled schedules") @click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)") @@ -173,6 +176,9 @@ def schedule( @docstring(schedule.__doc__) def main(**kwargs): """Manage database-backed scheduled crawls processed by the crawl runner.""" + only_new = kwargs.pop("only_new", None) + if only_new is not None: + kwargs["config"] = {"ONLY_NEW": bool(only_new)} schedule(**kwargs) diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index 0d39d403..5fb2c5dc 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -15,11 +15,193 @@ from rich import print from archivebox.misc.util import docstring, enforce_types +import re as _re + +_IPV4_RE = _re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$") +_IPV6_CHARS_RE = _re.compile(r"^[0-9a-fA-F:.]+$") +_LOCAL_BIND_HOSTS = frozenset({"0.0.0.0", "::", "::0", "127.0.0.1", "::1"}) + + +def _is_ipv4_literal(host: str) -> bool: + return bool(_IPV4_RE.match(host)) + + +def _is_ipv6_literal(host: str) -> bool: + # Bracketed (e.g. ``[2001:db8::1]``) or bare form. Require at least two + # colons so we don't catch random strings with one ``:``. + stripped = host.strip("[]") + return stripped.count(":") >= 2 and bool(_IPV6_CHARS_RE.match(stripped)) + + +def _bind_host_looks_like_ip(host: str) -> bool: + if not host or host in _LOCAL_BIND_HOSTS: + return False + return _is_ipv4_literal(host) or _is_ipv6_literal(host) + + +def _split_bind_spec(spec: str) -> tuple[str, str]: + """Split a ``host:port`` / ``host`` / ``port`` spec into ``(host, port)``. + + The empty strings stand in for "not provided"; the caller fills in + defaults. Bracketed IPv6 literals like ``[::1]:8000`` are handled. + """ + spec = (spec or "").strip() + if not spec: + return "", "" + if spec.startswith("["): + # Bracketed IPv6: ``[::1]`` or ``[::1]:8000`` + end = spec.find("]") + if end == -1: + return spec, "" # malformed; let validator reject it + host = spec[: end + 1] + rest = spec[end + 1 :] + if rest.startswith(":"): + return host, rest[1:] + return host, "" + if ":" in spec: + host, _, port = spec.rpartition(":") + return host, port + # Bare token: digits = port, anything else = host + if spec.isdigit(): + return "", spec + return spec, "" + + +def _parse_and_validate_bind_spec(spec: str) -> tuple[str, str]: + """Resolve a CLI/config bind spec to ``(host, port)`` or hard-error. + + Accepts only IP literals (v4 or v6) or the special string ``localhost`` + (normalized to ``127.0.0.1``). Bare hostnames are rejected because the + bind address feeds Daphne, which has to listen on a numeric address; + public hostnames belong in ``BASE_URL`` instead. Empty values fall back + to ``127.0.0.1`` / ``8000``. + """ + raw_host, raw_port = _split_bind_spec(spec) + host = raw_host.strip() + port = (raw_port or "").strip() or "8000" + + if host == "" or host.lower() == "localhost": + host = "127.0.0.1" + elif _is_ipv4_literal(host) or _is_ipv6_literal(host): + pass + else: + print( + f"[red][X] Invalid BIND_ADDR host {host!r}: must be an IP literal or 'localhost'.[/red]", + ) + print( + "[red] Hostnames like archive.example.com are not valid bind addresses — Daphne[/red]", + ) + print( + "[red] listens on numeric addresses only. Bind to 0.0.0.0 and set BASE_URL instead:[/red]", + ) + print( + f"[red] BASE_URL=https://{host} archivebox server 0.0.0.0:{port}[/red]", + ) + sys.exit(1) + + try: + port_int = int(port) + except ValueError: + print(f"[red][X] Invalid BIND_ADDR port {port!r}: must be an integer 1-65535.[/red]") + sys.exit(1) + if not (0 < port_int < 65536): + print(f"[red][X] Invalid BIND_ADDR port {port_int}: must be 1-65535.[/red]") + sys.exit(1) + + return host, port + + +def _print_server_startup_warnings(config, host: str, *, base_url_explicit: bool) -> None: + """Print startup-time security / routing warnings for the server command. + + Runs only from ``archivebox server`` so other entry points (manage shell, + plugin lookups, etc.) don't repeat this banner on every config load. + """ + if config.IS_LOWER_SECURITY_MODE: + print( + f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={config.SERVER_SECURITY_MODE}[/yellow]", + ) + print("[yellow] Archived pages may share an origin with privileged app routes in this mode.[/yellow]") + print("[yellow] To switch to the safer isolated setup:[/yellow]") + print("[yellow] 1. Set SERVER_SECURITY_MODE=safe-subdomains-fullreplay[/yellow]") + print("[yellow] 2. Point *.archivebox.localhost (or your chosen base domain) at this server[/yellow]") + print( + "[yellow] 3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]", + ) + print() + + if base_url_explicit: + return + + # If the user is upgrading from 0.7.3 and already had + # CSRF_TRUSTED_ORIGINS set, get_base_url() will silently use that as the + # implicit BASE_URL. Surface what we picked so the user knows where their + # links / redirects are going — and tell them how to make it explicit. + from archivebox.core.host_utils import derive_base_url_from_csrf + + csrf_derived = derive_base_url_from_csrf(config) + if csrf_derived: + print( + f"[yellow][!] BASE_URL is not set; auto-derived [bold]{csrf_derived}[/bold] from a single CSRF_TRUSTED_ORIGINS entry.[/yellow]", + ) + print( + "[yellow] Links / redirects / cookies will use that origin. To silence this hint, set BASE_URL[/yellow]", + ) + print( + f"[yellow] explicitly: [bold]BASE_URL={csrf_derived}[/bold] (matches your existing CSRF_TRUSTED_ORIGINS).[/yellow]", + ) + print() + return + + # BASE_URL was not set explicitly. The host_utils derivation gives one of + # three results, with very different risk profiles — show a tailored hint + # so new users coming from the 0.7.x single-domain world know whether the + # default is fine for them or needs attention. + if _bind_host_looks_like_ip(host): + # Real IP literal: subdomain routing can't work, URLs leak the IP. + # This is the most urgent case. + print( + f"[yellow][!] WARNING: BASE_URL is not set and BIND_ADDR resolves to an IP literal ({host}).[/yellow]", + ) + print( + "[yellow] Snapshot / admin / api URLs will be generated with the IP, and subdomain[/yellow]", + ) + print( + "[yellow] routing cannot work against an IP address. Set BASE_URL explicitly, e.g.[/yellow]", + ) + print( + "[yellow] BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]", + ) + if config.USES_SUBDOMAIN_ROUTING: + print( + "[yellow] Or switch SERVER_SECURITY_MODE to a one-domain mode if you can't run a hostname.[/yellow]", + ) + print() + else: + # Loopback / wildcard bind. The host_utils default of + # http://archivebox.localhost:PORT works in a browser on the same + # machine, but anything else (reverse proxy, k8s ingress, LAN client) + # needs BASE_URL set. (Real hostnames can't reach this branch — the + # bind validator rejects them upfront.) + print( + "[yellow][!] BASE_URL is not set. Generated URLs will fall back to http://archivebox.localhost:.[/yellow]", + ) + print( + "[yellow] That's fine for local browsing on this machine. Set BASE_URL when running behind[/yellow]", + ) + print( + "[yellow] a reverse proxy / ingress / public hostname, e.g.[/yellow]", + ) + print( + "[yellow] BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]", + ) + print() + + @enforce_types def server( runserver_args: Iterable[str] | None = None, reload: bool = False, - init: bool = False, debug: bool = False, daemonize: bool = False, nothreading: bool = False, @@ -30,12 +212,6 @@ def server( config = get_config() runserver_args = list(runserver_args or (config.BIND_ADDR,)) - if init: - from archivebox.cli.archivebox_init import init as archivebox_init - - archivebox_init(quick=True) - print() - run_in_debug = config.DEBUG or debug or reload if debug or reload: os.environ["DEBUG"] = "True" @@ -50,20 +226,12 @@ def server( print(" [green]archivebox manage createsuperuser[/green]") print() - host = "127.0.0.1" - port = "8000" - - try: - host_and_port = [arg for arg in runserver_args if arg.replace(".", "").replace(":", "").isdigit()][0] - if ":" in host_and_port: - host, port = host_and_port.split(":") - else: - if "." in host_and_port: - host = host_and_port - else: - port = host_and_port - except IndexError: - pass + # First non-empty positional arg is the bind spec; otherwise inherit from + # config (which defaults to "127.0.0.1:8000"). _parse_and_validate_bind_spec + # hard-errors on hostnames so the rest of the server can assume a numeric + # bind host. + bind_spec = next((arg for arg in runserver_args if arg), "") + host, port = _parse_and_validate_bind_spec(bind_spec) if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1": from archivebox.config import CONSTANTS @@ -137,6 +305,15 @@ def server( ) print(" > Writing ArchiveBox error log to ./logs/errors.log") print() + + # Reload config after we've set os.environ["BIND_ADDR"] above so the + # security-mode + base-url warnings see the effective values. + runtime_config = get_config() + _print_server_startup_warnings( + runtime_config, + host, + base_url_explicit=bool(os.environ.get("BASE_URL", "").strip()), + ) bind_url = f"http://{host}:{port}" command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url) @@ -196,7 +373,6 @@ def server( @click.option("--reload", is_flag=True, help="Enable auto-reloading when code or templates change") @click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors") @click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode") -@click.option("--init", is_flag=True, help="Run a full archivebox init/upgrade before starting the server") @click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon") @docstring(server.__doc__) def main(**kwargs): diff --git a/archivebox/config/common.py b/archivebox/config/common.py index c670cc55..dfc455da 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -33,7 +33,6 @@ PluginSchemaDocuments = dict[str, dict[str, Any]] _STDOUT_CONSOLE = Console() _STDERR_CONSOLE = Console(stderr=True) -_WARNED_SERVER_SECURITY_MODES: set[str] = set() _WARNED_ARCHIVING_CONFIGS: set[tuple[int, bool]] = set() @@ -126,13 +125,9 @@ class StorageConfig(BaseConfigSet): CUSTOM_TEMPLATES_DIR: Path = Field(default=CONSTANTS.CUSTOM_TEMPLATES_DIR) OUTPUT_PERMISSIONS: str = Field(default="644") - RESTRICT_FILE_NAMES: str = Field(default="windows") ENFORCE_ATOMIC_WRITES: bool = Field(default=True) ALLOW_NO_UNIX_SOCKETS: bool = Field(default=False, alias="ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS") - # not supposed to be user settable: - DIR_OUTPUT_PERMISSIONS: str = Field(default="755") # computed from OUTPUT_PERMISSIONS - class GeneralConfig(BaseConfigSet): toml_section_header: str = "GENERAL_CONFIG" @@ -158,7 +153,6 @@ class ServerConfig(BaseConfigSet): SERVER_SECURITY_MODE: str = Field(default="safe-subdomains-fullreplay") SNAPSHOTS_PER_PAGE: int = Field(default=40) - PREVIEW_ORIGINALS: bool = Field(default=True) FOOTER_INFO: str = Field( default="Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.", ) @@ -241,39 +235,6 @@ class DatabaseConfig(BaseConfigSet): SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0) -def _print_server_security_mode_warning(config: ServerConfig) -> None: - if not config.IS_LOWER_SECURITY_MODE: - return - if config.SERVER_SECURITY_MODE in _WARNED_SERVER_SECURITY_MODES: - return - - rprint( - f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={config.SERVER_SECURITY_MODE}[/yellow]", - file=sys.stderr, - ) - rprint( - "[yellow] Archived pages may share an origin with privileged app routes in this mode.[/yellow]", - file=sys.stderr, - ) - rprint( - "[yellow] To switch to the safer isolated setup:[/yellow]", - file=sys.stderr, - ) - rprint( - "[yellow] 1. Set SERVER_SECURITY_MODE=safe-subdomains-fullreplay[/yellow]", - file=sys.stderr, - ) - rprint( - "[yellow] 2. Point *.archivebox.localhost (or your chosen base domain) at this server[/yellow]", - file=sys.stderr, - ) - rprint( - "[yellow] 3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]", - file=sys.stderr, - ) - _WARNED_SERVER_SECURITY_MODES.add(config.SERVER_SECURITY_MODE) - - class ArchivingConfig(BaseConfigSet): toml_section_header: str = "ARCHIVING_CONFIG" @@ -285,17 +246,10 @@ class ArchivingConfig(BaseConfigSet): default="", description="Comma-separated plugin selection override used by the UI and API.", ) - ENABLED_EXTRACTORS: str = Field( - default="", - description="Legacy comma-separated plugin selection override.", - ) ONLY_NEW: bool = Field(default=True) - OVERWRITE: bool = Field(default=False) TIMEOUT: int = Field(default=60) - MAX_URL_ATTEMPTS: int = Field(default=50) - MAX_DEPTH: int = Field(default=0) CRAWL_MAX_URLS: int = Field(default=0) CRAWL_MAX_SIZE: int = Field(default=0) CRAWL_TIMEOUT: int = Field(default=0, description="Maximum total crawl runtime in seconds (0 = unlimited).") @@ -315,9 +269,6 @@ class ArchivingConfig(BaseConfigSet): URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST") URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST") - SAVE_ALLOWLIST: dict[str, list[str]] = Field(default={}) # mapping of regex patterns to list of archive methods - SAVE_DENYLIST: dict[str, list[str]] = Field(default={}) - DEFAULT_PERSONA: str = Field(default="Default") PERMISSIONS: str = Field( default="public", @@ -375,30 +326,6 @@ class ArchivingConfig(BaseConfigSet): def URL_DENYLIST_PTN(self) -> re.Pattern: return re.compile(self.URL_DENYLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS) - @property - def SAVE_ALLOWLIST_PTNS(self) -> dict[re.Pattern, list[str]]: - return ( - { - # regexp: methods list - re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val - for key, val in self.SAVE_ALLOWLIST.items() - } - if self.SAVE_ALLOWLIST - else {} - ) - - @property - def SAVE_DENYLIST_PTNS(self) -> dict[re.Pattern, list[str]]: - return ( - { - # regexp: methods list - re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val - for key, val in self.SAVE_DENYLIST.items() - } - if self.SAVE_DENYLIST - else {} - ) - def parse_delete_after(value) -> timedelta | None: if value is None: @@ -435,11 +362,7 @@ def parse_delete_after(value) -> timedelta | None: class SearchBackendConfig(BaseConfigSet): toml_section_header: str = "SEARCH_BACKEND_CONFIG" - USE_INDEXING_BACKEND: bool = Field(default=True) - USE_SEARCHING_BACKEND: bool = Field(default=True) - SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep") - SEARCH_PROCESS_HTML: bool = Field(default=True) def _plugin_user_config_value(value: Any) -> str: @@ -517,7 +440,6 @@ class ArchiveBoxBaseConfig( DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR) ABX_RUNTIME: str = Field(default="archivebox") CRAWL_DIR: Path | None = Field(default=None) - CRAWL_OUTPUT_DIR: Path | None = Field(default=None) SNAP_DIR: Path | None = Field(default=None) computed_config_keys: ClassVar[tuple[str, ...]] = COMPUTED_CONFIG_KEYS @@ -653,13 +575,8 @@ def get_config( scope_overrides.update(crawl.config) if crawl is not None: - crawl_output_dir = None - if not overrides or "CRAWL_OUTPUT_DIR" not in overrides or "CRAWL_DIR" not in overrides: - crawl_output_dir = crawl.output_dir - if not overrides or "CRAWL_OUTPUT_DIR" not in overrides: - scope_overrides["CRAWL_OUTPUT_DIR"] = crawl_output_dir if not overrides or "CRAWL_DIR" not in overrides: - scope_overrides["CRAWL_DIR"] = crawl_output_dir + scope_overrides["CRAWL_DIR"] = crawl.output_dir if snapshot is not None and snapshot.config: scope_overrides.update(snapshot.config) @@ -704,7 +621,6 @@ def get_config( if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS: config.warn_if_invalid() _WARNED_ARCHIVING_CONFIGS.add(archiving_warning_key) - _print_server_security_mode_warning(config) return config diff --git a/archivebox/config/configset.py b/archivebox/config/configset.py index b4c15efe..4fdb940b 100644 --- a/archivebox/config/configset.py +++ b/archivebox/config/configset.py @@ -22,8 +22,6 @@ COMPUTED_CONFIG_KEYS = ( "IS_LOWER_SECURITY_MODE", "URL_ALLOWLIST_PTN", "URL_DENYLIST_PTN", - "SAVE_ALLOWLIST_PTNS", - "SAVE_DENYLIST_PTNS", ) diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index 474abd34..481f0ed4 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -1605,7 +1605,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): from archivebox.cli.archivebox_add import add - add(urls=urls, bg=True) + # "Archive Now" is an explicit user re-archive — force ONLY_NEW=False + # on the resulting crawl so existing snapshots don't cause the crawl to + # seal immediately with zero new snapshots (the default ONLY_NEW=True + # would skip any URLs that have ever been archived before). + add(urls=urls, bg=True, config={"ONLY_NEW": False}) messages.success( request, diff --git a/archivebox/core/context_processors.py b/archivebox/core/context_processors.py new file mode 100644 index 00000000..623265e5 --- /dev/null +++ b/archivebox/core/context_processors.py @@ -0,0 +1,9 @@ +from archivebox.config import VERSION +from archivebox.config.version import get_COMMIT_HASH + + +def archivebox_globals(request): + return { + "VERSION": VERSION, + "STATIC_CACHE_KEY": (get_COMMIT_HASH() or VERSION or "dev").strip(), + } diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py index f698b74e..87a707d3 100644 --- a/archivebox/core/forms.py +++ b/archivebox/core/forms.py @@ -44,6 +44,7 @@ PLUGIN_GROUP_DEFINITIONS = ( "wget", "archivedotorg", "chrome_mhtml", + "archivewebpage", ), ), ( @@ -166,11 +167,9 @@ def get_plugin_choice_label(plugin_name: str, plugin_configs: dict[str, dict]) - icon_html = get_plugin_icon(plugin_name) return format_html( - '{}{}{}', + '{}{}', icon_html, plugin_name, - plugin_name, - description, ) @@ -288,6 +287,7 @@ class PluginConfigFormMixin: *PLUGIN_GROUP_DEFINITIONS, ("other_plugins", "Other", "", "", "", other_plugins), ) + binary_url_lookup = _build_required_binary_url_lookup(plugin_configs, runtime_config) self.plugin_groups = [ { "field_name": field_name, @@ -296,7 +296,7 @@ class PluginConfigFormMixin: "dom_id": dom_id, "select_all_group": select_all_group, "show_selectors": field_name in self.fields, - "plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config), + "plugins": self._build_plugin_cards(field_name, plugin_names, plugin_configs, runtime_config, binary_url_lookup), } for field_name, title, note, dom_id, select_all_group, plugin_names in group_specs if any(plugin in all_plugins for plugin in plugin_names) @@ -308,6 +308,7 @@ class PluginConfigFormMixin: plugin_names: Iterable[str], plugin_configs: dict[str, dict[str, Any]], runtime_config: Mapping[str, Any], + binary_url_lookup: Mapping[str, str] | None = None, ) -> list[dict[str, Any]]: if field_name in self.fields: choices = list(get_choice_field(self, field_name).choices) @@ -341,7 +342,11 @@ class PluginConfigFormMixin: "source_url": f"https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/{plugin_name}", "docs_url": f"https://archivebox.github.io/abx-plugins/#{plugin_name}", "required_plugins": [str(item) for item in schema.get("required_plugins") or []], - "required_binaries_count": len(schema.get("required_binaries") or []), + "required_binary_links": _build_required_binary_links( + schema.get("required_binaries") or [], + runtime_config, + binary_url_lookup, + ), "config_fields": config_fields, "config_count": len(config_fields), }, @@ -476,6 +481,96 @@ class PluginConfigFormMixin: } +_BINARY_TEMPLATE_PATTERN = re.compile(r"\{([A-Z_][A-Z0-9_]*)\}") + + +def _resolve_required_binary_name(template_name: str, runtime_config: Mapping[str, Any]) -> str: + if "{" not in template_name: + return template_name + + def _replace(match: re.Match[str]) -> str: + key = match.group(1) + try: + value = runtime_config.get(key) + except Exception: + value = None + if value is None or value == "": + return match.group(0) + return str(value) + + resolved = _BINARY_TEMPLATE_PATTERN.sub(_replace, template_name).strip() + if not resolved: + return template_name + return Path(resolved).name if "/" in resolved else resolved + + +def _iter_required_binary_names( + required_binaries: Iterable[Any], + runtime_config: Mapping[str, Any], +) -> Iterable[str]: + for item in required_binaries or []: + if not isinstance(item, dict): + continue + raw_name = str(item.get("name") or "").strip() + if not raw_name: + continue + resolved = _resolve_required_binary_name(raw_name, runtime_config) + if resolved: + yield resolved + + +def _build_required_binary_url_lookup( + plugin_configs: Mapping[str, dict[str, Any]], + runtime_config: Mapping[str, Any], +) -> dict[str, str]: + """Resolve admin URLs for every required binary across all plugin schemas in a single DB query.""" + from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url + from archivebox.machine.models import Binary, Machine + + resolved_names: set[str] = set() + for schema in plugin_configs.values(): + for name in _iter_required_binary_names(schema.get("required_binaries") or [], runtime_config): + resolved_names.add(name) + + if not resolved_names: + return {} + + machine = Machine.current() + name_to_binary: dict[str, Binary] = {} + for binary in ( + Binary.objects.filter(machine=machine, name__in=resolved_names) + .exclude(abspath="") + .exclude(abspath__isnull=True) + .order_by("-modified_at") + ): + key = binary.name.lower() + if key not in name_to_binary: + name_to_binary[key] = binary + + return { + name: (get_installed_binary_change_url(name, name_to_binary.get(name.lower())) or get_environment_binary_url(name)) + for name in resolved_names + } + + +def _build_required_binary_links( + required_binaries: list[dict[str, Any]], + runtime_config: Mapping[str, Any], + binary_url_lookup: Mapping[str, str] | None = None, +) -> list[dict[str, str]]: + from archivebox.config.views import get_environment_binary_url + + links: list[dict[str, str]] = [] + seen: set[str] = set() + for resolved in _iter_required_binary_names(required_binaries, runtime_config): + if resolved in seen: + continue + seen.add(resolved) + url = (binary_url_lookup or {}).get(resolved) or get_environment_binary_url(resolved) + links.append({"name": resolved, "url": url}) + return links + + def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str, str]: from archivebox.config.views import get_environment_binary_url, get_installed_binary_change_url from archivebox.machine.models import Binary, Machine diff --git a/archivebox/core/host_utils.py b/archivebox/core/host_utils.py index bfbe90c3..5648370a 100644 --- a/archivebox/core/host_utils.py +++ b/archivebox/core/host_utils.py @@ -9,6 +9,7 @@ from archivebox.config.common import get_config _SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$") _SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P[0-9a-fA-F]{12})$") +_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api", "public") def split_host_port(host: str) -> tuple[str, str | None]: @@ -29,13 +30,84 @@ def _normalize_base_url(value: str | None) -> str: parsed = urlparse(base) if not parsed.netloc: return "" - return f"{parsed.scheme}://{parsed.netloc}" + # Accept ``*.`` as a synonym for ```` so users can paste the + # wildcard-friendly form (e.g. from the banner suggestion) without it + # leaking ``*.`` into every downstream URL. Subdomain routing already + # prepends the appropriate role label (admin/web/api/snap-*) at build + # time, so the bare base host is what we want to store. + netloc = parsed.netloc + while netloc.startswith("*."): + netloc = netloc[2:] + if not netloc: + return "" + return f"{parsed.scheme}://{netloc}" def normalize_base_url(value: str | None) -> str: return _normalize_base_url(value) +def _csrf_trusted_origins(config) -> list[str]: + raw = (config.CSRF_TRUSTED_ORIGINS or "").strip() + if not raw: + return [] + seen: list[str] = [] + for entry in raw.split(","): + normalized = _normalize_base_url(entry.strip()) + if normalized and normalized not in seen: + seen.append(normalized) + return seen + + +def _allowed_hosts(config) -> set[str]: + raw = (config.ALLOWED_HOSTS or "").strip() + if not raw: + return set() + return {entry.strip().lower() for entry in raw.split(",") if entry.strip() and entry.strip() != "*"} + + +def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + """Pick a single CSRF_TRUSTED_ORIGINS entry to act as the implicit BASE_URL. + + 0.7.3 → 0.9.0 upgrade path: any reverse-proxied 0.7.3 deployment already + had ``CSRF_TRUSTED_ORIGINS=https://archive.example.com`` set (required for + admin login to work). On upgrade, ``BASE_URL`` is the new knob — but it + defaults to empty, and falling through to ``BIND_ADDR`` produces an + unreachable URL like ``http://0.0.0.0:8000``. If the user has exactly one + CSRF origin we treat it as the implicit BASE_URL so links/redirects keep + pointing at the public hostname they already configured. + + Returns ``""`` when the inference is ambiguous (multiple origins) or + impossible (none set) so callers fall through to their next strategy. + """ + config = config or get_config(**config_kwargs) + origins = _csrf_trusted_origins(config) + if len(origins) == 1: + return origins[0] + return "" + + +def request_host_is_explicitly_allowed(request_host: str, config) -> bool: + """True if the incoming Host is in ALLOWED_HOSTS or matches a CSRF origin. + + Used by ``get_base_url`` to honour the request's own Host header when the + operator hasn't pinned ``BASE_URL`` explicitly. The check is intentionally + strict — we won't trust arbitrary Host headers, only ones the user has + already opted into via ALLOWED_HOSTS / CSRF_TRUSTED_ORIGINS. + """ + if not request_host: + return False + host, _ = split_host_port(request_host) + allowed = _allowed_hosts(config) + if host in allowed: + return True + for origin in _csrf_trusted_origins(config): + origin_host, _ = split_host_port(urlparse(origin).netloc) + if origin_host == host: + return True + return False + + def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) return (config.BIND_ADDR or "").strip() @@ -50,10 +122,48 @@ def _with_port(host: str, port: str | None) -> str: return f"{host}:{port}" if port else host +def strip_role_subdomain(host: str) -> str: + """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``public.`` / ``snap-*.`` + labels from a host (preserving the port). Strips repeatedly so an + already-compounded host like ``snap-X.snap-X.`` reduces all the + way down to ````. + + Used when we want to recover the canonical base host from a request that + arrived on a role subdomain — otherwise builders that prepend their own + role label (e.g. ``snap-X.``) compound onto the existing prefix and you + get ``snap-X.snap-X.snap-X.`` on every click. + """ + if not host: + return "" + hostname, port = split_host_port(host) + while hostname and "." in hostname: + head, _sep, rest = hostname.partition(".") + if head in _ROLE_SUBDOMAIN_LABELS or _SNAPSHOT_SUBDOMAIN_RE.match(head): + hostname = rest + continue + break + return _with_port(hostname, port) + + def _is_local_bind_host(host: str) -> bool: return host in {"", "0.0.0.0", "::", "127.0.0.1", "::1", "localhost"} +def canonical_base_host_for_request(request_host: str) -> str: + """Strip role subdomains and remap loopback hostnames to ``archivebox.localhost``. + + Used by the banner suggestion and the in-browser pin endpoint: when the + user is hitting the server on raw ``localhost:9292`` or ``127.0.0.1:9292`` + we want to suggest the wildcard-friendly ``archivebox.localhost`` family + instead, so the eventual pinned ``BASE_URL`` plays nicely with subdomain + routing without forcing the user to add a /etc/hosts entry. + """ + hostname, port = split_host_port(strip_role_subdomain(request_host or "")) + if _is_local_bind_host(hostname): + hostname = "archivebox.localhost" + return _with_port(hostname, port) + + def _root_host_from_listen(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) listen_host, listen_port = get_listen_parts(config=config) @@ -67,6 +177,13 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw if override: return override + # A) Implicit BASE_URL from a single CSRF_TRUSTED_ORIGINS entry. Catches + # 0.7.3 → 0.9.0 upgrades where users already set CSRF_TRUSTED_ORIGINS + # for their reverse-proxy login but never set BASE_URL. + csrf_derived = derive_base_url_from_csrf(config) + if csrf_derived: + return csrf_derived + scheme = request.scheme if request else "http" if request: req_host, req_port = split_host_port(request.get_host()) @@ -74,6 +191,17 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw return f"{scheme}://{_with_port('archivebox.localhost', req_port)}" if _is_local_bind_host(req_host): return f"{scheme}://{_with_port('archivebox.localhost', req_port)}" + # C) Per-request fallback: when ``BASE_URL`` is unset and CSRF didn't + # give us a single origin, trust the request's Host header — but first + # peel off any ``admin.`` / ``web.`` / ``api.`` / ``public.`` / + # ``snap-*.`` label. Otherwise the URL builders below prepend their own + # role label onto a host that already carries one, producing the + # ``snap-X.snap-X.snap-X.`` compounding bug. Django has already + # admitted the host via ALLOWED_HOSTS; the misconfig banner surfaces + # the case where the resulting URL doesn't match what the operator + # probably intended. + canonical_host = strip_role_subdomain(request.get_host()) + return f"{scheme}://{canonical_host}" root_host = _root_host_from_listen(config=config) return f"{scheme}://{root_host}" if root_host else "" diff --git a/archivebox/core/middleware.py b/archivebox/core/middleware.py index d5df2a36..0c0a654e 100644 --- a/archivebox/core/middleware.py +++ b/archivebox/core/middleware.py @@ -21,6 +21,7 @@ from archivebox.core.host_utils import ( build_web_url, get_api_host, get_admin_host, + get_base_host, get_listen_host, get_listen_subdomain, get_public_host, @@ -36,14 +37,32 @@ ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in" def _admin_login_hint_cookie_domain(config) -> str | None: + """Resolve the parent domain to scope the cross-subdomain login hint. + + NOTE: this cookie carries only the single bit "user is logged in on + admin somewhere"; it MUST NOT be confused with the session cookie, + which stays admin-host-scoped (see core/settings.py + SESSION_COOKIE_DOMAIN comment — admin/public is a security boundary). + + Returns the hostname portion of ``get_base_host`` (which respects + ``BASE_URL`` and falls back to the local-bind mapping). Strips the + port — cookie ``Domain=`` attributes don't include ports. Returns + ``None`` when subdomain routing is off, the base host is empty, or + the base host is an IP / bare ``localhost`` (browsers reject + cross-host cookies for those). + """ if not config.USES_SUBDOMAIN_ROUTING: return None - listen_host, _listen_port = split_host_port(get_listen_host(config=config)) + base_host = get_base_host(config=config) + if not base_host: + return None + hostname, _port = split_host_port(base_host) + if not hostname or hostname == "localhost": + return None try: - ipaddress.ip_address(listen_host) + ipaddress.ip_address(hostname) except ValueError: - if listen_host and listen_host != "localhost": - return listen_host + return hostname return None @@ -166,12 +185,32 @@ def HostRoutingMiddleware(get_response): if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}: return get_response(request) + # In subdomain mode with no explicit BASE_URL we can't safely emit + # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the + # request's own Host (via the request-host fallback in get_base_url), + # so prepending ``admin.`` to whatever the client sent produces a + # redirect chain of ``admin.admin.admin.``. Pass the request + # through; the misconfig banner on the rendered page tells the user + # to pin BASE_URL so the redirects can resume. + if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL: + return get_response(request) + if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host): + # ``/add`` is admin-only unless ``PUBLIC_ADD_VIEW`` is on. Without + # this redirect, hitting it on public.* falls into AddView's + # auth check, bounces through ``/accounts/login/?next=/add/`` → + # ``/admin/login/?next=/add/``, and Django admin's LoginView + # silently drops ``next`` when the user already has an admin + # session — dumping the user on the admin homepage instead of + # the add form. Routing the request to admin.* directly lets + # AddView run on the host where the session lives. + add_should_redirect = not config.PUBLIC_ADD_VIEW and (request.path == "/add" or request.path.startswith("/add/")) if ( request.path == "/admin" or request.path.startswith("/admin/") or request.path == "/accounts" or request.path.startswith("/accounts/") + or add_should_redirect ): target = build_admin_url(request.path, request=request) if request.META.get("QUERY_STRING"): @@ -262,7 +301,13 @@ def HostRoutingMiddleware(get_response): target = f"{target}?{request.META['QUERY_STRING']}" return redirect(target) - if admin_host or web_host: + if (admin_host or web_host) and config.BASE_URL: + # Only force a canonical-host redirect when BASE_URL was set + # explicitly. If BASE_URL is empty (e.g. 0.7.3 → 0.9.0 upgrade + # where the user has CSRF_TRUSTED_ORIGINS but never set BASE_URL), + # the subdomain we'd redirect to may not actually resolve in the + # user's reverse proxy — serve the request as-is instead and let + # the misconfig banner surface the problem in the page. target = build_web_url(request.path, request=request) if target: if request.META.get("QUERY_STRING"): diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 974046c3..d2994db3 100755 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -278,12 +278,6 @@ class SnapshotQuerySet(models.QuerySet): def search(self, patterns: list[str]) -> "SnapshotQuerySet": """Search snapshots using the configured search backend""" from archivebox.search import query_search_index - from archivebox.misc.logging import stderr - - if not get_config().USE_SEARCHING_BACKEND: - stderr() - stderr("[X] The search backend is not enabled, set config.USE_SEARCHING_BACKEND = True", color="red") - raise SystemExit(2) qsearch = self.none() for pattern in patterns: @@ -3060,8 +3054,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW from archivebox.misc.logging_util import printable_filesize output_dir = Path(out_dir) if out_dir is not None else self.output_dir - config = get_config() - SAVE_ARCHIVE_DOT_ORG = config.get("SAVE_ARCHIVE_DOT_ORG", True) TITLE_LOADING_MSG = "Not yet archived..." preview_priority = [ @@ -3104,8 +3096,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW "status": "archived" if is_archived else "not yet archived", "status_color": "success" if is_archived else "danger", "oldest_archive_date": ts_to_date_str(self.oldest_archive_date), - "SAVE_ARCHIVE_DOT_ORG": SAVE_ARCHIVE_DOT_ORG, - "PREVIEW_ORIGINALS": config.PREVIEW_ORIGINALS, "best_preview_path": best_preview_path, "best_result": best_result, "archiveresults": outputs, diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 22e328ea..50ce16ff 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -210,6 +210,7 @@ TEMPLATES = [ "django.template.context_processors.request", "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", + "archivebox.core.context_processors.archivebox_globals", ], }, }, @@ -371,12 +372,17 @@ api_base_url = normalize_base_url(get_api_base_url()) if api_base_url and api_base_url not in CSRF_TRUSTED_ORIGINS: CSRF_TRUSTED_ORIGINS.append(api_base_url) -# automatically fix case when user sets ALLOWED_HOSTS (e.g. to archivebox.example.com) -# but forgets to add https://archivebox.example.com to CSRF_TRUSTED_ORIGINS +# Auto-extend CSRF_TRUSTED_ORIGINS with the explicit ALLOWED_HOSTS entries so +# users who set ALLOWED_HOSTS=archivebox.example.com don't have to also list +# https://archivebox.example.com under CSRF_TRUSTED_ORIGINS. (The previous +# per-host WARNING print was just noise — the auto-append below is the actual +# fix, and the effective CSRF_TRUSTED_ORIGINS gets surfaced once at startup +# from archivebox_server.py.) for hostname in ALLOWED_HOSTS: + if hostname == "*": + continue https_endpoint = f"https://{hostname}" - if hostname != "*" and https_endpoint not in CSRF_TRUSTED_ORIGINS: - print(f"[!] WARNING: {https_endpoint} from ALLOWED_HOSTS should be added to CSRF_TRUSTED_ORIGINS") + if https_endpoint not in CSRF_TRUSTED_ORIGINS: CSRF_TRUSTED_ORIGINS.append(https_endpoint) SECURE_BROWSER_XSS_FILTER = True @@ -386,6 +392,12 @@ SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin" CSRF_COOKIE_SECURE = False SESSION_COOKIE_SECURE = False SESSION_COOKIE_HTTPONLY = True +# Auth cookies are intentionally scoped to the exact host that set them so +# the admin session is NOT readable from public.* / web.* / api.* — that +# split is a security boundary, not a UX choice. Subdomains that need to +# render an "is the user logged in?" indicator must use the single-bit +# `archivebox_admin_logged_in` hint cookie set by core/middleware.py +# (which IS scoped to the listen-host parent), never widen these. SESSION_COOKIE_DOMAIN = None CSRF_COOKIE_DOMAIN = None SESSION_COOKIE_AGE = 1209600 # 2 weeks diff --git a/archivebox/core/settings_logging.py b/archivebox/core/settings_logging.py index 77d95755..dc52dcae 100644 --- a/archivebox/core/settings_logging.py +++ b/archivebox/core/settings_logging.py @@ -4,7 +4,6 @@ import re import os import tempfile import logging -from pathlib import Path from archivebox.config import CONSTANTS @@ -21,39 +20,6 @@ IGNORABLE_URL_PATTERNS = [ ] -SENSITIVE_QUERY_PARAM_RE = re.compile(r"(?i)([?&](?:api_key|token|access_token|password|secret)=)([^&#\s]+)") -WEBREQUEST_RE = re.compile(r"]*\bmethod=(?P[A-Z]+)\s+uri=(?P\S+)") -RUNNING_AT_RE = re.compile(r"running at\s+([^>]+:\d+)") - - -def _redact_url(url: str) -> str: - return SENSITIVE_QUERY_PARAM_RE.sub(r"\1[REDACTED]", url) - - -def _short_code_path(path: str) -> str: - try: - return str(Path(path).resolve().relative_to(Path.cwd().resolve())) - except (OSError, ValueError): - parts = Path(path).parts - return "/".join(parts[-4:]) if len(parts) > 4 else path - - -def _resolve_view_name(url: str) -> str: - try: - from django.urls import resolve - - match = resolve(url.split("?", 1)[0]) - if match.view_name: - return match.view_name - view_func = match.func - view_class = getattr(view_func, "view_class", None) - if view_class is not None: - return f"{view_class.__module__}.{view_class.__name__}" - return f"{view_func.__module__}.{view_func.__name__}" - except Exception: - return "unknown" - - class NoisyRequestsFilter(logging.Filter): def filter(self, record) -> bool: logline = record.getMessage() @@ -77,26 +43,27 @@ class NoisyRequestsFilter(logging.Filter): class DaphneCloseTimeoutFilter(logging.Filter): + """Drop daphne's noisy "killed slow response after client disconnect" warning. + + Daphne emits this whenever a request handler is still running when the + client disconnects (e.g. iframe gets navigated away mid-response while + fetching favicon / screenshot / preview html). For our use case these are + always benign — the disconnect is the browser cancelling a request, not a + server-side fault — so we suppress them outright rather than spamming + WARNING. Other daphne.server lines pass through unchanged. + """ + def filter(self, record) -> bool: if record.name != "daphne.server": return True logline = record.getMessage() - if not ( + if ( "Application instance" in logline and "for connection QuerySet[Tag]: sort = normalize_tag_sort(sort) has_snapshots = normalize_has_snapshots_filter(has_snapshots) - needs_snapshot_counts = with_snapshot_counts or sort.startswith("snapshots_") or has_snapshots != "all" + needs_snapshot_counts = sort.startswith("snapshots_") queryset = Tag.objects.select_related("created_by") if needs_snapshot_counts: @@ -83,10 +83,13 @@ def get_matching_tags( if year: queryset = queryset.filter(created_at__year=int(year)) + if has_snapshots != "all" and not needs_snapshot_counts: + queryset = queryset.annotate(has_snapshot=Exists(SnapshotTag.objects.filter(tag_id=OuterRef("pk")))) + if has_snapshots == "yes": - queryset = queryset.filter(num_snapshots__gt=0) + queryset = queryset.filter(num_snapshots__gt=0) if needs_snapshot_counts else queryset.filter(has_snapshot=True) elif has_snapshots == "no": - queryset = queryset.filter(num_snapshots=0) + queryset = queryset.filter(num_snapshots=0) if needs_snapshot_counts else queryset.filter(has_snapshot=False) if sort == "name_asc": queryset = queryset.order_by(Lower("name"), "id") @@ -252,12 +255,12 @@ def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = No "name": tag.name, "slug": tag.slug, "num_snapshots": count, - "filter_url": f"{reverse('admin:core_snapshot_changelist')}?tags__id__exact={tag.pk}", - "edit_url": reverse("admin:core_tag_change", args=[tag.pk]), - "export_urls_url": reverse("api-1:tag_urls_export", args=[tag.pk]), - "export_jsonl_url": reverse("api-1:tag_snapshots_export", args=[tag.pk]), - "rename_url": reverse("api-1:rename_tag", args=[tag.pk]), - "delete_url": reverse("api-1:delete_tag", args=[tag.pk]), + "filter_url": f"/admin/core/snapshot/?tags__id__exact={tag.pk}", + "edit_url": f"/admin/core/tag/{tag.pk}/change/", + "export_urls_url": f"/api/v1/core/tag/{tag.pk}/urls.txt", + "export_jsonl_url": f"/api/v1/core/tag/{tag.pk}/snapshots.jsonl", + "rename_url": f"/api/v1/core/tag/{tag.pk}/rename", + "delete_url": f"/api/v1/core/tag/{tag.pk}/", "snapshots": snapshot_previews or [], } @@ -274,7 +277,7 @@ def build_tag_cards( ) -> list[dict[str, Any]]: sort = normalize_tag_sort(sort) has_snapshots = normalize_has_snapshots_filter(has_snapshots) - needs_snapshot_count_annotation = sort.startswith("snapshots_") or has_snapshots != "all" + needs_snapshot_count_annotation = sort.startswith("snapshots_") queryset = get_matching_tags( query=query, sort=sort, diff --git a/archivebox/core/templatetags/core_tags.py b/archivebox/core/templatetags/core_tags.py index 4b024b6d..f4689440 100644 --- a/archivebox/core/templatetags/core_tags.py +++ b/archivebox/core/templatetags/core_tags.py @@ -13,6 +13,7 @@ from archivebox.hooks import ( get_plugin_name, ) from archivebox.core.host_utils import ( + canonical_base_host_for_request, get_admin_base_url, get_public_base_url, get_web_base_url, @@ -26,6 +27,7 @@ register = template.Library() _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log") _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif") _MHTML_PREVIEW_EXTS = (".mhtml", ".mht") +_WACZ_PREVIEW_EXTS = (".wacz", ".warc", ".warc.gz") _MEDIA_FILE_EXTS = { ".mp4", @@ -204,7 +206,12 @@ def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None, if _is_root_snapshot_output_path(path): return _build_snapshot_files_url(snapshot_id, request=request, config=config) url = build_snapshot_url(str(snapshot_id), path, request=request, config=config) - if not (_is_text_preview_path(path) or _is_image_preview_path(path) or (path or "").lower().endswith(_MHTML_PREVIEW_EXTS)): + if not ( + _is_text_preview_path(path) + or _is_image_preview_path(path) + or (path or "").lower().endswith(_MHTML_PREVIEW_EXTS) + or (path or "").lower().endswith(_WACZ_PREVIEW_EXTS) + ): return url separator = "&" if "?" in url else "?" return f"{url}{separator}preview=1" @@ -317,6 +324,77 @@ def result_list_tag(parser, token): ) +@register.inclusion_tag("security_mode_banner.html", takes_context=True) +def security_mode_banner(context): + """Render the top-of-page warning banner for one of two conditions: + + 1. ``mode="unconfigured"`` — ``BASE_URL`` is empty. The server is running + on whatever host the operator happens to be hitting; CSRF auto-derive + and the request-host fallback in ``get_base_url`` keep things working, + but the operator should pin ``BASE_URL`` explicitly so links stay + stable across hosts (and the misconfig banner goes away). + 2. ``mode="unsafe"`` — ``SERVER_SECURITY_MODE`` is a non-subdomain mode. + Archived pages share an origin with privileged routes. + + Both conditions can hold; we show the ``unconfigured`` banner first + because pinning ``BASE_URL`` is the more immediately actionable fix. + """ + config = context.get("CONFIG") + if config is None: + from archivebox.config.common import get_config + + config = get_config(resolve_plugins=False) + + if not config.BASE_URL: + return _unconfigured_banner_context(context.get("request")) + if not config.USES_SUBDOMAIN_ROUTING: + return {"mode": "unsafe"} + return {"mode": ""} + + +def _unconfigured_banner_context(request) -> dict: + """Build the banner payload for the unset-BASE_URL case. + + Always returns ``mode="unconfigured"`` — the user explicitly asked for + the banner to render whenever ``BASE_URL`` is empty, regardless of + whether the request host happens to match a CSRF-derived value. The + ``suggested_base_url`` is derived from the current request when one is + available so the user can copy/paste the right value straight into + their config. + """ + if request is None: + return { + "mode": "unconfigured", + "actual_host": "", + "suggested_base_url": "", + "machine_admin_url": "", + } + scheme = request.scheme or "http" + actual_full_host = request.get_host() or "" + canonical_host = canonical_base_host_for_request(actual_full_host) + # Suggest the wildcard form ``http://*.`` so the value lands in the + # operator's clipboard already aligned with subdomain routing. The config + # parser strips the leading ``*.`` so users can paste it verbatim. + suggested_base_url = f"{scheme}://*.{canonical_host}" if canonical_host else "" + user = getattr(request, "user", None) + is_superuser = bool(user and user.is_authenticated and user.is_superuser) + machine_admin_url = "" + if is_superuser: + try: + from archivebox.machine.models import Machine + + machine = Machine.current() + machine_admin_url = f"/admin/machine/machine/{machine.id}/change/" + except Exception: + machine_admin_url = "" + return { + "mode": "unconfigured", + "actual_host": actual_full_host, + "suggested_base_url": suggested_base_url, + "machine_admin_url": machine_admin_url, + } + + @register.simple_tag(takes_context=True) def url_replace(context, **kwargs): dict_ = context["request"].GET.copy() @@ -324,6 +402,16 @@ def url_replace(context, **kwargs): return dict_.urlencode() +@register.simple_tag +def has_real_admin_users() -> bool: + """True if any non-``system`` superuser exists. Used by the login page to + only show the bootstrap hint (createsuperuser / ADMIN_USERNAME env vars) + when the collection still has no real admin.""" + from django.contrib.auth.models import User + + return User.objects.filter(is_superuser=True).exclude(username="system").exists() + + @register.simple_tag(takes_context=True) def admin_base_url(context) -> str: return get_admin_base_url(request=context.get("request"), config=context.get("CONFIG")) diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py index 819789f0..eca2f2e2 100644 --- a/archivebox/core/urls.py +++ b/archivebox/core/urls.py @@ -66,10 +66,15 @@ urlpatterns = [ ), path("admin/core/snapshot/add/", RedirectView.as_view(url="/add/")), path("add/", AddView.as_view(), name="add"), - path("accounts/login/", RedirectView.as_view(url="/admin/login/")), - path("accounts/logout/", RedirectView.as_view(url="/admin/logout/")), + # ``query_string=True`` preserves the ``?next=…`` param that Django's + # auth/login mixins append, so e.g. ``UserPassesTestMixin`` redirecting + # an unauthenticated ``/add`` visitor to ``/accounts/login/?next=/add/`` + # carries the ``next`` through to ``/admin/login/`` and lands them at + # ``/add/`` after login instead of the admin homepage. + path("accounts/login/", RedirectView.as_view(url="/admin/login/", query_string=True)), + path("accounts/logout/", RedirectView.as_view(url="/admin/logout/", query_string=True)), path("accounts/", include("django.contrib.auth.urls")), - path("admin/live-progress/", archivebox_admin.admin_view(live_progress_view), name="live_progress"), + path("progress.json", live_progress_view, name="live_progress"), path("admin/", archivebox_admin.urls), path("api/", include("archivebox.api.urls"), name="api"), path("health/", HealthCheckView.as_view(), name="healthcheck"), diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 7556a691..bf2f344e 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -36,7 +36,14 @@ from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, VERSION from archivebox.config.common import get_config, get_all_configs from archivebox.config.configset import BaseConfigSet from archivebox.misc.paginators import CountlessPaginator -from archivebox.misc.util import base_url, htmlencode, ts_to_date_str, urldecode, without_fragment +from archivebox.misc.util import ( + base_url, + filter_queryset_by_uuid_substring, + htmlencode, + ts_to_date_str, + urldecode, + without_fragment, +) from archivebox.misc.serve_static import serve_static_with_byterange_support from archivebox.misc.logging_util import printable_filesize from archivebox.search import ( @@ -55,6 +62,7 @@ from archivebox.core.permissions import ( can_view_snapshot, direct_snapshots_queryset, filter_personas_by_permissions, + get_snapshot_permissions, is_admin_user, public_snapshots_queryset, ) @@ -63,6 +71,7 @@ from archivebox.core.host_utils import ( build_snapshot_url, build_web_url, get_admin_host, + get_api_base_url, get_snapshot_host, get_snapshot_lookup_key, get_web_host, @@ -159,25 +168,30 @@ class SnapshotView(View): @staticmethod def find_snapshots_for_url(path: str): - """Return a queryset of snapshots matching a URL-ish path.""" + """Return a queryset of snapshots matching a URL-ish path. URL only — never tries ID matching. + + Use ``find_snapshots_for_id`` separately if you also want to match by snapshot UUID. + """ def _fragmentless_url_query(url: str) -> Q: + # Use a range comparison (url >= 'canonical#' AND url < 'canonical#\U0010ffff') + # instead of LIKE/__startswith — SQLite's case-insensitive LIKE bypasses the + # url index and forces a full-table scan over ~1M rows (~250ms). The range + # form lets SQLite use a MULTI-INDEX OR and stays under 1ms. canonical = without_fragment(url) - return Q(url=canonical) | Q(url__startswith=f"{canonical}#") + return Q(url=canonical) | (Q(url__gte=f"{canonical}#") & Q(url__lt=f"{canonical}#\U0010ffff")) normalized = without_fragment(path) if path.startswith(("http://", "https://")): - # try exact match on full url / ID first - qs = Snapshot.objects.filter(_fragmentless_url_query(path) | Q(id__icontains=path) | Q(id__icontains=normalized)) + # exact url match (indexed) — fastest path + qs = Snapshot.objects.filter(_fragmentless_url_query(path)) if qs.exists(): return qs normalized = normalized.split("://", 1)[1] - # try exact match on full url / ID (without scheme) + # try exact match on full url (without scheme) qs = Snapshot.objects.filter( - _fragmentless_url_query("http://" + normalized) - | _fragmentless_url_query("https://" + normalized) - | Q(id__icontains=normalized), + _fragmentless_url_query("http://" + normalized) | _fragmentless_url_query("https://" + normalized), ) if qs.exists(): return qs @@ -193,28 +207,26 @@ class SnapshotView(View): # fall back to matching base_url as prefix return Snapshot.objects.filter(Q(url__startswith="http://" + base) | Q(url__startswith="https://" + base)) + @staticmethod + def find_snapshots_for_id(slug: str): + """Return a queryset of snapshots matching a (possibly truncated) UUID via prefix or suffix. + + Strips non-hex characters from ``slug`` (so input with or without hyphens both work). + Requires at least 8 hex chars — shorter inputs return an empty queryset to avoid + scanning the entire snapshots table on too-broad matches. + """ + return filter_queryset_by_uuid_substring(Snapshot.objects.all(), slug) + @staticmethod def render_live_index(request, snapshot): TITLE_LOADING_MSG = "Not yet archived..." from archivebox.core.widgets import TagEditorWidget - crawl = getattr(snapshot, "crawl", None) - runtime_config = getattr(request, "archivebox_config", None) - page_config_keys = { - "PREVIEW_ORIGINALS", - "BIND_ADDR", - "USES_SUBDOMAIN_ROUTING", - "BASE_URL", - "PERMISSIONS", - "SERVER_SECURITY_MODE", - } - scoped_config_keys = set((getattr(snapshot, "config", None) or {}).keys()) - scoped_config_keys.update((getattr(crawl, "config", None) or {}).keys()) - needs_scoped_config = bool(scoped_config_keys & page_config_keys) - if runtime_config is None or needs_scoped_config: - runtime_config = get_config(snapshot=snapshot, resolve_plugins=False) - request.archivebox_config = runtime_config + # Reuse the middleware-attached config; never re-bootstrap from env + plugin + # schemas just to render a snapshot page (that pays ~30ms for no reason). + runtime_config = _get_request_config(request) snapshot._runtime_config = runtime_config + snapshot_permissions = get_snapshot_permissions(snapshot) hidden_card_plugins = {"archivedotorg", "favicon", "title"} outputs = [ out @@ -253,8 +265,17 @@ class SnapshotView(View): best_result = archiveresults[result_type] break - related_snapshots_qs = SnapshotView.find_snapshots_for_url(snapshot.url) - related_snapshots = list(related_snapshots_qs.exclude(id=snapshot.id).order_by("-bookmarked_at", "-created_at", "-timestamp")[:25]) + related_snapshots_qs = ( + SnapshotView.find_snapshots_for_url(snapshot.url) + .select_related("crawl", "crawl__created_by") + .annotate( + num_outputs_cached=Count("archiveresult", filter=Q(archiveresult__status="succeeded")), + num_failures_cached=Count("archiveresult", filter=Q(archiveresult__status="failed")), + ) + ) + related_snapshots = list( + related_snapshots_qs.exclude(id=snapshot.id).order_by("-bookmarked_at", "-created_at", "-timestamp")[:25], + ) related_years_map: dict[int, list[Snapshot]] = {} for snap in [snapshot, *related_snapshots]: snap_dt = snap.bookmarked_at or snap.created_at or snap.downloaded_at @@ -295,32 +316,53 @@ class SnapshotView(View): compact_outputs = [out for out in ordered_outputs if out.get("is_compact") or out.get("is_metadata")] tag_widget = TagEditorWidget() output_size = sum(int(out.get("size") or 0) for out in ordered_outputs) - is_archived = bool(ordered_outputs or snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED) + has_outputs = bool(ordered_outputs) + is_archived = has_outputs or snapshot.status == Snapshot.StatusChoices.SEALED + snapshot_status = str(snapshot.status or "").lower() + status_label_by_state = { + "queued": ("queued", "info"), + "started": ("running", "warning"), + "paused": ("paused", "default"), + "sealed": ("archived", "success"), + } + if has_outputs and not is_archived: + status_label, status_color = ("partial", "warning") + elif has_outputs: + status_label, status_color = ("archived", "success") + else: + status_label, status_color = status_label_by_state.get(snapshot_status, ("not yet archived", "danger")) + + # One canonical progress endpoint, same-origin to whichever host the page is on. + # The id is always carried explicitly in the query string (derived from the page + # context, never the host) so this works in every routing/security mode. + progress_endpoint = f"/progress.json?snapshot_id={snapshot.id}" context = { "id": str(snapshot.id), "snapshot_id": str(snapshot.id), + "progress_endpoint": progress_endpoint, "url": snapshot.url, "archive_path": snapshot.archive_path_from_db, "title": htmlencode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)), "extension": snapshot.extension or "html", "tags": snapshot.tags_str() or "untagged", - "size": printable_filesize(output_size) if output_size else "pending", - "status": "archived" if is_archived else "not yet archived", - "status_color": "success" if is_archived else "danger", - "snapshot_permissions": str(runtime_config.PERMISSIONS).strip().lower(), + "size": printable_filesize(output_size) if output_size else "—", + "status": status_label, + "status_color": status_color, + "snapshot_state": snapshot_status, + "has_outputs": has_outputs, + "snapshot_permissions": snapshot_permissions, "snapshot_permissions_icon": { "public": "👥", "unlisted": "🔗", "private": "🔒", - }[str(runtime_config.PERMISSIONS).strip().lower()], + }.get(snapshot_permissions, "👥"), "bookmarked_date": snapshot.bookmarked_date, "downloaded_datestr": snapshot.downloaded_datestr, "num_outputs": snapshot.num_outputs, "num_failures": snapshot.num_failures, "oldest_archive_date": ts_to_date_str(snapshot.oldest_archive_date), "warc_path": warc_path, - "PREVIEW_ORIGINALS": runtime_config.PREVIEW_ORIGINALS, "archiveresults": [*non_compact_outputs, *compact_outputs], "best_result": best_result, "snapshot": snapshot, # Pass the snapshot object for template tags @@ -467,10 +509,20 @@ class SnapshotView(View): status=404, ) - # slug is a URL + # slug is either a URL or a (possibly truncated) snapshot UUID + def _resolve_snapshots_for_slug(slug: str): + # full URLs go straight to the url-only path (fast, indexed) + if "://" in slug: + return SnapshotView.find_snapshots_for_url(slug) + # short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first + id_qs = SnapshotView.find_snapshots_for_id(slug) + if id_qs.exists(): + return id_qs + return SnapshotView.find_snapshots_for_url(slug) + try: try: - snapshot = direct_snapshots_queryset(request, SnapshotView.find_snapshots_for_url(path)).get() + snapshot = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path)).get() except Snapshot.DoesNotExist: raise except Snapshot.DoesNotExist: @@ -491,7 +543,7 @@ class SnapshotView(View): status=404, ) except Snapshot.MultipleObjectsReturned: - snapshots = direct_snapshots_queryset(request, SnapshotView.find_snapshots_for_url(path)) + snapshots = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path)) snapshot_hrefs = mark_safe("
").join( format_html( '{} {} {} {} {}', @@ -548,15 +600,8 @@ class SnapshotPathView(View): snapshot = None snapshots_qs = direct_snapshots_queryset(request, Snapshot.objects.select_related("crawl", "crawl__created_by")) if snapshot_id: - try: - snapshot = snapshots_qs.get(pk=snapshot_id) - except Snapshot.DoesNotExist: - try: - snapshot = snapshots_qs.get(id__startswith=snapshot_id) - except Snapshot.DoesNotExist: - snapshot = None - except Snapshot.MultipleObjectsReturned: - snapshot = snapshots_qs.filter(id__startswith=snapshot_id).first() + matches = list(filter_queryset_by_uuid_substring(snapshots_qs, snapshot_id)[:2]) + snapshot = matches[0] if matches else None else: # fuzzy lookup by date + domain/url (most recent) username_lookup = "system" if username == "web" else username @@ -830,8 +875,35 @@ def _serve_responses_path(request, responses_root: Path, rel_path: str, show_ind def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""): request_config = get_config(snapshot=snapshot, resolve_plugins=False) request.archivebox_config = request_config + request.archivebox_snapshot_url = snapshot.url snapshot._runtime_config = request_config rel_path = path or "" + + # POLICY EXCEPTION: archivebox normally does not depend on any specific + # plugin. The WACZ/WARC embedded-replay viewer needs ``/replay/sw.js`` and + # ``/replay/ui.js`` served same-origin on each snapshot host so its + # service worker can register. Until plugins can register their own URL + # routes generically, we conditionally import the archivewebpage plugin's + # ``replay_preview`` module here and let it handle these paths. If the + # plugin is not installed the import fails and the path falls through to + # the regular snapshot file lookup (which 404s, the expected behavior). + if rel_path.startswith("replay/") or rel_path == "replay": + try: + from abx_plugins.plugins.archivewebpage import replay_preview as _awp_preview + except ImportError: + _awp_preview = None + if _awp_preview is not None: + response = _awp_preview.serve_replay_asset(rel_path, request_config) + if response is not None: + return response + + if rel_path == "progress.json": + # Host routing forwards every snap-* path to SnapshotHostView, so we forward + # /progress.json on through to the same view used everywhere else. The caller + # passes snapshot_id explicitly in the query string — we don't read it from the + # subdomain (this keeps the endpoint identical across all security modes). + return live_progress_view(request) + is_directory_request = bool(path) and path.endswith("/") show_indexes = bool(request.GET.get("files")) or (request_config.USES_SUBDOMAIN_ROUTING and is_directory_request) if not show_indexes and (not rel_path or rel_path == "index.html"): @@ -1150,6 +1222,7 @@ class AddView(UserPassesTestMixin, FormView): "title": "Create Crawl", # We can't just call request.build_absolute_uri in the template, because it would include query parameters "absolute_add_path": self.request.build_absolute_uri(self.request.path), + "web_base_url": build_web_url("", request=self.request), "VERSION": VERSION, "FOOTER_INFO": request_config.FOOTER_INFO, "required_search_plugin": required_search_plugin, @@ -1411,18 +1484,47 @@ def live_progress_view(request): from archivebox.core.models import Snapshot, ArchiveResult from archivebox.machine.models import Process, Machine - if not request.user.is_authenticated or not request.user.is_active or not request.user.is_staff: - return JsonResponse({"error": "Permission denied"}, status=403) + snapshot_id_filter = (request.GET.get("snapshot_id") or "").strip() + crawl_id_filter = (request.GET.get("crawl_id") or "").strip() + is_admin = is_admin_user(request) + + scoped_snapshot = None + if snapshot_id_filter: + import uuid as _uuid + + try: + _uuid.UUID(snapshot_id_filter) + except (TypeError, ValueError): + return JsonResponse({"error": "Invalid snapshot_id"}, status=400) + scoped_snapshot = Snapshot.objects.filter(id=snapshot_id_filter).select_related("crawl").first() + if scoped_snapshot is None or not can_view_snapshot(request, scoped_snapshot): + return JsonResponse({"error": "Permission denied"}, status=403) + elif crawl_id_filter: + # Crawl-only scope still requires staff: there's no per-crawl ACL helper, + # and a crawl can mix snapshot permissions levels. + if not is_admin: + return JsonResponse({"error": "Permission denied"}, status=403) + else: + if not is_admin: + return JsonResponse({"error": "Permission denied"}, status=403) request_config = request.archivebox_config now = timezone.now() crawl_scope = Crawl.objects.all() snapshot_scope = Snapshot.objects.all() archiveresult_scope = ArchiveResult.objects.all() - if not request.user.is_superuser: + if is_admin and not request.user.is_superuser: crawl_scope = crawl_scope.filter(created_by=request.user) snapshot_scope = snapshot_scope.filter(crawl__created_by=request.user) archiveresult_scope = archiveresult_scope.filter(snapshot__crawl__created_by=request.user) + if scoped_snapshot is not None: + snapshot_scope = Snapshot.objects.filter(id=scoped_snapshot.id) + crawl_scope = Crawl.objects.filter(id=scoped_snapshot.crawl_id) + archiveresult_scope = ArchiveResult.objects.filter(snapshot_id=scoped_snapshot.id) + elif crawl_id_filter: + snapshot_scope = snapshot_scope.filter(crawl_id=crawl_id_filter) + crawl_scope = crawl_scope.filter(id=crawl_id_filter) + archiveresult_scope = archiveresult_scope.filter(snapshot__crawl_id=crawl_id_filter) def is_current_run_timestamp(event_ts, run_started_at) -> bool: if run_started_at is None: @@ -1550,6 +1652,8 @@ def live_progress_view(request): url = str(url or "") return url if len(url) <= 96 else f"{url[:93]}..." + api_base = get_api_base_url(request=request, config=request_config) if scoped_snapshot is not None else "" + def screencast_frame_url(crawl_id: str, crawl_dir: Path) -> str: frame_path = crawl_dir / "chrome_screencast" / "latest.jpg" try: @@ -1560,7 +1664,8 @@ def live_progress_view(request): return "" if now.timestamp() - frame_stat.st_mtime > 15: return "" - return f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}" + rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}" + return f"{api_base}{rel}" if api_base else rel machine_id = Machine.current().id orchestrator_proc = ( @@ -2263,6 +2368,11 @@ def live_progress_view(request): ) payload = { + "is_admin": is_admin, + "scope": { + "snapshot_id": str(scoped_snapshot.id) if scoped_snapshot is not None else "", + "crawl_id": crawl_id_filter, + }, "orchestrator_running": orchestrator_running, "orchestrator_pid": orchestrator_pid, "total_workers": total_workers, diff --git a/archivebox/crawls/admin.py b/archivebox/crawls/admin.py index 48f30d81..55cd70ec 100644 --- a/archivebox/crawls/admin.py +++ b/archivebox/crawls/admin.py @@ -63,8 +63,10 @@ def render_snapshots_list(snapshots_qs, request=None, crawl=None, page_size=50, filtered_qs = snapshots_qs if query: - id_query = query.replace("-", "") - filtered_qs = filtered_qs.filter(Q(id__icontains=id_query) | Q(url__icontains=query) | Q(title__icontains=query)) + from archivebox.misc.util import filter_queryset_by_uuid_substring + + id_match_pks = list(filter_queryset_by_uuid_substring(Snapshot.objects.all(), query).values_list("pk", flat=True)[:100]) + filtered_qs = filtered_qs.filter(Q(pk__in=id_match_pks) | Q(url__icontains=query) | Q(title__icontains=query)) if status_filter in valid_statuses: filtered_qs = filtered_qs.filter(status=status_filter) @@ -1182,7 +1184,7 @@ class CrawlScheduleAdmin(BaseModelAdmin): return super().change_view(request, object_id, form_url, extra_context) def add_view(self, request, form_url="", extra_context=None): - return redirect("/add/?focus=schedule") + return redirect("/add/#schedule") def get_fieldsets(self, request, obj=None): if obj is None: diff --git a/archivebox/crawls/models.py b/archivebox/crawls/models.py index 5fc77fbb..ba2eab48 100755 --- a/archivebox/crawls/models.py +++ b/archivebox/crawls/models.py @@ -865,7 +865,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith crawl_tag_names = self.current_tag_names() tags_by_name: dict[str, Tag] = {} config = get_config(crawl=self) - only_new_urls = bool(config.ONLY_NEW) and not bool(config.OVERWRITE) + only_new_urls = bool(config.ONLY_NEW) for line in self.urls.splitlines(): if not line.strip(): @@ -1039,7 +1039,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith if not deduped_records: return [] - existing_scope = Snapshot.objects if bool(config.ONLY_NEW) and not bool(config.OVERWRITE) else self.snapshot_set + existing_scope = Snapshot.objects if bool(config.ONLY_NEW) else self.snapshot_set existing_urls = set(existing_scope.filter(url__in=deduped_records.keys()).values_list("url", flat=True)) urls = [url for url in deduped_records.keys() if url not in existing_urls] remaining = self.remaining_snapshot_capacity() diff --git a/archivebox/hooks.py b/archivebox/hooks.py index 54c96f3d..81dfe266 100644 --- a/archivebox/hooks.py +++ b/archivebox/hooks.py @@ -53,6 +53,7 @@ from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, TypedDict from abx_plugins import get_plugins_dir from django.utils.safestring import mark_safe from archivebox.config.constants import CONSTANTS +from archivebox.config.version import VERSION from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url if TYPE_CHECKING: @@ -400,6 +401,7 @@ def run_hook( env["DATA_DIR"] = str(resolved_config.DATA_DIR) env["ARCHIVE_DIR"] = str(resolved_config.ARCHIVE_DIR) env["ABX_RUNTIME"] = "archivebox" + env["LIBRARY_VERSION"] = VERSION env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID)) resolved_output_dir = output_dir.resolve() @@ -679,13 +681,10 @@ def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any return [str(plugin).strip() for plugin in value if str(plugin).strip()] return [str(value).strip()] if str(value).strip() else [] - # Support explicit ENABLED_PLUGINS override (legacy) + # Support explicit ENABLED_PLUGINS override enabled_plugins = config.get("ENABLED_PLUGINS") if enabled_plugins: return normalize_enabled_plugins(enabled_plugins) - enabled_extractors = config.get("ENABLED_EXTRACTORS") - if enabled_extractors: - return normalize_enabled_plugins(enabled_extractors) # Filter all plugins by enabled status all_plugins = get_plugins() diff --git a/archivebox/misc/checks.py b/archivebox/misc/checks.py index 3ead47d9..5f99eb68 100644 --- a/archivebox/misc/checks.py +++ b/archivebox/misc/checks.py @@ -61,7 +61,7 @@ def check_migrations(*, blocking: bool = True, auto_apply: bool = False, cancel_ from archivebox.misc.db import apply_migrations, migration_state, pending_migrations pending, missing_from_code, rollback_targets = migration_state() - is_migrating = any(arg in sys.argv for arg in ["makemigrations", "migrate", "init"]) + is_migrating = any(arg in sys.argv for arg in ["makemigrations", "migrate", "init"]) or os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" if missing_from_code: print( @@ -240,7 +240,9 @@ def check_data_dir_permissions(config=None, **config_kwargs): # Check /lib dir permissions check_lib_dir(lib_dir, throw=False, must_exist=True, config=config) - os.umask(0o777 - int(config.DIR_OUTPUT_PERMISSIONS, base=8)) + # Derive directory mode from file mode by OR-ing the execute bits (matches + # the old DIR_OUTPUT_PERMISSIONS=755 vs OUTPUT_PERMISSIONS=644 convention). + os.umask(0o777 - (int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)) def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs): diff --git a/archivebox/misc/db.py b/archivebox/misc/db.py index 7221d032..997ae05f 100644 --- a/archivebox/misc/db.py +++ b/archivebox/misc/db.py @@ -19,6 +19,91 @@ from archivebox.config import DATA_DIR from archivebox.misc.util import enforce_types +def run_db_analyze_batch( + remaining: list[str] | None, + *, + max_seconds_per_table: float = 120.0, +) -> list[str]: + """Advance one step of a batched SQLite ``ANALYZE`` sweep. + + Without periodic ANALYZE the optimizer's table stats go stale as + snapshot/archiveresult tables grow, causing it to start large joins from + ``auth_user`` instead of using the indexed url column and blowing snapshot + detail page render time from ~50ms to ~500ms+. + + The whole sweep is spread across many calls instead of running as one + blocking ``ANALYZE``: pass ``None`` to start a fresh sweep (this call + enumerates user tables and runs ``ANALYZE`` on the first one); pass the + returned list to advance one more table on each subsequent call. An + empty return value means the sweep is complete (or has been aborted) and + the next caller should pass ``None`` again. Caller is responsible for + throttling new sweeps (orchestrator starts at most one per 24hr while + idle) and enforcing a hard upper bound on total sweep wall time. + + Safety guarantees: + + - **Never raises**: every database call is wrapped; on any failure the + function returns ``[]`` (abandoning the rest of the sweep) so the + orchestrator never crashes on maintenance errors. + - **Bounded per-call wall time**: a SQLite progress handler aborts the + current ``ANALYZE`` statement once ``max_seconds_per_table`` is + exceeded, so a single pathological table cannot wedge the call. + - **Never leaves the db locked**: each ``ANALYZE`` runs as a single + statement transaction that auto-commits (or rolls back on + abort/error). The cursor and progress handler are always cleaned up + in ``finally`` blocks even if Python raises mid-call. + - Silent no-op on non-SQLite backends. + + WAL journal mode (set in Django settings) keeps readers fully unblocked + throughout; the writer lock is only held for the brief ``sqlite_stat*`` + flush after each table completes. + """ + from django.db import connection + + if connection.vendor != "sqlite": + return [] + + if remaining is None: + try: + with connection.cursor() as cursor: + cursor.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + ) + remaining = [row[0] for row in cursor.fetchall()] + except Exception: + return [] + + if not remaining: + return [] + + next_table, *rest = remaining + raw_conn = getattr(connection, "connection", None) + progress_handler_set = False + if raw_conn is not None and max_seconds_per_table > 0: + deadline = time.monotonic() + max_seconds_per_table + try: + raw_conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 10000) + progress_handler_set = True + except Exception: + progress_handler_set = False + + try: + with connection.cursor() as cursor: + cursor.execute(f'ANALYZE "{next_table}"') + except Exception: + # Aborted by progress handler, locked db, or any other failure — skip + # this table and continue the sweep. ANALYZE is idempotent so we can + # retry on the next 24hr sweep. + pass + finally: + if progress_handler_set and raw_conn is not None: + try: + raw_conn.set_progress_handler(None, 0) + except Exception: + pass + return rest + + def compact_command(cmdline: list[str] | None, fallback: str = "") -> str: parts = [str(part) for part in (cmdline or []) if str(part)] if not parts: diff --git a/archivebox/misc/monkey_patches.py b/archivebox/misc/monkey_patches.py index 53045083..a2f99e32 100644 --- a/archivebox/misc/monkey_patches.py +++ b/archivebox/misc/monkey_patches.py @@ -59,7 +59,7 @@ class ModifiedAccessLogGenerator(access.AccessLogGenerator): return if "GET /health/" in request: return - if "GET /admin/live-progress/" in request and (time_taken is None or time_taken < 1.0): + if "GET /progress.json" in request and (time_taken is None or time_taken < 1.0): return if "GET /api/v1/crawls/crawl/" in request and "/files/chrome_screencast/latest.jpg" in request: return diff --git a/archivebox/misc/serve_static.py b/archivebox/misc/serve_static.py index 72e9ed35..506d0c09 100644 --- a/archivebox/misc/serve_static.py +++ b/archivebox/misc/serve_static.py @@ -792,6 +792,9 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ bool(request.GET.get("preview")) and content_type.startswith("image/") and not content_type.startswith("image/svg+xml") ) preview_as_mhtml_html = bool(request.GET.get("preview")) and fullpath.suffix.lower() in {".mhtml", ".mht"} + preview_as_archivewebpage_html = bool(request.GET.get("preview")) and ( + fullpath.suffix.lower() in {".wacz", ".warc"} or fullpath.name.lower().endswith(".warc.gz") + ) # Respect the If-Modified-Since header for non-markdown responses. if not (content_type.startswith("text/plain") or content_type.startswith("text/html")): @@ -859,6 +862,50 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ except Exception: pass + if preview_as_archivewebpage_html: + # POLICY EXCEPTION: archivebox normally does not depend on any specific + # plugin. The WACZ/WARC embedded-replay viewer (ui.js + sw.js + the + # rendered preview HTML) is plugin-owned, but it has to be served at + # plugin-defined paths on the snapshot host so the same-origin service + # worker registration works. There is no clean generic "plugin + # contributes a preview handler" extension hook in archivebox yet, so + # we conditionally import the archivewebpage plugin's + # ``replay_preview`` module here. If the plugin is not installed, the + # import fails and we fall through to default static-file serving. + try: + from abx_plugins.plugins.archivewebpage import replay_preview as _awp_preview + except ImportError: + _awp_preview = None + if _awp_preview is not None: + try: + raw_query = request.GET.copy() + raw_query.pop("preview", None) + raw_output_path = request.path + if raw_query: + raw_output_path = f"{raw_output_path}?{raw_query.urlencode()}" + snapshot_url_fallback = getattr(request, "archivebox_snapshot_url", "") or "" + rendered = _awp_preview.render_preview_html( + fullpath.name, + raw_output_path, + wacz_path=fullpath, + fallback_url=snapshot_url_fallback, + ) + response = HttpResponse(rendered, content_type="text/html; charset=utf-8") + response.headers["Last-Modified"] = http_date(statobj.st_mtime) + if etag: + response.headers["ETag"] = etag + response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable" + else: + response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300" + response.headers["Content-Disposition"] = f'inline; filename="{fullpath.stem}.html"' + for key, value in _awp_preview.preview_response_headers().items(): + response.headers[key] = value + if encoding: + response.headers["Content-Encoding"] = encoding + return response + except Exception: + pass + if preview_as_mhtml_html: try: raw_query = request.GET.copy() diff --git a/archivebox/misc/util.py b/archivebox/misc/util.py index 6ddfe567..452306b2 100644 --- a/archivebox/misc/util.py +++ b/archivebox/misc/util.py @@ -20,6 +20,27 @@ from base32_crockford import encode as base32_encode from .logging import COLOR_DICT +def filter_queryset_by_uuid_substring(queryset, slug: str, field: str = "id"): + """Filter a queryset to UUID-column matches by prefix or suffix (case-insensitive). + + Avoids ``id__icontains`` (an unindexed full-table scan over the UUID column) by + stripping non-hex chars from ``slug`` and matching with ``istartswith`` / + ``iendswith``. Returns an empty queryset for inputs with fewer than 8 hex chars + to avoid overly broad matches. A full 32-char hex string falls back to an + exact-equality lookup. + """ + from django.db.models import Q + + normalized = re.sub(r"[^0-9a-fA-F]", "", slug or "").lower() + if len(normalized) < 8: + return queryset.none() + if len(normalized) == 32: + return queryset.filter(**{field: normalized}) + prefix = f"{field}__istartswith" + suffix = f"{field}__iendswith" + return queryset.filter(Q(**{prefix: normalized}) | Q(**{suffix: normalized})) + + def detect_encoding(rawdata): try: import chardet # type:ignore diff --git a/archivebox/search/__init__.py b/archivebox/search/__init__.py index b76319a9..1764c893 100644 --- a/archivebox/search/__init__.py +++ b/archivebox/search/__init__.py @@ -241,9 +241,6 @@ def query_search_index( from archivebox.core.models import Snapshot config = config or get_config(**config_kwargs) - if not config.USE_SEARCHING_BACKEND: - return Snapshot.objects.none() - search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config) search_mode_base = get_search_mode_base(search_mode, config=config) if search_mode_base == "meta": @@ -262,9 +259,6 @@ def iter_query_search_ids( ): """Yield snapshot IDs from configured search backends as soon as each backend produces them.""" config = config or get_config(**config_kwargs) - if not config.USE_SEARCHING_BACKEND: - return - search_mode = "contents" if search_mode is None else get_search_mode(search_mode, config=config) search_mode_base = get_search_mode_base(search_mode, config=config) forced_backend = get_search_mode_backend(search_mode, config=config) @@ -342,7 +336,7 @@ def flush_search_index(snapshots: QuerySet, config: dict[str, Any] | None = None Remove snapshots from the search index. """ config = config or get_config(**config_kwargs) - if not config.USE_INDEXING_BACKEND or not snapshots: + if not snapshots: return backend = get_backend(config=config) diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 5ff2dd5f..c0c6a282 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -60,6 +60,7 @@ from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelled from archivebox.config.configset import BaseConfigSet from archivebox.core.recovery_util import recover_orchestrator_state +from archivebox.misc.db import run_db_analyze_batch from archivebox.core.shutdown_util import foreground_shutdown_signals from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS @@ -290,6 +291,14 @@ class CrawlRunner: if self._signal_abort_requested: return True + if self.allow_maintenance_on_inactive_crawl: + # SEALED is the normal terminal state of a finished crawl, not a + # cancellation signal for maintenance work on its already-sealed + # snapshots (search backend backfill, fs migration, etc.). When the + # runner is invoked with explicit snapshot_ids + selected_plugins, + # treat sealed as completed rather than cancelled so the requested + # maintenance hooks can actually run. + return False return await Crawl.objects.filter(id=self.crawl.id, status=Crawl.StatusChoices.SEALED).aexists() async def crawl_is_paused(self) -> bool: @@ -311,7 +320,16 @@ class CrawlRunner: return filter_plugins(self.plugins, self.selected_plugins, include_providers=True) if self.selected_plugins else self.plugins @property - def allow_paused_snapshot_maintenance(self) -> bool: + def allow_maintenance_on_inactive_crawl(self) -> bool: + """Run the requested hooks on a snapshot whose parent crawl is paused or sealed. + + Maintenance entry paths — direct ``snapshot_ids + selected_plugins`` invocations + for search backend backfill, fs migration, plugin-targeted updates — are + legitimately allowed to operate on finished/paused crawls. Without this gate, + ``crawl_is_cancelled`` would treat a SEALED parent as a cancellation signal + and short-circuit every guard before any hook ran, leaving the queued + ArchiveResult rows stuck and the orchestrator looping on them. + """ return bool(self.initial_snapshot_ids and self.selected_plugins) async def run(self) -> None: @@ -384,7 +402,7 @@ class CrawlRunner: async def enqueue_snapshot(self, snapshot_id: str, crawl_start_event: CrawlStartEvent | None = None) -> None: if await self.crawl_is_cancelled(): return - if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance: + if await self.crawl_is_paused() and not self.allow_maintenance_on_inactive_crawl: return task = self.snapshot_tasks.get(snapshot_id) if task is not None and not task.done(): @@ -462,7 +480,7 @@ class CrawlRunner: task_errors.append(err) stop_scheduling = True if self.snapshot_tasks and ( - await self.crawl_is_cancelled() or (await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance) + await self.crawl_is_cancelled() or (await self.crawl_is_paused() and not self.allow_maintenance_on_inactive_crawl) ): stop_scheduling = True if not stop_scheduling: @@ -520,7 +538,7 @@ class CrawlRunner: return if await self.crawl_is_cancelled(): return - if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance: + if await self.crawl_is_paused() and not self.allow_maintenance_on_inactive_crawl: return await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)() @@ -725,7 +743,7 @@ class CrawlRunner: from archivebox.hooks import collect_urls_from_plugins await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)() - if self.crawl.is_paused and not self.allow_paused_snapshot_maintenance: + if self.crawl.is_paused and not self.allow_maintenance_on_inactive_crawl: return if int(snapshot_payload["depth"]) >= self.crawl.max_depth: return @@ -844,7 +862,7 @@ class CrawlRunner: break if await self.crawl_is_cancelled(): break - if await self.crawl_is_paused() and not self.allow_paused_snapshot_maintenance: + if await self.crawl_is_paused() and not self.allow_maintenance_on_inactive_crawl: break await self.enqueue_snapshot(snapshot_id) await self.wait_for_snapshot_tasks() @@ -855,7 +873,9 @@ class CrawlRunner: cancel_watcher = asyncio.create_task(self.watch_for_cancelled_crawl(event)) try: try: - if not await self.crawl_is_cancelled() and (not await self.crawl_is_paused() or self.allow_paused_snapshot_maintenance): + if not await self.crawl_is_cancelled() and ( + not await self.crawl_is_paused() or self.allow_maintenance_on_inactive_crawl + ): await _run_event_now( event.emit( CrawlSetupEvent( @@ -868,7 +888,9 @@ class CrawlRunner: ), crawl_setup_phase_timeout, ) - if not await self.crawl_is_cancelled() and (not await self.crawl_is_paused() or self.allow_paused_snapshot_maintenance): + if not await self.crawl_is_cancelled() and ( + not await self.crawl_is_paused() or self.allow_maintenance_on_inactive_crawl + ): crawl_start_event = CrawlStartEvent( url=snapshot["url"], snapshot_id=snapshot["id"], @@ -1720,6 +1742,10 @@ def run_pending_crawls( ) last_recovery_at = 0.0 last_retention_at = 0.0 + last_analyze_at = 0.0 + analyze_queue: list[str] | None = None + analyze_sweep_started_at = 0.0 + orchestrator_started_at = time.monotonic() while True: now_monotonic = time.monotonic() if now_monotonic - last_retention_at >= (60.0 if daemon else 1.0): @@ -1873,6 +1899,39 @@ def run_pending_crawls( if now_monotonic - last_recovery_at >= 30.0: recover_orchestrator_state() last_recovery_at = now_monotonic + # SQLite query plans degrade as the snapshot/archiveresult tables grow + # past their last ANALYZE — stale stats make the optimizer start large + # joins from auth_user/crawl instead of using the url index, blowing the + # snapshot detail page out to ~500ms. Refresh stats at most once per + # 24hr while the queue is idle, and only after the orchestrator has + # been alive for at least an hour so short server boots / one-off work + # never pay the cost. The sweep is batched one table per idle tick; + # individual table ANALYZE statements abort after 2min (progress + # handler) and the whole sweep is hard-capped at 5min so a + # pathological table cannot wedge maintenance forever. Any failure + # inside the maintenance hook is swallowed — orchestrator must never + # be taken down by stats refresh. + try: + if ( + analyze_queue is None + and now_monotonic - orchestrator_started_at >= 3600.0 + and now_monotonic - last_analyze_at >= 86400.0 + ): + analyze_sweep_started_at = now_monotonic + analyze_queue = run_db_analyze_batch(None) + elif analyze_queue and now_monotonic - analyze_sweep_started_at >= 300.0: + # Sweep blew past the 5min hard cap — abandon what's left + # and don't retry until the next 24hr window. + analyze_queue = None + last_analyze_at = now_monotonic + elif analyze_queue: + analyze_queue = run_db_analyze_batch(analyze_queue) + if analyze_queue is not None and not analyze_queue: + analyze_queue = None + last_analyze_at = now_monotonic + except Exception: + analyze_queue = None + last_analyze_at = now_monotonic time.sleep(2.0) continue return 0 diff --git a/archivebox/templates/admin/base.html b/archivebox/templates/admin/base.html index 3e951cd2..7ce687d5 100644 --- a/archivebox/templates/admin/base.html +++ b/archivebox/templates/admin/base.html @@ -1660,6 +1660,7 @@ + {% security_mode_banner %} {% include 'progressbar.html' %}
diff --git a/archivebox/templates/admin/core/tag/change_list.html b/archivebox/templates/admin/core/tag/change_list.html index 6c5b33f7..7af180f3 100644 --- a/archivebox/templates/admin/core/tag/change_list.html +++ b/archivebox/templates/admin/core/tag/change_list.html @@ -247,6 +247,13 @@ color: #075985; font-size: 11px; font-weight: 700; + text-decoration: none; + } + + .tag-card__count:hover { + background: #bae6fd; + color: #0c4a6e; + text-decoration: none; } .tag-card__actions { @@ -308,58 +315,6 @@ font-size: 12px; } - .tag-card__snapshots { - display: grid; - gap: 8px; - grid-template-columns: repeat(auto-fit, minmax(130px, 1fr)); - } - - .tag-snapshot-badge { - display: flex; - align-items: center; - gap: 8px; - min-width: 0; - padding: 6px 8px; - border-radius: 12px; - border: 1px solid #dbe4ee; - background: rgba(255, 255, 255, 0.86); - text-decoration: none; - color: #0f172a; - } - - .tag-snapshot-badge img { - width: 16px; - height: 16px; - border-radius: 4px; - flex: 0 0 auto; - background: #f8fafc; - } - - .tag-snapshot-badge span { - min-width: 0; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - font-size: 11px; - font-weight: 600; - } - - .tag-card__empty { - padding: 14px; - border-radius: 14px; - border: 1px dashed #cbd5e1; - background: #f8fafc; - color: #64748b; - font-size: 13px; - } - - .tag-card__empty[href] { - display: block; - color: #075985; - font-weight: 600; - text-decoration: none; - } - .tag-toast { position: sticky; top: 12px; @@ -626,17 +581,6 @@ document.addEventListener('DOMContentLoaded', function () { } grid.innerHTML = cards.map(function (card) { - const snapshotCount = Number(card.num_snapshots || 0); - const snapshotHtml = (card.snapshots || []).length - ? card.snapshots.map(function (snapshot) { - return '' + - '' + - '' + - '' + escapeHtml(snapshot.title) + '' + - ''; - }).join('') - : '' + escapeHtml(snapshotCount) + ' snapshot' + (snapshotCount === 1 ? '' : 's') + ' tagged'; - return '' + '
' + '
' + @@ -655,10 +599,9 @@ document.addEventListener('DOMContentLoaded', function () { '' + '' + '' + - '' + escapeHtml(card.num_snapshots) + '' + + '' + escapeHtml(card.num_snapshots) + '' + '
' + '
' + - '
' + snapshotHtml + '
' + ''; }).join(''); } @@ -849,8 +792,8 @@ document.addEventListener('DOMContentLoaded', function () { grid.addEventListener('click', async function (event) { const actionButton = event.target.closest('[data-action]'); - const snapshotLink = event.target.closest('.tag-snapshot-badge'); - if (snapshotLink) return; + const filterLink = event.target.closest('.tag-card__count'); + if (filterLink) return; const cardEl = event.target.closest('.tag-card'); if (!cardEl) return; diff --git a/archivebox/templates/admin/login.html b/archivebox/templates/admin/login.html index 98283f80..df5fc77d 100644 --- a/archivebox/templates/admin/login.html +++ b/archivebox/templates/admin/login.html @@ -1,5 +1,5 @@ {% extends "admin/base_site.html" %} -{% load i18n static %} +{% load i18n static core_tags %} {% block extrastyle %}{{ block.super }} {{ form.media }} @@ -7,11 +7,11 @@ {% block bodyclass %}{{ block.super }} login{% endblock %} + {% block branding %}

ArchiveBox Admin

{% endblock %} - -{% block usertools %} +
- Back to Main Index +{% block usertools %} {% endblock %} {% block nav-global %}{% endblock %} @@ -53,22 +53,16 @@
{% csrf_token %} -
+
{{ form.username.errors }} {{ form.username.label_tag }} {{ form.username }}
-
+
{{ form.password.errors }} {{ form.password.label_tag }} {{ form.password }}
- {% url 'admin_password_reset' as password_reset_url %} - {% if password_reset_url %} -
- {% trans 'Forgotten your password or username?' %} -
- {% endif %} -
+
@@ -77,22 +71,20 @@


- If you forgot your password, reset it here or run:
+ To create a new admin user or reset a password, run:
-archivebox manage changepassword USERNAME
+cd data/   # run commands inside your data folder
+archivebox manage createsuperuser <username>
+archivebox manage changepassword <username>
- -

-
-
- To create a new admin user, run the following: 
-archivebox manage createsuperuser
-
-
-
- (cd into your archive folder before running commands) + + {% has_real_admin_users as real_admins_exist %} + {% if not real_admins_exist %} + (or set env vars ADMIN_USERNAME + ADMIN_PASSWORD) + {% endif %} +
diff --git a/archivebox/templates/admin/progress_monitor.html b/archivebox/templates/admin/progress_monitor.html index 9d568eb6..4c6c6f39 100644 --- a/archivebox/templates/admin/progress_monitor.html +++ b/archivebox/templates/admin/progress_monitor.html @@ -187,6 +187,17 @@ #progress-monitor.collapsed .progress-content { display: none; } + /* Hide admin-only controls when viewer is not staff, or when the monitor is + embedded on a non-admin host (snap-* subdomain) where cross-origin POSTs + to /api would violate the subdomain isolation model. */ + #progress-monitor.is-guest .crawl-action-btn, + #progress-monitor.is-guest .cancel-item-btn, + #progress-monitor.is-guest .pause-item-btn, + #progress-monitor[data-progress-scope="snapshot"] .crawl-action-btn, + #progress-monitor[data-progress-scope="snapshot"] .cancel-item-btn, + #progress-monitor[data-progress-scope="snapshot"] .pause-item-btn { + display: none !important; + } /* Chrome Screencast */ #progress-monitor .screencast-panel { @@ -1037,7 +1048,9 @@ -