From 05350846aba81103a8fdaf70d72eaea87b57fcbd Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 25 Jul 2026 14:48:55 -0700 Subject: [PATCH] Sync ArchiveBox UI and setup updates --- .github/workflows/deploy-publicsite.yml | 27 + .gitignore | 2 + archivebox/base_models/admin.py | 89 +- archivebox/config/common.py | 91 +- archivebox/config/views.py | 51 +- archivebox/core/admin_archiveresults.py | 112 +-- archivebox/core/admin_site.py | 23 +- archivebox/core/context_processors.py | 14 +- archivebox/core/forms.py | 13 +- archivebox/core/models.py | 224 +++-- archivebox/core/routes_util.py | 19 +- archivebox/core/setup_wizard.py | 68 ++ archivebox/core/templatetags/core_tags.py | 141 ++-- archivebox/core/views.py | 213 +++-- archivebox/crawls/admin.py | 43 +- archivebox/machine/admin.py | 42 +- archivebox/mcp/server.py | 683 +++++++++------ archivebox/misc/serve_static.py | 147 ++-- archivebox/plugins/views.py | 91 +- archivebox/templates/admin/actions.html | 4 +- .../admin/archivebox_change_form.html | 260 ++++++ .../admin/auth/user/change_form.html | 41 +- archivebox/templates/admin/base.html | 94 ++- .../templates/admin/change_list_panel.html | 2 +- .../admin/core/archiveresult/change_list.html | 2 +- .../templates/admin/core/tag/change_form.html | 2 +- .../admin/crawls/crawl/change_form.html | 46 +- .../admin/personas/persona/change_form.html | 2 +- .../templates/admin/snapshots_grid.html | 46 +- archivebox/templates/core/add.html | 2 +- archivebox/templates/core/base.html | 2 +- archivebox/templates/core/navigation.html | 47 +- archivebox/templates/core/public_index.html | 76 +- archivebox/templates/core/setup_wizard.html | 121 +++ archivebox/templates/core/snapshot.html | 603 ++++++++++--- .../core/system_warnings_banner.html | 160 ++-- archivebox/templates/static/add.css | 75 ++ archivebox/templates/static/admin.css | 796 +++++++++++++++--- .../static/admin/crawls/crawl_change.css | 22 + archivebox/templates/static/setup_wizard.css | 56 ++ archivebox/templates/static/setup_wizard.js | 405 +++++++++ archivebox/tests/test_migrations_07_to_09.py | 181 +++- archivebox/tests/test_ui_add_view.py | 48 +- .../tests/test_ui_admin_config_widget.py | 230 +++++ archivebox/tests/test_ui_admin_crawl.py | 60 +- bin/collect_ui_screenshots.sh | 485 +++++++++++ bin/generate_ui_screenshot_gallery.py | 263 ++++++ bin/setup_ui_screenshot_data.js | 120 +++ bin/take_screenshot.js | 124 ++- docker-compose.yml | 4 +- docs/Configuration.md | 16 +- docs/Screenshots.md | 725 ++++++++++++++++ docs/Security-Overview.md | 2 +- docs/_Sidebar.md | 1 + publicsite/index.html | 2 +- 55 files changed, 5838 insertions(+), 1380 deletions(-) mode change 100755 => 100644 archivebox/core/models.py create mode 100644 archivebox/core/setup_wizard.py create mode 100644 archivebox/templates/admin/archivebox_change_form.html create mode 100644 archivebox/templates/core/setup_wizard.html create mode 100644 archivebox/templates/static/setup_wizard.css create mode 100644 archivebox/templates/static/setup_wizard.js create mode 100755 bin/collect_ui_screenshots.sh create mode 100755 bin/generate_ui_screenshot_gallery.py create mode 100755 bin/setup_ui_screenshot_data.js create mode 100644 docs/Screenshots.md diff --git a/.github/workflows/deploy-publicsite.yml b/.github/workflows/deploy-publicsite.yml index c42ded47..3813dbd4 100644 --- a/.github/workflows/deploy-publicsite.yml +++ b/.github/workflows/deploy-publicsite.yml @@ -6,6 +6,13 @@ on: - dev paths: - publicsite/** + - archivebox/** + - bin/collect_ui_screenshots.* + - bin/generate_ui_screenshot_gallery.py + - bin/take_screenshot.js + - docs/Screenshots.md + - pyproject.toml + - uv.lock - .github/workflows/deploy-publicsite.yml workflow_dispatch: @@ -30,6 +37,26 @@ jobs: with: fetch-depth: 1 + - name: Checkout screenshot plugin + uses: actions/checkout@v4 + with: + repository: ArchiveBox/abx-plugins + path: .ui-screenshot-deps/abx-plugins + + - name: Setup Python and uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Install ArchiveBox and screenshot dependencies + run: uv sync --frozen + + - name: Build UI screenshot gallery + run: bin/collect_ui_screenshots.sh + env: + ABX_PLUGINS_DIR: ${{ github.workspace }}/.ui-screenshot-deps/abx-plugins/abx_plugins/plugins + BASE_URL: http://archivebox.localhost:8000 + - name: Setup Pages uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 diff --git a/.gitignore b/.gitignore index 1f6cdd50..b9c50741 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ __pycache__/ .mypy_cache/ .eggs/ tests/out/ +/docs/screenshots/ +/publicsite/screenshots/ # Coverage .coverage diff --git a/archivebox/base_models/admin.py b/archivebox/base_models/admin.py index 887dae39..e7eaaf3b 100644 --- a/archivebox/base_models/admin.py +++ b/archivebox/base_models/admin.py @@ -5,11 +5,11 @@ __package__ = "archivebox.base_models" import json import uuid from collections.abc import Mapping -from typing import NotRequired, TypedDict, cast +from typing import ClassVar, NotRequired, TypedDict, cast from django import forms from django.contrib import admin -from django.db import models +from django.db import DatabaseError, models from django.forms.renderers import BaseRenderer from django.http import HttpRequest, QueryDict from django.urls import path, register_converter @@ -68,10 +68,10 @@ class KeyValueWidget(forms.Widget): template_name = "" # We render manually class Media: - css = { + css: ClassVar[dict[str, list[str]]] = { "all": [], } - js = [] + js: ClassVar[list[str]] = [] def _get_config_options(self) -> dict[str, ConfigOption]: """Get available config options from plugins.""" @@ -94,7 +94,7 @@ class KeyValueWidget(forms.Widget): option[schema_key] = schema[schema_key] options[key] = option return options - except Exception: + except (ImportError, KeyError, TypeError, ValueError): return {} def _parse_value(self, value: object) -> dict[str, object]: @@ -644,41 +644,64 @@ class KeyValueWidget(forms.Widget): window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id}; - function focusConfigKeyFromHash_{widget_id}() {{ - // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly - // to (or creates) the matching row in this editor. Used by - // the in-banner "pin via admin" link and the - // ``→ Edit in Machine.config`` shortcut on the live - // config detail page. - var hash = (window.location.hash || '').replace(/^#/, '').trim(); - if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{ - return; - }} + function configRowForKey_{widget_id}(key) {{ var container = document.getElementById('{widget_id}_rows'); if (!container) {{ - return; + return null; }} var match = null; container.querySelectorAll('.key-value-row').forEach(function(row) {{ if (match) {{ return; }} var keyInput = row.querySelector('.kv-key'); - if (keyInput && keyInput.value.trim() === hash) {{ + if (keyInput && keyInput.value.trim() === key) {{ match = row; }} }}); if (!match) {{ - // No existing row for this key — prepopulate one with the - // key filled in but value left blank so the operator just - // types/pastes the value and hits save. window.addKeyValueRow_{widget_id}(); var rows = container.querySelectorAll('.key-value-row'); match = rows[rows.length - 1]; var keyInput = match.querySelector('.kv-key'); if (keyInput) {{ - keyInput.value = hash; + keyInput.value = key; keyInput.dispatchEvent(new Event('input', {{ bubbles: true }})); }} }} + return match; + }} + + function prefillConfigFromQuery_{widget_id}() {{ + var params = new URLSearchParams(window.location.search); + var consumedConfigKeys = []; + params.forEach(function(value, key) {{ + if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{ + return; + }} + consumedConfigKeys.push(key); + var match = configRowForKey_{widget_id}(key); + var valueInput = match && match.querySelector('.kv-value'); + if (valueInput) {{ + valueInput.value = value; + valueInput.dispatchEvent(new Event('input', {{ bubbles: true }})); + }} + }}); + updateHiddenField_{widget_id}(); + if (consumedConfigKeys.length) {{ + var cleanUrl = new URL(window.location.href); + consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }}); + window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash); + }} + }} + + function focusConfigKeyFromHash_{widget_id}() {{ + // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly + // to (or creates) the matching row in this editor. Used by + // the setup wizard and live-config detail pages. + var hash = (window.location.hash || '').replace(/^#/, '').trim(); + if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{ + return; + }} + var match = configRowForKey_{widget_id}(hash); if (!match) {{ return; }} @@ -700,12 +723,14 @@ class KeyValueWidget(forms.Widget): // Initialize on load document.addEventListener('DOMContentLoaded', function() {{ initializeRows_{widget_id}(); + prefillConfigFromQuery_{widget_id}(); updateHiddenField_{widget_id}(); focusConfigKeyFromHash_{widget_id}(); }}); // Also run immediately in case DOM is already ready if (document.readyState !== 'loading') {{ initializeRows_{widget_id}(); + prefillConfigFromQuery_{widget_id}(); updateHiddenField_{widget_id}(); focusConfigKeyFromHash_{widget_id}(); }} @@ -826,7 +851,7 @@ class ConfigEditorMixin(admin.ModelAdmin): if change and obj.pk and obj.config is not None: try: stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {} - except Exception: + except (AttributeError, DatabaseError, TypeError, ValueError): stored = {} if isinstance(stored, dict): new_config = dict(obj.config or {}) @@ -845,6 +870,20 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin): list_display = ("id", "created_at", "created_by") readonly_fields = ("id", "created_at", "modified_at") show_search_mode_selector = False + change_form_template = "admin/archivebox_change_form.html" + + def get_admin_toolbar_actions(self, request, obj): + """Return extra action button dicts for the shared change-form toolbar. + + See ``templates/admin/includes/archivebox_toolbar.html`` for the + accepted keys. Default: no extras (toolbar renders Save/History/Delete + plus any ``django-object-actions`` change_actions). + """ + return [] + + def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None): + context.setdefault("archivebox_admin_actions", self.get_admin_toolbar_actions(request, obj)) + return super().render_change_form(request, context, add=add, change=change, form_url=form_url, obj=obj) def get_default_search_mode(self) -> str: # The shared changelist template always asks every admin for a default @@ -877,9 +916,9 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin): """ info = self.opts.app_label, self.opts.model_name object_routes = [ - path("/history/", self.admin_site.admin_view(self.history_view), name="%s_%s_history" % info), - path("/delete/", self.admin_site.admin_view(self.delete_view), name="%s_%s_delete" % info), - path("/change/", self.admin_site.admin_view(self.change_view), name="%s_%s_change" % info), + path("/history/", self.admin_site.admin_view(self.history_view), name="{}_{}_history".format(*info)), + path("/delete/", self.admin_site.admin_view(self.delete_view), name="{}_{}_delete".format(*info)), + path("/change/", self.admin_site.admin_view(self.change_view), name="{}_{}_change".format(*info)), ] # Append after super().get_urls() so our patterns are the # *last-registered* ones with the canonical admin URL names — Django's diff --git a/archivebox/config/common.py b/archivebox/config/common.py index 01c4c9ec..69eed925 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -2,32 +2,31 @@ from __future__ import annotations __package__ = "archivebox.config" +import inspect import json import os import re import secrets -import sys import shutil -import inspect -from functools import lru_cache +import sys from collections.abc import Mapping from datetime import timedelta -from typing import Any, ClassVar, cast +from functools import cache, lru_cache from pathlib import Path -from urllib.parse import quote +from typing import Any, ClassVar, cast +from urllib.parse import quote, urlparse -from rich.console import Console +from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator from pydantic_settings import SettingsConfigDict -from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs +from rich.console import Console -from archivebox.config.configset import BaseConfigSet, IniConfigSettingsSource -from archivebox.config.configset import COMPUTED_CONFIG_KEYS +from archivebox.config.configset import COMPUTED_CONFIG_KEYS, BaseConfigSet, IniConfigSettingsSource from .constants import CONSTANTS from .ldap import LDAPConfig -from .version import get_COMMIT_HASH, get_BUILD_TIME, VERSION from .permissions import IN_DOCKER +from .version import VERSION, get_BUILD_TIME, get_COMMIT_HASH ConfigOverrides = Mapping[str, object] ConfigPayload = dict[str, object] @@ -67,6 +66,40 @@ def permissions_from_legacy_public_flags(raw_config: Mapping[str, object]) -> st return None +def base_url_from_legacy_server_config(raw_config: Mapping[str, object]) -> str: + """Derive the current canonical URL from the pre-0.9 server settings.""" + + def normalize(value: object) -> str: + raw = str(value or "").strip() + if not raw: + return "" + parsed = urlparse(raw if "://" in raw else f"http://{raw}") + return f"{parsed.scheme}://{parsed.netloc}" if parsed.netloc else "" + + for key in ("ARCHIVE_BASE_URL", "ADMIN_BASE_URL"): + candidate = normalize(raw_config.get(key)) + if candidate: + return candidate + + listen_host = str(raw_config.get("LISTEN_HOST") or "").strip() + if listen_host and listen_host.lower() != "archivebox.localhost:8000": + return normalize(listen_host) + + csrf_origins = [normalize(origin) for origin in str(raw_config.get("CSRF_TRUSTED_ORIGINS") or "").split(",")] + csrf_origins = [origin for origin in csrf_origins if origin] + nondefault_csrf_origins = [origin for origin in csrf_origins if origin != "http://admin.archivebox.localhost:8000"] + if len(nondefault_csrf_origins) == 1: + return nondefault_csrf_origins[0] + + allowed_hosts = [host.strip() for host in str(raw_config.get("ALLOWED_HOSTS") or "").split(",") if host.strip() != "*"] + if len(allowed_hosts) == 1: + return normalize(allowed_hosts[0]) + + if len(csrf_origins) == 1: + return csrf_origins[0] + return normalize(listen_host) + + def resolve_delete_after_config_value(*configs: Mapping[str, Any] | None) -> str: for config in configs: if config is None: @@ -272,6 +305,7 @@ class ServerConfig(BaseConfigSet): _scope: str = PrivateAttr(default=_SCOPE_SERVER) SERVER_SECURITY_MODES: ClassVar[tuple[str, ...]] = ( + "auto", "safe-subdomains-fullreplay", "safe-onedomain-nojsreplay", "unsafe-onedomain-noadmin", @@ -283,7 +317,7 @@ class ServerConfig(BaseConfigSet): BASE_URL: str = Field(default="") ALLOWED_HOSTS: str = Field(default="*") CSRF_TRUSTED_ORIGINS: str = Field(default="") - SERVER_SECURITY_MODE: str = Field(default="safe-subdomains-fullreplay") + SERVER_SECURITY_MODE: str = Field(default="auto") SNAPSHOTS_PER_PAGE: int = Field(default=50, ge=1) FOOTER_INFO: str = Field( @@ -308,11 +342,21 @@ class ServerConfig(BaseConfigSet): @property def USES_SUBDOMAIN_ROUTING(self) -> bool: - return self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay" + if self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay": + return True + if self.SERVER_SECURITY_MODE != "auto": + return False + + base_host = (urlparse(self.BASE_URL if "://" in self.BASE_URL else f"//{self.BASE_URL}").hostname or "").lower() + if base_host: + return base_host.endswith(".localhost") + + bind_host = (urlparse(f"//{self.BIND_ADDR}").hostname or "").lower() + return bind_host in {"", "0.0.0.0", "127.0.0.1", "::", "::1", "localhost"} @property def ENABLES_FULL_JS_REPLAY(self) -> bool: - return self.SERVER_SECURITY_MODE in ( + return self.USES_SUBDOMAIN_ROUTING or self.SERVER_SECURITY_MODE in ( "safe-subdomains-fullreplay", "unsafe-onedomain-noadmin", "danger-onedomain-fullreplay", @@ -328,7 +372,7 @@ class ServerConfig(BaseConfigSet): @property def SHOULD_NEUTER_RISKY_REPLAY(self) -> bool: - return self.SERVER_SECURITY_MODE == "safe-onedomain-nojsreplay" + return self.SERVER_SECURITY_MODE in ("auto", "safe-onedomain-nojsreplay") @property def IS_UNSAFE_MODE(self) -> bool: @@ -643,7 +687,7 @@ class ArchiveBoxBaseConfig( @classmethod def _plugin_field_scope(cls, key: str) -> str | None: scope = None - for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items(): + for schema in PLUGIN_CONFIG_SCHEMAS.values(): properties = schema.get("properties") if isinstance(schema, dict) else None if not isinstance(properties, dict) or key not in properties: continue @@ -655,7 +699,7 @@ class ArchiveBoxBaseConfig( return scope @classmethod - @lru_cache(maxsize=None) + @cache def scope_for_key(cls, key: str) -> str: for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items(): properties = schema.get("properties") if isinstance(schema, dict) else None @@ -995,8 +1039,12 @@ def get_request_config(request: Any, *, resolve_plugins: bool = False) -> Archiv request_config_resolves_plugins = bool(request_state.get("_archivebox_config_resolves_plugins", False)) if request_config is None or (resolve_plugins and not request_config_resolves_plugins): request_config = get_config(resolve_plugins=resolve_plugins) - request.archivebox_config = request_config request._archivebox_config_resolves_plugins = resolve_plugins + if request_config.SERVER_SECURITY_MODE == "auto": + request_host = (urlparse(f"//{request.get_host()}").hostname or "").lower().rstrip(".") + effective_mode = "safe-subdomains-fullreplay" if request_host.endswith(".localhost") else "safe-onedomain-nojsreplay" + request_config = request_config.model_copy(update={"SERVER_SECURITY_MODE": effective_mode}) + request.archivebox_config = request_config return request_config @@ -1036,7 +1084,7 @@ def get_config( from archivebox.machine.models import Machine machine = Machine.current() - except Exception: + except (ImportError, RuntimeError, TypeError, ValueError): machine = None if persona is None and crawl is not None: @@ -1062,9 +1110,14 @@ def get_config( config_data.update( normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False), ) - legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ}) + legacy_config = {**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ} + legacy_permissions = permissions_from_legacy_public_flags(legacy_config) if legacy_permissions: config_data["PERMISSIONS"] = legacy_permissions + if not str(config_data.get("BASE_URL") or "").strip(): + legacy_base_url = base_url_from_legacy_server_config(legacy_config) + if legacy_base_url: + config_data["BASE_URL"] = legacy_base_url scope_overrides: ConfigPayload = {} diff --git a/archivebox/config/views.py b/archivebox/config/views.py index 5ca38483..9d19cd7c 100644 --- a/archivebox/config/views.py +++ b/archivebox/config/views.py @@ -1,25 +1,32 @@ __package__ = "archivebox.config" -import os import inspect +import os from pathlib import Path from typing import Any from urllib.parse import quote, urlencode + +from admin_data_views.typing import ItemContext, SectionData, TableContext +from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view from django.http import HttpRequest from django.utils import timezone from django.utils.html import format_html from django.utils.safestring import mark_safe -from admin_data_views.typing import TableContext, ItemContext, SectionData -from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink - from archivebox.config import CONSTANTS -from archivebox.misc.util import parse_date - from archivebox.machine.models import Binary +from archivebox.misc.util import parse_date ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/" INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/" +LOG_DETAIL_TAIL_BYTES = 100_000 + + +def _read_text_tail(path: Path, max_bytes: int = LOG_DETAIL_TAIL_BYTES) -> str: + with path.open("rb") as log_file: + size = path.stat().st_size + log_file.seek(max(size - max_bytes, 0)) + return log_file.read(max_bytes).decode("utf-8", errors="replace") def is_superuser(request: HttpRequest) -> bool: @@ -87,7 +94,7 @@ def obj_to_yaml(obj: Any, indent: int = 0) -> str: return f" {obj}" elif isinstance(obj, (int, float, bool)): - return f" {str(obj)}" + return f" {obj!s}" elif callable(obj): source = ( @@ -98,7 +105,7 @@ def obj_to_yaml(obj: Any, indent: int = 0) -> str: return f" {indent_str} " + source.replace("\n", f"\n{indent_str} ") else: - return f" {str(obj)}" + return f" {obj!s}" def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]: @@ -300,7 +307,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: pid_to_process_id[row.pid] = str(row.id) pid_to_process_type[row.pid] = row.process_type - except Exception: + except (ImportError, RuntimeError, TypeError, ValueError): pass def _pid_cell(pid_value: int | None, uptime_str: str = ""): @@ -341,7 +348,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: hours, remainder = divmod(seconds, 3600) minutes, secs = divmod(remainder, 60) supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}" - except Exception: + except (ImportError, RuntimeError, TypeError, ValueError): try: from archivebox.machine.models import Machine, Process @@ -360,7 +367,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: hours, remainder = divmod(seconds, 3600) minutes, secs = divmod(remainder, 60) supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}" - except Exception: + except (RuntimeError, TypeError, ValueError): pass rows["PID"].append(_pid_cell(supervisor_pid if isinstance(supervisor_pid, int) else None, supervisor_uptime)) @@ -415,7 +422,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: assert is_superuser(request), "Must be a superuser to view configuration settings." - from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker, get_sock_file, CONFIG_FILE_NAME + from archivebox.workers.supervisord_util import CONFIG_FILE_NAME, get_existing_supervisord_process, get_sock_file, get_worker SOCK_FILE = get_sock_file() CONFIG_FILE = SOCK_FILE.parent / CONFIG_FILE_NAME @@ -438,8 +445,12 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: if key == "supervisord": relevant_config = CONFIG_FILE.read_text() - relevant_logs = str(supervisor.readLog(0, 10_000_000)) - start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0] + supervisor_log = CONSTANTS.LOGS_DIR / "supervisord.log" + with supervisor_log.open("rb") as log_file: + startup_logs = log_file.read(LOG_DETAIL_TAIL_BYTES).decode("utf-8", errors="replace") + relevant_logs = _read_text_tail(supervisor_log) + startup_lines = [line for line in startup_logs.split("\n") if "RPC interface 'supervisor' initialized" in line] + start_ts = startup_lines[-1].split(",", 1)[0] if startup_lines else "" start_dt = parse_date(start_ts) uptime = str(timezone.now() - start_dt).split(".")[0] if start_dt else "" supervisor_state = supervisor.getState() @@ -458,7 +469,7 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: worker_data = get_worker(supervisor, key) proc = worker_data if isinstance(worker_data, dict) else {} relevant_config = next((config for config in all_config if config.get("name") == key), {}) - log_result = supervisor.tailProcessStdoutLog(key, 0, 10_000_000) + log_result = supervisor.tailProcessStdoutLog(key, -LOG_DETAIL_TAIL_BYTES, LOG_DETAIL_TAIL_BYTES) relevant_logs = str(log_result[0] if isinstance(log_result, tuple) else log_result) section: SectionData = { @@ -474,7 +485,7 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: "Logfile": str(proc.get("stdout_logfile") or ""), "Uptime": str(str(proc.get("description") or "").split("uptime ", 1)[-1]), "Config": obj_to_yaml(relevant_config) if isinstance(relevant_config, dict) else str(relevant_config), - "Logs": relevant_logs, + "Recent Logs (last 100 KB)": relevant_logs, }, "help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"}, } @@ -528,9 +539,9 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext: def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: assert is_superuser(request), "Must be a superuser to view configuration settings." - log_file = [logfile for logfile in CONSTANTS.LOGS_DIR.glob("*.log") if key in logfile.name][0] + log_file = next(logfile for logfile in CONSTANTS.LOGS_DIR.glob("*.log") if key in logfile.name) - log_text = log_file.read_text() + log_text = _read_text_tail(log_file) log_stat = log_file.stat() section: SectionData = { @@ -540,8 +551,8 @@ def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: "Path": str(log_file), "Size": f"{log_stat.st_size // 1000} kb", "Last Updated": format_parsed_datetime(log_stat.st_mtime), - "Tail": "\n".join(log_text[-10_000:].split("\n")[-20:]), - "Full Log": log_text, + "Tail": "\n".join(log_text.split("\n")[-20:]), + "Recent Log (last 100 KB)": log_text, }, } diff --git a/archivebox/core/admin_archiveresults.py b/archivebox/core/admin_archiveresults.py index c4b153b6..2ebd004f 100644 --- a/archivebox/core/admin_archiveresults.py +++ b/archivebox/core/admin_archiveresults.py @@ -4,31 +4,29 @@ import html import json import os import shlex -from pathlib import Path -from urllib.parse import quote from functools import reduce from operator import and_ +from pathlib import Path +from urllib.parse import quote from django.contrib import admin +from django.core.exceptions import ValidationError from django.db.models import Min, Prefetch, Q, TextField from django.db.models.functions import Cast +from django.urls import resolve, reverse +from django.utils import timezone from django.utils.html import format_html from django.utils.safestring import mark_safe -from django.core.exceptions import ValidationError -from django.urls import reverse, resolve -from django.utils import timezone from django.utils.text import smart_split -from archivebox.misc.paginators import AcceleratedPaginator from archivebox.base_models.admin import BaseModelAdmin -from archivebox.plugins.discovery import get_plugin_icon -from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL +from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.routes_util import build_snapshot_url from archivebox.core.widgets import InlineTagEditorWidget from archivebox.machine.env_util import env_to_shell_exports - - -from archivebox.core.models import ArchiveResult, Snapshot +from archivebox.misc.paginators import AcceleratedPaginator +from archivebox.plugins.discovery import get_plugin_icon +from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL def _get_replay_source_url(result: ArchiveResult) -> str: @@ -120,15 +118,13 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None): status = result.status or "queued" color, bg = status_colors.get(status, ("#6b7280", "#f3f4f6")) output_files = result.output_files or {} - if isinstance(output_files, dict): - output_file_count = len(output_files) - elif isinstance(output_files, (list, tuple, set)): + if isinstance(output_files, (dict, list, tuple, set)): output_file_count = len(output_files) elif isinstance(output_files, str): try: parsed = json.loads(output_files) output_file_count = len(parsed) if isinstance(parsed, (dict, list, tuple, set)) else 0 - except Exception: + except (TypeError, ValueError): output_file_count = 0 else: output_file_count = 0 @@ -258,7 +254,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
- ID: {str(result.id)} + ID: {result.id!s} Version: {version} PWD: {pwd_text}
@@ -413,7 +409,6 @@ class ArchiveResultAdmin(BaseModelAdmin): list_display_links = None sort_fields = ("id", "created_at", "plugin", "status") readonly_fields = ( - "admin_actions", "cmd", "cmd_version", "pwd", @@ -437,16 +432,9 @@ class ArchiveResultAdmin(BaseModelAdmin): "output_json", "process__cmd", ) - autocomplete_fields = ["snapshot"] + autocomplete_fields = ("snapshot",) fieldsets = ( - ( - "Actions", - { - "fields": ("admin_actions",), - "classes": ("card", "wide"), - }, - ), ( "Snapshot", { @@ -485,14 +473,14 @@ class ArchiveResultAdmin(BaseModelAdmin): ) list_filter = ("status", "plugin", "start_ts") - ordering = ["-start_ts"] + ordering = ("-start_ts",) list_per_page = 50 paginator = AcceleratedPaginator save_on_top = True show_full_result_count = False - actions = ["delete_selected"] + actions = ("delete_selected",) class Meta: verbose_name = "Archive Result" @@ -502,6 +490,36 @@ class ArchiveResultAdmin(BaseModelAdmin): self.request = request return super().change_view(request, object_id, form_url, extra_context) + def get_admin_toolbar_actions(self, request, obj): + if obj is None: + return [] + self.request = request + return [ + { + "label": "View Output", + "icon": "📄", + "url": self.get_output_view_url(obj), + "title": "Open the archived output for this result", + }, + { + "label": "Output files", + "icon": "📁", + "url": self.get_output_files_url(obj), + "title": "Browse the output files for this result", + }, + { + "label": "Download Zip", + "icon": "⬇", + "url": self.get_output_zip_url(obj), + "kind": "accent", + "css_classes": "archivebox-zip-button", + "onclick": "return window.archiveboxHandleZipClick(this, event);", + "extra_attrs": [("data-loading-label", "Preparing...")], + "title": "Download all output files as a zip", + }, + {"label": "Snapshot", "icon": "🗂", "url": self.get_snapshot_view_url(obj), "title": "Open the parent snapshot view"}, + ] + def changelist_view(self, request, extra_context=None): self.request = request saved_list_per_page = self.list_per_page @@ -749,46 +767,6 @@ class ArchiveResultAdmin(BaseModelAdmin): output_text, ) - @admin.display(description="") - def admin_actions(self, result): - return format_html( - """ - - """, - self.get_output_view_url(result), - self.get_output_files_url(result), - self.get_output_zip_url(result), - self.get_snapshot_view_url(result), - ) - def output_summary(self, result): snapshot_dir = Path(result.snapshot.output_dir) output_html = format_html( diff --git a/archivebox/core/admin_site.py b/archivebox/core/admin_site.py index 36ef51ba..e87c88a8 100644 --- a/archivebox/core/admin_site.py +++ b/archivebox/core/admin_site.py @@ -2,6 +2,15 @@ __package__ = "archivebox.core" from typing import TYPE_CHECKING, Any +from admin_data_views.admin import ( + admin_data_index_view as adv_admin_data_index_view, +) +from admin_data_views.admin import ( + get_admin_data_urls as adv_get_admin_data_urls, +) +from admin_data_views.admin import ( + get_app_list as adv_get_app_list, +) from django.contrib import admin from django.contrib.auth import REDIRECT_FIELD_NAME from django.contrib.auth.decorators import login_not_required @@ -12,23 +21,17 @@ from django.urls import reverse from django.utils.decorators import method_decorator from django.utils.translation import gettext as _ from django.views.decorators.cache import never_cache -from admin_data_views.admin import ( - admin_data_index_view as adv_admin_data_index_view, - get_admin_data_urls as adv_get_admin_data_urls, - get_app_list as adv_get_app_list, -) from archivebox.config import VERSION -from archivebox.config.version import get_COMMIT_HASH +from archivebox.core.context_processors import get_static_cache_key from archivebox.core.routes_util import is_allowed_archivebox_redirect_url if TYPE_CHECKING: + from admin_data_views.typing import AppDict from django.http import HttpRequest from django.template.response import TemplateResponse from django.urls import URLPattern, URLResolver - from admin_data_views.typing import AppDict - class ArchiveBoxLoginView(LoginView): def get_redirect_url(self) -> str: @@ -50,7 +53,7 @@ class ArchiveBoxAdmin(admin.AdminSite): def each_context(self, request: "HttpRequest") -> dict[str, Any]: context = super().each_context(request) context["VERSION"] = VERSION - context["STATIC_CACHE_KEY"] = (get_COMMIT_HASH() or VERSION or "dev").strip() + context["STATIC_CACHE_KEY"] = get_static_cache_key() return context @staticmethod @@ -183,9 +186,9 @@ def register_admin_site(): # Register admin views for each app # (Previously handled by ABX plugin system, now called directly) + from archivebox.api.admin import register_admin as register_api_admin from archivebox.core.admin import register_admin as register_core_admin from archivebox.crawls.admin import register_admin as register_crawls_admin - from archivebox.api.admin import register_admin as register_api_admin from archivebox.machine.admin import register_admin as register_machine_admin from archivebox.personas.admin import register_admin as register_personas_admin from archivebox.workers.admin import register_admin as register_workers_admin diff --git a/archivebox/core/context_processors.py b/archivebox/core/context_processors.py index 623265e5..09b55099 100644 --- a/archivebox/core/context_processors.py +++ b/archivebox/core/context_processors.py @@ -1,9 +1,21 @@ +from pathlib import Path + from archivebox.config import VERSION from archivebox.config.version import get_COMMIT_HASH +def get_static_cache_key() -> str: + """Version the admin stylesheet even when the checkout has uncommitted edits.""" + base_key = (get_COMMIT_HASH() or VERSION or "dev").strip() + admin_css_path = Path(__file__).resolve().parent.parent / "templates" / "static" / "admin.css" + try: + return f"{base_key}-{admin_css_path.stat().st_mtime_ns}" + except OSError: + return base_key + + def archivebox_globals(request): return { "VERSION": VERSION, - "STATIC_CACHE_KEY": (get_COMMIT_HASH() or VERSION or "dev").strip(), + "STATIC_CACHE_KEY": get_static_cache_key(), } diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py index 94f973e6..45f8ab5f 100644 --- a/archivebox/core/forms.py +++ b/archivebox/core/forms.py @@ -1,16 +1,17 @@ __package__ = "archivebox.core" import re -from decimal import Decimal, InvalidOperation, ROUND_CEILING +from decimal import ROUND_CEILING, Decimal, InvalidOperation from django import forms -from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes from archivebox.base_models.admin import KeyValueWidget -from archivebox.crawls.schedule_util import validate_schedule from archivebox.config.common import get_config, parse_delete_after from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget +from archivebox.crawls.schedule_util import validate_schedule +from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes +from archivebox.personas.models import Persona from archivebox.plugins.discovery import get_plugins from archivebox.plugins.forms import ( PLUGIN_GROUP_DEFINITIONS, @@ -18,7 +19,6 @@ from archivebox.plugins.forms import ( PluginConfigFormMixin, get_choice_field, ) -from archivebox.personas.models import Persona DEPTH_CHOICES = ( ("0", "depth = 0 (archive just these URLs)"), @@ -101,12 +101,11 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form): attrs={ "data-url-regex": URL_REGEX.pattern, "placeholder": ( - "\n", - "Enter URL(s) to archive. Any format is ok: one per line, CSV, JSON, embedded in text, etc." + "Enter URL(s) to archive. Any format is ok: one per line, CSV, JSON, embedded in text, etc.\n\n" "Examples:\n\n" "https://example.com\n\n" "https://news.ycombinator.com,https://news.google.com\n\n" - "Or any text-based content [containing URLs](https://github.com/ArchiveBox/ArchiveBox)...", + "Or any text-based content [containing URLs](https://github.com/ArchiveBox/ArchiveBox)..." ), }, ), diff --git a/archivebox/core/models.py b/archivebox/core/models.py old mode 100755 new mode 100644 index c710574a..1dd17944 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -1,68 +1,72 @@ __package__ = "archivebox.core" -from typing import TYPE_CHECKING, Optional, Any -from collections.abc import Iterable, Mapping, Sequence -import uuid -from archivebox.uuid_compat import CompactUUIDField, uuid7 -from datetime import datetime, timedelta - -import os import json +import os +import uuid +from collections.abc import Iterable, Mapping, Sequence +from datetime import UTC, datetime, timedelta from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar, Optional from urllib.parse import urlparse -from statemachine import State, registry - +from django.conf import settings +from django.contrib import admin +from django.core.cache import cache +from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError from django.db import models, transaction from django.db.models import Case, F, Q, QuerySet, Sum, Value, When -from django.db.models.functions import Coalesce, Concat from django.db.models.fields.json import KT -from django.utils.functional import cached_property -from django.utils.text import slugify -from django.utils import timezone -from django.core.cache import cache +from django.db.models.functions import Coalesce, Concat from django.urls import reverse_lazy -from django.contrib import admin -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.utils import timezone +from django.utils.functional import cached_property from django.utils.safestring import mark_safe +from django.utils.text import slugify +from statemachine import State, registry +from archivebox.base_models.models import ( + ModelWithConfig, + ModelWithDeleteAfter, + ModelWithHealthStats, + ModelWithNotes, + ModelWithOutputDir, + ModelWithUUID, + get_or_create_system_user_pk, +) from archivebox.config import CONSTANTS from archivebox.config.common import get_config, rprint +from archivebox.crawls.models import Crawl +from archivebox.machine.models import Binary from archivebox.misc.system import atomic_write from archivebox.misc.util import ( - parse_date, domain as url_domain, +) +from archivebox.misc.util import ( + htmldecode, + parse_date, + sanitize_html_text, to_json, ts_to_date_str, - urlencode, - htmldecode, - sanitize_html_text, urldecode, + urlencode, validate_url, ) from archivebox.plugins.discovery import ( - get_plugins, - get_plugin_name, get_plugin_icon, + get_plugin_name, + get_plugins, ) -from archivebox.base_models.models import ( - ModelWithUUID, - ModelWithDeleteAfter, - ModelWithOutputDir, - ModelWithConfig, - ModelWithNotes, - ModelWithHealthStats, - get_or_create_system_user_pk, -) -from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS, RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine -from archivebox.crawls.models import Crawl -from archivebox.machine.models import Binary +from archivebox.uuid_compat import CompactUUIDField, uuid7 +from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS, RETRY_AT_MAX, BaseStateMachine, ModelWithStateMachine if TYPE_CHECKING: from archivebox.config.common import ArchiveBoxBaseConfig +class SnapshotMigrationError(RuntimeError): + """Raised when a snapshot filesystem migration fails validation.""" + + class UngroupedSubquery(models.Subquery): """Scalar subquery that should not be copied into the outer GROUP BY.""" @@ -156,7 +160,7 @@ class SnapshotTag(models.Model): class Meta: app_label = "core" db_table = "core_snapshot_tags" - unique_together = [("snapshot", "tag")] + unique_together: ClassVar[list[tuple[str, str]]] = [("snapshot", "tag")] class SnapshotQuerySet(models.QuerySet): @@ -226,7 +230,7 @@ class SnapshotQuerySet(models.QuerySet): field_name = pk_field ordering.append(f"-{field_name}" if descending else field_name) - ordered_field_names = [term[1:] if term.startswith("-") else term for term in ordering] + ordered_field_names = [term.removeprefix("-") for term in ordering] try: if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names): offset = 0 @@ -237,7 +241,7 @@ class SnapshotQuerySet(models.QuerySet): yield from batch offset += chunk_size return - except Exception: + except (AttributeError, FieldDoesNotExist): offset = 0 while True: batch = list(self[offset : offset + chunk_size]) @@ -292,7 +296,7 @@ class SnapshotQuerySet(models.QuerySet): # Filtering Methods # ========================================================================= - FILTER_TYPES = { + FILTER_TYPES: ClassVar[dict[str, Any]] = { "exact": lambda pattern: models.Q(url=pattern), "substring": lambda pattern: models.Q(url__icontains=pattern), "regex": lambda pattern: models.Q(url__iregex=pattern), @@ -337,7 +341,6 @@ class SnapshotQuerySet(models.QuerySet): return self.filter(q_filter) def search(self, **kwargs) -> "SnapshotQuerySet": - from datetime import timezone as dt_timezone from archivebox.core.snapshot_status import filter_snapshots_by_status from archivebox.search.query import apply_snapshot_search @@ -364,9 +367,9 @@ class SnapshotQuerySet(models.QuerySet): if kwargs.get("tag"): queryset = queryset.filter(tags__name__iexact=kwargs["tag"]) if kwargs.get("before") is not None: - queryset = queryset.filter(bookmarked_at__lt=datetime.fromtimestamp(float(kwargs["before"]), tz=dt_timezone.utc)) + queryset = queryset.filter(bookmarked_at__lt=datetime.fromtimestamp(float(kwargs["before"]), tz=UTC)) if kwargs.get("after") is not None: - queryset = queryset.filter(bookmarked_at__gt=datetime.fromtimestamp(float(kwargs["after"]), tz=dt_timezone.utc)) + queryset = queryset.filter(bookmarked_at__gt=datetime.fromtimestamp(float(kwargs["after"]), tz=UTC)) if query: queryset = apply_snapshot_search( @@ -399,7 +402,8 @@ class SnapshotQuerySet(models.QuerySet): def to_json(self, with_headers: bool = False) -> str: """Generate JSON index from snapshots""" import sys - from datetime import datetime, timezone as tz + from datetime import datetime + from archivebox.config import VERSION config = get_config() @@ -430,7 +434,7 @@ class SnapshotQuerySet(models.QuerySet): output = { **MAIN_INDEX_HEADER, "num_links": len(snapshot_dicts), - "updated": datetime.now(tz.utc), + "updated": datetime.now(UTC), "last_run_cmd": sys.argv, "links": snapshot_dicts, } @@ -447,8 +451,10 @@ class SnapshotQuerySet(models.QuerySet): def to_html(self, with_headers: bool = True) -> str: """Generate main index HTML from snapshots""" - from datetime import datetime, timezone as tz + from datetime import datetime + from django.template.loader import render_to_string + from archivebox.config import VERSION from archivebox.config.version import get_COMMIT_HASH @@ -463,8 +469,8 @@ class SnapshotQuerySet(models.QuerySet): "version": VERSION, "git_sha": get_COMMIT_HASH() or VERSION, "num_links": str(len(snapshot_list)), - "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"), - "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"), + "date_updated": datetime.now(UTC).strftime("%Y-%m-%d"), + "time_updated": datetime.now(UTC).strftime("%Y-%m-%d %H:%M"), "links": snapshot_list, "FOOTER_INFO": config.FOOTER_INFO, }, @@ -603,11 +609,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW app_label = "core" verbose_name = "Snapshot" verbose_name_plural = "Snapshots" - indexes = [ + indexes: ClassVar[list[models.Index]] = [ models.Index(fields=["-bookmarked_at", "-created_at"], name="snapshot_public_order_idx"), models.Index(fields=["crawl", "status", "modified_at"], name="snapshot_progress_idx"), ] - constraints = [ + constraints: ClassVar[list[models.BaseConstraint]] = [ # Allow same URL in different crawls, but not duplicates within same crawl models.UniqueConstraint(fields=["url", "crawl"], name="unique_url_per_crawl"), # Global timestamp uniqueness for 1:1 symlink mapping @@ -899,9 +905,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW from archivebox.core.permissions import PERMISSIONS_PUBLIC, PERMISSIONS_VALUES, normalize_permissions if permission not in PERMISSIONS_VALUES: - if self.crawl_id: - if not crawl_permissions: - crawl_permissions = Crawl.objects.filter(pk=self.crawl_id).values_list("permissions", flat=True).first() + if self.crawl_id and not crawl_permissions: + crawl_permissions = Crawl.objects.filter(pk=self.crawl_id).values_list("permissions", flat=True).first() config["PERMISSIONS"] = normalize_permissions( crawl_permissions, default=PERMISSIONS_PUBLIC, @@ -940,9 +945,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW crawl_config_for_save = crawl_row.get("config") or {} crawl_permissions_for_save = crawl_row.get("permissions") - if self.ensure_permissions_config(crawl_permissions=crawl_permissions_for_save): - if update_fields is not None: - kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"])) + if self.ensure_permissions_config(crawl_permissions=crawl_permissions_for_save) and update_fields is not None: + kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"])) if validate_url_field: self.validate_url_for_archiving(config=crawl_config_for_save if self.crawl_id else None) @@ -1191,9 +1195,16 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW new_file = new_dir / rel_path # Skip if already copied - if new_file.exists(): - if new_file.stat().st_size == old_file.stat().st_size and filecmp.cmp(old_file, new_file, shallow=False): - continue + if ( + new_file.exists() + and new_file.stat().st_size == old_file.stat().st_size + and filecmp.cmp( + old_file, + new_file, + shallow=False, + ) + ): + continue new_file.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(old_file, new_file) @@ -1206,15 +1217,15 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW missing = old_files.keys() - new_files.keys() missing.discard(Path(CONSTANTS.JSON_INDEX_FILENAME)) if missing: - raise Exception(f"Migration incomplete: missing {missing}") + raise SnapshotMigrationError(f"Migration incomplete: missing {missing}") for rel_path, old_size in old_files.items(): if rel_path == Path(CONSTANTS.JSON_INDEX_FILENAME): continue if new_files.get(rel_path) != old_size: - raise Exception(f"Migration incomplete: size mismatch for {rel_path}") + raise SnapshotMigrationError(f"Migration incomplete: size mismatch for {rel_path}") if not filecmp.cmp(old_dir / rel_path, new_dir / rel_path, shallow=False): - raise Exception(f"Migration incomplete: content mismatch for {rel_path}") + raise SnapshotMigrationError(f"Migration incomplete: content mismatch for {rel_path}") # Convert index.json to index.jsonl in the new directory. self.convert_index_json_to_jsonl(output_dir=new_dir) @@ -1225,14 +1236,14 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW """ Delete old directory and create symlink after successful migration. """ - import shutil import logging + import shutil # Delete old directory if old_dir.exists() and not old_dir.is_symlink(): try: shutil.rmtree(old_dir) - except Exception as e: + except OSError as e: logging.getLogger("archivebox.migration").warning( f"Could not remove old migration directory {old_dir}: {e}", ) @@ -1246,7 +1257,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW if not symlink_path.exists(): try: symlink_path.symlink_to(new_dir, target_is_directory=True) - except Exception as e: + except OSError as e: logging.getLogger("archivebox.migration").warning( f"Could not create symlink from {symlink_path} to {new_dir}: {e}", ) @@ -1282,7 +1293,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW return parsed.scheme else: return "unknown" - except Exception: + except (TypeError, ValueError): return "unknown" def get_storage_path_for_version(self, version: str) -> Path: @@ -1722,6 +1733,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW """Create ArchiveResult if not already in DB.""" from dateutil import parser from django.db import transaction + from archivebox.machine.models import Machine, Process # Support both old 'extractor' and new 'plugin' keys for backwards compat @@ -1909,11 +1921,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW """ from archivebox.machine.models import Process from archivebox.misc.jsonl import ( - TYPE_SNAPSHOT, TYPE_ARCHIVERESULT, - TYPE_BINARYREQUEST, TYPE_BINARY, + TYPE_BINARYREQUEST, TYPE_PROCESS, + TYPE_SNAPSHOT, ) output_dir = Path(output_dir) if output_dir is not None else Path(self.output_dir) @@ -2059,10 +2071,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW Used by: archivebox update (when encountering invalid directories) """ - from datetime import datetime import shutil - invalid_dir = CONSTANTS.DATA_DIR / "invalid" / datetime.now().strftime("%Y%m%d") + invalid_dir = CONSTANTS.DATA_DIR / "invalid" / datetime.now(UTC).strftime("%Y%m%d") invalid_dir.mkdir(parents=True, exist_ok=True) dest = invalid_dir / snapshot_dir.name @@ -2073,8 +2084,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW try: shutil.move(str(snapshot_dir), str(dest)) - except Exception: - pass + except OSError: + return @classmethod def find_and_merge_duplicates(cls) -> int: @@ -2098,8 +2109,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW try: cls._merge_snapshots(snapshots) merged += 1 - except Exception: - pass + except OSError: + continue return merged @@ -2134,8 +2145,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW try: shutil.rmtree(dup_dir) - except Exception: - pass + except OSError: + continue # Merge tags for tag in dup.tags.all(): @@ -2200,6 +2211,29 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW running = int(progress_stats.get("running") or 0) completed = succeeded + failed + skipped + noresults percent = int((completed / total * 100) if total > 0 else 0) + successful_plugins = sorted(self.__dict__.get("_icons_archive_results") or ()) + visible_plugins = successful_plugins[:8] + successful_icons = format_html( + '
{}
', + mark_safe( + "".join( + str(format_html('{}', plugin, mark_safe(get_plugin_icon(plugin)))) + for plugin in visible_plugins + if str(get_plugin_icon(plugin)).strip() + ) + + ( + str( + format_html( + '+{}', + len(successful_plugins) - 8, + len(successful_plugins) - 8, + ), + ) + if len(successful_plugins) > 8 + else "" + ), + ), + ) return format_html( '
' '
' @@ -2212,6 +2246,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW '
' "✓{} ✗{} ⏳{}" "
" + "{}" "
", completed, total, @@ -2221,6 +2256,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW succeeded, failed, running, + successful_icons, ) precomputed_archive_results = self.__dict__.get("_icons_archive_results") @@ -2363,7 +2399,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW try: data = json.loads(hashes_path.read_text(encoding="utf-8")) - except Exception: + except (json.JSONDecodeError, OSError, TypeError, ValueError): return {} index: dict[str, dict[str, Any]] = {} @@ -2524,7 +2560,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW output_dir = Path(self.output_dir).resolve() try: rel_users_path = output_dir.relative_to(CONSTANTS.USERS_DIR) - except Exception: + except ValueError: rel_users_path = None if rel_users_path: @@ -2541,7 +2577,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW try: rel_path = output_dir.relative_to(CONSTANTS.DATA_DIR) - except Exception: + except ValueError: return self.legacy_archive_path parts = rel_path.parts @@ -2687,7 +2723,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW Snapshot instance or None """ import re + from django.utils import timezone + from archivebox.base_models.models import get_or_create_system_user_pk config = get_config() @@ -2712,9 +2750,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW continue # Special parsing for date fields - if field_name in ("bookmarked_at", "retry_at", "created_at", "modified_at"): - if value and isinstance(value, str): - value = parse_date(value) + if field_name in ("bookmarked_at", "retry_at", "created_at", "modified_at") and value and isinstance(value, str): + value = parse_date(value) # Update field if value is provided and different if value is not None and getattr(snapshot, field_name) != value: @@ -2758,8 +2795,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW crawl = parent_snapshot.crawl else: # Auto-create a single-URL crawl - from archivebox.crawls.models import Crawl from archivebox.config import CONSTANTS + from archivebox.crawls.models import Crawl timestamp_str = timezone.now().strftime("%Y-%m-%d__%H-%M-%S") sources_file = CONSTANTS.SOURCES_DIR / f"{timestamp_str}__auto_crawl.txt" @@ -2876,8 +2913,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW return [] if hooks is None: - from archivebox.plugins.hooks import discover_hooks from archivebox.config.common import get_config + from archivebox.plugins.hooks import discover_hooks # Compatibility path for direct model callers. The runner passes its # abx-dl hook inventory explicitly so queued rows match execution. @@ -3127,9 +3164,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW def latest_outputs(self, status: str | None = None) -> dict[str, Any]: """Get the latest output that each plugin produced""" - from archivebox.plugins.discovery import get_plugins from django.db.models import Q + from archivebox.plugins.discovery import get_plugins + latest: dict[str, Any] = {} for plugin in get_plugins(): results = self.archiveresult_set.filter(plugin=plugin) @@ -3388,6 +3426,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW def write_html_details(self, out_dir: Path | str | None = None) -> None: """Write HTML detail page for this snapshot to its output directory""" from django.template.loader import render_to_string + from archivebox.core.widgets import TagEditorWidget from archivebox.misc.logging_util import printable_filesize @@ -3427,7 +3466,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW "snapshot": self, "title": htmldecode(self.resolved_title or (self.base_url if is_archived else TITLE_LOADING_MSG)), "url_str": htmldecode(urldecode(self.base_url)), - "archive_url": urlencode(f"warc/{self.timestamp}" or (self.domain if is_archived else "")) or "about:blank", + "archive_url": urlencode(f"warc/{self.timestamp}") or "about:blank", "extension": self.extension or "html", "tags": self.tags_str() or "untagged", "size": printable_filesize(output_size) if output_size else "pending", @@ -3861,12 +3900,12 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes): app_label = "core" verbose_name = "Archive Result" verbose_name_plural = "Archive Results Log" - indexes = [ + indexes: ClassVar[list[models.Index]] = [ models.Index(fields=["snapshot", "status"], name="archiveresult_snap_status_idx"), models.Index(fields=["status", "snapshot"], name="archiveresult_status_snap_idx"), models.Index(fields=["-start_ts", "-id"], name="archiveresult_start_idx"), ] - constraints = [ + constraints: ClassVar[list[models.BaseConstraint]] = [ models.UniqueConstraint(fields=["snapshot", "plugin", "hook_name"], name="unique_archiveresult_per_snapshot_hook"), ] @@ -4228,6 +4267,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes): def update_output_metadata_from_filesystem(self, snapshot_dir: Path | None = None, save: bool = True) -> bool: from collections import defaultdict + from abx_dl.output_files import guess_mimetype if self.plugin == "title": @@ -4582,10 +4622,12 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes): """ from collections import defaultdict from pathlib import Path - from django.utils import timezone + from abx_dl.output_files import guess_mimetype - from archivebox.plugins.hooks import process_hook_records, extract_records_from_process + from django.utils import timezone + from archivebox.machine.models import Process + from archivebox.plugins.hooks import extract_records_from_process, process_hook_records plugin_dir = Path(self.pwd) if self.pwd else None if not plugin_dir or not plugin_dir.exists(): @@ -4637,8 +4679,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes): from archivebox.plugins.hooks import is_background_hook is_background = bool(self.hook_name and is_background_hook(self.hook_name)) - except Exception: - pass + except (ImportError, TypeError, ValueError): + is_background = False if is_background or (process and process.exit_code == 0): self.status = self.StatusChoices.SKIPPED diff --git a/archivebox/core/routes_util.py b/archivebox/core/routes_util.py index b47f6a26..60278480 100644 --- a/archivebox/core/routes_util.py +++ b/archivebox/core/routes_util.py @@ -6,8 +6,7 @@ from urllib.parse import urlparse from django.utils.http import url_has_allowed_host_and_scheme -from archivebox.config.common import get_config - +from archivebox.config.common import get_config, get_request_config _SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$") _SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P[0-9a-fA-F]{12})$") @@ -153,7 +152,7 @@ def _root_host_from_listen(config: dict[str, Any] | None = None, **config_kwargs def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: - config = config or get_config(**config_kwargs) + config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs)) override = _normalize_base_url(config.BASE_URL) if override: return override @@ -281,9 +280,7 @@ def host_matches(request_host: str, target_host: str) -> bool: target_host_only, target_port = split_host_port(target_host) if req_host != target_host_only: return False - if target_port and req_port and target_port != req_port: - return False - return True + return not (target_port and req_port and target_port != req_port) def is_allowed_archivebox_redirect_url(url: str | None, request=None, config: dict[str, Any] | None = None) -> bool: @@ -352,28 +349,28 @@ def _build_base_url_for_host(host: str, request=None, config: dict[str, Any] | N def get_admin_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: - config = config or get_config(**config_kwargs) + config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs)) if not config.USES_SUBDOMAIN_ROUTING: return get_base_url(request=request, config=config) return _build_base_url_for_host(_build_base_host("admin", request=request, config=config), request=request, config=config) def get_web_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: - config = config or get_config(**config_kwargs) + config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs)) if not config.USES_SUBDOMAIN_ROUTING: return get_base_url(request=request, config=config) return _build_base_url_for_host(_build_base_host("web", request=request, config=config), request=request, config=config) def get_api_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: - config = config or get_config(**config_kwargs) + config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs)) if not config.USES_SUBDOMAIN_ROUTING: return get_base_url(request=request, config=config) return _build_base_url_for_host(_build_base_host("api", request=request, config=config), request=request, config=config) def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: - config = config or get_config(**config_kwargs) + config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs)) if not config.USES_SUBDOMAIN_ROUTING: return _build_url(get_web_base_url(request=request, config=config), f"/snapshot/{str(snapshot_id).replace('-', '')}") return _build_base_url_for_host( @@ -384,7 +381,7 @@ def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] def get_original_base_url(domain: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: - config = config or get_config(**config_kwargs) + config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs)) if not config.USES_SUBDOMAIN_ROUTING: return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}") return _build_base_url_for_host(_build_base_host(domain, request=request, config=config), request=request, config=config) diff --git a/archivebox/core/setup_wizard.py b/archivebox/core/setup_wizard.py new file mode 100644 index 00000000..36020c33 --- /dev/null +++ b/archivebox/core/setup_wizard.py @@ -0,0 +1,68 @@ +from typing import Any + +from archivebox.core.routes_util import canonical_base_host_for_request, get_base_url, split_host_port + + +def get_setup_wizard_context(request, config) -> dict[str, Any]: + """Build the first-run setup context when ``BASE_URL`` is unset.""" + context = { + "mode": "unconfigured", + "canonical_host": "", + "display_host": "", + "suggested_base_url": "", + "machine_admin_url": "", + "can_configure": False, + "public_index": config.PUBLIC_INDEX, + "public_add_view": config.PUBLIC_ADD_VIEW, + "permissions": config.PERMISSIONS, + } + if request is None: + return context + + scheme = request.scheme or "http" + canonical_host = canonical_base_host_for_request(request.get_host() or "") + display_hostname, display_port = split_host_port(canonical_host) + display_host = canonical_host + if (scheme, display_port) in (("http", "80"), ("https", "443")): + display_host = display_hostname + + user = request.user + is_superuser = bool(user and user.is_authenticated and user.is_superuser) + machine_admin_url = "" + if is_superuser: + try: + from archivebox.machine.models import Machine + + machine_admin_url = f"/admin/machine/machine/{Machine.current().id}/change/" + except (ImportError, RuntimeError, TypeError, ValueError): + machine_admin_url = "" + + context.update( + canonical_host=canonical_host, + display_host=display_host, + suggested_base_url=f"{scheme}://{canonical_host}" if canonical_host else "", + machine_admin_url=machine_admin_url, + can_configure=is_superuser, + ) + return context + + +def get_base_url_mismatch_context(request, config) -> dict[str, str] | None: + """Describe a request origin that does not resolve to the configured base.""" + if request is None or not config.BASE_URL: + return None + + scheme = request.scheme or "http" + browser_url = f"{scheme}://{request.get_host()}".rstrip("/") + browser_base_url = browser_url + if config.USES_SUBDOMAIN_ROUTING: + browser_base_url = f"{scheme}://{canonical_base_host_for_request(request.get_host())}".rstrip("/") + configured_base_url = get_base_url(config=config).rstrip("/") + if browser_base_url.lower() == configured_base_url.lower(): + return None + + return { + "mode": "base_url_mismatch", + "browser_url": browser_url, + "configured_base_url": configured_base_url, + } diff --git a/archivebox/core/templatetags/core_tags.py b/archivebox/core/templatetags/core_tags.py index 11825af8..30ecd6ee 100644 --- a/archivebox/core/templatetags/core_tags.py +++ b/archivebox/core/templatetags/core_tags.py @@ -1,36 +1,33 @@ +import os +from pathlib import Path from typing import Any +from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target from django import template from django.contrib.admin.templatetags.base import InclusionAdminNode -from django.utils.safestring import mark_safe -from django.utils.html import escape from django.templatetags.static import static from django.utils import timezone +from django.utils.html import escape +from django.utils.safestring import mark_safe from django.utils.text import Truncator -from pathlib import Path - -from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target - +from archivebox.core.routes_util import ( + build_snapshot_url, + get_admin_base_url, + get_snapshot_base_url, + get_web_base_url, +) +from archivebox.core.setup_wizard import get_base_url_mismatch_context, get_setup_wizard_context from archivebox.plugins.discovery import ( get_plugin_icon, - get_plugin_template, get_plugin_name, + get_plugin_template, ) -from archivebox.core.routes_util import ( - canonical_base_host_for_request, - get_admin_base_url, - get_web_base_url, - get_snapshot_base_url, - build_snapshot_url, -) - register = template.Library() _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log") _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif") -_MHTML_PREVIEW_EXTS = (".mhtml", ".mht") _MEDIA_FILE_EXTS = { ".mp4", @@ -94,15 +91,15 @@ def _coerce_output_file_size(value: Any) -> int | None: def _count_media_files(result) -> int: try: output_files = _normalize_output_files(result.output_files or {}) - except Exception: + except (AttributeError, TypeError, ValueError): output_files = {} if output_files: - return sum(1 for path in output_files.keys() if Path(path).suffix.lower() in _MEDIA_FILE_EXTS) + return sum(1 for path in output_files if Path(path).suffix.lower() in _MEDIA_FILE_EXTS) try: plugin_dir = Path(result.snapshot_dir) / result.plugin - except Exception: + except (AttributeError, TypeError, ValueError): return 0 if not plugin_dir.exists(): @@ -126,7 +123,7 @@ def _list_media_files(result) -> list[dict]: media_files: list[dict] = [] try: plugin_dir = Path(result.snapshot_dir) / result.plugin - except Exception: + except (AttributeError, TypeError, ValueError): return media_files output_files = _normalize_output_files(result.output_files or {}) @@ -184,7 +181,7 @@ def _resolve_snapshot_output_file(snapshot_dir: str | Path | None, raw_output_pa snap_dir = Path(snapshot_dir).resolve() if snap_dir not in output_file.parents and output_file != snap_dir: return None - except Exception: + except (OSError, RuntimeError, TypeError, ValueError): return None if output_file.exists() and output_file.is_file(): @@ -209,17 +206,17 @@ def _build_snapshot_files_url(snapshot_id: str, request=None, config=None) -> st return build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config) -def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None, config=None) -> str: +def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None, config=None, plugin: str = "") -> str: if path == "about:blank": return path if _is_root_snapshot_output_path(path): return _build_snapshot_files_url(snapshot_id, request=request, config=config) url = build_snapshot_url(str(snapshot_id), path, request=request, config=config) + path_parts = Path(path).parts + plugin = get_plugin_name(plugin) if plugin else (path_parts[0] if len(path_parts) > 1 else "") + has_plugin_preview = bool(plugin and get_plugin_template(plugin, "full", fallback=False)) if not ( - _is_text_preview_path(path) - or _is_image_preview_path(path) - or (path or "").lower().endswith(_MHTML_PREVIEW_EXTS) - or is_archivewebpage_replay_target(path or "") + _is_text_preview_path(path) or _is_image_preview_path(path) or has_plugin_preview or is_archivewebpage_replay_target(path or "") ): return url separator = "&" if "?" in url else "?" @@ -268,7 +265,7 @@ def _render_text_file_preview(snapshot_dir: str | Path | None, raw_output_path: lines = text.splitlines()[:6] snippet = "\n".join(lines) return _render_text_preview(plugin, icon_html, snippet) - except Exception: + except (OSError, UnicodeDecodeError, ValueError): return None @@ -281,12 +278,12 @@ def split(value, separator: str = ","): def index(value, position): try: return value[int(position)] - except Exception: + except (IndexError, TypeError, ValueError): return None @register.filter -def file_size(num_bytes: int | float) -> str: +def file_size(num_bytes: float) -> str: for count in ["Bytes", "KB", "MB", "GB"]: if num_bytes > -1024.0 and num_bytes < 1024.0: return f"{num_bytes:3.1f} {count}" @@ -359,7 +356,7 @@ def _machine_health_stats() -> dict: from archivebox.machine.detect import get_host_stats stats = get_host_stats() or {} - except Exception: + except (ImportError, RuntimeError, TypeError, ValueError): stats = {} _health_cache["checked_at"] = now @@ -375,13 +372,15 @@ def system_warnings_banner(context): 1. ``mode="unconfigured"``— ``BASE_URL`` is empty. Security/correctness issue: until it's pinned, generated URLs can echo any Host the client sends, and admin/web/api routing has no canonical anchor. - 2. ``mode="unsafe"`` — ``SERVER_SECURITY_MODE`` is a non-subdomain - mode. Archived pages share an origin with privileged routes. - 3. ``mode="low_disk"`` — ``DATA_DIR`` has <1 GiB free; new archive + 2. ``mode="base_url_mismatch"`` — the browser reached ArchiveBox through + an origin that differs from the configured canonical ``BASE_URL``. + 3. ``mode="unsafe"`` — ``SERVER_SECURITY_MODE`` explicitly enables + a lower-security replay mode. + 4. ``mode="low_disk"`` — ``DATA_DIR`` has <1 GiB free; new archive jobs will start failing on ENOSPC. - 4. ``mode="high_memory"`` — virtual memory utilization at/above 95%; the + 5. ``mode="high_memory"`` — virtual memory utilization at/above 95%; the host is one OOM-kill from a crash. - 5. ``mode="high_load"`` — 15-minute load average exceeds 3 × CPU count + 6. ``mode="high_load"`` — 15-minute load average exceeds 3 × CPU count (the kernel's own sustained-load EMA, so no rolling buffer of ours is needed). @@ -397,8 +396,11 @@ def system_warnings_banner(context): config = get_config(resolve_plugins=False) if not config.BASE_URL: - return _unconfigured_banner_context(context.get("request")) - if not config.USES_SUBDOMAIN_ROUTING: + return get_setup_wizard_context(context.get("request"), config) + mismatch = get_base_url_mismatch_context(context.get("request"), config) + if mismatch: + return mismatch + if config.IS_LOWER_SECURITY_MODE: return {"mode": "unsafe"} stats = _machine_health_stats() @@ -417,6 +419,8 @@ def system_warnings_banner(context): # "sustained for 15min" and the kernel already maintains that EMA. load_15 = cpu_load[2] if isinstance(cpu_load, (list, tuple)) and len(cpu_load) >= 3 else None if isinstance(load_15, (int, float)) and load_15 > _HIGH_LOAD_MULTIPLE * cpu_count: + if os.environ.get("UI_SCREENSHOT_HIDE_HIGH_LOAD_WARNING") == "1": + return {"mode": ""} return { "mode": "high_load", "load_15": f"{load_15:.2f}", @@ -427,49 +431,6 @@ def system_warnings_banner(context): return {"mode": ""} -def _unconfigured_banner_context(request) -> dict: - """Build the banner payload for the unset-BASE_URL case. - - Always returns ``mode="unconfigured"`` — the user explicitly asked for - the banner to render whenever ``BASE_URL`` is empty, regardless of - whether the request host happens to match a CSRF-derived value. The - ``suggested_base_url`` is derived from the current request when one is - available so the user can copy/paste the right value straight into - their config. - """ - if request is None: - return { - "mode": "unconfigured", - "actual_host": "", - "suggested_base_url": "", - "machine_admin_url": "", - } - scheme = request.scheme or "http" - actual_full_host = request.get_host() or "" - canonical_host = canonical_base_host_for_request(actual_full_host) - # Suggest the wildcard form ``http://*.`` so the value lands in the - # operator's clipboard already aligned with subdomain routing. The config - # parser strips the leading ``*.`` so users can paste it verbatim. - suggested_base_url = f"{scheme}://*.{canonical_host}" if canonical_host else "" - user = request.user - is_superuser = bool(user and user.is_authenticated and user.is_superuser) - machine_admin_url = "" - if is_superuser: - try: - from archivebox.machine.models import Machine - - machine = Machine.current() - machine_admin_url = f"/admin/machine/machine/{machine.id}/change/" - except Exception: - machine_admin_url = "" - return { - "mode": "unconfigured", - "actual_host": actual_full_host, - "suggested_base_url": suggested_base_url, - "machine_admin_url": machine_admin_url, - } - - @register.simple_tag(takes_context=True) def url_replace(context, **kwargs): dict_ = context["request"].GET.copy() @@ -581,7 +542,7 @@ def snapshot_index_row(context, link) -> str: url = getattr(link, "url", "") or "" title = getattr(link, "title", "") or "" is_pending = status in {"queued", "started", "backoff"} - title_text = title or ("Loading..." if is_pending else url) + title_text = title or url tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "") tag_html = "".join(f'{escape(tag)}' for tag in (tags_str or "").split(",") if tag) if tag_html: @@ -682,6 +643,7 @@ def snapshot_index_row(context, link) -> str: {escape(url)} + Saved {escape(date_text)} at {escape(time_text)} {tag_cell} @@ -706,9 +668,16 @@ def snapshot_index_row(context, link) -> str: @register.simple_tag(takes_context=True) -def snapshot_preview_url(context, snapshot, path: str = "") -> str: +def snapshot_preview_url(context, snapshot, path: str = "", result=None) -> str: snapshot_id = _snapshot_id(snapshot) - return _build_snapshot_preview_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG")) + plugin = getattr(result, "plugin", "") if result else "" + return _build_snapshot_preview_url( + str(snapshot_id), + path, + request=context.get("request"), + config=context.get("CONFIG"), + plugin=plugin, + ) @register.simple_tag @@ -789,8 +758,8 @@ def plugin_card(context, result) -> str: # Only return non-empty content (strip whitespace to check) if rendered.strip(): return mark_safe(rendered) - except Exception: - pass + except (template.TemplateSyntaxError, AttributeError, TypeError, ValueError): + rendered = "" if force_text_preview: preview = _render_text_file_preview(result.snapshot_dir, raw_output_path, plugin, icon_html) @@ -865,7 +834,7 @@ def plugin_full(context, result) -> str: if rendered.strip(): return mark_safe(rendered) return "" - except Exception: + except (template.TemplateSyntaxError, AttributeError, TypeError, ValueError): return "" diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 857d7170..d154ac25 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -3,71 +3,50 @@ __package__ = "archivebox.core" import json import os import posixpath -from glob import glob, escape -from django.utils import timezone -from typing import cast +from glob import escape, glob from pathlib import Path +from typing import ClassVar, cast from urllib.parse import quote, urlparse -from django.shortcuts import render, redirect -from django.http import HttpRequest, HttpResponse, Http404, HttpResponseForbidden, QueryDict -from django.utils.html import format_html, format_html_join -from django.utils.safestring import mark_safe -from django.views import View -from django.views.generic.list import ListView -from django.views.generic import FormView -from django.db.models import Case, IntegerField, Q, Value, When -from django.core.paginator import InvalidPage -from django.contrib import messages +from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay +from admin_data_views.typing import ItemContext, SectionData, TableContext +from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view +from django import template from django.conf import settings +from django.contrib import messages from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model from django.contrib.auth.mixins import UserPassesTestMixin from django.contrib.sessions.models import Session from django.core import signing -from django.views.decorators.csrf import csrf_exempt +from django.core.paginator import InvalidPage +from django.db.models import Case, IntegerField, Q, Value, When +from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidden, QueryDict +from django.shortcuts import redirect, render +from django.utils import timezone from django.utils.decorators import method_decorator - -from admin_data_views.typing import TableContext, ItemContext, SectionData -from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink - -from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay +from django.utils.html import format_html, format_html_join +from django.utils.safestring import mark_safe +from django.views import View +from django.views.decorators.csrf import csrf_exempt +from django.views.generic import FormView +from django.views.generic.list import ListView from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, VERSION from archivebox.config.common import ( + PLUGIN_CONFIG_SCHEMAS, SENSITIVE_CONFIG_VALUE_REDACTED, + _plugin_config_properties, find_config_default, find_config_section, find_config_source, find_config_type, - get_config, get_all_configs, + get_config, get_request_config, - _plugin_config_properties, redact_sensitive_config, ) -from archivebox.config.common import PLUGIN_CONFIG_SCHEMAS from archivebox.config.configset import BaseConfigSet -from archivebox.misc.paginators import AcceleratedPaginator -from archivebox.misc.util import ( - base_url, - filter_queryset_by_uuid_substring, - htmldecode, - sanitize_html_text, - ts_to_date_str, - urldecode, - validate_url, - without_fragment, -) -from archivebox.misc.serve_static import serve_static_with_byterange_support -from archivebox.misc.logging_util import printable_filesize -from archivebox.search.config import ( - get_search_mode, - get_search_mode_backend, - get_search_mode_base, - get_search_mode_options, -) -from archivebox.search.views import get_cached_public_search_state - +from archivebox.core.forms import AddLinkForm from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag from archivebox.core.permissions import ( PERMISSIONS_PRIVATE, @@ -90,11 +69,31 @@ from archivebox.core.routes_util import ( get_web_host, host_matches, ) -from archivebox.core.forms import AddLinkForm -from archivebox.plugins.forms import get_plugin_config_binary_urls from archivebox.crawls.models import Crawl +from archivebox.misc.logging_util import printable_filesize +from archivebox.misc.paginators import AcceleratedPaginator +from archivebox.misc.serve_static import serve_static_with_byterange_support +from archivebox.misc.util import ( + base_url, + filter_queryset_by_uuid_substring, + htmldecode, + sanitize_html_text, + ts_to_date_str, + urldecode, + validate_url, + without_fragment, +) +from archivebox.plugins.discovery import get_plugin_name, get_plugin_template +from archivebox.plugins.forms import get_plugin_config_binary_urls from archivebox.plugins.views import get_config_definition_link from archivebox.progressmonitor.views import live_progress_view, progress_endpoint +from archivebox.search.config import ( + get_search_mode, + get_search_mode_backend, + get_search_mode_base, + get_search_mode_options, +) +from archivebox.search.views import get_cached_public_search_state def _files_index_target(snapshot: Snapshot, archivefile: str | None) -> str: @@ -169,7 +168,7 @@ def _replay_payload_is_valid(payload: dict, snapshot: Snapshot) -> bool: user_id = str(session_data.get(SESSION_KEY) or "") auth_hash = str(session_data.get(HASH_SESSION_KEY) or "") user = get_user_model().objects.get(pk=user_id) - except Exception: + except (Session.DoesNotExist, get_user_model().DoesNotExist, KeyError, TypeError, ValueError): return False return ( str(payload.get("user_id")) == user_id @@ -343,7 +342,11 @@ class SnapshotView(View): for out in snapshot.discover_outputs(include_filesystem_fallback=True) if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins ] - archiveresults = {out["name"]: out for out in outputs} + archiveresults = {} + for output in outputs: + current = archiveresults.get(output["name"]) + if current is None or (output.get("size") or 0) > (current.get("size") or 0): + archiveresults[output["name"]] = output hash_index = snapshot.hashes_index accounted_entries: set[str] = set() for output in outputs: @@ -446,6 +449,7 @@ class SnapshotView(View): "id": str(snapshot.id), "snapshot_id": str(snapshot.id), "progress_endpoint": progress_endpoint("snapshot", snapshot.id), + "progress_auto_expand": snapshot_status in {"queued", "started", "paused"}, "url": snapshot.url, "archive_path": snapshot.archive_path_from_db, "title": htmldecode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)), @@ -632,10 +636,7 @@ class SnapshotView(View): return SnapshotView.find_snapshots_for_url(slug) try: - try: - snapshot = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path)).get() - except Snapshot.DoesNotExist: - raise + snapshot = direct_snapshots_queryset(request, _resolve_snapshots_for_slug(path)).get() except Snapshot.DoesNotExist: return HttpResponse( format_html( @@ -827,21 +828,23 @@ def _safe_archive_relpath(path: str) -> str | None: return cleaned -def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[str, bool]: +def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[str, ArchiveResult | None]: """Resolve plugin-relative output paths through ArchiveResult.output_files.""" parts = Path(rel_path).parts if len(parts) < 2: - return rel_path, False + return rel_path, None plugin = parts[0] plugin_relpath = posixpath.join(*parts[1:]) result = ( ArchiveResult.objects.filter(snapshot=snapshot, plugin=plugin, status=ArchiveResult.StatusChoices.SUCCEEDED) - .only("output_files") + .only("plugin", "output_files") .first() ) - if not result or not result.output_files: - return rel_path, False + if not result: + return rel_path, None + if not result.output_files: + return rel_path, result output_files = result.output_files or {} for candidate in (plugin_relpath, rel_path): @@ -849,10 +852,79 @@ def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[s if not isinstance(file_info, dict): continue if file_info.get("root_relative"): - return candidate, True - return rel_path, True + return candidate, result + return rel_path, result - return rel_path, True + return rel_path, result + + +def _plugin_full_preview_response( + request: HttpRequest, + snapshot: Snapshot, + rel_path: str, + result: ArchiveResult | None, +) -> HttpResponse | None: + """Render an explicit plugin full template as a trusted preview wrapper.""" + if not request.GET.get("preview"): + return None + + path_parts = Path(rel_path).parts + plugin = get_plugin_name(result.plugin) if result else (path_parts[0] if len(path_parts) > 1 else "") + if not plugin: + return None + + # ReplayWeb.page needs plugin-owned WACZ inspection and service-worker + # context, so it remains the one narrow preview exception below. + if plugin == "archivewebpage" and archivewebpage_replay.is_replay_target(rel_path): + return None + + template_str = get_plugin_template(plugin, "full", fallback=False) + if not template_str: + return None + + raw_query = request.GET.copy() + raw_query.pop("preview", None) + output_url = request.path + if raw_query: + output_url = f"{output_url}?{raw_query.urlencode()}" + + rendered = ( + template.Engine(debug=False) + .from_string(template_str) + .render( + template.Context( + { + "result": result, + "snapshot": snapshot, + "output_path": output_url, + "output_path_raw": rel_path, + "plugin": plugin, + "preview_base": f"{request.path.rsplit('/', 1)[0]}/", + }, + ), + ) + ) + response = HttpResponse(rendered, content_type="text/html; charset=utf-8") + response.headers["Content-Disposition"] = f'inline; filename="{Path(rel_path).stem}.html"' + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["X-ArchiveBox-Security-Mode"] = request.archivebox_config.SERVER_SECURITY_MODE + response.headers["Referrer-Policy"] = "no-referrer" + response.headers["Content-Security-Policy"] = ( + "default-src 'self' data: blob:; " + "script-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:; " + "style-src 'self' 'unsafe-inline' data: blob:; " + "connect-src 'self' data: blob:; " + "img-src 'self' data: blob:; " + "media-src 'self' data: blob:; " + "font-src 'self' data: blob:; " + "frame-src 'self' data: blob:; " + "worker-src 'self' blob:; " + "object-src 'none'; " + "base-uri 'none'; " + "form-action 'none'; " + "frame-ancestors 'self';" + ) + return response def _coerce_sort_timestamp(value: str | float | None) -> float: @@ -872,7 +944,7 @@ def _snapshot_sort_key(match_path: str, cache: dict[str, float]) -> tuple[float, idx = parts.index("snapshots") date_str = parts[idx + 1] snapshot_id = parts[idx + 3] - except Exception: + except (IndexError, ValueError): return (_coerce_sort_timestamp(date_str), match_path) if snapshot_id not in cache: @@ -1048,7 +1120,11 @@ def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = if rel_path is None: raise Http404 - rel_path, _is_known_plugin_output = _resolve_archiveresult_relpath(snapshot, rel_path) + rel_path, archive_result = _resolve_archiveresult_relpath(snapshot, rel_path) + + plugin_preview = _plugin_full_preview_response(request, snapshot, rel_path, archive_result) + if plugin_preview is not None: + return plugin_preview try: return serve_static_with_byterange_support( @@ -1172,7 +1248,7 @@ class OriginalDomainReplayView(View): class PublicIndexView(ListView): template_name = "public_index.html" model = Snapshot - ordering = ["-bookmarked_at", "-created_at"] + ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"] paginator_class = AcceleratedPaginator public_page_scan_chunk_size = 50 @@ -1634,13 +1710,16 @@ class WebAddView(AddView): return redirect(f"/{snapshot.url_path}") request_host = (request.get_host() or "").lower() - if request.user.is_authenticated and not get_request_config(request).PUBLIC_ADD_VIEW and host_matches(request_host, get_web_host()): + request_config = get_request_config(request) + web_host = get_web_host(config=request_config) + admin_host = get_admin_host(config=request_config) + if request.user.is_authenticated and not request_config.PUBLIC_ADD_VIEW and host_matches(request_host, web_host): return redirect(build_admin_url(request.get_full_path(), request=request)) if not self.test_func(): - if host_matches(request_host, get_web_host()): + if host_matches(request_host, web_host): return redirect(build_admin_url(request.get_full_path(), request=request)) - if host_matches(request_host, get_admin_host()): + if host_matches(request_host, admin_host): next_url = quote(request.get_full_path(), safe="/:?=&") return redirect(f"{build_admin_url('/admin/login/', request=request)}?next={next_url}") return HttpResponse( @@ -1733,7 +1812,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext: } for section_id, section in reversed(list(CONFIGS.items())): - for key in dict(section).keys(): + for key in dict(section): rows["Section"].append(section_id) # section.replace('_', ' ').title().replace(' Config', '') rows["Key"].append(ItemLink(key, key=key)) rows["Type"].append(format_html("{}", find_config_type(key))) @@ -1756,7 +1835,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext: ) section = "CONSTANT" - for key in CONSTANTS_CONFIG.keys(): + for key in CONSTANTS_CONFIG: rows["Section"].append(section) # section.replace('_', ' ').title().replace(' Config', '') rows["Key"].append(ItemLink(key, key=key)) rows["Type"].append(format_html("{}", type(CONSTANTS_CONFIG[key]).__name__)) diff --git a/archivebox/crawls/admin.py b/archivebox/crawls/admin.py index 9f43c48d..dd9eaa14 100644 --- a/archivebox/crawls/admin.py +++ b/archivebox/crawls/admin.py @@ -1,27 +1,25 @@ __package__ = "archivebox.crawls" -from copy import copy import json +from copy import copy +from typing import ClassVar from urllib.parse import urlencode, urlparse from django import forms -from django.core.paginator import Paginator -from django.http import JsonResponse, HttpRequest, HttpResponseBadRequest, HttpResponseNotAllowed -from django.shortcuts import get_object_or_404, redirect -from django.template.response import TemplateResponse -from django.template.loader import render_to_string -from django.urls import path, reverse -from django.utils.html import escape, format_html, format_html_join -from django.utils import timezone -from django.utils.safestring import mark_safe from django.contrib import admin, messages +from django.core.paginator import Paginator from django.db.models import Count, F, Q - - +from django.http import HttpRequest, HttpResponseBadRequest, HttpResponseNotAllowed, JsonResponse +from django.shortcuts import get_object_or_404, redirect +from django.template.loader import render_to_string +from django.template.response import TemplateResponse +from django.urls import path, reverse +from django.utils import timezone +from django.utils.html import escape, format_html, format_html_join +from django.utils.safestring import mark_safe from django_object_actions import action from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin - from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.permissions import ( PERMISSIONS_CHOICES, @@ -397,7 +395,7 @@ class CrawlAdminForm(forms.ModelForm): class Meta: model = Crawl fields = "__all__" - widgets = { + widgets: ClassVar[dict[str, forms.Widget]] = { "urls": forms.Textarea( attrs={ "rows": 8, @@ -704,16 +702,16 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): ) list_filter = (MaxDepthListFilter, "schedule", "created_by", "status", "retry_at") - ordering = ["-created_at", "-retry_at"] + ordering = ("-created_at", "-retry_at") list_per_page = 50 - actions = [ + actions = ( "pause_selected_crawls", "resume_selected_crawls", "seal_selected_crawls", "delete_selected_batched", "set_crawl_permissions", - ] - change_actions = ["recrawl"] + ) + change_actions = ("recrawl",) def __init__(self, model, admin_site): super().__init__(model, admin_site) @@ -721,8 +719,8 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin): self.stop_reason_cache = {} class Media: - css = {"all": ("admin/crawls/crawl_change.css",)} - js = ("admin/crawls/crawl_admin.js",) + css: ClassVar[dict[str, tuple[str, ...]]] = {"all": ("admin/crawls/crawl_change.css",)} + js: ClassVar[tuple[str, ...]] = ("admin/crawls/crawl_admin.js",) def changelist_view(self, request, extra_context=None): self.request = request @@ -1219,6 +1217,7 @@ class CrawlScheduleAdmin(BaseModelAdmin): search_fields = ("id", "created_by__username", "label", "notes", "schedule_id", "template_id", "template__urls") readonly_fields = ("created_at", "modified_at", "crawls", "snapshots") + autocomplete_fields = ("template", "created_by") fieldsets = ( ( @@ -1259,9 +1258,9 @@ class CrawlScheduleAdmin(BaseModelAdmin): ) list_filter = ("created_by",) - ordering = ["-created_at"] + ordering = ("-created_at",) list_per_page = 100 - actions = ["delete_selected"] + actions = ("delete_selected",) def get_queryset(self, request): self.request = request diff --git a/archivebox/machine/admin.py b/archivebox/machine/admin.py index 889f6769..ef9a4171 100644 --- a/archivebox/machine/admin.py +++ b/archivebox/machine/admin.py @@ -5,6 +5,7 @@ import shlex from pathlib import Path from django.contrib import admin, messages +from django.db import DatabaseError from django.db.models import DurationField, ExpressionWrapper, F from django.db.models.functions import Coalesce, Now from django.shortcuts import redirect @@ -14,9 +15,9 @@ from django.utils.safestring import mark_safe from django_object_actions import action from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin -from archivebox.misc.logging_util import printable_filesize from archivebox.machine.env_util import env_to_dotenv_text -from archivebox.machine.models import Machine, NetworkInterface, Binary, Process +from archivebox.machine.models import Binary, Machine, NetworkInterface, Process +from archivebox.misc.logging_util import printable_filesize def _render_copy_block(text: str, *, multiline: bool = False): @@ -102,6 +103,21 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin): "os_release", "hw_uuid", ) + search_fields = ( + "id", + "guid", + "hostname", + "networkinterface__ip_public", + "networkinterface__ip_local", + "hw_manufacturer", + "hw_product", + "hw_uuid", + "os_platform", + "os_family", + "os_arch", + "os_release", + "os_kernel", + ) readonly_fields = ("guid", "created_at", "modified_at", "ips") @@ -162,9 +178,9 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin): ) list_filter = ("hw_in_docker", "hw_in_vm", "os_arch", "os_family", "os_platform") - ordering = ["-created_at"] + ordering = ("-created_at",) list_per_page = 100 - actions = ["delete_selected"] + actions = ("delete_selected",) @admin.display(description="Public IP", ordering="networkinterface__ip_public") def ips(self, machine): @@ -192,7 +208,7 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin): try: current_id = str(Machine.current().pk) - except Exception: + except (DatabaseError, RuntimeError, TypeError, ValueError): current_id = None machine_id = str(machine.pk) @@ -297,9 +313,9 @@ class NetworkInterfaceAdmin(BaseModelAdmin): ) list_filter = ("isp", "country", "region") - ordering = ["-created_at"] + ordering = ("-created_at",) list_per_page = 100 - actions = ["delete_selected"] + actions = ("delete_selected",) @admin.display(description="Machine", ordering="machine__id") def machine_info(self, iface): @@ -370,9 +386,9 @@ class BinaryAdmin(BaseModelAdmin): ) list_filter = ("name", "binprovider", "status", "machine_id") - ordering = ["-created_at"] + ordering = ("-created_at",) list_per_page = 100 - actions = ["delete_selected"] + actions = ("delete_selected",) @admin.display(description="Machine", ordering="machine__id") def machine_info(self, binary): @@ -492,10 +508,10 @@ class ProcessAdmin(BaseModelAdmin): ) list_filter = ("status", "exit_code", "machine_id") - ordering = ["-created_at"] + ordering = ("-created_at",) list_per_page = 100 - actions = ["kill_processes", "delete_selected"] - change_actions = ["kill_process"] + actions = ("kill_processes", "delete_selected") + change_actions = ("kill_process",) def get_queryset(self, request): return ( @@ -694,7 +710,7 @@ class ProcessAdmin(BaseModelAdmin): if isinstance(output_files, str): try: output_files = json.loads(output_files) - except Exception: + except (json.JSONDecodeError, TypeError, ValueError): output_files = {} file_count = 0 diff --git a/archivebox/mcp/server.py b/archivebox/mcp/server.py index 26196b79..5888f710 100644 --- a/archivebox/mcp/server.py +++ b/archivebox/mcp/server.py @@ -1,13 +1,13 @@ -""" -Model Context Protocol (MCP) server implementation for ArchiveBox. +"""Model Context Protocol server for ArchiveBox. -Dynamically exposes all ArchiveBox CLI commands as MCP tools by introspecting -Click command metadata. Handles JSON-RPC 2.0 requests over stdio transport. +The server exposes five agent-friendly workflow tools backed by ArchiveBox's +existing Click CLI and newline-delimited JSON-RPC. """ -import sys import json +import sys import traceback +from dataclasses import dataclass from typing import Any import click @@ -15,388 +15,553 @@ from click.testing import CliRunner from archivebox.config.version import VERSION +PROTOCOL_VERSION = "2025-11-25" +PUBLIC_TOOLS = ("add", "search", "crawl", "snapshot", "archiveresult", "shell") +ACTION_TOOLS = {"crawl", "snapshot", "archiveresult"} +READ_ONLY_ACTIONS = {"help", "list", "search", "status", "version"} +DESTRUCTIVE_ACTIONS = {"delete", "remove"} + class MCPJSONEncoder(json.JSONEncoder): - """Custom JSON encoder that handles Click sentinel values and other special types""" + """JSON encoder for UUIDs, paths, Click sentinels, and other CLI values.""" - def default(self, o): - # Handle Click's sentinel values + def default(self, value): sentinel_type = getattr(click.core, "_SentinelClass", None) - if isinstance(sentinel_type, type) and isinstance(o, sentinel_type): + if isinstance(sentinel_type, type) and isinstance(value, sentinel_type): return None - - # Handle tuples (convert to lists) - if isinstance(o, tuple): - return list(o) - - # Handle any other non-serializable objects + if isinstance(value, tuple): + return list(value) try: - return super().default(o) + return super().default(value) except TypeError: - return str(o) + return str(value) + + +@dataclass(frozen=True) +class MCPTool: + """A discovered leaf Click command and its ArchiveBox command path.""" + + name: str + command_path: tuple[str, ...] + command: click.Command -# Type mapping from Click types to JSON Schema types def click_type_to_json_schema_type(click_type: click.ParamType) -> dict[str, Any]: - """Convert a Click parameter type to JSON Schema type definition""" + """Convert a Click parameter type to JSON Schema.""" if isinstance(click_type, click.types.StringParamType): return {"type": "string"} - elif isinstance(click_type, click.types.IntParamType): + if isinstance(click_type, click.types.IntParamType): return {"type": "integer"} - elif isinstance(click_type, click.types.FloatParamType): + if isinstance(click_type, click.types.FloatParamType): return {"type": "number"} - elif isinstance(click_type, click.types.BoolParamType): + if isinstance(click_type, click.types.BoolParamType): return {"type": "boolean"} - elif isinstance(click_type, click.types.Choice): - return {"type": "string", "enum": list(click_type.choices)} - elif isinstance(click_type, click.types.Path): + if isinstance(click_type, click.types.Choice): + schema: dict[str, Any] = {"enum": list(click_type.choices)} + if all(isinstance(choice, str) for choice in click_type.choices): + schema["type"] = "string" + return schema + if isinstance(click_type, (click.types.Path, click.types.File)): return {"type": "string", "description": "File or directory path"} - elif isinstance(click_type, click.types.File): - return {"type": "string", "description": "File path"} - elif isinstance(click_type, click.types.Tuple): - # Multiple arguments of same type - return {"type": "array", "items": {"type": "string"}} - else: - # Default to string for unknown types - return {"type": "string"} + if isinstance(click_type, click.types.Tuple): + return { + "type": "array", + "prefixItems": [click_type_to_json_schema_type(item) for item in click_type.types], + "minItems": len(click_type.types), + "maxItems": len(click_type.types), + } + return {"type": "string"} -def click_command_to_mcp_tool(cmd_name: str, click_command: click.Command) -> dict[str, Any]: - """ - Convert a Click command to an MCP tool definition with JSON Schema. +def command_accepts_stdin(click_command: click.Command) -> bool: + """Return whether a command documents a stdin/JSONL input path.""" - Introspects the Click command's parameters to automatically generate - the input schema without manual definition. - """ + command_help = " ".join( + value + for value in ( + click_command.help, + click_command.short_help, + click_command.callback.__doc__ if click_command.callback else None, + ) + if value + ).lower() + return "stdin" in command_help + + +def tool_annotations(command_path: tuple[str, ...]) -> dict[str, Any]: + """Describe a command's side effects to MCP clients.""" + + action = command_path[-1] + root = command_path[0] + read_only = action in READ_ONLY_ACTIONS + destructive = action in DESTRUCTIVE_ACTIONS + return { + "title": "ArchiveBox " + " ".join(command_path), + "readOnlyHint": read_only, + "destructiveHint": destructive, + "idempotentHint": read_only or destructive, + "openWorldHint": root in {"add", "extract", "oneshot", "run", "update"} or (root in {"crawl", "snapshot"} and action == "create"), + } + + +def click_command_to_mcp_tool(tool: MCPTool) -> dict[str, Any]: + """Convert a leaf Click command to an MCP tool definition.""" properties: dict[str, dict[str, Any]] = {} required: list[str] = [] - # Extract parameters from Click command - for param in click_command.params: - # Skip internal parameters - if param.name is None or param.name in ("help", "version"): + for param in tool.command.params: + if param.name is None or param.name in {"help", "version"}: continue param_schema = click_type_to_json_schema_type(param.type) - - # Add description from Click help text help_text = getattr(param, "help", None) if help_text: param_schema["description"] = help_text - # Handle default values - if param.default is not None and param.default != (): - param_schema["default"] = param.default + default = param.default + sentinel_type = getattr(click.core, "_SentinelClass", None) + is_sentinel = isinstance(sentinel_type, type) and isinstance(default, sentinel_type) + if default is not None and default != () and not is_sentinel: + param_schema["default"] = default - # Handle multiple values (like multiple URLs) - if param.multiple: - properties[param.name] = { - "type": "array", - "items": param_schema, - "description": param_schema.get("description", f"Multiple {param.name} values"), - } + if param.multiple or param.nargs != 1: + item_schema = param_schema + if isinstance(param.type, click.types.Tuple): + properties[param.name] = param_schema + else: + properties[param.name] = { + "type": "array", + "items": item_schema, + "description": help_text or f"One or more {param.name.replace('_', ' ')} values", + } else: properties[param.name] = param_schema - # Mark as required if Click requires it if param.required: required.append(param.name) + if command_accepts_stdin(tool.command): + properties["records"] = { + "type": "array", + "items": {"type": "object"}, + "description": "Records to pass to the command as JSONL stdin. Use this instead of building a CLI pipeline.", + } + properties["stdin"] = { + "type": "string", + "description": "Raw stdin text. Prefer records for JSONL commands.", + } + + command_name = "archivebox " + " ".join(tool.command_path) + description = tool.command.help or tool.command.short_help or f"Run {command_name}" return { - "name": cmd_name, - "description": click_command.help or click_command.short_help or f"Run archivebox {cmd_name} command", + "name": tool.name, + "title": " ".join(part.title() for part in tool.command_path), + "description": f"{description}\n\nEquivalent CLI command: `{command_name}`.", "inputSchema": { "type": "object", "properties": properties, "required": required, + "additionalProperties": False, + }, + "outputSchema": tool_output_schema(), + "annotations": tool_annotations(tool.command_path), + } + + +def tool_output_schema() -> dict[str, Any]: + """Return the shared envelope schema for all CLI-backed tools.""" + + return { + "type": "object", + "properties": { + "command": {"type": "string"}, + "success": {"type": "boolean"}, + "error": {"type": ["string", "null"]}, + "exitCode": {"type": "integer"}, + "records": {"type": "array", "items": {}}, + "stdout": {"type": "string"}, + "stderr": {"type": "string"}, + }, + "required": ["command", "success", "error", "exitCode", "records", "stdout", "stderr"], + } + + +def click_group_to_mcp_tool(group_name: str, actions: list[MCPTool]) -> dict[str, Any]: + """Expose a Click command group as one MCP tool with an action selector.""" + + action_names = [tool.command_path[-1] for tool in actions] + properties: dict[str, dict[str, Any]] = { + "action": { + "type": "string", + "enum": action_names, + "description": f"{group_name.title()} action to run.", + }, + } + action_help = [] + for action_tool in actions: + action_def = click_command_to_mcp_tool(action_tool) + action_name = action_tool.command_path[-1] + action_help.append(f"- {action_name}: {action_tool.command.help or action_tool.command.short_help}") + for name, schema in action_def["inputSchema"]["properties"].items(): + if name not in properties: + properties[name] = dict(schema) + elif properties[name].get("default") != schema.get("default"): + properties[name].pop("default", None) + + return { + "name": group_name, + "title": f"ArchiveBox {group_name.title()}", + "description": (f"Manage ArchiveBox {group_name} records through the existing CLI.\n\nActions:\n{chr(10).join(action_help)}"), + "inputSchema": { + "type": "object", + "properties": properties, + "required": ["action"], + "additionalProperties": False, + }, + "outputSchema": tool_output_schema(), + "annotations": {"title": f"ArchiveBox {group_name.title()}"}, + } + + +def shell_to_mcp_tool() -> dict[str, Any]: + """Expose ``archivebox shell -c`` as one explicit Python escape hatch.""" + + return { + "name": "shell", + "title": "ArchiveBox Python Shell", + "description": ( + "Run arbitrary Python with ArchiveBox and Django initialized. " + "Equivalent CLI command: `archivebox shell --plain --quiet-load -c CODE`. " + "This has full access to the collection database and filesystem." + ), + "inputSchema": { + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "Python source to execute in the initialized ArchiveBox Django shell.", + }, + }, + "required": ["code"], + "additionalProperties": False, + }, + "outputSchema": tool_output_schema(), + "annotations": { + "title": "ArchiveBox Python Shell", + "readOnlyHint": False, + "destructiveHint": True, + "idempotentHint": False, + "openWorldHint": False, }, } -def execute_click_command(cmd_name: str, click_command: click.Command, arguments: dict) -> dict: - """ - Execute a Click command programmatically with given arguments. +def _option_args(param: click.Option, value: Any) -> list[str]: + """Serialize one MCP argument using the option's real Click spellings.""" - Returns MCP-formatted result with captured output and error status. - """ + primary = next((opt for opt in param.opts if opt.startswith("--")), param.opts[0]) + if param.is_bool_flag: + if value: + return [primary] + if param.secondary_opts: + secondary = next((opt for opt in param.secondary_opts if opt.startswith("--")), param.secondary_opts[0]) + return [secondary] + return [] - # Setup Django for archive commands (commands that need database access) - from archivebox.cli import ArchiveBoxGroup + values = value if isinstance(value, list) and param.multiple else [value] + args: list[str] = [] + for item in values: + args.extend((primary, str(item))) + return args - if cmd_name in ArchiveBoxGroup.archive_commands: - try: - from archivebox.config.django import setup_django - from archivebox.misc.checks import check_data_folder - setup_django() - check_data_folder() - except Exception as e: - # If Django setup fails, return error (unless it's manage/shell which handle this themselves) - if cmd_name not in ("manage", "shell"): - return { - "content": [ - { - "type": "text", - "text": f"Error setting up Django: {str(e)}\n\nMake sure you're running the MCP server from inside an ArchiveBox data directory.", - }, - ], - "isError": True, - } +def arguments_to_cli( + click_command: click.Command, + arguments: dict[str, Any], +) -> tuple[list[str], str | None]: + """Convert MCP JSON arguments into Click argv and optional stdin.""" - # Use Click's test runner to invoke command programmatically - runner = CliRunner() + supplied = dict(arguments) + records = supplied.pop("records", None) + stdin_text = supplied.pop("stdin", None) + if records is not None and stdin_text is not None: + raise ValueError("Pass either records or stdin, not both") + if records is not None: + stdin_text = "".join(json.dumps(record, cls=MCPJSONEncoder) + "\n" for record in records) - # Build a map of parameter names to their Click types (Argument vs Option) param_map = {param.name: param for param in click_command.params} + unknown = sorted(set(supplied) - set(param_map)) + if unknown: + raise ValueError(f"Unknown argument(s): {', '.join(unknown)}") - # Convert arguments dict to CLI args list - args = [] - positional_args = [] - - for key, value in arguments.items(): - param_name = key.replace("_", "-") # Click uses dashes - param = param_map.get(key) - - # Check if this is a positional Argument (not an Option) - is_argument = isinstance(param, click.Argument) - - if is_argument: - # Positional arguments - add them without dashes - if isinstance(value, list): - positional_args.extend([str(v) for v in value]) - elif value is not None: - positional_args.append(str(value)) + option_args: list[str] = [] + positional_args: list[str] = [] + for key, value in supplied.items(): + if value is None: + continue + param = param_map[key] + if isinstance(param, click.Argument): + values = value if isinstance(value, list) else [value] + positional_args.extend(str(item) for item in values) else: - # Options - add with dashes - if isinstance(value, bool): - if value: - args.append(f"--{param_name}") - elif isinstance(value, list): - # Multiple values for an option (rare) - for item in value: - args.append(f"--{param_name}") - args.append(str(item)) - elif value is not None: - args.append(f"--{param_name}") - args.append(str(value)) + option_args.extend(_option_args(param, value)) - # Add positional arguments at the end - args.extend(positional_args) + return [*option_args, *positional_args], stdin_text - # Execute the command + +def parse_structured_records(stdout: str) -> list[Any]: + """Parse a CLI JSON array or JSONL stream without guessing at human output.""" + + stripped = stdout.strip() + if not stripped: + return [] try: - result = runner.invoke(click_command, args, catch_exceptions=False) + parsed = json.loads(stripped) + except json.JSONDecodeError: + records = [] + for line in stripped.splitlines(): + try: + records.append(json.loads(line)) + except json.JSONDecodeError: + return [] + return records + return parsed if isinstance(parsed, list) else [parsed] - # Format output as MCP content - content = [] - if result.output: - content.append( - { - "type": "text", - "text": result.output, - }, - ) +def execute_click_command(tool: MCPTool, arguments: dict[str, Any]) -> dict[str, Any]: + """Execute one discovered CLI command and return MCP structured content.""" - if result.stderr_bytes: - stderr_text = result.stderr_bytes.decode("utf-8", errors="replace") - if stderr_text.strip(): - content.append( - { - "type": "text", - "text": f"[stderr]\n{stderr_text}", - }, - ) + from archivebox.cli import cli - # Check exit code - is_error = result.exit_code != 0 + try: + cli_args, stdin_text = arguments_to_cli(tool.command, arguments) + result = CliRunner().invoke( + cli, + [*tool.command_path, *cli_args], + input=stdin_text, + prog_name="archivebox", + catch_exceptions=False, + ) + return _tool_result( + tool, + exit_code=result.exit_code, + stdout=result.stdout, + stderr=result.stderr, + is_error=result.exit_code != 0, + ) + except (click.ClickException, click.UsageError, OSError, SystemExit, ValueError) as err: + print(traceback.format_exc(), file=sys.stderr) + return _tool_result( + tool, + exit_code=1, + stderr=f"Could not invoke archivebox {' '.join(tool.command_path)}: {err}", + is_error=True, + ) - if is_error and not content: - content.append( - { - "type": "text", - "text": f"Command failed with exit code {result.exit_code}", - }, - ) - return { - "content": content or [{"type": "text", "text": "(no output)"}], - "isError": is_error, - } - - except Exception as e: - # Capture any exceptions during execution - error_trace = traceback.format_exc() - return { - "content": [ - { - "type": "text", - "text": f"Error executing {cmd_name}: {str(e)}\n\n{error_trace}", - }, - ], - "isError": True, - } +def _tool_result( + tool: MCPTool, + *, + exit_code: int, + stdout: str = "", + stderr: str = "", + is_error: bool, +) -> dict[str, Any]: + error = None + if is_error: + error = stderr.strip() or stdout.strip() or f"Command failed with exit code {exit_code}" + structured = { + "command": "archivebox " + " ".join(tool.command_path), + "success": not is_error, + "error": error, + "exitCode": exit_code, + "records": parse_structured_records(stdout), + "stdout": stdout, + "stderr": stderr, + } + return { + "content": [{"type": "text", "text": json.dumps(structured, cls=MCPJSONEncoder)}], + "structuredContent": structured, + "isError": is_error, + } class MCPServer: - """ - Model Context Protocol server for ArchiveBox. - - Provides JSON-RPC 2.0 interface over stdio, dynamically exposing - all Click commands as MCP tools. - """ + """ArchiveBox MCP server using JSON-RPC 2.0 over stdio.""" def __init__(self): - # Import here to avoid circular imports from archivebox.cli import ArchiveBoxGroup self.cli_group = ArchiveBoxGroup() - self.protocol_version = "2025-11-25" - self._tool_cache = {} # Cache loaded Click commands + self.protocol_version = PROTOCOL_VERSION + self._tools: dict[str, MCPTool] | None = None - def get_click_command(self, cmd_name: str) -> click.Command | None: - """Get a Click command by name, with caching""" - if cmd_name not in self._tool_cache: - if cmd_name not in self.cli_group.all_subcommands: - return None - self._tool_cache[cmd_name] = self.cli_group.get_command(click.Context(self.cli_group), cmd_name) - return self._tool_cache[cmd_name] + def _discover_command( + self, + command: click.Command, + command_path: tuple[str, ...], + tools: dict[str, MCPTool], + ) -> None: + if isinstance(command, click.Group): + context = click.Context(command) + for child_name in command.list_commands(context): + child = command.get_command(context, child_name) + if child is not None: + self._discover_command(child, (*command_path, child_name), tools) + return + + tool_name = "_".join(command_path).replace("-", "_") + tools[tool_name] = MCPTool(tool_name, command_path, command) + + def get_tools(self) -> dict[str, MCPTool]: + """Discover and cache the CLI commands backing the five public tools.""" + + if self._tools is None: + tools: dict[str, MCPTool] = {} + context = click.Context(self.cli_group) + for command_name in PUBLIC_TOOLS: + command = self.cli_group.get_command(context, command_name) + if command is not None: + self._discover_command(command, (command_name,), tools) + self._tools = tools + return self._tools + + def get_public_tool_definitions(self) -> list[dict[str, Any]]: + """Build the small agent-facing surface from the existing Click tree.""" + + commands = self.get_tools() + definitions = [ + shell_to_mcp_tool() if name == "shell" else click_command_to_mcp_tool(commands[name]) + for name in PUBLIC_TOOLS + if name not in ACTION_TOOLS + ] + for group_name in PUBLIC_TOOLS: + if group_name not in ACTION_TOOLS: + continue + actions = sorted( + (tool for tool in commands.values() if tool.command_path[0] == group_name), + key=lambda tool: tool.command_path, + ) + definitions.append(click_group_to_mcp_tool(group_name, actions)) + return definitions def handle_initialize(self, params: dict) -> dict: - """Handle MCP initialize request""" return { "protocolVersion": self.protocol_version, - "capabilities": { - "tools": {}, - }, + "capabilities": {"tools": {"listChanged": False}}, "serverInfo": { "name": "archivebox-mcp", + "title": "ArchiveBox", "version": VERSION, }, + "instructions": ( + "Use add to archive URLs and search for deep search. Use crawl, snapshot, or archiveresult " + "with action=create/list/update/delete to manage records. Pass returned records directly " + "to update/delete actions through the records argument. Use shell only when the curated " + "tools cannot express the operation; it runs arbitrary Python with full collection access." + ), } def handle_tools_list(self, params: dict) -> dict: - """Handle MCP tools/list request - returns all available CLI commands as tools""" - tools = [] - - for cmd_name in self.cli_group.all_subcommands.keys(): - click_cmd = self.get_click_command(cmd_name) - if click_cmd: - try: - tool_def = click_command_to_mcp_tool(cmd_name, click_cmd) - tools.append(tool_def) - except Exception as e: - # Log but don't fail - skip problematic commands - print(f"Warning: Could not generate tool for {cmd_name}: {e}", file=sys.stderr) - - return {"tools": tools} + return {"tools": self.get_public_tool_definitions()} def handle_tools_call(self, params: dict) -> dict: - """Handle MCP tools/call request - executes a CLI command""" tool_name = params.get("name") - arguments = params.get("arguments", {}) - if not tool_name: raise ValueError("Missing required parameter: name") - - click_cmd = self.get_click_command(tool_name) - if not click_cmd: + if tool_name not in PUBLIC_TOOLS: raise ValueError(f"Unknown tool: {tool_name}") + arguments = params.get("arguments", {}) + if not isinstance(arguments, dict): + raise TypeError("Tool arguments must be an object") + arguments = dict(arguments) + command_name = tool_name + if tool_name == "shell": + unknown = sorted(set(arguments) - {"code"}) + if unknown: + raise ValueError(f"Unknown shell argument(s): {', '.join(unknown)}") + code = arguments.get("code") + if not isinstance(code, str) or not code.strip(): + raise ValueError("shell requires a non-empty code string") + arguments = {"args": ["--plain", "--quiet-load", "-c", code]} + elif tool_name in ACTION_TOOLS: + action = arguments.pop("action", None) + available_actions = sorted(tool.command_path[-1] for tool in self.get_tools().values() if tool.command_path[0] == tool_name) + if action not in available_actions: + raise ValueError( + f"Unknown {tool_name} action: {action!r}. Choose one of: {', '.join(available_actions)}", + ) + command_name = f"{tool_name}_{action}" + tool = self.get_tools().get(command_name) + if tool is None: + raise ValueError(f"ArchiveBox CLI command is unavailable: {command_name}") + return execute_click_command(tool, arguments) - # Execute the command and return MCP-formatted result - return execute_click_command(tool_name, click_cmd, arguments) - - def handle_request(self, request: dict) -> dict: - """ - Handle a JSON-RPC 2.0 request and return response. - - Supports MCP methods: initialize, tools/list, tools/call - """ - + def handle_request(self, request: dict) -> dict | None: method = request.get("method") params = request.get("params", {}) request_id = request.get("id") + is_notification = "id" not in request + + if is_notification: + return None try: - # Route to appropriate handler if method == "initialize": result = self.handle_initialize(params) + elif method == "ping": + result = {} elif method == "tools/list": result = self.handle_tools_list(params) elif method == "tools/call": result = self.handle_tools_call(params) else: - # Method not found return { "jsonrpc": "2.0", "id": request_id, - "error": { - "code": -32601, - "message": f"Method not found: {method}", - }, + "error": {"code": -32601, "message": f"Method not found: {method}"}, } - - # Success response + return {"jsonrpc": "2.0", "id": request_id, "result": result} + except (TypeError, ValueError) as err: return { "jsonrpc": "2.0", "id": request_id, - "result": result, + "error": {"code": -32602, "message": str(err)}, } - - except Exception as e: - # Error response - error_trace = traceback.format_exc() + except (click.ClickException, click.UsageError, OSError, RuntimeError) as err: return { "jsonrpc": "2.0", "id": request_id, "error": { "code": -32603, - "message": str(e), - "data": error_trace, + "message": str(err), + "data": traceback.format_exc(), }, } - def run_stdio_server(self): - """ - Run the MCP server in stdio mode. + def run_stdio_server(self) -> None: + """Read and write one UTF-8 JSON-RPC message per line.""" - Reads JSON-RPC requests from stdin (one per line), - writes JSON-RPC responses to stdout (one per line). - """ - - # Read requests from stdin line by line for line in sys.stdin: - line = line.strip() - if not line: + if not line.strip(): continue - try: - # Parse JSON-RPC request request = json.loads(line) - - # Handle request response = self.handle_request(request) - - # Write response to stdout (use custom encoder for Click types) - print(json.dumps(response, cls=MCPJSONEncoder), flush=True) - - except json.JSONDecodeError as e: - # Invalid JSON - error_response = { + if response is not None: + print(json.dumps(response, cls=MCPJSONEncoder), flush=True) + except json.JSONDecodeError as err: + response = { "jsonrpc": "2.0", "id": None, - "error": { - "code": -32700, - "message": "Parse error", - "data": str(e), - }, + "error": {"code": -32700, "message": "Parse error", "data": str(err)}, } - print(json.dumps(error_response, cls=MCPJSONEncoder), flush=True) + print(json.dumps(response, cls=MCPJSONEncoder), flush=True) -def run_mcp_server(): - """Main entry point for MCP server""" - server = MCPServer() - server.run_stdio_server() +def run_mcp_server() -> None: + """Start the ArchiveBox MCP stdio server.""" + + MCPServer().run_stdio_server() diff --git a/archivebox/misc/serve_static.py b/archivebox/misc/serve_static.py index b0fc4a0d..5edf4658 100644 --- a/archivebox/misc/serve_static.py +++ b/archivebox/misc/serve_static.py @@ -1,36 +1,35 @@ -import html -import json -import re -import os -import sys -import stat import asyncio -import posixpath -import mimetypes +import html import importlib +import json +import mimetypes +import os +import posixpath import queue +import re +import stat +import sys import threading import time import zipfile -from datetime import datetime from collections.abc import Callable +from datetime import UTC, datetime from pathlib import Path from urllib.parse import urlencode -from django import template -from django.core.handlers.asgi import ASGIRequest +from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay from django.contrib.staticfiles import finders +from django.core.handlers.asgi import ASGIRequest +from django.http import Http404, HttpResponse, HttpResponseNotModified, StreamingHttpResponse from django.template import TemplateDoesNotExist, loader -from django.views import static -from django.http import StreamingHttpResponse, Http404, HttpResponse, HttpResponseNotModified from django.utils._os import safe_join from django.utils.http import http_date from django.utils.translation import gettext as _ -from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay +from django.views import static + from archivebox.config.common import get_config from archivebox.misc.logging_util import printable_filesize - _HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {} @@ -49,7 +48,7 @@ def _load_hash_map(snapshot_dir: Path) -> dict[str, str] | None: try: data = json.loads(hashes_path.read_text(encoding="utf-8")) - except Exception: + except (json.JSONDecodeError, OSError, TypeError, ValueError): return None file_map = {str(entry.get("path")): entry.get("hash") for entry in data.get("files", []) if entry.get("path")} @@ -99,28 +98,9 @@ def _cache_policy(config=None, **config_kwargs) -> str: return "private" if config.PERMISSIONS == "private" else "public" -def _render_mhtml_preview_document(filename: str, output_path: str) -> str: - from archivebox.plugins.discovery import get_plugin_template - - template_str = get_plugin_template("chrome_mhtml", "full", fallback=False) - if not template_str: - raise FileNotFoundError("chrome_mhtml/templates/full.html") - - tpl = template.Engine(debug=False).from_string(template_str) - return tpl.render( - template.Context( - { - "output_path": output_path, - "output_path_raw": filename, - "plugin": "chrome_mhtml", - }, - ), - ) - - def _format_direntry_timestamp(stat_result: os.stat_result) -> str: timestamp = stat_result.st_birthtime if sys.platform == "darwin" else stat_result.st_mtime - return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M") + return datetime.fromtimestamp(timestamp, tz=UTC).strftime("%Y-%m-%d %H:%M") def _safe_zip_stem(name: str) -> str: @@ -190,7 +170,7 @@ def _build_directory_zip_response( rel_parts = entry.relative_to(fullpath).parts arcname = Path(root_name, *rel_parts).as_posix() zip_file.write(entry, arcname) - except BaseException as err: + except (OSError, RuntimeError, TypeError, ValueError, zipfile.BadZipFile) as err: output_queue.put(err) finally: output_queue.put(sentinel) @@ -490,7 +470,7 @@ def _render_markdown_fallback(text: str) -> str: extensions=["extra", "toc", "sane_lists"], output_format="html", ) - except Exception: + except (ImportError, RuntimeError, ValueError): pass lines = text.splitlines() @@ -668,7 +648,7 @@ def _is_risky_replay_document(fullpath: Path, content_type: str) -> bool: # so one-domain no-JS mode still catches HTML/SVG documents. try: head = fullpath.read_bytes()[:4096].decode("utf-8", errors="ignore").lower() - except Exception: + except (OSError, UnicodeDecodeError): return False return any(marker in head for marker in RISKY_REPLAY_MARKERS) @@ -690,7 +670,9 @@ def _apply_archive_replay_headers( config = config or get_config(resolve_plugins=False, **config_kwargs) response.headers.setdefault("X-ArchiveBox-Security-Mode", config.SERVER_SECURITY_MODE) - if config.SHOULD_NEUTER_RISKY_REPLAY and _is_risky_replay_document(fullpath, content_type): + is_risky_replay = _is_risky_replay_document(fullpath, content_type) + + if config.SHOULD_NEUTER_RISKY_REPLAY and is_risky_replay and "Content-Security-Policy" not in response.headers: response.headers["Content-Security-Policy"] = ( "sandbox; " "default-src 'self' data: blob:; " @@ -797,19 +779,20 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ preview_as_image_html = ( bool(request.GET.get("preview")) and content_type.startswith("image/") and not content_type.startswith("image/svg+xml") ) - preview_as_mhtml_html = bool(request.GET.get("preview")) and fullpath.suffix.lower() in {".mhtml", ".mht"} preview_as_archivewebpage_html = bool(request.GET.get("preview")) and archivewebpage_replay.is_replay_target(fullpath.name) # Respect the If-Modified-Since header for non-markdown responses. - if not (content_type.startswith("text/plain") or content_type.startswith("text/html")): - if not static.was_modified_since(request.META.get("HTTP_IF_MODIFIED_SINCE"), statobj.st_mtime): - return _apply_archive_replay_headers( - HttpResponseNotModified(), - fullpath=fullpath, - content_type=content_type, - is_archive_replay=is_archive_replay, - config=config, - ) + if not content_type.startswith(("text/plain", "text/html")) and not static.was_modified_since( + request.META.get("HTTP_IF_MODIFIED_SINCE"), + statobj.st_mtime, + ): + return _apply_archive_replay_headers( + HttpResponseNotModified(), + fullpath=fullpath, + content_type=content_type, + is_archive_replay=is_archive_replay, + config=config, + ) # Wrap text-like outputs in HTML when explicitly requested for iframe previewing. if preview_as_text_html: @@ -835,8 +818,8 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ is_archive_replay=is_archive_replay, config=config, ) - except Exception: - pass + except (OSError, UnicodeDecodeError, ValueError): + preview_as_text_html = False if preview_as_image_html: try: @@ -863,8 +846,8 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ is_archive_replay=is_archive_replay, config=config, ) - except Exception: - pass + except (OSError, ValueError): + preview_as_image_html = False if preview_as_archivewebpage_html: try: @@ -890,46 +873,19 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ response = HttpResponse(body, content_type=preview_content_type) for key, value in headers.items(): response.headers[key] = value - return response - except Exception: - pass - - if preview_as_mhtml_html: - try: - raw_query = request.GET.copy() - raw_query.pop("preview", None) - raw_output_path = request.path - if raw_query: - raw_output_path = f"{raw_output_path}?{raw_query.urlencode()}" - rendered = _render_mhtml_preview_document(fullpath.name, raw_output_path) - response = HttpResponse(rendered, content_type="text/html; charset=utf-8") - response.headers["Last-Modified"] = http_date(statobj.st_mtime) - if etag: - response.headers["ETag"] = etag - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=31536000, immutable" - else: - response.headers["Cache-Control"] = f"{_cache_policy(config=config)}, max-age=60, stale-while-revalidate=300" - response.headers["Content-Disposition"] = f'inline; filename="{fullpath.stem}.html"' - response.headers["X-Content-Type-Options"] = "nosniff" - response.headers["Content-Security-Policy"] = ( - "default-src 'self' data: blob:; " - "script-src 'unsafe-inline'; " - "style-src 'unsafe-inline' data: blob:; " - "connect-src 'self'; " - "frame-src 'self' data: blob:; " - "object-src 'none'; " - "base-uri 'none'; " - "form-action 'none';" + return _apply_archive_replay_headers( + response, + fullpath=fullpath, + content_type=preview_content_type, + is_archive_replay=is_archive_replay, + config=config, ) - if encoding: - response.headers["Content-Encoding"] = encoding - return response - except Exception: - pass + except (OSError, RuntimeError, ValueError): + preview_as_archivewebpage_html = False # Heuristic fix: some archived HTML outputs (e.g. mercury content.html) # are stored with HTML-escaped markup or markdown sources. If so, render sensibly. - if content_type.startswith("text/plain") or content_type.startswith("text/html"): + if content_type.startswith(("text/plain", "text/html")): try: max_unescape_size = 10 * 1024 * 1024 # 10MB cap to avoid heavy memory use if statobj.st_size <= max_unescape_size: @@ -977,11 +933,11 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ is_archive_replay=is_archive_replay, config=config, ) - except Exception: + except (OSError, UnicodeDecodeError, ValueError): pass # setup response object - ranged_file = RangedFileReader(open(fullpath, "rb")) + ranged_file = RangedFileReader(fullpath.open("rb")) response = StreamingHttpResponse( _stream_ranged_file_async(ranged_file) if _is_asgi_request(request) else ranged_file, content_type=content_type, @@ -1018,7 +974,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_ return HttpResponse(status=416) ranged_file.start = start ranged_file.stop = stop - response["Content-Range"] = "bytes %d-%d/%d" % (start, stop - 1, size) + response["Content-Range"] = f"bytes {start}-{stop - 1}/{size}" response["Content-Length"] = stop - start response.status_code = 206 if encoding: @@ -1053,7 +1009,7 @@ def serve_static(request, path, **kwargs): if not absolute_path: if path.endswith("/") or path == "": raise Http404("Directory indexes are not allowed here.") - raise Http404("'%s' could not be found" % path) + raise Http404(f"'{path}' could not be found") document_root, path = os.path.split(absolute_path) return serve_static_with_byterange_support(request, path, document_root=document_root, **kwargs) @@ -1085,8 +1041,7 @@ def parse_range_header(header, resource_size): # suffix-byte-range-spec: this form specifies the last N bytes of an # entity-body start = resource_size + int(val) - if start < 0: - start = 0 + start = max(start, 0) stop = resource_size else: # byte-range-spec: first-byte-pos "-" [last-byte-pos] diff --git a/archivebox/plugins/views.py b/archivebox/plugins/views.py index e23a89c4..e5dc1744 100644 --- a/archivebox/plugins/views.py +++ b/archivebox/plugins/views.py @@ -3,22 +3,20 @@ __package__ = "archivebox.plugins" import html import json import re -from typing import Any from collections.abc import Callable +from typing import Any from urllib.parse import quote -from django.http import HttpRequest +from admin_data_views.typing import TableContext +from admin_data_views.utils import ItemLink, render_with_table_view +from django.http import HttpRequest, HttpResponseRedirect from django.utils.html import format_html from django.utils.safestring import mark_safe -from admin_data_views.typing import ItemContext, SectionData, TableContext -from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view - from archivebox.config.common import get_live_config_url from archivebox.config.views import get_environment_binary_url, is_superuser from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, discover_plugin_configs, iter_plugin_dirs - ABX_PLUGINS_DOCS_BASE_URL = "https://archivebox.github.io/abx-plugins/" ABX_PLUGINS_GITHUB_BASE_URL = "https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/" LIVE_PLUGIN_BASE_URL = "/admin/environment/plugins/" @@ -372,83 +370,8 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext: ) -@render_with_item_view -def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: +def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> HttpResponseRedirect: assert is_superuser(request), "Must be a superuser to view configuration settings." - plugins = get_filesystem_plugins() - - plugin = plugins.get(key) - if not plugin: - return ItemContext( - slug=key, - title=f"Plugin not found: {key}", - data=[], - ) - - docs_url = get_plugin_docs_url(plugin["name"]) - machine_admin_url = get_machine_admin_url() - fields = { - "id": plugin["id"], - "name": plugin["name"], - "source": plugin["source"], - } - - sections: list[SectionData] = [ - { - "name": plugin["name"], - "description": format_html( - '{}
ABX Plugin Docs', - plugin["path"], - docs_url, - ), - "fields": fields, - "help_texts": {}, - }, - ] - - if plugin["hooks"]: - sections.append( - { - "name": "Hooks", - "description": mark_safe(render_hook_links_html(plugin["name"], plugin["hooks"], plugin["source"])), - "fields": {}, - "help_texts": {}, - }, - ) - - if plugin.get("config"): - sections.append( - { - "name": "Plugin Metadata", - "description": mark_safe(render_plugin_metadata_html(plugin["config"])), - "fields": {}, - "help_texts": {}, - }, - ) - - sections.append( - { - "name": "config.json", - "description": mark_safe(render_highlighted_json_block(plugin["config"])), - "fields": {}, - "help_texts": {}, - }, - ) - - config_properties = plugin["config"].get("properties", {}) - if config_properties: - sections.append( - { - "name": "Config Properties", - "description": mark_safe(render_config_properties_html(config_properties, machine_admin_url)), - "fields": {}, - "help_texts": {}, - }, - ) - - return ItemContext( - slug=key, - title=plugin["name"], - data=sections, - ) + plugin_name = key.removeprefix("builtin.").removeprefix("user.") + return HttpResponseRedirect(get_plugin_docs_url(plugin_name)) diff --git a/archivebox/templates/admin/actions.html b/archivebox/templates/admin/actions.html index 9015f8b2..2638fe96 100644 --- a/archivebox/templates/admin/actions.html +++ b/archivebox/templates/admin/actions.html @@ -24,11 +24,10 @@ selected - {% if cl.opts.model_name == 'snapshot' %} {% if cl.full_result_count and cl.full_result_count != cl.result_count %} {{ cl.result_count|intcomma }} - / + filtered / {{ cl.full_result_count|intcomma }} total {% else %} @@ -36,7 +35,6 @@ total {% endif %} - {% endif %} {% if cl.result_count != cl.result_list|length %}
+{% endif %} +{% endblock %} + +{% block submit_buttons_top %}{% endblock %} +{% block submit_buttons_bottom %}{% endblock %} diff --git a/archivebox/templates/admin/auth/user/change_form.html b/archivebox/templates/admin/auth/user/change_form.html index b825a830..ca20d1bb 100644 --- a/archivebox/templates/admin/auth/user/change_form.html +++ b/archivebox/templates/admin/auth/user/change_form.html @@ -1,26 +1,17 @@ -{% extends "admin/change_form.html" %} +{% extends "admin/archivebox_change_form.html" %} {% load core_tags %} -{% block extrastyle %} -{{ block.super }} +{% block extrastyle %}{{ block.super }} {% endblock %} -{% block object-tools-items %} +{% block toolbar_extras %} {% if original %} {% api_token as api_token %} -
  • - - RSS +
    + + Snapshot Feed -
    • +
    {% endif %} -{{ block.super }} {% endblock %} diff --git a/archivebox/templates/admin/base.html b/archivebox/templates/admin/base.html index 16ea008b..707c639e 100644 --- a/archivebox/templates/admin/base.html +++ b/archivebox/templates/admin/base.html @@ -65,6 +65,8 @@ flex-wrap: wrap; gap: 20px; align-items: stretch; + width: 100%; + box-sizing: border-box; } /* Each fieldset becomes a card */ @@ -355,6 +357,40 @@ background: transparent !important; } + #content-main form fieldset.actions-card, + #content form fieldset.actions-card, + #content-main form fieldset:has(.field-snapshot_summary), + #content form fieldset:has(.field-snapshot_summary) { + flex: 1 1 calc(50% - 10px) !important; + max-width: calc(50% - 10px) !important; + min-width: min(520px, 100%); + } + + .archivebox-snapshot-summary { + display: flex; + gap: 16px; + align-items: flex-start; + min-width: 0; + } + + .archivebox-snapshot-summary-preview { + display: block; + flex: 0 1 220px; + width: 220px; + min-width: 140px; + } + + .archivebox-snapshot-summary-meta { + flex: 1 1 180px; + min-width: 160px; + } + + .archivebox-snapshot-summary-path { + display: inline; + overflow-wrap: anywhere; + word-break: normal; + } + /* Readonly fields styling */ #content-main form fieldset .readonly, #content form fieldset .readonly { @@ -605,26 +641,78 @@ } /* Responsive: 2 columns on medium screens */ - @media (max-width: 1400px) { + @media (max-width: 1500px) { #content-main form fieldset, #content form fieldset { max-width: calc(50% - 10px); flex: 1 1 320px; } + + #content-main form fieldset.actions-card, + #content form fieldset.actions-card, + #content-main form fieldset:has(.field-snapshot_summary), + #content form fieldset:has(.field-snapshot_summary) { + flex-basis: 520px; + min-width: min(520px, 100%); + } } /* Responsive: stack on smaller screens */ @media (max-width: 900px) { + body.change-form #content, + body.change-form #content-main, + body.change-form #content-main > form, + body.change-form #content-main > form > div { + width: 100%; + max-width: 100%; + min-width: 0; + box-sizing: border-box; + } + #content-main form fieldset, #content form fieldset { flex: 1 1 100%; max-width: 100%; - min-width: auto; + min-width: 0; + width: 100%; + } + + #content-main form fieldset fieldset, + #content form fieldset fieldset, + #content-main form fieldset .flex-container, + #content form fieldset .flex-container, + #content-main form .module fieldset, + #content form .module fieldset { + width: 100% !important; + max-width: 100% !important; + } + + #content-main form fieldset.actions-card, + #content form fieldset.actions-card, + #content-main form fieldset:has(.field-snapshot_summary), + #content form fieldset:has(.field-snapshot_summary) { + flex-basis: 100% !important; + max-width: 100% !important; + min-width: 0; } #content { padding: 16px; } + + .archivebox-snapshot-summary { + flex-wrap: wrap; + } + + .archivebox-snapshot-summary-preview { + flex-basis: 100%; + width: 100%; + max-width: 320px; + } + + .archivebox-snapshot-summary-meta { + flex-basis: 100%; + } } /* Module content padding */ @@ -1693,7 +1781,7 @@

    - ArchiveBox + ArchiveBox

    diff --git a/archivebox/templates/admin/change_list_panel.html b/archivebox/templates/admin/change_list_panel.html index edea90a5..caad164a 100644 --- a/archivebox/templates/admin/change_list_panel.html +++ b/archivebox/templates/admin/change_list_panel.html @@ -63,7 +63,7 @@ {% if cl.has_filters and not embedded_changelist %} diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html index 72476798..d5c06afe 100644 --- a/archivebox/templates/core/snapshot.html +++ b/archivebox/templates/core/snapshot.html @@ -39,6 +39,8 @@ background-color: #aa1e55; order: 1; flex: 0 0 auto; + container-type: inline-size; + container-name: snapshot-header; } small { font-weight: 200; @@ -254,6 +256,14 @@ align-items: center; gap: 6px; } + .header-mobile-badges { + display: none; + } + .badge-label, + .badge-mobile-text, + .header-date-label { + display: none; + } .header-year-badges { display: flex; flex-wrap: wrap; @@ -314,7 +324,7 @@ margin-top: -13px; margin-left: 4px; } - @media(max-width: 900px) { + @container snapshot-header (max-width: 900px) { .header-top .header-nav { grid-template-columns: 1fr; gap: 8px; @@ -331,12 +341,49 @@ margin-left: 0; } } - @media(max-width: 600px) { + @container snapshot-header (max-width: 900px) { .header-top { + padding: 10px 12px 12px; font-size: 14px; + text-align: left; + } + .header-top .header-nav { + gap: 10px; + } + .header-top .header-left { + order: 1; + padding-bottom: 2px; + } + .header-top .header-main { + order: 2; + gap: 8px; + } + .header-top .header-meta { + order: 3; + gap: 8px; + } + .header-top .header-right { + order: 4; + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 8px; + padding: 0 !important; + } + .header-archivebox { + color: #fff !important; + font-size: 16px; + font-weight: 600; + } + .header-archivebox img { + width: 28px; + height: 28px; + margin: 0; } .header-top .header-url { - font-size: 16px; + font-size: 15px; + background: rgba(0, 0, 0, 0.14); + border: 1px solid rgba(255, 255, 255, 0.12); + border-radius: 7px; } .header-title-text, .header-top .header-url a.header-url-text { @@ -346,14 +393,357 @@ -webkit-line-clamp: 2; -webkit-box-orient: vertical; } - .header-badges, - .header-tags, - .header-year-badges { + .header-top .header-url a.header-url-text { + padding: 7px 9px; + line-height: 1.35; + } + .header-title-line { + display: grid; + grid-template-columns: 22px minmax(0, 1fr) 28px; + gap: 7px; + align-items: center; + color: #fff; + } + .header-title-text { + grid-column: 2; + grid-row: 1; + color: #fff; + font-size: 14px; + font-weight: 500; + line-height: 1.35; + } + .header-top .favicon { + grid-column: 1; + grid-row: 1; + width: 22px; + height: 22px; + margin: 0; + } + .header-tags { + grid-column: 1 / -1; + grid-row: 2; + width: 100%; justify-content: flex-start; } + .header-tags .tag-pill { + padding: 4px 8px; + font-size: 11px; + background: rgba(255, 255, 255, 0.15); + color: #fff; + } .header-toggle { - font-size: 46px; - vertical-align: -6px; + grid-column: 3; + grid-row: 1; + display: inline-flex; + align-items: center; + justify-content: center; + width: 28px; + height: 28px; + margin: 0; + border-radius: 6px; + background: rgba(0, 0, 0, 0.14); + color: #fff !important; + font-size: 24px; + line-height: 1; + } + .header-badges { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(min(100%, 8rem), 1fr)); + width: 100%; + gap: 6px; + } + header .header-badges .badge { + display: flex; + align-items: center; + justify-content: space-between; + min-width: 0; + min-height: 34px; + margin: 0; + padding: 6px 8px; + border: 1px solid rgba(255, 255, 255, 0.16); + font-family: inherit; + font-size: 12px; + font-weight: 600; + } + .badge-label, + .header-date-label { + display: inline; + margin-right: 6px; + color: currentColor; + font-size: 10px; + font-weight: 600; + letter-spacing: 0.04em; + opacity: 0.72; + text-transform: uppercase; + } + .badge-desktop-icon { + display: none; + } + .badge-mobile-text { + display: inline; + } + .header-year-badges { + width: 100%; + margin: 0; + padding: 1px 0 0; + justify-content: flex-start; + } + .header-year-badges::before { + content: "Years"; + color: rgba(255, 255, 255, 0.7); + font-size: 10px; + font-weight: 600; + letter-spacing: 0.04em; + text-transform: uppercase; + } + header .header-year-badges .badge { + margin: 0; + padding: 4px 7px; + font-size: 11px; + } + .header-right .header-date, + .header-right .snapshot-variants { + display: block; + width: 100%; + min-width: 0; + white-space: normal; + } + .header-right .snapshot-date-summary, + .header-right > .header-date { + display: flex; + align-items: center; + justify-content: flex-start; + min-height: 34px; + padding: 6px 8px; + border: 1px solid rgba(255, 255, 255, 0.16); + border-radius: 6px; + background: rgba(0, 0, 0, 0.12); + color: #fff !important; + gap: 7px; + } + .header-right br { + display: none; + } + header .external-links { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 6px; + width: 100%; + padding: 0; + color: #fff; + font-size: 12px; + text-align: left; + overflow: visible; + } + header .external-links a { + display: inline-flex; + align-items: center; + min-height: 32px; + padding: 6px 9px; + border: 1px solid rgba(255, 255, 255, 0.16); + border-radius: 6px; + background: rgba(0, 0, 0, 0.12); + color: #fff; + } + .external-links-separator { + display: none; + } + .snapshot-variants-list { + right: auto; + left: 0; + width: min(360px, calc(100vw - 24px)); + min-width: 0; + } + } + @container snapshot-header (max-width: 480px) { + .header-top { + padding: 6px 8px 7px; + font-size: 12px; + } + .header-top .header-nav { + grid-template-columns: 22px minmax(0, 1fr); + gap: 6px; + } + .header-top .header-left { + grid-column: 1; + grid-row: 1; + min-height: 0; + padding-bottom: 0; + line-height: 1 !important; + } + .header-top .header-main { + display: contents; + } + .header-top .header-meta { + grid-column: 1 / -1; + grid-row: 3; + gap: 4px; + } + .header-top .header-right { + grid-column: 1 / -1; + grid-row: 4; + grid-template-columns: repeat(2, minmax(0, 1fr)); + align-items: stretch; + gap: 4px; + } + .header-archivebox { + display: flex; + align-items: center; + justify-content: center; + width: 22px; + height: 22px; + overflow: hidden; + font-size: 0; + line-height: 1; + } + .header-archivebox img { + display: block; + width: auto; + height: auto; + max-width: 20px; + max-height: 20px; + margin: 0; + object-fit: contain; + } + .header-top .header-url { + grid-column: 2; + grid-row: 1; + font-size: 13px; + } + .header-top .header-url a.header-url-text { + display: block; + padding: 5px 7px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + .header-title-line { + grid-column: 1 / -1; + grid-row: 2; + grid-template-columns: 22px minmax(0, 1fr) 24px; + gap: 5px; + } + .header-top .favicon { + width: 18px; + height: 18px; + justify-self: center; + } + .header-title-text { + display: block; + font-size: 13px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + .header-toggle { + width: 24px; + height: 24px; + font-size: 20px; + } + .header-tags { + gap: 3px; + } + .header-tags .tag-pill { + padding: 2px 6px; + font-size: 10px; + } + .header-badges { + display: none; + } + .header-mobile-badges { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 4px; + width: 100%; + } + .header-mobile-badge { + display: flex; + align-items: center; + justify-content: space-between; + min-width: 0; + min-height: 28px; + padding: 4px 7px; + border: 1px solid rgba(255, 255, 255, 0.2); + border-radius: 6px; + background: rgba(255, 255, 255, 0.14); + color: #fff !important; + font-size: 11px; + font-weight: 600; + line-height: 1; + } + .header-mobile-badge:last-child { + background: rgba(212, 237, 218, 0.96); + color: #315d3b !important; + } + .header-mobile-badge-separator { + margin: 0 5px; + opacity: 0.45; + } + .badge-label, + .header-date-label, + .header-year-badges::before { + font-size: 9px; + } + .header-year-badges { + display: none; + } + .header-right .snapshot-date-summary, + .header-right > .header-date { + height: 100%; + min-height: 32px; + padding: 4px 6px; + gap: 5px; + font-size: 11px; + font-family: inherit; + font-weight: 600; + line-height: 1; + } + .header-right .snapshot-date-summary > * { + display: inline-flex; + align-items: center; + height: 16px; + margin: 0; + font-family: inherit; + font-size: 11px; + font-weight: 600; + line-height: 1; + letter-spacing: 0; + text-transform: none; + opacity: 1; + } + .header-right > .snapshot-variants, + .header-right > .header-date { + grid-column: 2; + grid-row: 1; + height: 100%; + } + .header-right .snapshot-count-badge { + min-width: 16px; + height: 16px; + padding: 0 4px; + font-size: 9px; + } + header .external-links { + grid-column: 1; + grid-row: 1; + display: grid; + grid-template-columns: auto minmax(0, 1fr); + align-items: stretch; + gap: 4px; + font-size: 11px; + flex-wrap: nowrap; + height: 100%; + } + header .external-links a { + justify-content: center; + height: 100%; + min-height: 32px; + font-family: inherit; + font-size: 11px; + font-weight: 600; + line-height: 1; + padding: 4px 7px; } } @@ -666,30 +1056,14 @@ object-position: top center; } .thumb-grid { - display: block; - column-gap: 6px; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(clamp(180px, 14vw, 250px), 1fr)); + gap: 6px; align-content: start; width: 100%; max-width: 100%; margin-left: 0; padding: 0 6px; - column-fill: balance; - column-count: 2; - } - @media (min-width: 720px) { - .thumb-grid { column-count: 3; } - } - @media (min-width: 1024px) { - .thumb-grid { column-count: 4; } - } - @media (min-width: 1280px) { - .thumb-grid { column-count: 5; } - } - @media (min-width: 1600px) { - .thumb-grid { column-count: 6; } - } - @media (min-width: 1920px) { - .thumb-grid { column-count: 7; } } .thumb-card { box-shadow: 2px 2px 7px 0px rgba(0, 0, 0, 0.1); @@ -718,8 +1092,13 @@ contain-intrinsic-size: 46px; } .thumb-card .thumb-body { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + align-items: center; + gap: 6px; font-size: 14px; - padding: 3px 8px; + min-height: 30px; + padding: 4px 6px 4px 8px; line-height: 1.2; word-wrap: break-word; overflow: hidden; @@ -730,22 +1109,37 @@ position: relative; } .thumb-actions { - position: absolute; - top: 2px; - right: 6px; + position: static; + grid-column: 2; + grid-row: 1; display: flex; - gap: 6px; + align-items: center; + gap: 3px; font-size: 12px; line-height: 1; - opacity: 0.7; + opacity: 0.8; } .thumb-actions a { + display: inline-flex; + align-items: center; + justify-content: center; + width: 22px; + height: 22px; + border-radius: 5px; + background: #e7ebef; text-decoration: none; color: inherit; } .thumb-actions a:hover { + background: #d9e0e7; opacity: 1; } + .thumb-card .thumb-body > a:not(.thumb-actions a), + .thumb-card .thumb-body > h4 { + grid-column: 1; + grid-row: 1; + min-width: 0; + } .thumb-card .thumb-body h4 { font-size: 0.8em; text-transform: uppercase; @@ -839,9 +1233,15 @@ flex: 0 0 auto; } .thumb-card:has([data-compact]) .thumb-body { - padding: 2px 6px; + min-height: 22px; + padding: 2px 4px 2px 6px; font-size: 12px; - max-height: 20px; + max-height: 24px; + } + .thumb-card:has([data-compact]) .thumb-actions a { + width: 18px; + height: 18px; + border-radius: 4px; } .thumb-card:has([data-compact]) .thumb-body h4 { font-size: 0.9em; @@ -1175,27 +1575,45 @@
    - {{num_outputs}} - {% if num_failures %} - + {{num_failures}} errors - {% endif %} + Outputs + + {{num_outputs}} + {% if num_failures %} + + {{num_failures}} errors + {% endif %} +
    - + Size + {{size}}
    - - ✏️ + Manage + + ✏️Edit
    - + Status + {{status|upper}}
    +
    + + {{num_outputs}} Files + | + {{size}} + + + {{status|capfirst}} + | + Edit + +
    {% if related_years %}
    {% for entry in related_years %} @@ -1227,7 +1645,8 @@ {{ related_snapshots|length }} - {{oldest_archive_date|default:downloaded_datestr|default:bookmarked_date}} + Captures + {{oldest_archive_date|default:downloaded_datestr|default:bookmarked_date|slice:":10"}}
    @@ -1240,13 +1659,14 @@ {% else %} - {{oldest_archive_date|default:downloaded_datestr|default:bookmarked_date}} + Captures + {{oldest_archive_date|default:downloaded_datestr|default:bookmarked_date|slice:":10"}} {% endif %}
    - 📁   - See all files...  |  + 📁 Files + | 🏛️ Archive.org @@ -1261,8 +1681,8 @@ {% for result in archiveresults %} {% with display_path=result.path display_url='' preview_url='' %} {% if display_path %}{% snapshot_url snapshot display_path as display_url %}{% endif %} - {% if display_path %}{% snapshot_preview_url snapshot display_path as preview_url %}{% endif %} -
    + {% if display_path %}{% snapshot_preview_url snapshot display_path result.result as preview_url %}{% endif %} +
    📁 @@ -1617,7 +2037,15 @@ if (!hashValue) { return null } - return [...document.querySelectorAll('a[target=preview]')].find((link) => getPreviewHashValue(link) == hashValue) + const previewLinks = [...document.querySelectorAll('a[target=preview]')] + const pathMatch = previewLinks.find((link) => getPreviewHashValue(link) == hashValue) + if (pathMatch) { + return pathMatch + } + const pluginCard = [...document.querySelectorAll('.thumb-card[data-plugin-name]')].find( + (card) => (card.dataset.pluginName || '').toLowerCase() == hashValue, + ) + return pluginCard && pluginCard.querySelector('a[target=preview]') } function selectInitialPreview() { @@ -1655,80 +2083,6 @@ } } - function getBaseThumbGridColumnCount() { - const width = window.innerWidth || document.documentElement.clientWidth || 0 - if (width >= 1920) return 7 - if (width >= 1600) return 6 - if (width >= 1280) return 5 - if (width >= 1024) return 4 - if (width >= 720) return 3 - return 2 - } - - function measureThumbGridRightGap(grid) { - const cards = [...grid.querySelectorAll('.thumb-card')] - if (!cards.length) { - return 0 - } - const gridRect = grid.getBoundingClientRect() - let maxRight = gridRect.left - for (const card of cards) { - const rect = card.getBoundingClientRect() - if (rect.width > 0) { - maxRight = Math.max(maxRight, rect.right) - } - } - return Math.max(0, gridRect.right - maxRight) - } - - function adjustThumbGridColumns() { - const grid = document.querySelector('.thumb-grid') - if (!grid) { - return - } - - const cards = [...grid.querySelectorAll('.thumb-card')] - if (!cards.length) { - grid.style.removeProperty('column-count') - return - } - - const baseCount = Math.min(getBaseThumbGridColumnCount(), cards.length) - const candidates = [...new Set([ - Math.max(1, baseCount - 1), - baseCount, - Math.min(cards.length, baseCount + 1), - ])] - - let bestCount = baseCount - let bestScore = Number.POSITIVE_INFINITY - - for (const count of candidates) { - grid.style.columnCount = String(count) - const gap = measureThumbGridRightGap(grid) - const score = gap + (Math.abs(count - baseCount) * 12) - if (score < bestScore) { - bestScore = score - bestCount = count - } - } - - grid.style.columnCount = String(bestCount) - } - - let thumbGridLayoutFrame = null - function scheduleThumbGridAdjustment() { - if (thumbGridLayoutFrame) { - cancelAnimationFrame(thumbGridLayoutFrame) - } - thumbGridLayoutFrame = requestAnimationFrame(() => { - thumbGridLayoutFrame = null - adjustThumbGridColumns() - }) - } - - window.addEventListener('resize', scheduleThumbGridAdjustment) - function hideSnapshotHeader() { console.log('Collapsing Snapshot header...') jQuery('.header-toggle').text('▸') @@ -1779,7 +2133,6 @@ // check URL for hash e.g. #git and load relevant preview selectInitialPreview() prepareThumbnailPriorities() - scheduleThumbGridAdjustment() loadSnapshotHeaderState() diff --git a/archivebox/templates/core/system_warnings_banner.html b/archivebox/templates/core/system_warnings_banner.html index 9db4e112..895b89cd 100644 --- a/archivebox/templates/core/system_warnings_banner.html +++ b/archivebox/templates/core/system_warnings_banner.html @@ -1,115 +1,51 @@ {% comment %} - Fixed red badge that hangs off the top center of the page. Five trigger - conditions (see ``system_warnings_banner`` in core_tags.py); precedence is - config/security first, then host-health: - - * mode="unconfigured" — BASE_URL is not set. Always shown until the - operator pins it explicitly, even when CSRF - auto-derive or request-host fallback are keeping - the server functional. - * mode="unsafe" — server is in a non-subdomain SERVER_SECURITY_MODE. - Archived content shares an origin with the admin UI. - * mode="low_disk" — free space on DATA_DIR's filesystem is below 1GiB. - Archive jobs will fail until the operator frees space. - * mode="high_memory" — virtual memory utilization above 95%; one OOM-kill - from a crash. - * mode="high_load" — 15-min loadavg > 3 × cpu_count; saturated host. + Render exactly one system warning. Configuration/security warnings take + precedence over host-health warnings; superusers get the full setup wizard + while BASE_URL is unset. {% endcomment %} -{% if mode == "low_disk" %} - -{% elif mode == "high_memory" %} - -{% elif mode == "high_load" %} - -{% elif mode == "unsafe" %} - -{% elif mode == "unconfigured" %} - +{% if mode == "unconfigured" and can_configure and machine_admin_url %} + {% include "setup_wizard.html" %} +{% elif mode %} + {% endif %} diff --git a/archivebox/templates/static/add.css b/archivebox/templates/static/add.css index 55ed039e..81b0b74e 100755 --- a/archivebox/templates/static/add.css +++ b/archivebox/templates/static/add.css @@ -4,6 +4,21 @@ header { color: white; height: 30px; } + +.add-page { + width: 100%; + max-width: 1440px; + min-width: 0; + margin: auto; + float: none; +} + +#add-form, +#add-form .form-section, +#add-form .form-field { + min-width: 0; + max-width: 100%; +} .header-top { color: white; } @@ -658,6 +673,10 @@ select { margin-bottom: 0; } +.url-filters-column { + min-width: 0; +} + .url-filter-label-main { font-weight: 600; white-space: nowrap; @@ -737,6 +756,19 @@ select { border-color: #c3e6cb; } +@media (max-width: 1100px) { + .url-filter-label-row { + flex-wrap: wrap; + } + + .url-filter-label-note { + width: 100%; + margin-left: 0; + text-align: left; + white-space: normal; + } +} + @media (max-width: 1020px) { .tags-persona-row { grid-template-columns: 1fr; @@ -927,6 +959,49 @@ input:focus, select:focus, textarea:focus, button:focus { /* Responsive layout */ @media (max-width: 768px) { + .add-page { + overflow-x: hidden; + } + + #add-form > center h1 { + font-size: 26px; + line-height: 1.2; + } + + .crawl-explanation, + .form-section { + padding: 12px; + } + + .crawl-tip > span[style*="float"] { + display: block; + float: none !important; + margin-bottom: 8px; + } + + .crawl-tip-url { + display: block; + max-width: 100%; + margin-top: 6px; + white-space: normal; + overflow-wrap: anywhere; + } + + .url-editor-shell textarea[name="url"] { + max-width: 100%; + } + + .url-filter-label-row { + flex-wrap: wrap; + } + + .url-filter-label-note { + width: 100%; + margin-left: 0; + text-align: left; + white-space: normal; + } + .crawl-limit-grid { grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); } diff --git a/archivebox/templates/static/admin.css b/archivebox/templates/static/admin.css index 581fdcb7..9df2a65b 100755 --- a/archivebox/templates/static/admin.css +++ b/archivebox/templates/static/admin.css @@ -42,6 +42,10 @@ border-radius: 4px; padding: 2px 6px; } +.navbar-separator-wide { + display: inline-block; + width: 1em; +} div.breadcrumbs { background: #772948; @@ -156,28 +160,88 @@ body.change-list #content .object-tools { background-blend-mode: difference; } +@media (max-width: 900px) { + body.change-form div.breadcrumbs { + overflow-wrap: anywhere; + } + + body.change-form #content .object-tools { + display: flex; + float: none; + clear: both; + width: 100%; + max-width: 100%; + margin: 0 0 16px; + padding: 0; + gap: 8px; + flex-wrap: wrap; + justify-content: flex-start; + box-sizing: border-box; + } + + body.change-form #content .object-tools li { + min-width: 0; + margin: 0; + } + + body.change-form #content .object-tools a, + body.change-form #content .object-tools form button { + white-space: normal; + text-align: center; + } +} + #content #changelist #toolbar { padding: 0px; background: none; margin-bottom: 10px; border-top: 0px; border-bottom: 0px; - display: inline-flex; - width: auto; + display: block; + width: 100%; max-width: 100%; box-sizing: border-box; position: relative; z-index: 1; } +body.change-list:not(.model-snapshot) #content #changelist #toolbar .changelist-toolbar-row { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 8px; + align-items: start; + width: 100%; + min-width: 0; +} + #content #changelist #toolbar #changelist-search, #content #changelist #toolbar #changelist-search > div { - width: auto; + width: 100%; + min-width: 0; max-width: 100%; - flex: 0 1 auto; box-sizing: border-box; } +#content #changelist #toolbar #changelist-search > div { + display: grid !important; + grid-template-columns: minmax(0, 1fr); + align-items: center; +} + +#content #changelist #toolbar .changelist-search-submit { + position: absolute !important; + width: 1px !important; + height: 1px !important; + min-width: 0 !important; + margin: -1px !important; + padding: 0 !important; + overflow: hidden !important; + clip: rect(0 0 0 0) !important; + clip-path: inset(50%) !important; + border: 0 !important; + white-space: nowrap !important; +} + #content #changelist #toolbar #searchbar { width: clamp(180px, 32vw, 420px); max-width: 100%; @@ -422,25 +486,18 @@ body.change-list:not(.model-snapshot) #changelist #changelist-form > .results { body.change-list:not(.model-snapshot) #changelist #toolbar #changelist-search > div { display: grid !important; - grid-template-columns: 24px minmax(0, 1fr) auto; + grid-template-columns: minmax(0, 1fr) auto; gap: 7px; align-items: center; } - body.change-list:not(.model-snapshot) #content #changelist #toolbar label[for="searchbar"] { + body.change-list:not(.model-snapshot) #content #changelist #toolbar .search-input-wrap { grid-column: 1; - justify-content: center; - } - - body.change-list:not(.model-snapshot) #content #changelist #toolbar #searchbar { - grid-column: 2; - flex: none !important; - width: 100% !important; min-width: 0; } - body.change-list:not(.model-snapshot) #content #changelist #toolbar form input[type="submit"] { - grid-column: 3; + body.change-list:not(.model-snapshot) #content #changelist #toolbar .changelist-search-submit { + grid-column: 2; flex: none !important; width: auto !important; min-width: 76px; @@ -555,6 +612,263 @@ body.change-list:not(.model-snapshot) #changelist #changelist-form > .results { } +@media (max-width: 520px) { + #container { + width: 100%; + min-width: 0 !important; + } + + #header { + height: auto !important; + min-height: 0; + flex-shrink: 0 !important; + flex-direction: column !important; + align-items: center !important; + gap: 4px; + } + + #branding { + flex: none !important; + width: 100%; + text-align: center; + } + + #user-tools { + width: 100%; + margin: 0 !important; + padding: 0 !important; + line-height: 1.8; + text-align: center; + white-space: normal !important; + overflow-wrap: anywhere; +} + +@media (max-width: 430px) { + #user-tools { + display: flex; + flex-wrap: wrap; + align-items: center; + justify-content: center; + gap: 6px; + line-height: 1.2; + overflow-wrap: normal; + } + + #user-tools .navbar-item { + display: inline-flex; + align-items: center; + justify-content: center; + min-height: 32px; + padding: 5px 8px; + border-radius: 5px; + white-space: nowrap; + } + + #user-tools a.navbar-add-link, + #user-tools a.navbar-add-link:link, + #user-tools a.navbar-add-link:visited { + padding: 5px 8px; + } + + #user-tools .navbar-item:where( + .navbar-log, + .navbar-tags, + .navbar-docs, + .navbar-api, + .navbar-admin, + .navbar-user, + .navbar-account + ), + #user-tools .navbar-separator { + display: none; + } +} + +body.change-list:not(.model-snapshot) #changelist #toolbar, + body.change-list:not(.model-snapshot) #changelist #toolbar #changelist-search, + body.change-list:not(.model-snapshot) #changelist #toolbar #changelist-search > div { + width: 100%; + min-width: 0; + } + + body.change-list:not(.model-snapshot) #changelist #toolbar .search-input-wrap { + height: 40px; + } + + body.change-list:not(.model-snapshot) #content #changelist #toolbar #searchbar { + height: 100%; + } +} + +/* Keep the global admin navigation on one compact row at tablet/mobile sizes. */ +@media (max-width: 1180px) { + #archivebox-system-warning-banner { + position: relative !important; + inset: auto !important; + transform: none !important; + width: 100%; + max-width: none !important; + padding: 6px 8px !important; + border-radius: 0 !important; + box-sizing: border-box; + text-align: center; + } + + #header { + display: grid !important; + grid-template-columns: 30px minmax(0, 1fr); + align-items: center !important; + column-gap: clamp(4px, 1vw, 12px); + width: 100%; + height: auto !important; + min-height: 42px; + padding: 5px 8px; + box-sizing: border-box; + } + + #branding { + width: auto !important; + min-width: 0; + margin: 0; + padding: 0; + text-align: left; + } + + #site-name { + margin: 0; + line-height: 1; + } + + #site-name a { + display: flex; + align-items: center; + justify-content: center; + width: 30px; + height: 30px; + overflow: hidden; + } + + #site-name .branding-label { + display: none; + } + + #site-name #logo, + #site-name img { + display: block; + width: auto; + height: auto !important; + max-width: 28px; + max-height: 28px; + margin: 0; + padding: 0; + object-fit: contain; + } + + #user-tools { + display: flex !important; + flex-wrap: nowrap !important; + align-items: center; + justify-content: flex-end; + gap: clamp(3px, 0.7vw, 10px); + width: 100%; + min-width: 0; + margin: 0 !important; + padding: 0 !important; + overflow: hidden; + line-height: 1.2; + text-align: right; + white-space: nowrap !important; + } + + #user-tools .navbar-item { + flex: 0 0 auto; + min-width: 0; + white-space: nowrap; + } + + #user-tools .navbar-user { + display: inline-flex; + flex: 0 1 min(18vw, 180px); + min-width: 0; + max-width: min(18vw, 180px); + gap: 4px; + overflow: hidden; + } + + #user-tools .navbar-user-label { + display: none; + } + + #user-tools .navbar-username { + display: block; + flex: 1 1 auto; + width: 100%; + min-width: 0; + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + #user-tools .navbar-separator { + display: none; + } +} + +@media (max-width: 520px) { + #header { + grid-template-columns: 26px minmax(0, 1fr); + column-gap: 4px; + min-height: 38px; + padding: 4px 6px; + } + + #site-name a, + #site-name #logo, + #site-name img { + width: 26px; + height: 26px !important; + } + + #site-name #logo, + #site-name img { + width: auto; + height: auto !important; + max-width: 24px; + max-height: 24px; + } + + #user-tools { + justify-content: space-between; + gap: 2px; + font-size: 11px; + } + + #user-tools .navbar-item { + min-height: 28px; + padding: 4px 3px; + border-radius: 4px; + } + + #user-tools a.navbar-add-link, + #user-tools a.navbar-add-link:link, + #user-tools a.navbar-add-link:visited { + padding: 4px 5px; + } + + #user-tools .navbar-item:where( + .navbar-log, + .navbar-tags, + .navbar-docs, + .navbar-api, + .navbar-admin, + .navbar-user, + .navbar-account + ) { + display: none; + } +} + /* Filter Sidebar - Improved Layout */ #content #changelist-filter { background: #fff; @@ -674,6 +988,7 @@ body.change-list:not(.model-snapshot) #changelist #changelist-form > .results { max-width: 100px; max-height: 100px; object-fit: cover; + object-position: top center; } #content th.field-preview_icon, @@ -731,6 +1046,32 @@ body.change-list:not(.model-snapshot) #changelist #changelist-form > .results { padding-right: 2px; } +body.change-list:not(.model-snapshot) #result_list th.action-checkbox-column, +body.change-list:not(.model-snapshot) #result_list td.action-checkbox { + position: sticky; + left: 0; + z-index: 2; + width: 40px; + min-width: 40px; + max-width: 40px; + padding-left: 10px !important; + padding-right: 10px !important; + box-sizing: border-box; + text-align: center; + background: inherit; +} + +body.change-list:not(.model-snapshot) #result_list th.action-checkbox-column { + z-index: 3; + background: #f8fafc; +} + +body.change-list:not(.model-snapshot) #result_list th.action-checkbox-column input, +body.change-list:not(.model-snapshot) #result_list td.action-checkbox input { + display: block; + margin: 0 auto; +} + #content th.field-preview_icon, #content td.field-preview_icon { padding-left: 2px; @@ -1164,6 +1505,14 @@ body.model-archiveresult.change-list #result_list { width: 100%; } +body.model-archiveresult.change-list #result_list th, +body.model-archiveresult.change-list #result_list td { + box-sizing: border-box; + padding-left: 8px; + padding-right: 8px; + overflow: hidden; +} + body.model-archiveresult.change-list #result_list th.column-cmd_str, body.model-archiveresult.change-list #result_list td.field-cmd_str { width: 180px !important; @@ -1188,8 +1537,10 @@ body.model-archiveresult.change-list #result_list td.field-zip_link { body.model-archiveresult.change-list #result_list th.column-created_at, body.model-archiveresult.change-list #result_list td.field-created_at { - width: 96px; - max-width: 96px; + width: 128px; + max-width: 128px; + white-space: nowrap; + text-overflow: ellipsis; } body.model-archiveresult.change-list #result_list th.column-snapshot_info, @@ -1385,7 +1736,7 @@ tbody .output-link:hover {opacity: 1;} border-radius: 4px; } -@media (max-width: 1180px) { +@media (max-width: 1500px) { .model-snapshot.change-list #result_list { display: block; min-width: 0 !important; @@ -1406,12 +1757,12 @@ tbody .output-link:hover {opacity: 1;} .model-snapshot.change-list #result_list tbody tr { display: grid; - grid-template-columns: 24px minmax(0, 1fr) auto; + grid-template-columns: 24px 84px minmax(0, 1fr) auto; grid-template-areas: - "check date status" - "check title title" - "check tags tags" - "check files size"; + "check preview date status" + "check preview title title" + "check preview tags tags" + "check preview files size"; gap: 5px 8px; align-items: start; min-width: 0; @@ -1454,7 +1805,19 @@ tbody .output-link:hover {opacity: 1;} } .model-snapshot.change-list #result_list tbody td.field-preview_icon { - display: none; + display: grid; + grid-area: preview; + align-self: stretch; + place-items: center; + min-height: 84px; + overflow: hidden; + } + + .model-snapshot.change-list #result_list tbody td.field-preview_icon img.snapshot-preview.screenshot { + width: 80px; + height: 80px; + max-width: 80px; + max-height: 80px; } .model-snapshot.change-list #result_list tbody td.field-title_str { @@ -1667,14 +2030,21 @@ tbody .output-link:hover {opacity: 1;} } .model-snapshot.change-list #result_list tbody tr { - grid-template-columns: 24px minmax(0, 1fr); + grid-template-columns: 24px 72px minmax(0, 1fr); grid-template-areas: - "check date" - "check title" - "check tags" - "check status" - "check files" - "check size"; + "check preview date" + "check preview title" + "check preview tags" + "check preview status" + "check preview files" + "check preview size"; + } + + .model-snapshot.change-list #result_list tbody td.field-preview_icon img.snapshot-preview.screenshot { + width: 68px; + height: 68px; + max-width: 68px; + max-height: 68px; } .model-snapshot.change-list #result_list tbody td.field-status_with_progress, @@ -1961,7 +2331,7 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { } } -@media (max-width: 900px) { +@media (max-width: 1180px) { .model-snapshot.change-list #changelist .changelist-form-container > div { grid-template-columns: minmax(0, 1fr); } @@ -1976,10 +2346,6 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { grid-template-columns: repeat(2, minmax(0, 1fr)); } - .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary { - overflow-x: auto; - max-width: 100%; - } } @media (max-width: 520px) { @@ -2002,23 +2368,51 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { #toolbar .search-input-wrap { position: relative; display: block; + width: 100%; min-width: 0; + height: 40px; + border: 1px solid #cbd5e1; + border-radius: 8px; + background: #ffffff; + overflow: hidden; + box-sizing: border-box; +} + +#toolbar .search-input-wrap:focus-within { + border-color: #93c5fd; + box-shadow: 0 0 0 1px #93c5fd; +} + +#toolbar .filter-pane-toggle { + display: inline-flex; + align-self: start; + align-items: center; + justify-content: center; + height: 40px; + margin: 0; + border: 1px solid #cbd5e1; + border-radius: 8px; + box-shadow: none; + background: #ffffff; + box-sizing: border-box; + line-height: 1; } #toolbar .search-input-addon { position: absolute; - left: 1px; - top: 1px; - bottom: 1px; + left: 0; + top: 0; + bottom: 0; z-index: 2; display: inline-flex; align-items: center; + justify-content: center; gap: 4px; box-sizing: border-box; max-width: calc(100% - 2px); padding: 0 8px; border-right: 1px solid #cbd5e1; - border-radius: 7px 0 0 7px; + border-radius: 0; background: #f8fafc; } @@ -2062,7 +2456,14 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { #toolbar .search-input-wrap #searchbar, .model-snapshot.change-list #content #changelist #toolbar .search-input-wrap #searchbar { - padding-left: 42px !important; + height: 100%; + margin: 0 !important; + border: 0; + border-radius: 0; + box-shadow: none; + background: transparent; + box-sizing: border-box; + padding-left: 48px !important; } #toolbar .search-input-wrap-with-mode #searchbar, @@ -2090,22 +2491,11 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { box-sizing: border-box; } -#content form .search-input-addon { - position: absolute !important; - left: 1px !important; - top: 1px !important; - bottom: 1px !important; - width: 112px !important; - padding: 0 8px !important; -} - -#content form .search-mode-select { - position: static !important; - width: auto !important; - max-width: 82px !important; - min-width: 0 !important; - height: 28px !important; - padding-left: 0 !important; +#content form .search-input-wrap:not(.search-input-wrap-with-mode) .search-input-addon { + width: 46px !important; + min-width: 46px !important; + max-width: 46px !important; + padding: 0 !important; } #content form .search-input-wrap #searchbar { @@ -2264,7 +2654,61 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { display: none !important; } +body.change-list #changelist #changelist-form > .actions-top .action-summary { + display: inline-flex; + flex-direction: column; + flex-wrap: nowrap; + align-items: flex-end; + gap: 2px; + margin-left: auto; + min-width: 0; + text-align: right; +} + +body.change-list #changelist #changelist-form > .actions-top .action-selected-count, +body.change-list #changelist #changelist-form > .actions-top .action-total-count { + display: inline-flex; + flex: 0 0 auto !important; + align-items: baseline; + gap: 3px; + width: auto !important; + white-space: nowrap; +} + +body.change-list #changelist #changelist-form > .actions-top .action-selected-current { + flex: 0 0 auto !important; + width: auto !important; +} + @media (max-width: 1180px) { + .model-snapshot.change-list #changelist #changelist-form > .actions-top .actions-left { + display: grid !important; + grid-template-columns: minmax(0, 1fr); + width: 100%; + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-summary { + grid-column: 1; + grid-row: 1; + width: 100%; + max-width: 100%; + flex-direction: column; + flex-wrap: nowrap; + align-items: flex-end; + justify-content: flex-start; + justify-self: stretch; + gap: 2px; + overflow: visible; + } + + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-selected-count, + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-total-count { + display: inline-flex !important; + flex: 0 0 auto !important; + width: auto !important; + overflow: visible; + } + .model-snapshot.change-list #changelist #changelist-form > .actions-top .action-buttons { grid-template-columns: repeat(4, minmax(78px, 1fr)); } @@ -2311,9 +2755,9 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { justify-content: center; gap: 2px; height: 100%; - width: 46px; - min-width: 46px; - max-width: 46px; + width: 54px; + min-width: 54px; + max-width: 54px; padding: 0; border-right: 1px solid #e2e8f0; border-radius: 0; @@ -2327,7 +2771,7 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { position: absolute; left: 12px; top: 10px; - z-index: 4; + z-index: 6; display: block; width: 11px; height: 11px; @@ -2342,7 +2786,7 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { position: absolute; left: 25px; top: 24px; - z-index: 4; + z-index: 6; display: block; width: 7px; height: 2px; @@ -2379,11 +2823,11 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { -webkit-appearance: none; position: absolute; inset: 0 auto 0 0; - width: 46px !important; - min-width: 46px !important; - max-width: 46px !important; + width: 54px !important; + min-width: 54px !important; + max-width: 54px !important; height: 100%; - padding: 0; + padding: 0 26px 0 10px !important; border: 0; border-radius: 0; background: transparent; @@ -2395,7 +2839,7 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { opacity: 1; text-indent: -9999px; cursor: pointer; - z-index: 3; + z-index: 5; } .model-snapshot.change-list #toolbar .search-mode-select option { @@ -2408,15 +2852,20 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { .model-snapshot.change-list #toolbar .search-mode-caret { position: absolute; - right: 9px; - top: 50%; + right: 8px; + top: 17px; display: block; - z-index: 2; - color: #64748b; - font-size: 10px; - line-height: 1; + z-index: 6; + width: 0; + height: 0; + border-left: 4px solid transparent; + border-right: 4px solid transparent; + border-top: 5px solid #475569; + color: transparent; + font-size: 0; + line-height: 0; pointer-events: none; - transform: translateY(-45%); + transform: none; } .model-snapshot.change-list #toolbar .search-input-wrap #searchbar, @@ -2440,19 +2889,11 @@ body.embedded-change-list.model-snapshot.change-list #changelist .actions { .model-snapshot.change-list #toolbar .search-input-wrap #searchbar:focus { outline: 0; - box-shadow: inset 0 0 0 1px #93c5fd; + box-shadow: none; } .model-snapshot.change-list #toolbar form input[type="submit"] { - grid-column: 2; - grid-row: 1; - height: 40px; - min-width: 86px; - padding: 0 16px; - margin: 0; - border-radius: 8px; - font-size: 14px; - font-weight: 600; + position: absolute !important; } #content #changelist #toolbar .changelist-search-submit { @@ -2464,16 +2905,7 @@ body.change-list:not(.model-snapshot) #changelist #toolbar .changelist-search-su } .model-snapshot.change-list #toolbar .changelist-search-submit { - grid-column: 2; - grid-row: 1; - height: 40px; - min-width: 86px; - padding: 0 16px; - margin: 0; - border-radius: 8px; - font-size: 14px; - font-weight: 600; - white-space: nowrap; + position: absolute !important; } .snapshot-changelist-widget.model-snapshot.change-list #toolbar .changelist-toolbar-row { @@ -2585,10 +3017,10 @@ body.change-list:not(.model-snapshot) #changelist #toolbar .changelist-search-su .model-crawl.change-list #content #changelist #toolbar #changelist-search > div, .model-crawl.change-list #toolbar #changelist-search > div { display: grid !important; - grid-template-columns: minmax(0, 1fr) auto; + grid-template-columns: minmax(0, 1fr); grid-template-areas: - "search submit" - "count count"; + "search" + "count"; gap: 8px; align-items: center; width: 100% !important; @@ -2692,9 +3124,7 @@ body.change-list:not(.model-snapshot) #changelist #toolbar .changelist-search-su } .model-crawl.change-list #toolbar .changelist-search-submit { - grid-area: submit; - min-width: 86px; - padding: 0 16px; + position: absolute !important; } .model-crawl.change-list #content #changelist #toolbar .small.quiet { @@ -2828,12 +3258,12 @@ body.change-list:not(.model-snapshot) #changelist #toolbar .changelist-search-su .model-crawl.change-list #changelist #changelist-form > .actions-top .action-summary { display: inline-flex; - align-items: center; + align-items: flex-end; justify-content: center; - height: 40px; + height: auto; min-height: 40px; - padding: 0 4px; - margin-left: 2px; + padding: 3px 4px; + margin-left: auto; color: #475569; font-size: 14px; font-weight: 500; @@ -2845,7 +3275,7 @@ body.change-list:not(.model-snapshot) #changelist #toolbar .changelist-search-su .model-crawl.change-list #changelist #changelist-form > .actions-top .action-summary .action-selected-count { display: inline-flex; align-items: center; - height: 40px; + height: auto; color: inherit; font: inherit; line-height: 1; @@ -2961,12 +3391,12 @@ body.change-list:not(.model-snapshot) #changelist #toolbar .changelist-search-su @media (max-width: 720px) { .model-crawl.change-list #toolbar .changelist-toolbar-row { - grid-template-columns: minmax(0, 1fr); + grid-template-columns: minmax(0, 1fr) 84px; } .model-crawl.change-list #toolbar .filter-pane-toggle { - width: 100%; - min-width: 0; + width: 84px; + min-width: 84px; } } @@ -2974,3 +3404,153 @@ body.change-list #changelist #changelist-form > .actions-top .action-buttons[hid body.change-list #changelist #changelist-form > .actions-top .actions-tags-with-buttons[hidden] { display: none !important; } + +/* Keep the search controls and the clickable selection totals in one compact + toolbar row on every changelist. The action form is rendered as a sibling of + #toolbar, so the shared wrapper owns the two-column layout. */ +body.change-list:not(.embedded-change-list) #changelist .changelist-form-container > div { + display: grid !important; + grid-template-columns: minmax(0, 1fr) max-content !important; + grid-template-areas: + "toolbar counts" + "results results" + "footer footer" !important; + gap: 10px 8px !important; + align-items: start; +} + +body.change-list:not(.embedded-change-list) #changelist #toolbar { + grid-area: toolbar !important; + width: 100% !important; + min-width: 0 !important; + margin: 0 !important; +} + +/* The action summary already exposes selected, filtered, and total counts with + clickable values. Django's older search-result count duplicates that data + below the field and makes the toolbar row taller on large result sets. */ +body.change-list:not(.embedded-change-list) #changelist #toolbar .small.quiet { + display: none !important; +} + +body.change-list:not(.embedded-change-list) #changelist #changelist-form > .actions-top { + grid-area: counts !important; + display: flex !important; + align-items: center !important; + align-self: start !important; + width: auto !important; + min-width: 0 !important; + height: 40px !important; + min-height: 40px !important; + margin: 0 !important; + padding: 0 !important; + border: 0 !important; + border-radius: 0 !important; + background: transparent !important; + overflow: visible !important; +} + +body.change-list:not(.embedded-change-list) #changelist #changelist-form > .actions-top .actions-left { + display: flex !important; + align-items: center !important; + justify-content: flex-end !important; + width: auto !important; + min-width: 0 !important; + height: 40px !important; + padding: 0 !important; + gap: 4px !important; + overflow: visible !important; +} + +body.change-list:not(.embedded-change-list) #changelist #changelist-form > .actions-top .action-summary { + display: inline-flex !important; + flex: 0 0 auto !important; + flex-direction: column !important; + flex-wrap: nowrap !important; + align-items: flex-end !important; + justify-content: center !important; + width: auto !important; + min-width: 0 !important; + height: auto !important; + min-height: 0 !important; + margin: 0 !important; + padding: 0 !important; + gap: 0 !important; + color: #475569; + font-size: 12px !important; + font-weight: 500 !important; + line-height: 1.15 !important; + text-align: right !important; + white-space: nowrap !important; +} + +body.change-list:not(.embedded-change-list) #changelist #changelist-form > .actions-top .action-summary a, +body.change-list:not(.embedded-change-list) #changelist #changelist-form > .actions-top .action-summary span { + font-size: inherit !important; + line-height: inherit !important; +} + +body.change-list:not(.embedded-change-list) #changelist #changelist-form > .cards, +body.change-list:not(.embedded-change-list) #changelist #changelist-form > .results, +body.change-list:not(.embedded-change-list) #changelist #changelist-form > .changelist-footer, +body.change-list:not(.embedded-change-list) #changelist #changelist-form > .actions-bottom, +body.change-list:not(.embedded-change-list) #changelist .paginator, +body.change-list:not(.embedded-change-list) #changelist .xfull { + grid-column: 1 / -1 !important; +} + +/* Keep the grouped search control's outline above its absolutely positioned + icon/mode segment. Without this overlay that segment can paint over the + rounded upper-left border. */ +#toolbar .search-input-wrap::after { + content: ""; + position: absolute; + inset: 0; + z-index: 10; + border: 1px solid #cbd5e1; + border-radius: 8px; + pointer-events: none; +} + +#toolbar .search-input-wrap:focus-within::after { + border-color: #93c5fd; +} + +/* Django's stock change-form stylesheet is loaded after the card rules in the + template. Reassert the shared mobile geometry here so every admin detail + view fills the viewport instead of retaining the desktop 320px card width. */ +@media (max-width: 900px) { + body.change-form #content, + body.change-form #content-main, + body.change-form #content-main > form, + body.change-form #content-main > form > div { + width: 100% !important; + max-width: 100% !important; + min-width: 0 !important; + box-sizing: border-box; + } + + body.change-form #content-main form fieldset, + body.change-form #content form fieldset, + body.change-form #content-main form .module:not(.inline-group), + body.change-form #content form .module:not(.inline-group) { + flex: 1 1 100% !important; + width: 100% !important; + max-width: 100% !important; + min-width: 0 !important; + } + + body.change-form #content-main form fieldset > div, + body.change-form #content form fieldset > div, + body.change-form #content-main form fieldset .form-row, + body.change-form #content form fieldset .form-row, + body.change-form #content-main form fieldset .form-row > div, + body.change-form #content form fieldset .form-row > div, + body.change-form #content-main form fieldset .flex-container, + body.change-form #content form fieldset .flex-container { + width: 100% !important; + max-width: 100% !important; + min-width: 0 !important; + box-sizing: border-box; + } +} diff --git a/archivebox/templates/static/admin/crawls/crawl_change.css b/archivebox/templates/static/admin/crawls/crawl_change.css index fe039862..e691de66 100644 --- a/archivebox/templates/static/admin/crawls/crawl_change.css +++ b/archivebox/templates/static/admin/crawls/crawl_change.css @@ -281,6 +281,21 @@ body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots . } @media (max-width: 900px) { + body.model-crawl.change-form #content .object-tools .archivebox-crawl-resume-tool { + display: flex; + flex: 1 1 100%; + min-width: 0; + margin: 0; + gap: 8px; + justify-content: flex-start; + } + + body.model-crawl.change-form #content .object-tools .crawl-stop-reason-inline { + min-width: 0; + white-space: normal; + overflow-wrap: anywhere; + } + body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-multiline, body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview .form-row.field-notes.field-tags_editor .form-multiline, body.model-crawl.change-form #content-main form fieldset.crawl-admin-config .key-value-rows { @@ -291,4 +306,11 @@ body.model-crawl.change-form #content-main form fieldset.crawl-admin-snapshots . grid-template-columns: 1fr; align-items: stretch !important; } + + body.model-crawl.change-form #content-main form fieldset.crawl-admin-config *, + body.model-crawl.change-form #content-main form fieldset.crawl-admin-overview * { + min-width: 0; + max-width: 100%; + box-sizing: border-box; + } } diff --git a/archivebox/templates/static/setup_wizard.css b/archivebox/templates/static/setup_wizard.css new file mode 100644 index 00000000..78c369b3 --- /dev/null +++ b/archivebox/templates/static/setup_wizard.css @@ -0,0 +1,56 @@ +#archivebox-setup-wizard { position:fixed; inset:0; z-index:2147483647; display:flex; align-items:center; justify-content:center; padding:24px; background:rgba(15,23,42,.68); color:#0f172a; font:14px/1.5 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Arial,sans-serif; } +#archivebox-setup-wizard .abx-setup-panel { width:min(1040px,100%); max-height:calc(100vh - 48px); overflow:auto; background:#fff; border-radius:12px; box-shadow:0 24px 70px rgba(0,0,0,.38); padding:24px; } +#archivebox-setup-wizard .abx-setup-eyebrow { color:#aa1e55; font-size:12px; font-weight:800; text-transform:uppercase; letter-spacing:.08em; } +#archivebox-setup-title { margin:4px 0 6px; font-size:24px; line-height:1.2; } +#archivebox-setup-wizard .abx-setup-row { margin-top:18px; padding:16px; border:1px solid #dbe3ee; border-radius:9px; background:#fff; } +#archivebox-setup-wizard .abx-setup-label { display:block; margin-bottom:7px; font-size:15px; font-weight:800; } +#archivebox-setup-wizard .abx-question { margin:18px 0 0; padding:16px; border:1px solid #dbe3ee; border-radius:9px; } +#archivebox-setup-wizard .abx-question legend { padding:0 6px; font-size:15px; font-weight:800; } +#archivebox-setup-wizard .abx-question-options { display:grid; gap:9px; } +#archivebox-setup-wizard .abx-question-option { display:flex; align-items:flex-start; gap:10px; padding:11px 12px; border:1px solid #dbe3ee; border-radius:7px; background:#f8fafc; color:#334155; cursor:pointer; } +#archivebox-setup-wizard .abx-question-option:has(input:checked) { border-color:#15803d; background:#f0fdf4; box-shadow:0 0 0 1px #15803d; } +#archivebox-setup-wizard .abx-question-option input { width:18px; height:18px; margin-top:2px; flex:0 0 auto; accent-color:#15803d; } +#archivebox-setup-wizard .abx-question-option small { display:block; margin-top:2px; color:#64748b; } +#archivebox-setup-title code { font-size:inherit; line-height:inherit; } +#archivebox-setup-wizard .abx-setup-help { display:block; margin-top:6px; color:#64748b; } +#archivebox-setup-wizard .abx-url-comparison { display:grid; gap:4px; margin-top:10px; padding:10px 12px; border-radius:7px; background:#f8fafc; color:#475569; } +#archivebox-setup-wizard .abx-url-comparison-status.is-match { color:#15803d; } +#archivebox-setup-wizard .abx-url-comparison-status.is-warning { color:#b45309; } +#archivebox-setup-wizard .abx-question-status { margin-top:9px; padding:9px 11px; border-radius:7px; background:#f8fafc; color:#475569; font-weight:700; } +#archivebox-setup-wizard .abx-setup-control { box-sizing:border-box; width:100%; height:44px !important; min-height:44px !important; margin:0; padding:0 12px !important; border:1px solid #b8c4d4 !important; border-radius:7px; background:#fff !important; color:#0f172a !important; font:14px/normal -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Arial,sans-serif !important; opacity:1 !important; } +#archivebox-setup-wizard select.abx-setup-control { appearance:auto !important; -webkit-appearance:menulist !important; } +#archivebox-setup-wizard select.abx-setup-control option { color:#0f172a !important; background:#fff !important; } +#archivebox-setup-wizard .abx-mode-list { margin:12px 0 0; padding-left:22px; color:#475569; } +#archivebox-setup-wizard .abx-mode-list li { margin:7px 0; padding-left:3px; } +#archivebox-setup-effective-mode { margin-top:10px; padding:11px 12px; border-radius:7px; background:#eef6ff; color:#1e3a5f; font-weight:700; } +#archivebox-setup-wizard .abx-permission-grid { display:grid; grid-template-columns:1fr 1fr; gap:10px 24px; } +#archivebox-setup-wizard .abx-permissions-field { grid-column:1/-1; } +#archivebox-setup-wizard .abx-permissions-label { display:block; margin-bottom:6px; font-weight:700; } +#archivebox-setup-wizard .abx-check { display:flex; align-items:flex-start; gap:9px; color:#334155; } +#archivebox-setup-wizard .abx-check input { width:18px; height:18px; margin-top:2px; } +#archivebox-setup-wizard .abx-preview { margin-top:22px; padding-top:20px; border-top:2px solid #e2e8f0; } +#archivebox-setup-preview-title { margin:0 0 5px; font-size:18px; } +#archivebox-setup-wizard .abx-preview-copy { margin:0 0 12px; color:#64748b; } +#archivebox-setup-wizard .abx-preview-row { display:grid; grid-template-columns:170px minmax(240px,1fr) 220px; gap:10px; align-items:center; margin:8px 0; } +#archivebox-setup-wizard .abx-preview-row label { font-weight:750; } +#archivebox-setup-wizard .abx-preview-url { box-sizing:border-box; width:100%; height:36px; padding:0 9px; border:1px solid #d6dee9; border-radius:6px; background:#f8fafc; color:#334155; font:12px ui-monospace,SFMono-Regular,Menlo,Consolas,monospace; } +#archivebox-setup-wizard .abx-status-row { margin:8px 0; padding:10px 12px; border-radius:7px; background:#f8fafc; color:#334155; } +#archivebox-setup-wizard .abx-preview-statuses { margin-top:16px; } +#archivebox-setup-wizard .abx-exposure-title { margin:18px 0 4px; font-size:16px; } +#archivebox-setup-wizard .abx-exposure-note { margin:12px 0 0; color:#64748b; font-size:12px; } +#archivebox-setup-wizard .abx-risk-list { margin:12px 0 0; padding:0; list-style:none; } +#archivebox-setup-wizard .abx-risk-list li { margin:7px 0; padding:10px 12px; border-radius:7px; background:#fff7ed; color:#7c2d12; } +#archivebox-setup-wizard #archivebox-setup-review:disabled { cursor:not-allowed !important; background:#94a3b8 !important; opacity:.75; } +#archivebox-setup-wizard .abx-setup-actions { display:flex; flex-wrap:wrap; align-items:center; justify-content:space-between; gap:10px; margin-top:22px; } +#archivebox-setup-validation { flex:1; min-width:260px; padding:10px 12px; border-radius:7px; background:#fff7ed; color:#7c2d12; font-weight:700; } +#archivebox-setup-validation.is-testing { background:#eef6ff; color:#1e3a5f; } +#archivebox-setup-validation.is-success { background:#ecfdf5; color:#065f46; } +#archivebox-setup-wizard .abx-button { border:1px solid #94a3b8; border-radius:7px; padding:9px 14px; background:#fff; color:#334155; font-weight:700; cursor:pointer; } +#archivebox-setup-wizard .abx-button-primary { border:0; padding:10px 16px; background:#aa1e55; color:#fff; font-weight:800; } +#archivebox-setup-wizard code { padding:1px 4px; border-radius:4px; background:#eef2f7; color:#334155; } +@media (max-width:760px) { + #archivebox-setup-wizard { padding:8px !important; } + #archivebox-setup-wizard .abx-setup-panel { max-height:calc(100vh - 16px); padding:16px; } + #archivebox-setup-wizard .abx-permission-grid { grid-template-columns:1fr; } + #archivebox-setup-wizard .abx-preview-row { grid-template-columns:1fr; } +} diff --git a/archivebox/templates/static/setup_wizard.js b/archivebox/templates/static/setup_wizard.js new file mode 100644 index 00000000..c13ed38f --- /dev/null +++ b/archivebox/templates/static/setup_wizard.js @@ -0,0 +1,405 @@ +(function () { + var wizard = document.getElementById('archivebox-setup-wizard'); + if (!wizard) return; + + var baseUrlInput = document.getElementById('archivebox-setup-base-url'); + var securityModeInput = document.getElementById('archivebox-setup-security-mode'); + var publicIndexInput = document.getElementById('archivebox-setup-public-index'); + var publicAddInput = document.getElementById('archivebox-setup-public-add'); + var permissionsInput = document.getElementById('archivebox-setup-permissions'); + var hostingInputs = document.querySelectorAll('input[name="archivebox-hosting-location"]'); + var dnsInputs = document.querySelectorAll('input[name="archivebox-dns-mode"]'); + var tlsInputs = document.querySelectorAll('input[name="archivebox-tls-mode"]'); + var reviewButton = document.getElementById('archivebox-setup-review'); + var validationStatus = document.getElementById('archivebox-setup-validation'); + var probeTimer = null; + var probeGeneration = 0; + var currentPreview = null; + var machineAdminUrl = new URL(wizard.dataset.machineAdminUrl, window.location.origin); + if (window.location.pathname === machineAdminUrl.pathname && new URLSearchParams(window.location.search).has('BASE_URL')) { + wizard.remove(); + return; + } + publicIndexInput.checked = wizard.dataset.publicIndex === 'true'; + publicAddInput.checked = wizard.dataset.publicAddView === 'true'; + permissionsInput.value = wizard.dataset.permissions || 'public'; + + function selectedValue(inputs) { + var selected = Array.prototype.find.call(inputs, function(input) { return input.checked; }); + return selected ? selected.value : ''; + } + + function selectValue(inputs, value) { + Array.prototype.forEach.call(inputs, function(input) { input.checked = input.value === value; }); + } + + var detectedUrl = new URL(baseUrlInput.value); + var detectedLocalhost = detectedUrl.hostname === 'localhost' || detectedUrl.hostname.endsWith('.localhost'); + if (detectedLocalhost) { + selectValue(hostingInputs, 'localhost'); + selectValue(dnsInputs, 'localhost'); + selectValue(tlsInputs, 'localhost'); + securityModeInput.value = 'auto'; + } else { + selectValue(dnsInputs, 'single'); + selectValue(tlsInputs, detectedUrl.protocol === 'https:' ? 'single' : 'none'); + securityModeInput.value = 'auto'; + } + + function setPreviewValue(id, value) { + document.getElementById(id).value = value; + } + + function updateUrlComparison(configuredUrl, expectedAdminOrigin) { + var browserUrl = window.location.origin; + var status = document.getElementById('archivebox-setup-url-match'); + document.getElementById('archivebox-setup-browser-url').textContent = browserUrl; + document.getElementById('archivebox-setup-configured-url').textContent = configuredUrl; + if (browserUrl.toLowerCase() === configuredUrl.toLowerCase()) { + status.className = 'abx-url-comparison-status is-match'; + status.textContent = '✅ Browser URL matches BASE_URL.'; + } else if (expectedAdminOrigin && browserUrl.toLowerCase() === expectedAdminOrigin.toLowerCase()) { + status.className = 'abx-url-comparison-status is-match'; + status.textContent = '✅ Browser URL matches admin.BASE_URL as expected.'; + } else { + status.className = 'abx-url-comparison-status is-warning'; + status.textContent = '⚠️ Browser URL does not match BASE_URL' + (expectedAdminOrigin ? ' or its expected admin URL.' : '.'); + } + } + + function updatePreview() { + var parsed; + try { + parsed = new URL(baseUrlInput.value.trim() || baseUrlInput.placeholder); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') throw new Error('Unsupported URL scheme'); + } catch (error) { + currentPreview = null; + updateUrlComparison(baseUrlInput.value.trim() || '(invalid BASE_URL)', ''); + reviewButton.disabled = true; + validationStatus.textContent = 'Enter a valid http:// or https:// BASE_URL to continue.'; + ['archivebox-preview-admin', 'archivebox-preview-api', 'archivebox-preview-index', 'archivebox-preview-snapshot', 'archivebox-preview-save', 'archivebox-preview-last'].forEach(function(id) { setPreviewValue(id, 'Enter a valid http:// or https:// BASE_URL'); }); + ['archivebox-preview-admin-status', 'archivebox-preview-api-status', 'archivebox-preview-index-status', 'archivebox-preview-snapshot-status', 'archivebox-preview-save-status', 'archivebox-preview-last-status'].forEach(function(id) { document.getElementById(id).textContent = '❌ Invalid BASE_URL'; }); + return; + } + + var hostname = parsed.hostname.toLowerCase(); + var isLocalhost = hostname === 'localhost' || hostname.endsWith('.localhost'); + var selectedHosting = selectedValue(hostingInputs); + var selectedDnsMode = selectedValue(dnsInputs); + var selectedTlsMode = selectedValue(tlsInputs); + var selectedLocalhost = selectedHosting === 'localhost' && selectedDnsMode === 'localhost' && selectedTlsMode === 'localhost'; + var usesSubdomains = securityModeInput.value === 'safe-subdomains-fullreplay' || (securityModeInput.value === 'auto' && selectedLocalhost); + var fullJsReplay = usesSubdomains || securityModeInput.value === 'unsafe-onedomain-noadmin' || securityModeInput.value === 'danger-onedomain-fullreplay'; + var controlPlaneEnabled = securityModeInput.value !== 'unsafe-onedomain-noadmin'; + var baseHost = parsed.host; + var originFor = function(role) { return parsed.protocol + '//' + (usesSubdomains ? role + '.' + baseHost : baseHost); }; + var adminOrigin = originFor('admin'); + var webOrigin = originFor('web'); + var apiOrigin = originFor('api'); + var snapshotOrigin = usesSubdomains ? parsed.protocol + '//snap-456789abcdef.' + baseHost : webOrigin + '/snapshot/0123456789abcdef0123456789abcdef'; + var originalOrigin = usesSubdomains ? parsed.protocol + '//reddit.com.' + baseHost : webOrigin + '/original/reddit.com'; + var permission = permissionsInput.value; + var httpsReady = selectedTlsMode === 'wildcard' || selectedTlsMode === 'single' || selectedTlsMode === 'localhost'; + var effectiveMode = document.getElementById('archivebox-setup-effective-mode'); + + document.getElementById('archivebox-setup-wildcard-example').textContent = 'https://*.' + parsed.host; + document.getElementById('archivebox-setup-base-url-example').textContent = parsed.origin; + updateUrlComparison(parsed.origin, usesSubdomains ? adminOrigin : parsed.origin); + + if (securityModeInput.value === 'auto') { + effectiveMode.textContent = selectedLocalhost + ? 'Effective result for this BASE_URL: isolated full replay. Localhost needs no DNS or HTTPS setup.' + : 'Effective result for this BASE_URL: one-domain replay with archived JavaScript disabled. Only one DNS record is needed; HTTPS unlocks service-worker replay viewers.'; + } else if (securityModeInput.value === 'safe-onedomain-nojsreplay') { + effectiveMode.textContent = 'Effective result: one-domain replay with archived JavaScript disabled. Only one DNS record is needed; ' + (httpsReady ? 'high-fidelity replay viewers are available.' : 'add HTTPS to enable service-worker replay viewers.'); + } else if (securityModeInput.value === 'safe-subdomains-fullreplay') { + effectiveMode.textContent = selectedLocalhost + ? 'Effective result: isolated full replay with no manual DNS or HTTPS setup.' + : 'Effective result: isolated full replay. Configure wildcard DNS and a wildcard HTTPS certificate for all replay features.'; + } else if (securityModeInput.value === 'unsafe-onedomain-noadmin') { + effectiveMode.textContent = 'Effective result: full replay on one shared domain, with login, admin, API, submissions, and mutations disabled.'; + } else { + effectiveMode.textContent = 'Effective result: full replay and privileged UI/API share one domain. Malicious archived JavaScript can compromise the archive and your browser session.'; + } + + setPreviewValue('archivebox-preview-admin', adminOrigin + '/admin/'); + setPreviewValue('archivebox-preview-api', apiOrigin + '/api/v1/docs'); + setPreviewValue('archivebox-preview-index', webOrigin + '/public/'); + setPreviewValue('archivebox-preview-snapshot', snapshotOrigin + '/'); + setPreviewValue('archivebox-preview-save', webOrigin + '/web/https://example.com'); + setPreviewValue('archivebox-preview-last', originalOrigin + '/r/somesubpage'); + + var routeSemantics = { + admin: controlPlaneEnabled ? 'Admin login required' : 'Disabled in this mode', + api: controlPlaneEnabled ? 'Available; access-controlled' : 'Disabled in this mode', + index: publicIndexInput.checked ? 'Anonymous index enabled' : 'Sign-in required', + snapshot: permission === 'private' ? (controlPlaneEnabled ? 'Admin only by default' : 'Private content unavailable') : (permission === 'unlisted' ? 'Anyone with URL' : 'Anonymous and listed'), + save: controlPlaneEnabled ? (publicAddInput.checked ? 'Anonymous submissions' : 'Admin only') : 'Submissions disabled', + last: permission === 'private' ? (controlPlaneEnabled ? 'Admin only by default' : 'Private content unavailable') : 'Anonymous direct access', + }; + document.getElementById('archivebox-preview-admin-status').textContent = '⏳ Testing · ' + routeSemantics.admin; + document.getElementById('archivebox-preview-api-status').textContent = '⏳ Testing · ' + routeSemantics.api; + document.getElementById('archivebox-preview-index-status').textContent = '⏳ Testing · ' + routeSemantics.index; + document.getElementById('archivebox-preview-snapshot-status').textContent = '⏳ Testing host · ' + routeSemantics.snapshot; + document.getElementById('archivebox-preview-save-status').textContent = '⏳ Testing web host · ' + routeSemantics.save; + document.getElementById('archivebox-preview-last-status').textContent = '⏳ Testing host · ' + routeSemantics.last; + + document.getElementById('archivebox-preview-dns').textContent = selectedDnsMode === 'localhost' + ? '✅ No manual DNS setup is needed with localhost.' + : (selectedDnsMode === 'wildcard' ? '✱ Configure wildcard DNS for the base hostname and all *.hostname subdomains.' : (selectedDnsMode === 'single' ? '🔢 Configure one A/AAAA/CNAME or /etc/hosts entry for the base hostname.' : '❌ Choose a DNS mode.')); + document.getElementById('archivebox-preview-tls').textContent = selectedTlsMode === 'localhost' + ? '✅ No HTTPS setup is needed with localhost.' + : (selectedTlsMode === 'wildcard' ? '✱ Configure a browser-trusted wildcard HTTPS certificate.' : (selectedTlsMode === 'single' ? '🔒 Configure one browser-trusted HTTPS certificate for the base hostname.' : (selectedTlsMode === 'none' ? '⚠️ Direct HTTP selected; in-browser WARC viewing will not work.' : '❌ Choose an ingress and TLS mode.'))); + document.getElementById('archivebox-preview-warc').textContent = httpsReady ? '✅ Available' : '❌ Requires HTTPS (or localhost)'; + document.getElementById('archivebox-preview-js').textContent = fullJsReplay ? (usesSubdomains ? '✅ Available with per-snapshot isolation' : '⚠️ Available on the shared archive origin') : '❌ Archived JavaScript is disabled'; + + var risks = []; + if (usesSubdomains) { + risks.push('✅ Snapshot replay origins are isolated from the admin UI, API, and other snapshots.'); + } else if (securityModeInput.value === 'unsafe-onedomain-noadmin') { + risks.push('⚠️ Untrusted archived JavaScript can read any anonymous public or unlisted archive content on the shared origin. Admin UI, API, login, submissions, and all state-changing requests are disabled; private snapshots are unavailable because every visitor is anonymous.'); + } else if (securityModeInput.value === 'danger-onedomain-fullreplay') { + risks.push('🛑 Malicious archived JavaScript shares an origin with the archive index, every reachable snapshot, saved headers, admin UI, and REST API. It can trivially read sensitive data and use your authenticated browser to change configuration, install or invoke binaries, archive intranet URLs, or delete data. Use only on a disposable isolated server with no secrets or trusted browser session.'); + } else { + risks.push('⚠️ UI, API, and archive replay share one origin. CSP disables risky archived scripts, but a CSP or content-type bypass could expose the archive index, other snapshots, saved headers, admin pages, API data, and canonical-host mutations.'); + } + risks.push(publicIndexInput.checked + ? '⚠️ Anonymous visitors can enumerate public snapshot URLs and titles; saved URLs may contain private share tokens or other secrets.' + : '✅ Anonymous visitors cannot browse the snapshot index.'); + risks.push(!controlPlaneEnabled + ? '✅ URL submission and other state-changing requests are disabled for everyone in this replay-only mode.' + : (publicAddInput.checked ? '⚠️ Anonymous visitors can submit malicious or private/intranet URLs. A filtering or per-crawl configuration bypass could expose internal content or threaten the server.' : '✅ Only signed-in admins can submit new URLs.')); + risks.push(permission === 'public' + ? '⚠️ New snapshots are listed and readable anonymously. Replayed pages, metadata, headers, cookies, PII, and API keys captured in an archive may become public.' + : (permission === 'unlisted' ? '⚠️ New snapshots are hidden from listings but remain readable by anyone who discovers or receives their URL.' : (controlPlaneEnabled ? '✅ New snapshots require an authenticated ArchiveBox admin by default.' : '⚠️ Private snapshots cannot be viewed while the replay-only mode disables authentication.'))); + document.getElementById('archivebox-setup-risks').innerHTML = risks.map(function(risk) { return '
  • ' + risk + '
    • '; }).join(''); + + currentPreview = { + parsed: parsed, + isLocalhost: isLocalhost, + usesSubdomains: usesSubdomains, + controlPlaneEnabled: controlPlaneEnabled, + routeSemantics: routeSemantics, + adminUrl: adminOrigin + '/admin/login/', + apiUrl: apiOrigin + '/api/v1/docs', + indexUrl: webOrigin + '/public/', + webHealthUrl: webOrigin + '/health/', + snapshotHealthUrl: (usesSubdomains ? snapshotOrigin : webOrigin) + '/health/', + originalHealthUrl: (usesSubdomains ? originalOrigin : webOrigin) + '/health/', + wildcardHealthUrl: usesSubdomains ? parsed.protocol + '//abx-probe-' + Math.random().toString(36).slice(2, 12) + '.' + baseHost + '/health/' : webOrigin + '/health/', + expectedBrowserOrigin: usesSubdomains ? adminOrigin : parsed.origin, + }; + updateOptionGuidance(); + scheduleAccessChecks(); + } + + function updateOptionGuidance() { + var hosting = selectedValue(hostingInputs); + var dnsMode = selectedValue(dnsInputs); + var tlsMode = selectedValue(tlsInputs); + var desiredScheme = tlsMode === 'wildcard' || tlsMode === 'single' ? 'https://' : 'http://'; + var exampleBaseHost = 'archivebox.example.com'; + var exampleWildcardHost = '*.' + exampleBaseHost; + var exampleBaseUrl = desiredScheme + exampleBaseHost; + var exampleWildcardUrl = desiredScheme + exampleWildcardHost; + var exampleAdminUrl = desiredScheme + (dnsMode === 'wildcard' ? 'admin.' + exampleBaseHost : exampleBaseHost) + '/admin/'; + var localhostAdminUrl = 'http://admin.archivebox.localhost:8000/admin/'; + + document.getElementById('archivebox-setup-hosting-status').textContent = hosting === 'localhost' + ? '❌ Visit ' + localhostAdminUrl + ' from the same machine to continue setup.' + : (hosting === 'private' + ? '❌ Configure a LAN, VPN, Tailscale, or intranet hostname/IP such as ' + exampleBaseHost + ' pointing to this ArchiveBox server. The URL will have the shape ' + exampleBaseUrl + '; visit ' + exampleAdminUrl + ' to continue setup.' + : (hosting === 'public' + ? '❌ Point public DNS for a hostname such as ' + exampleBaseHost + ' to this ArchiveBox server or ingress. The URL will have the shape ' + exampleBaseUrl + '; visit ' + exampleAdminUrl + ' to continue setup.' + : '❌ Choose where this server is hosted. Your choice will not be changed automatically.')); + document.getElementById('archivebox-setup-dns-status').textContent = dnsMode === 'localhost' + ? '❌ Visit ' + localhostAdminUrl + ' from the same machine to continue setup. No DNS record is needed.' + : (dnsMode === 'wildcard' + ? '❌ Create DNS records for ' + exampleBaseHost + ' and ' + exampleWildcardHost + ' pointing to this ArchiveBox server or ingress. The wildcard URL will have the shape ' + exampleWildcardUrl + '; visit ' + exampleAdminUrl + ' to continue setup.' + : (dnsMode === 'single' + ? '❌ Create one A/AAAA/CNAME record or /etc/hosts entry for ' + exampleBaseHost + ' pointing to this ArchiveBox server. The URL will have the shape ' + exampleBaseUrl + '; visit ' + exampleAdminUrl + ' to continue setup.' + : '❌ Choose a DNS mode. Your choice will not be changed automatically.')); + document.getElementById('archivebox-setup-tls-status').textContent = tlsMode === 'localhost' + ? '❌ Visit ' + localhostAdminUrl + ' from this machine to continue setup. No certificate is needed.' + : (tlsMode === 'wildcard' + ? '❌ Configure your SSL ingress service in front of this ArchiveBox server with a browser-trusted certificate covering ' + exampleBaseHost + ' and ' + exampleWildcardHost + '. Visit ' + exampleAdminUrl + ' to continue setup.' + : (tlsMode === 'single' + ? '❌ Configure your SSL ingress service in front of this ArchiveBox server with a browser-trusted certificate for ' + exampleBaseHost + '. Visit ' + exampleAdminUrl + ' to continue setup.' + : (tlsMode === 'none' + ? '❌ Expose this ArchiveBox server directly over HTTP without a separate ingress or SSL termination service. Visit ' + exampleAdminUrl + ' to continue setup. In-browser WARC viewing will remain disabled unless browsing through localhost or HTTPS.' + : '❌ Choose an ingress and TLS mode. Your choice will not be changed automatically.'))); + document.getElementById('archivebox-setup-wildcard-help').hidden = dnsMode === 'localhost'; + } + + function probeUrl(url, generation) { + var controller = new AbortController(); + var timeout = window.setTimeout(function() { controller.abort(); }, 5000); + var target = new URL(url); + target.searchParams.set('archivebox_setup_probe', String(generation)); + return fetch(target.toString(), { + method: 'GET', + mode: 'no-cors', + credentials: 'omit', + cache: 'no-store', + redirect: 'follow', + signal: controller.signal, + }).then(function() { + window.clearTimeout(timeout); + return true; + }).catch(function() { + window.clearTimeout(timeout); + return false; + }); + } + + function setRouteCheck(id, reachable, semantic, noun) { + document.getElementById(id).textContent = (reachable ? '✅ ' + noun + ' reachable' : '❌ ' + noun + ' unreachable') + ' · ' + semantic; + } + + function setValidationState(state, message) { + reviewButton.disabled = state !== 'success'; + validationStatus.className = state ? 'is-' + state : ''; + validationStatus.textContent = message; + } + + function setInvalidSetup(message) { + setValidationState('', message); + ['archivebox-preview-admin-status', 'archivebox-preview-api-status', 'archivebox-preview-index-status', 'archivebox-preview-snapshot-status', 'archivebox-preview-save-status', 'archivebox-preview-last-status'].forEach(function(id) { + var status = document.getElementById(id); + if (status.textContent.indexOf('⏳') === 0) status.textContent = '⏸ Waiting for a matching browser URL and valid setup options'; + }); + } + + function scheduleAccessChecks() { + window.clearTimeout(probeTimer); + setValidationState('testing', 'Testing the generated ArchiveBox URLs from this browser…'); + probeTimer = window.setTimeout(runAccessChecks, 450); + } + + function runAccessChecks() { + var preview = currentPreview; + var hosting = selectedValue(hostingInputs); + var dnsMode = selectedValue(dnsInputs); + var tlsMode = selectedValue(tlsInputs); + if (!preview || !hosting || !dnsMode || !tlsMode) { + setInvalidSetup('Choose hosting, DNS, and HTTPS/ingress options to test this setup.'); + return; + } + if (tlsMode === 'single' && dnsMode !== 'single') { + setInvalidSetup('Single-domain HTTPS is only allowed with Single-domain DNS. Choose Single-domain DNS or use a TLS option that covers the selected DNS mode.'); + return; + } + if (preview.isLocalhost && hosting !== 'localhost') { + setInvalidSetup('The detected BASE_URL is local-only, but ' + (hosting === 'private' ? 'Private Server' : 'Public Server') + ' is selected. Finish the selected DNS or ingress setup, then visit the new ArchiveBox admin URL to continue.'); + return; + } + if (!preview.isLocalhost && hosting === 'localhost') { + setInvalidSetup('This page is not open through localhost. Choose the matching hosting option, or visit the intended *.localhost admin URL to continue.'); + return; + } + if (preview.isLocalhost && dnsMode !== 'localhost') { + setInvalidSetup('This page is open through localhost, but ' + (dnsMode === 'wildcard' ? 'wildcard DNS' : 'single-domain DNS') + ' is selected. Finish that DNS setup, then visit the new ArchiveBox admin URL to continue.'); + return; + } + if (!preview.isLocalhost && dnsMode === 'localhost') { + setInvalidSetup('This page is not open through localhost DNS. Choose the matching DNS option, or visit the intended *.localhost admin URL to continue.'); + return; + } + if (preview.isLocalhost && tlsMode !== 'localhost') { + setInvalidSetup('This page is open through localhost, but a separate ingress/TLS mode is selected. Finish that ingress setup, then visit the new ArchiveBox admin URL to continue.'); + return; + } + if (!preview.isLocalhost && tlsMode === 'localhost') { + setInvalidSetup('Localhost ingress mode only works with a .localhost BASE_URL. Choose the ingress/TLS setup used by ' + preview.parsed.hostname + '.'); + return; + } + if ((dnsMode === 'wildcard' || dnsMode === 'localhost') !== preview.usesSubdomains) { + setInvalidSetup('The selected SERVER_SECURITY_MODE does not match the selected DNS mode.'); + return; + } + if ((tlsMode === 'wildcard' || tlsMode === 'single') && preview.parsed.protocol !== 'https:') { + setInvalidSetup('The selected HTTPS mode requires an https:// BASE_URL.'); + return; + } + if (tlsMode === 'none' && preview.parsed.protocol !== 'http:') { + setInvalidSetup('Direct access without separate ingress / SSL termination requires an http:// BASE_URL.'); + return; + } + var expectedAdminUrl = preview.expectedBrowserOrigin + '/admin/'; + if (window.location.origin.toLowerCase() !== preview.expectedBrowserOrigin.toLowerCase()) { + setInvalidSetup('This wizard is open at ' + window.location.origin + ', but these settings are available at ' + preview.expectedBrowserOrigin + '. Finish the selected DNS, ingress, and TLS setup, then visit ' + expectedAdminUrl + ' to continue.'); + return; + } + + updateOptionGuidance(); + var dnsGuidance = document.getElementById('archivebox-setup-dns-status').textContent; + var tlsGuidance = document.getElementById('archivebox-setup-tls-status').textContent; + document.getElementById('archivebox-setup-hosting-status').textContent = '✅ Current browser URL matches this hosting choice.'; + document.getElementById('archivebox-setup-dns-status').textContent = dnsGuidance; + document.getElementById('archivebox-setup-tls-status').textContent = tlsGuidance; + + var generation = ++probeGeneration; + var checks = { + admin: probeUrl(preview.adminUrl, generation), + api: probeUrl(preview.apiUrl, generation), + index: probeUrl(preview.indexUrl, generation), + web: probeUrl(preview.webHealthUrl, generation), + snapshot: probeUrl(preview.snapshotHealthUrl, generation), + original: probeUrl(preview.originalHealthUrl, generation), + wildcard: probeUrl(preview.wildcardHealthUrl, generation), + }; + Promise.all(Object.keys(checks).map(function(key) { return checks[key].then(function(ok) { return [key, ok]; }); })).then(function(entries) { + if (generation !== probeGeneration || preview !== currentPreview) return; + var results = {}; + entries.forEach(function(entry) { results[entry[0]] = entry[1]; }); + setRouteCheck('archivebox-preview-admin-status', results.admin, preview.routeSemantics.admin, 'Admin URL'); + setRouteCheck('archivebox-preview-api-status', results.api, preview.routeSemantics.api, 'API URL'); + setRouteCheck('archivebox-preview-index-status', results.index, preview.routeSemantics.index, 'Public index'); + setRouteCheck('archivebox-preview-snapshot-status', results.snapshot, preview.routeSemantics.snapshot, 'Snapshot host'); + setRouteCheck('archivebox-preview-save-status', results.web, preview.routeSemantics.save, 'Web host'); + setRouteCheck('archivebox-preview-last-status', results.original, preview.routeSemantics.last, 'Last-save host'); + + var coreReachable = results.admin && results.api && results.index && results.web && results.snapshot && results.wildcard; + document.getElementById('archivebox-setup-dns-status').textContent = coreReachable + ? '✅ Browser requests reached the configured ' + (dnsMode === 'wildcard' || dnsMode === 'localhost' ? 'role and snapshot subdomains.' : 'single ArchiveBox hostname.') + : dnsGuidance + ' One or more configured ArchiveBox hosts are still unreachable.'; + if (tlsMode === 'wildcard' || tlsMode === 'single') { + document.getElementById('archivebox-setup-tls-status').textContent = coreReachable + ? '✅ Browser-trusted HTTPS reached every configured ArchiveBox URL.' + : tlsGuidance + ' HTTPS is still unreachable for one or more configured ArchiveBox URLs.'; + } else { + document.getElementById('archivebox-setup-tls-status').textContent = coreReachable ? '✅ Direct HTTP access reached every configured ArchiveBox URL.' : tlsGuidance + ' Direct HTTP access is still unreachable.'; + } + if (!coreReachable) { + setInvalidSetup('Setup is not reachable yet. Fix the failed URLs above and retry the access checks.'); + return; + } + + setValidationState('success', results.original + ? '✅ Required ArchiveBox URLs are reachable. Review and save these settings.' + : '✅ Core ArchiveBox URLs are reachable. The optional last-save domain shortcut did not respond; review its warning before saving.'); + }); + } + + [publicIndexInput, publicAddInput, permissionsInput].forEach(function(input) { + input.addEventListener('input', updatePreview); + }); + baseUrlInput.addEventListener('input', updatePreview); + securityModeInput.addEventListener('change', updatePreview); + [hostingInputs, dnsInputs, tlsInputs].forEach(function(inputs) { + Array.prototype.forEach.call(inputs, function(input) { input.addEventListener('change', updatePreview); }); + }); + document.getElementById('archivebox-setup-retry').addEventListener('click', function() { scheduleAccessChecks(); }); + document.getElementById('archivebox-setup-wildcard-help').hidden = detectedLocalhost; + updatePreview(); + document.getElementById('archivebox-setup-review').addEventListener('click', function () { + if (reviewButton.disabled) return; + machineAdminUrl.searchParams.set('BASE_URL', baseUrlInput.value.trim()); + machineAdminUrl.searchParams.set('SERVER_SECURITY_MODE', securityModeInput.value); + machineAdminUrl.searchParams.set('PUBLIC_INDEX', publicIndexInput.checked ? 'True' : 'False'); + machineAdminUrl.searchParams.set('PUBLIC_ADD_VIEW', publicAddInput.checked ? 'True' : 'False'); + machineAdminUrl.searchParams.set('PERMISSIONS', permissionsInput.value); + machineAdminUrl.hash = 'BASE_URL'; + window.location.assign(machineAdminUrl.pathname + machineAdminUrl.search + machineAdminUrl.hash); + }); +})(); diff --git a/archivebox/tests/test_migrations_07_to_09.py b/archivebox/tests/test_migrations_07_to_09.py index f3afa70f..db0ebc71 100644 --- a/archivebox/tests/test_migrations_07_to_09.py +++ b/archivebox/tests/test_migrations_07_to_09.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Migration tests from 0.7.x to 0.9.x. @@ -8,7 +7,9 @@ Migration tests from 0.7.x to 0.9.x. - AutoField primary keys """ +import json import sqlite3 +import zipfile import pytest @@ -124,6 +125,184 @@ def test_migration_preserves_archiveresults(archive_07): assert process_count == expected_count +def test_legacy_onedomain_server_upgrade_works_in_auto_mode(archive_07): + """A 0.7.3 LISTEN_HOST deployment should upgrade without new DNS/TLS config.""" + work_dir, _db_path, original_data = archive_07 + snapshot = original_data["snapshots"][0] + snapshot_dir = work_dir / "archive" / snapshot["timestamp"] + snapshot_dir.mkdir(parents=True, exist_ok=True) + + (work_dir / "ArchiveBox.conf").write_text( + """[SERVER_CONFIG] +LISTEN_HOST = archivebox.mydomain +ALLOWED_HOSTS = archivebox.mydomain +CSRF_TRUSTED_ORIGINS = http://admin.archivebox.localhost:8000 +PUBLIC_INDEX = True +PUBLIC_SNAPSHOTS = True +PUBLIC_ADD_VIEW = False +""", + encoding="utf-8", + ) + (snapshot_dir / "index.json").write_text( + json.dumps({"url": snapshot["url"], "timestamp": snapshot["timestamp"], "title": snapshot["title"]}), + encoding="utf-8", + ) + (snapshot_dir / "output.html").write_text( + "", + encoding="utf-8", + ) + (snapshot_dir / "wget").mkdir() + (snapshot_dir / "wget" / "index.html").write_text( + "", + encoding="utf-8", + ) + (snapshot_dir / "singlefile.html").write_text( + "", + encoding="utf-8", + ) + (snapshot_dir / "screenshot.png").write_bytes(b"\x89PNG\r\n\x1a\nlegacy") + (snapshot_dir / "output.pdf").write_bytes(b"%PDF-1.4\n%legacy\n") + (snapshot_dir / "video.mp4").write_bytes(b"\x00\x00\x00\x18ftypmp42legacy") + (snapshot_dir / "gallery.jpg").write_bytes(b"\xff\xd8\xff\xe0legacy\xff\xd9") + archivewebpage_dir = snapshot_dir / "archivewebpage" + archivewebpage_dir.mkdir() + with zipfile.ZipFile(archivewebpage_dir / "archivewebpage.wacz", "w") as wacz: + wacz.writestr( + "pages/pages.jsonl", + '{"format":"json-pages-1.0"}\n{"url":"https://example.com/page1"}\n', + ) + + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=60) + assert result.returncode == 0, f"Init failed: {result.stderr}" + migrated_config = (work_dir / "ArchiveBox.conf").read_text(encoding="utf-8") + assert "LISTEN_HOST = archivebox.mydomain" in migrated_config + assert "ALLOWED_HOSTS = archivebox.mydomain" in migrated_config + assert "BASE_URL = http://archivebox.mydomain" in migrated_config + + script = f""" +from django.contrib.auth import get_user_model +from django.test import Client +from archivebox.config.common import get_config, get_request_config +from archivebox.core.models import Snapshot +from archivebox.core.routes_util import build_admin_url, build_snapshot_url, build_web_url, get_base_url +from archivebox.machine.models import Machine + +def body(response): + return b''.join(response.streaming_content) if response.streaming else response.content + +snapshot = Snapshot.objects.get(id='{snapshot["id"]}') +snapshot_id = str(snapshot.id) +config = get_config(resolve_plugins=False) +assert config.SERVER_SECURITY_MODE == 'auto' +assert get_base_url(config=config) == 'http://archivebox.mydomain' +assert Machine.current().config['LISTEN_HOST'] == 'archivebox.mydomain' +assert Machine.current().config['BASE_URL'] == 'http://archivebox.mydomain' + +user = get_user_model().objects.get(username='admin') +user.set_password('testpassword') +user.save() +client = Client() + +for host in ('archivebox.mydomain', 'other-intranet-name'): + for app_path in ('/public/', '/admin/login/', '/api/v1/docs'): + app_response = client.get(app_path, HTTP_HOST=host) + assert app_response.status_code == 200 + assert "script-src 'none'" not in (app_response.headers.get('Content-Security-Policy') or '') + +alternate = client.get('/public/', HTTP_HOST='other-intranet-name') +alternate_config = get_request_config(alternate.wsgi_request) +assert alternate_config.SERVER_SECURITY_MODE == 'safe-onedomain-nojsreplay' +assert build_web_url('/public/', request=alternate.wsgi_request) == 'http://archivebox.mydomain/public/' +assert build_admin_url('/admin/login/', request=alternate.wsgi_request) == 'http://archivebox.mydomain/admin/login/' +assert build_snapshot_url(snapshot_id, 'output.html', request=alternate.wsgi_request).startswith( + f'http://archivebox.mydomain/snapshot/{{snapshot_id.replace("-", "")}}/' +) + +payload = '{{"username":"admin","password":"testpassword"}}' +canonical_post = client.post( + '/api/v1/auth/get_api_token', data=payload, content_type='application/json', HTTP_HOST='archivebox.mydomain' +) +assert canonical_post.status_code == 200 +assert canonical_post.json().get('token') +alternate_post = client.post( + '/api/v1/auth/get_api_token', data=payload, content_type='application/json', HTTP_HOST='other-intranet-name' +) +assert alternate_post.status_code == 403 +client.force_login(user) + +for path in ('output.html', 'wget/index.html'): + response = client.get(f'/snapshot/{{snapshot_id}}/{{path}}', HTTP_HOST='archivebox.mydomain') + assert response.status_code == 200, (path, response.status_code, response.headers) + assert response['X-ArchiveBox-Security-Mode'] == 'safe-onedomain-nojsreplay', (path, response.headers) + assert "script-src 'none'" in response['Content-Security-Policy'], (path, response.headers) + +singlefile = client.get(f'/snapshot/{{snapshot_id}}/singlefile.html', HTTP_HOST='archivebox.mydomain') +assert singlefile.status_code == 200 +assert 'sandbox;' in singlefile['Content-Security-Policy'] +assert "script-src 'none'" in singlefile['Content-Security-Policy'] + +for path, content_type in ( + ('screenshot.png', 'image/png'), + ('output.pdf', 'application/pdf'), + ('video.mp4', 'video/mp4'), + ('gallery.jpg', 'image/jpeg'), +): + response = client.get(f'/snapshot/{{snapshot_id}}/{{path}}', HTTP_HOST='archivebox.mydomain') + assert response.status_code == 200 + assert response['Content-Type'].startswith(content_type) + assert "script-src 'none'" not in (response.headers.get('Content-Security-Policy') or '') + +wacz = client.get( + f'/snapshot/{{snapshot_id}}/archivewebpage/archivewebpage.wacz?preview=1', HTTP_HOST='archivebox.mydomain' +) +assert wacz.status_code == 200 +assert 'archivebox.example.test:8000" in html + assert 'id="archivebox-setup-base-url"' in html + assert 'id="archivebox-setup-security-mode"' in html + assert 'id="archivebox-setup-public-index"' in html + assert 'id="archivebox-setup-public-add"' in html + assert 'id="archivebox-setup-permissions"' in html + assert 'id="archivebox-setup-effective-mode"' in html + assert 'id="archivebox-setup-browser-url"' in html + assert 'id="archivebox-setup-configured-url"' in html + assert 'id="archivebox-setup-url-match"' in html + assert 'name="archivebox-hosting-location"' in html + assert 'name="archivebox-dns-mode"' in html + assert 'name="archivebox-tls-mode"' in html + assert ( + html.index('name="archivebox-dns-mode" value="localhost"') + < html.index( + 'name="archivebox-dns-mode" value="single"', + ) + < html.index('name="archivebox-dns-mode" value="wildcard"') + ) + assert ( + html.index('name="archivebox-tls-mode" value="localhost"') + < html.index( + 'name="archivebox-tls-mode" value="none"', + ) + < html.index('name="archivebox-tls-mode" value="single"') + < html.index('name="archivebox-tls-mode" value="wildcard"') + ) + assert 'id="archivebox-setup-later"' not in html + assert 'id="archivebox-setup-review" disabled' in html + assert "as BASE_URL below" not in html + assert "Required:" not in html + assert "safe-onedomain-nojsreplay" in html + assert "simplest self-hosted setup" in html + assert "safe-subdomains-fullreplay" in html + assert "unsafe-onedomain-noadmin" in html + assert "danger-onedomain-fullreplay" in html + assert "Wildcard DNS" in html + assert "Modern browsers support wildcard *.archivebox.localhost with no DNS or HTTPS setup required." in html + assert "A archivebox.example.com" in html + assert "JavaScript still runs during capture" in html + assert "will not replay JavaScript unless wildcard DNS is used" in html + assert "Wildcard TLS" in html + assert "How will HTTPS traffic reach this ArchiveBox server?" in html + assert "This mode is not allowed unless also using Single-domain DNS." in html + assert "No separate ingress service / SSL termination" in html + assert "Cloudflare/AWS/Nginx/Caddy/Traefik" in html + assert "In-browser WARC viewing will be disabled unless using localhost or HTTPS" in html + assert "BASE_URL" in html + assert "SERVER_SECURITY_MODE" in html + assert "PUBLIC_INDEX" in html + assert "PUBLIC_ADD_VIEW" in html + assert "Be careful archiving any secret URLs or private share URLs" in html + assert "Beware capturing / viewing archives of URLs that contain malicious JS" in html + assert "PERMISSIONS" in html + assert "/admin/machine/machine/current/change/" in html + assert "setup_wizard.css" in html + assert "setup_wizard.js?v=20260723-2" in html + + +def test_setup_wizard_assets_enforce_selection_and_access_requirements(): + assert "border-color:#15803d; background:#f0fdf4" in SETUP_WIZARD_CSS + assert "accent-color:#15803d" in SETUP_WIZARD_CSS + assert "#archivebox-setup-title code { font-size:inherit; line-height:inherit; }" in SETUP_WIZARD_CSS + + assert "updateQuestionDefaults" not in SETUP_WIZARD_JS + assert "canonicalHost" not in SETUP_WIZARD_JS + assert "Browser URL matches BASE_URL." in SETUP_WIZARD_JS + assert "Browser URL matches admin.BASE_URL as expected." in SETUP_WIZARD_JS + assert "Browser URL does not match BASE_URL" in SETUP_WIZARD_JS + assert "saved URLs may contain private share tokens or other secrets." in SETUP_WIZARD_JS + assert "archive intranet URLs" in SETUP_WIZARD_JS + assert "tlsMode === 'single' && dnsMode !== 'single'" in SETUP_WIZARD_JS + assert "Single-domain HTTPS is only allowed with Single-domain DNS." in SETUP_WIZARD_JS + assert "expectedBrowserOrigin: usesSubdomains ? adminOrigin : parsed.origin" in SETUP_WIZARD_JS + assert "Waiting for a matching browser URL and valid setup options" in SETUP_WIZARD_JS + assert "Finish the selected DNS, ingress, and TLS setup" in SETUP_WIZARD_JS + + for target in ("adminUrl", "apiUrl", "indexUrl", "snapshotHealthUrl", "originalHealthUrl", "wildcardHealthUrl"): + assert f"probeUrl(preview.{target}" in SETUP_WIZARD_JS + assert "probeUrl(webOrigin + '/web/https://example.com'" not in SETUP_WIZARD_JS + assert "credentials: 'omit'" in SETUP_WIZARD_JS + + +@pytest.mark.parametrize( + ("request_host", "secure", "expected_display_host"), + ( + ("admin.archivebox.localhost:8000", False, "archivebox.localhost:8000"), + ("127.0.0.1:8000", False, "archivebox.localhost:8000"), + ("0.0.0.0:8000", False, "archivebox.localhost:8000"), + ("archivebox.io:80", False, "archivebox.io"), + ("archivebox.io:443", True, "archivebox.io"), + ("192.0.2.10:443", True, "192.0.2.10"), + ), +) +def test_unconfigured_banner_displays_canonical_url_host(request_host, secure, expected_display_host): + request = RequestFactory().get("/admin/", secure=secure, HTTP_HOST=request_host) + request.user = AnonymousUser() + + context = get_setup_wizard_context(request, get_config(include_machine=False)) + + assert context["display_host"] == expected_display_host + + +def test_unconfigured_banner_does_not_show_setup_wizard_to_non_superusers(): + html = render_to_string( + "core/system_warnings_banner.html", + { + "mode": "unconfigured", + "can_configure": False, + "suggested_base_url": "http://archivebox.example.test:8000", + }, + ) + + assert 'id="archivebox-setup-wizard"' not in html + assert html.count('id="archivebox-system-warning-banner"') == 1 + assert "Ask an ArchiveBox superuser to finish server setup" in html + + +@pytest.mark.parametrize( + ("context", "expected_text"), + ( + ({"mode": "low_disk", "free_gb": "0.50"}, "Only .*?
    ', response.content.decode(), flags=re.DOTALL) - assert submit_rows - for row in submit_rows: - assert 'name="_save"' not in row - assert 'name="_addanother"' not in row - assert 'value="Save and continue editing"' not in row - assert 'value="Save"' in row - assert 'name="_continue"' in row + toolbars = re.findall( + r'
    ', + response.content.decode(), + flags=re.DOTALL, + ) + assert toolbars + for toolbar in toolbars: + assert 'name="_save"' not in toolbar + assert 'name="_addanother"' not in toolbar + assert 'value="Save and continue editing"' not in toolbar + assert toolbar.count('name="_continue"') == 1 + assert "Save" in toolbar + assert '
    ' not in response.content.decode() -def test_admin_add_submit_row_hides_save_and_add_another(admin_client): +def test_admin_add_toolbar_hides_save_and_add_another(admin_client): response = admin_client.get( reverse("admin:crawls_crawl_add"), HTTP_HOST=ADMIN_TEST_HOST, ) assert response.status_code == 200 - submit_rows = re.findall(r'
    .*?
    ', response.content.decode(), flags=re.DOTALL) - assert submit_rows - assert all('name="_addanother"' not in row for row in submit_rows) + toolbars = re.findall( + r'
    ', + response.content.decode(), + flags=re.DOTALL, + ) + assert toolbars + for toolbar in toolbars: + assert 'name="_addanother"' not in toolbar + assert 'name="_saveasnew"' not in toolbar + assert toolbar.count('type="submit"') == 1 + assert '
    ' not in response.content.decode() def test_crawl_schedule_admin_add_redirects_to_add_page_schedule_field(admin_client): @@ -346,13 +359,7 @@ def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, adm def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(client, admin_user): crawl = Crawl.objects.create( - urls="\n".join( - [ - "https://cdn.example.com/asset.js", - "https://cdn.example.com/second.js", - "https://example.com/root", - ], - ), + urls=("https://cdn.example.com/asset.js\nhttps://cdn.example.com/second.js\nhttps://example.com/root"), created_by=admin_user, ) queued_snapshot = Snapshot.objects.create( @@ -399,14 +406,7 @@ def test_snapshot_from_json_trims_markdown_suffixes_on_discovered_urls(crawl): def test_create_snapshots_from_urls_skips_invalid_and_archivebox_internal_urls(admin_user): crawl = Crawl.objects.create( - urls="\n".join( - [ - "https://example.com/root", - "http://127.0.0.1:8765/page-001.html", - "not-a-url", - "http://admin.archivebox.localhost:8000/admin/", - ], - ), + urls=("https://example.com/root\nhttp://127.0.0.1:8765/page-001.html\nnot-a-url\nhttp://admin.archivebox.localhost:8000/admin/"), created_by=admin_user, ) diff --git a/bin/collect_ui_screenshots.sh b/bin/collect_ui_screenshots.sh new file mode 100755 index 00000000..4d9af43a --- /dev/null +++ b/bin/collect_ui_screenshots.sh @@ -0,0 +1,485 @@ +#!/usr/bin/env bash + +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DATA_DIR="${UI_SCREENSHOT_DATA_DIR:-$REPO_DIR/data}" +OUTPUT_DIR="${UI_SCREENSHOT_OUTPUT_DIR:-$REPO_DIR/docs/screenshots}" +PUBLIC_OUTPUT_DIR="${UI_SCREENSHOT_PUBLIC_OUTPUT_DIR:-$REPO_DIR/publicsite/screenshots}" +REQUESTED_PORT="${UI_SCREENSHOT_PORT:-}" +MAX_VIEWS="${UI_SCREENSHOT_MAX_VIEWS:-0}" +USERNAME="archivebox-screenshots-$$" +PASSWORD="archivebox-screenshots-$$" +SERVER_PID="" +ARCHIVE_PID="" +CREATED_TEMP_USER=0 +CREATE_API_TOKEN=0 +CREATE_WEBHOOK=0 +CAPTURE_ROOT="$(mktemp -d)" +MANIFEST_FILE="$CAPTURE_ROOT/manifest.jsonl" +PERSONAS_DIR="$CAPTURE_ROOT/personas" +ACTIVE_PERSONA="Screenshots" +CAPTURE_PROFILES=$'desktop|1600|1000\ntablet|1024|1366\nmobile|390|844' + +stop_background_runner() { + ( + cd "$DATA_DIR" + uv run --project "$REPO_DIR" archivebox manage shell --no-imports -c \ + 'from archivebox.workers.supervisord_util import get_existing_supervisord_process, stop_worker; supervisor=get_existing_supervisord_process(quiet=True); supervisor is not None and stop_worker(supervisor, "worker_runner")' + ) >/dev/null +} + +cleanup() { + if [[ -n "$ARCHIVE_PID" ]] && kill -0 "$ARCHIVE_PID" 2>/dev/null; then + archive_children="$(pgrep -P "$ARCHIVE_PID" 2>/dev/null || true)" + for child_pid in $archive_children; do + kill "$child_pid" 2>/dev/null || true + done + kill "$ARCHIVE_PID" 2>/dev/null || true + for _attempt in $(seq 1 50); do + archive_running=0 + kill -0 "$ARCHIVE_PID" 2>/dev/null && archive_running=1 + for child_pid in $archive_children; do + kill -0 "$child_pid" 2>/dev/null && archive_running=1 + done + [[ "$archive_running" == "0" ]] && break + sleep 0.1 + done + for child_pid in $archive_children; do + kill -KILL "$child_pid" 2>/dev/null || true + done + kill -KILL "$ARCHIVE_PID" 2>/dev/null || true + wait "$ARCHIVE_PID" 2>/dev/null || true + fi + if [[ -n "$SERVER_PID" ]]; then + server_children="$(pgrep -P "$SERVER_PID" 2>/dev/null || true)" + for child_pid in $server_children; do + kill "$child_pid" 2>/dev/null || true + done + kill "$SERVER_PID" 2>/dev/null || true + for _attempt in $(seq 1 50); do + server_running=0 + kill -0 "$SERVER_PID" 2>/dev/null && server_running=1 + for child_pid in $server_children; do + kill -0 "$child_pid" 2>/dev/null && server_running=1 + done + [[ "$server_running" == "0" ]] && break + sleep 0.1 + done + for child_pid in $server_children; do + kill -KILL "$child_pid" 2>/dev/null || true + done + kill -KILL "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + + if [[ "$CREATED_TEMP_USER" == "1" && -f "$DATA_DIR/index.sqlite3" ]]; then + ( + cd "$DATA_DIR" + UI_SCREENSHOT_USERNAME="$USERNAME" uv run --project "$REPO_DIR" archivebox manage shell --no-imports -c \ + 'import os; from django.contrib.auth import get_user_model; get_user_model().objects.filter(username=os.environ["UI_SCREENSHOT_USERNAME"]).delete()' + ) >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT INT TERM + +mkdir -p "$DATA_DIR" "$OUTPUT_DIR" "$PUBLIC_OUTPUT_DIR" + +echo "[*] Initializing the ArchiveBox collection at $DATA_DIR" +( + cd "$DATA_DIR" + uv run --project "$REPO_DIR" archivebox init --quick +) + +SEED_STATE="$( ( + cd "$DATA_DIR" + uv run --project "$REPO_DIR" archivebox manage shell --no-imports -c \ + 'from django.db.models import Count; from archivebox.core.models import Snapshot, ArchiveResult; from archivebox.crawls.models import CrawlSchedule; from archivebox.personas.models import Persona; from archivebox.api.models import APIToken; from signal_webhooks.utils import get_webhook_model; recent=list(Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED).order_by("-bookmarked_at").values_list("id", "status")[:100]); counts=dict(ArchiveResult.objects.filter(snapshot_id__in=[row[0] for row in recent], status="succeeded").values_list("snapshot_id").annotate(Count("id"))); screenshots=set(ArchiveResult.objects.filter(snapshot_id__in=[row[0] for row in recent], plugin="screenshot", status="succeeded").values_list("snapshot_id", flat=True)); useful=sum(counts.get(snapshot_id,0)>=8 and snapshot_id in screenshots for snapshot_id,status in recent)>=2; print(int(useful)); print(CrawlSchedule.objects.count()); print(Persona.objects.exclude(name="Default").count()); print(APIToken.objects.count()); print(get_webhook_model().objects.count())' +) | tail -5)" +HAS_USEFUL_SNAPSHOT="$(printf '%s\n' "$SEED_STATE" | sed -n '1p')" +HAS_SCHEDULE="$(printf '%s\n' "$SEED_STATE" | sed -n '2p')" +HAS_PERSONA="$(printf '%s\n' "$SEED_STATE" | sed -n '3p')" +HAS_API_TOKEN="$(printf '%s\n' "$SEED_STATE" | sed -n '4p')" +HAS_WEBHOOK="$(printf '%s\n' "$SEED_STATE" | sed -n '5p')" + +if [[ "$HAS_USEFUL_SNAPSHOT" == "0" ]]; then + echo "[*] Archiving real reference sites so snapshot views have meaningful outputs" + ( + cd "$DATA_DIR" + uv run --project "$REPO_DIR" archivebox add \ + --depth=0 \ + --overwrite \ + --tag=documentation,reference \ + --plugins=title,headers,wget,screenshot,pdf,dom,readability,htmltotext,hashes,dns \ + https://example.com https://archivebox.io + ) +fi + +if [[ "$HAS_SCHEDULE" == "0" ]]; then + echo "[*] Creating a real weekly documentation crawl schedule" + ( + cd "$DATA_DIR" + uv run --project "$REPO_DIR" archivebox schedule --every=weekly --depth=0 --tag=documentation https://archivebox.io/feed.xml + ) +fi + +if [[ "$HAS_PERSONA" == "0" ]]; then + echo "[*] Creating a real browser persona for the populated Persona views" + ( + cd "$DATA_DIR" + uv run --project "$REPO_DIR" archivebox persona create "Research Browser" + ) +fi + +[[ "$HAS_API_TOKEN" == "0" ]] && CREATE_API_TOKEN=1 +[[ "$HAS_WEBHOOK" == "0" ]] && CREATE_WEBHOOK=1 + +ROUTE_CONFIG="$( ( + cd "$DATA_DIR" + uv run --project "$REPO_DIR" archivebox manage shell --no-imports -c \ + 'from urllib.parse import urlparse; from archivebox.config.common import get_config; from archivebox.core.routes_util import get_admin_base_url, get_web_base_url; config = get_config(); admin = get_admin_base_url(config=config); web = get_web_base_url(config=config); parsed = urlparse(admin); print(admin); print(web); print(parsed.port or (443 if parsed.scheme == "https" else 80))' +) | tail -3)" +ADMIN_BASE_URL="$(printf '%s\n' "$ROUTE_CONFIG" | sed -n '1p')" +PUBLIC_BASE_URL="$(printf '%s\n' "$ROUTE_CONFIG" | sed -n '2p')" +PORT="$(printf '%s\n' "$ROUTE_CONFIG" | sed -n '3p')" + +if [[ -n "$REQUESTED_PORT" && "$REQUESTED_PORT" != "$PORT" ]]; then + echo "[!] UI_SCREENSHOT_PORT=$REQUESTED_PORT conflicts with the canonical admin origin $ADMIN_BASE_URL" >&2 + echo "[!] Update BASE_URL/BIND_ADDR together instead of capturing a login form on an origin that cannot submit it." >&2 + exit 1 +fi + +echo "[*] Creating a temporary screenshot admin" +( + cd "$DATA_DIR" + uv run --project "$REPO_DIR" archivebox manage shell --no-imports -c \ + 'from django.contrib.auth import get_user_model; get_user_model().objects.filter(username__startswith="archivebox-screenshots-").delete()' + UI_SCREENSHOT_USERNAME="$USERNAME" UI_SCREENSHOT_PASSWORD="$PASSWORD" \ + uv run --project "$REPO_DIR" archivebox manage shell --no-imports -c \ + 'import os; from django.contrib.auth import get_user_model; get_user_model().objects.create_superuser(username=os.environ["UI_SCREENSHOT_USERNAME"], password=os.environ["UI_SCREENSHOT_PASSWORD"])' +) +CREATED_TEMP_USER=1 + +echo "[*] Starting ArchiveBox on port $PORT" +( + cd "$DATA_DIR" + # Use the real server command so the runner, worker, and log views describe + # the same persistent runtime that an operator sees. + UI_SCREENSHOT_HIDE_HIGH_LOAD_WARNING=1 \ + exec uv run --project "$REPO_DIR" archivebox server "127.0.0.1:$PORT" +) >"$DATA_DIR/ui-screenshot-server.log" 2>&1 & +SERVER_PID=$! + +ready=0 +for _attempt in $(seq 1 60); do + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + echo "[!] ArchiveBox exited before becoming ready" >&2 + tail -100 "$DATA_DIR/ui-screenshot-server.log" >&2 + exit 1 + fi + if curl --fail --silent --show-error "$ADMIN_BASE_URL/admin/login/" >/dev/null; then + ready=1 + break + fi + sleep 1 +done + +if [[ "$ready" != "1" ]]; then + echo "[!] ArchiveBox did not become ready within 60 seconds" >&2 + tail -100 "$DATA_DIR/ui-screenshot-server.log" >&2 + exit 1 +fi + +# The persistent collection can contain old runnable work unrelated to this +# gallery. Keep the real server stack, but stop its general queue consumer; +# the Sweeting.me example below runs through its own real foreground runner. +stop_background_runner + +VIEWS=( + "Login|$ADMIN_BASE_URL/admin/login/|/admin/login/|archivebox/templates/admin/login.html" + "Public snapshot list|$PUBLIC_BASE_URL/public/|/public/|archivebox/core/views.py" +) + +capture_index=0 +while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do + view="${VIEWS[$capture_index]}" + IFS='|' read -r name url expected_path source capture_mode <<<"$view" + capture_index=$((capture_index + 1)) + slug="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | tr -cs '[:alnum:]' '-' | sed 's/^-//; s/-$//')" + + echo "[*] $name: $url" + if [[ "$capture_mode" == "snapshot-collapsed" ]]; then + NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \ + CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \ + SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \ + SCREENSHOT_SNAPSHOT_HEADER=collapsed \ + SCREENSHOT_WIDTH=1600 \ + SCREENSHOT_HEIGHT=1000 \ + node "$REPO_DIR/bin/take_screenshot.js" "$url" "$CAPTURE_ROOT/snapshot-header-state.png" >/dev/null + fi + while IFS='|' read -r profile viewport_width viewport_height; do + filename="$(printf '%02d' "$capture_index")-$slug-$profile.png" + capture_dir="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/$profile" + capture_env=( + "RESOLUTION=$viewport_width,$viewport_height" + "CHROME_RESOLUTION=$viewport_width,$viewport_height" + "PERSONAS_DIR=$PERSONAS_DIR" + "ACTIVE_PERSONA=$ACTIVE_PERSONA" + "SCREENSHOT_RESOLUTION=$viewport_width,$viewport_height" + "SCREENSHOT_TIMEOUT=120" + "CHROME_WAIT_FOR=load" + "SCREENSHOT_COLLAPSE_FILTERS=1" + ) + if [[ "$capture_mode" == wait-replay:* ]]; then + capture_env+=( + "SCREENSHOT_WAIT_FOR_TEXT=${capture_mode#wait-replay:}" + "SCREENSHOT_WAIT_FOR_FRAME_URL=/replay/w/" + ) + fi + + screenshot_path="" + if [[ "$capture_mode" == "live-progress" ]]; then + screenshot_path="$capture_dir/screenshot.png" + if [[ "$profile" == "desktop" ]]; then + mkdir -p \ + "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/desktop" \ + "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/tablet" \ + "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile" + live_variants="$(printf \ + '[{"path":"%s","width":1600,"height":1000},{"path":"%s","width":1024,"height":1366},{"path":"%s","width":390,"height":844}]' \ + "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/desktop/screenshot.png" \ + "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/tablet/screenshot.png" \ + "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile/screenshot.png")" + NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \ + CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \ + SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \ + SCREENSHOT_WIDTH=1600 \ + SCREENSHOT_HEIGHT=1000 \ + SCREENSHOT_VARIANTS_JSON="$live_variants" \ + SCREENSHOT_SNAPSHOT_HEADER=expanded \ + SCREENSHOT_EXPECT_LIVE_PROGRESS=1 \ + node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json" + fi + elif [[ "$profile" == "desktop" || "$capture_mode" == wait-replay:* || -z "${ABXPKG_LIB_DIR:-}" ]]; then + capture_log="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")-$profile-abx-dl.log" + if ! env "${capture_env[@]}" uv run --project "$REPO_DIR" abx-dl dl \ + --plugins=screenshot \ + --timeout=120 \ + --dir "$capture_dir" \ + "$url" >"$capture_log" 2>&1; then + echo "[!] abx-dl failed while capturing $profile $url" >&2 + tail -100 "$capture_log" >&2 + exit 1 + fi + + screenshot_path="$(find "$capture_dir" -type f -path '*/screenshot/screenshot.png' -print -quit)" + screenshot_metadata_path="$(find "$capture_dir" -type f -path '*/screenshot/screenshot.json' -print -quit)" + if [[ -z "$screenshot_metadata_path" || ! -s "$screenshot_metadata_path" ]]; then + echo "[!] Screenshot navigation metadata is missing for $profile $url" >&2 + exit 1 + fi + uv run --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" validate \ + "$screenshot_metadata_path" "$expected_path" + else + screenshot_path="$capture_dir/screenshot.png" + if [[ "$profile" == "tablet" ]]; then + mobile_capture_dir="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile" + mkdir -p "$capture_dir" "$mobile_capture_dir" + responsive_variants="$(printf \ + '[{"path":"%s","width":1024,"height":1366},{"path":"%s","width":390,"height":844}]' \ + "$capture_dir/screenshot.png" \ + "$mobile_capture_dir/screenshot.png")" + NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \ + CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \ + SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \ + SCREENSHOT_WIDTH=1024 \ + SCREENSHOT_HEIGHT=1366 \ + SCREENSHOT_VARIANTS_JSON="$responsive_variants" \ + SCREENSHOT_COLLAPSE_FILTERS=1 \ + node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json" + uv run --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" validate \ + "$capture_dir/report.json" "$expected_path" + fi + fi + if [[ -z "$screenshot_path" || ! -s "$screenshot_path" ]]; then + echo "[!] Screenshot plugin did not produce a $profile PNG for $url" >&2 + exit 1 + fi + cp "$screenshot_path" "$OUTPUT_DIR/$filename" + cp "$screenshot_path" "$PUBLIC_OUTPUT_DIR/$filename" + UI_SCREENSHOT_NAME="$name" UI_SCREENSHOT_URL="$url" UI_SCREENSHOT_SOURCE="$source" \ + UI_SCREENSHOT_FILENAME="$filename" UI_SCREENSHOT_PROFILE="$profile" \ + uv run --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" append \ + "$MANIFEST_FILE" "$OUTPUT_DIR/$filename" + done <<<"$CAPTURE_PROFILES" + + # Capture the public views before login because the real admin-login hint + # intentionally redirects authenticated personas away from /public/. + if [[ "$capture_index" == "2" ]]; then + ABXPKG_LIB_DIR="$(uv run --project "$REPO_DIR" abx-dl config --get ABXPKG_LIB_DIR | sed 's/^[^=]*=//; s/^"//; s/"$//')" + SCREENSHOT_CHROME_BINARY="$ABXPKG_LIB_DIR/bin/chromium" + if [[ ! -x "$SCREENSHOT_CHROME_BINARY" ]]; then + echo "[!] abx-dl managed Chromium was not found at $SCREENSHOT_CHROME_BINARY" >&2 + exit 1 + fi + echo "[*] Logging in through the real $ACTIVE_PERSONA browser persona" + NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \ + CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \ + SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \ + SCREENSHOT_LOGIN_USERNAME="$USERNAME" \ + SCREENSHOT_LOGIN_PASSWORD="$PASSWORD" \ + SCREENSHOT_WIDTH=1600 \ + SCREENSHOT_HEIGHT=1000 \ + SCREENSHOT_COLLAPSE_FILTERS=1 \ + node "$REPO_DIR/bin/take_screenshot.js" "$ADMIN_BASE_URL/admin/login/" "$CAPTURE_ROOT/persona-login.png" >/dev/null + + echo "[*] Populating empty configuration views through the real Django admin UI" + NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \ + CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \ + SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \ + CREATE_API_TOKEN="$CREATE_API_TOKEN" \ + CREATE_WEBHOOK="$CREATE_WEBHOOK" \ + node "$REPO_DIR/bin/setup_ui_screenshot_data.js" "$ADMIN_BASE_URL" "$USERNAME" + + SWEETING_CAPTURE_STARTED_AT="$( ( + cd "$DATA_DIR" + uv run --project "$REPO_DIR" archivebox manage shell --no-imports -c \ + 'from django.utils import timezone; print(timezone.now().isoformat())' + ) | tail -1)" + echo "[*] Starting a real Sweeting.me capture for the live progress view" + ( + cd "$DATA_DIR" + exec uv run --project "$REPO_DIR" archivebox add \ + --depth=0 \ + --overwrite \ + --tag=screenshot-gallery \ + https://sweeting.me + ) >"$CAPTURE_ROOT/sweeting-live-capture.log" 2>&1 & + ARCHIVE_PID=$! + + LIVE_SNAPSHOT_VIEW_URL="" + for _attempt in $(seq 1 120); do + LIVE_SNAPSHOT_RECORD="$( ( + cd "$DATA_DIR" + UI_SCREENSHOT_CAPTURE_STARTED_AT="$SWEETING_CAPTURE_STARTED_AT" uv run --project "$REPO_DIR" archivebox manage shell --no-imports -c \ + 'import os; from django.utils.dateparse import parse_datetime; from archivebox.core.models import Snapshot; from archivebox.core.routes_util import build_snapshot_url; started_at=parse_datetime(os.environ["UI_SCREENSHOT_CAPTURE_STARTED_AT"]); snapshot=Snapshot.objects.filter(url__startswith="https://sweeting.me",bookmarked_at__gte=started_at).order_by("-bookmarked_at").first(); print(build_snapshot_url(str(snapshot.id), "") if snapshot else "")' + ) | tail -1)" + if [[ -n "$LIVE_SNAPSHOT_RECORD" ]]; then + LIVE_SNAPSHOT_VIEW_URL="$LIVE_SNAPSHOT_RECORD" + break + fi + if ! kill -0 "$ARCHIVE_PID" 2>/dev/null; then + echo "[!] Sweeting.me capture exited before creating a snapshot" >&2 + tail -100 "$CAPTURE_ROOT/sweeting-live-capture.log" >&2 + exit 1 + fi + sleep 1 + done + if [[ -z "$LIVE_SNAPSHOT_VIEW_URL" ]]; then + echo "[!] Timed out waiting for the live Sweeting.me snapshot" >&2 + exit 1 + fi + VIEWS+=( + "Snapshot View (capture in progress)|$LIVE_SNAPSHOT_VIEW_URL|/|archivebox/templates/core/snapshot.html|live-progress" + ) + + RECORD_CONFIG="$( ( + cd "$DATA_DIR" + uv run --project "$REPO_DIR" archivebox manage shell --no-imports -c \ + 'from django.db.models import Count; from archivebox.core.models import Snapshot, ArchiveResult, Tag; from archivebox.crawls.models import Crawl, CrawlSchedule; from archivebox.personas.models import Persona; from archivebox.machine.models import Machine, NetworkInterface, Binary, Process; from archivebox.api.models import APIToken; from django.contrib.auth import get_user_model; from signal_webhooks.utils import get_webhook_model; from archivebox.core.routes_util import build_snapshot_url; recent=list(Snapshot.objects.filter(url__startswith="https://sweeting.me",status=Snapshot.StatusChoices.SEALED).order_by("-bookmarked_at").values_list("id", flat=True)[:1000]); counts=dict(ArchiveResult.objects.filter(snapshot_id__in=recent,status="succeeded").values_list("snapshot_id").annotate(Count("id"))); snapshot_id=str(max(recent,key=lambda item: counts.get(item,0))); snapshot=Snapshot.objects.get(id=snapshot_id); result=ArchiveResult.objects.filter(snapshot_id=snapshot_id,status="succeeded").order_by("-output_size").first() or ArchiveResult.objects.filter(snapshot_id=snapshot_id).first(); tag=snapshot.tags.first() or Tag.objects.first(); crawl=snapshot.crawl or Crawl.objects.order_by("-created_at").first(); schedule=CrawlSchedule.objects.order_by("-created_at").first(); persona=Persona.objects.exclude(name="Default").order_by("-created_at").first() or Persona.objects.first(); machine=Machine.objects.order_by("-modified_at").first(); interface=NetworkInterface.objects.order_by("-modified_at").first(); binary=Binary.objects.order_by("-modified_at").first(); process=Process.objects.order_by("-created_at").first(); token=APIToken.objects.order_by("-created_at").first(); webhook=get_webhook_model().objects.order_by("-created_at").first(); user=get_user_model().objects.get(username="'"$USERNAME"'"); values={"SNAPSHOT_ID":snapshot_id,"SNAPSHOT_VIEW_URL":build_snapshot_url(snapshot_id,""),"SNAPSHOT_FILES_URL":build_snapshot_url(snapshot_id,"/?files=1"),"ARCHIVERESULT_ID":str(result.id),"TAG_ID":str(tag.id),"USER_ID":str(user.id),"CRAWL_ID":str(crawl.id),"SCHEDULE_ID":str(schedule.id),"PERSONA_ID":str(persona.id),"MACHINE_ID":str(machine.id),"INTERFACE_ID":str(interface.id),"BINARY_ID":str(binary.id),"PROCESS_ID":str(process.id),"TOKEN_ID":str(token.id),"WEBHOOK_ID":str(webhook.id)}; [print(f"{key}={value}") for key,value in values.items()]' + ) | tail -15)" + eval "$RECORD_CONFIG" + + VIEWS+=( + "Add URLs|$ADMIN_BASE_URL/add/|/add/|archivebox/core/views.py" + "Admin dashboard|$ADMIN_BASE_URL/admin/|/admin/|archivebox/core/admin_site.py" + "AI agent|$ADMIN_BASE_URL/admin/agent/|/admin/agent/|abx_plugins/plugins/opencode/views.py" + "Snapshots table|$ADMIN_BASE_URL/admin/core/snapshot/|/admin/core/snapshot/|archivebox/core/admin_snapshots.py" + "Snapshots grid|$ADMIN_BASE_URL/admin/core/snapshot/grid/|/admin/core/snapshot/grid/|archivebox/templates/admin/snapshots_grid.html" + "Snapshot admin detail|$ADMIN_BASE_URL/admin/core/snapshot/$SNAPSHOT_ID/change/|/admin/core/snapshot/$SNAPSHOT_ID/change/|archivebox/core/admin_snapshots.py" + "Snapshot files|$SNAPSHOT_FILES_URL|/|archivebox/templates/core/static_index.html" + "Archive results|$ADMIN_BASE_URL/admin/core/archiveresult/|/admin/core/archiveresult/|archivebox/core/admin_archiveresults.py" + "Archive result detail|$ADMIN_BASE_URL/admin/core/archiveresult/$ARCHIVERESULT_ID/change/|/admin/core/archiveresult/$ARCHIVERESULT_ID/change/|archivebox/core/admin_archiveresults.py" + "Tags|$ADMIN_BASE_URL/admin/core/tag/|/admin/core/tag/|archivebox/core/admin_tags.py" + "Tag detail|$ADMIN_BASE_URL/admin/core/tag/$TAG_ID/change/|/admin/core/tag/$TAG_ID/change/|archivebox/core/admin_tags.py" + "Users|$ADMIN_BASE_URL/admin/auth/user/|/admin/auth/user/|archivebox/core/admin_users.py" + "User detail|$ADMIN_BASE_URL/admin/auth/user/$USER_ID/change/|/admin/auth/user/$USER_ID/change/|archivebox/core/admin_users.py" + "Crawls|$ADMIN_BASE_URL/admin/crawls/crawl/|/admin/crawls/crawl/|archivebox/crawls/admin.py" + "Crawl detail|$ADMIN_BASE_URL/admin/crawls/crawl/$CRAWL_ID/change/|/admin/crawls/crawl/$CRAWL_ID/change/|archivebox/crawls/admin.py" + "Crawl schedules|$ADMIN_BASE_URL/admin/crawls/crawlschedule/|/admin/crawls/crawlschedule/|archivebox/crawls/admin.py" + "Crawl schedule detail|$ADMIN_BASE_URL/admin/crawls/crawlschedule/$SCHEDULE_ID/change/|/admin/crawls/crawlschedule/$SCHEDULE_ID/change/|archivebox/crawls/admin.py" + "Personas|$ADMIN_BASE_URL/admin/personas/persona/|/admin/personas/persona/|archivebox/personas/admin.py" + "Persona detail|$ADMIN_BASE_URL/admin/personas/persona/$PERSONA_ID/change/|/admin/personas/persona/$PERSONA_ID/change/|archivebox/personas/admin.py" + "Machines|$ADMIN_BASE_URL/admin/machine/machine/|/admin/machine/machine/|archivebox/machine/admin.py" + "Machine detail|$ADMIN_BASE_URL/admin/machine/machine/$MACHINE_ID/change/|/admin/machine/machine/$MACHINE_ID/change/|archivebox/machine/admin.py" + "Network interfaces|$ADMIN_BASE_URL/admin/machine/networkinterface/|/admin/machine/networkinterface/|archivebox/machine/admin.py" + "Network interface detail|$ADMIN_BASE_URL/admin/machine/networkinterface/$INTERFACE_ID/change/|/admin/machine/networkinterface/$INTERFACE_ID/change/|archivebox/machine/admin.py" + "Binaries|$ADMIN_BASE_URL/admin/machine/binary/|/admin/machine/binary/|archivebox/machine/admin.py" + "Binary detail|$ADMIN_BASE_URL/admin/machine/binary/$BINARY_ID/change/|/admin/machine/binary/$BINARY_ID/change/|archivebox/machine/admin.py" + "Processes|$ADMIN_BASE_URL/admin/machine/process/|/admin/machine/process/|archivebox/machine/admin.py" + "Process detail|$ADMIN_BASE_URL/admin/machine/process/$PROCESS_ID/change/|/admin/machine/process/$PROCESS_ID/change/|archivebox/machine/admin.py" + "API tokens|$ADMIN_BASE_URL/admin/api/apitoken/|/admin/api/apitoken/|archivebox/api/admin.py" + "API token detail|$ADMIN_BASE_URL/admin/api/apitoken/$TOKEN_ID/change/|/admin/api/apitoken/$TOKEN_ID/change/|archivebox/api/admin.py" + "Webhooks|$ADMIN_BASE_URL/admin/api/outboundwebhook/|/admin/api/outboundwebhook/|archivebox/api/admin.py" + "Webhook detail|$ADMIN_BASE_URL/admin/api/outboundwebhook/$WEBHOOK_ID/change/|/admin/api/outboundwebhook/$WEBHOOK_ID/change/|archivebox/api/admin.py" + "Environment|$ADMIN_BASE_URL/admin/environment/|/admin/environment/|archivebox/core/settings.py" + "Configuration|$ADMIN_BASE_URL/admin/environment/config/|/admin/environment/config/|archivebox/core/views.py" + "Configuration detail|$ADMIN_BASE_URL/admin/environment/config/BASE_URL/|/admin/environment/config/BASE_URL/|archivebox/core/views.py" + "Dependencies|$ADMIN_BASE_URL/admin/environment/binaries/|/admin/environment/binaries/|archivebox/config/views.py" + "Dependency detail|$ADMIN_BASE_URL/admin/environment/binaries/abxbus/|/admin/environment/binaries/abxbus/|archivebox/config/views.py" + "Plugins|$ADMIN_BASE_URL/admin/environment/plugins/|/admin/environment/plugins/|archivebox/plugins/views.py" + "Workers|$ADMIN_BASE_URL/admin/environment/workers/|/admin/environment/workers/|archivebox/config/views.py" + "Worker detail|$ADMIN_BASE_URL/admin/environment/workers/supervisord/|/admin/environment/workers/supervisord/|archivebox/config/views.py" + "Logs|$ADMIN_BASE_URL/admin/environment/logs/|/admin/environment/logs/|archivebox/config/views.py" + "Log detail|$ADMIN_BASE_URL/admin/environment/logs/supervisord/|/admin/environment/logs/supervisord/|archivebox/config/views.py" + ) + + fi + + if [[ "$name" == "Snapshot View (capture in progress)" ]]; then + if ! wait "$ARCHIVE_PID"; then + echo "[!] Sweeting.me capture failed after the live progress screenshot" >&2 + tail -100 "$CAPTURE_ROOT/sweeting-live-capture.log" >&2 + exit 1 + fi + ARCHIVE_PID="" + stop_background_runner + + # Discover the selectable outputs only after the real foreground + # capture finishes. Loading another output-heavy snapshot page while + # its extractors are running can starve navigation on the same server. + SNAPSHOT_DISCOVERY_REPORT="$CAPTURE_ROOT/snapshot-output-discovery.json" + NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \ + CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \ + SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \ + SCREENSHOT_SNAPSHOT_HEADER=expanded \ + SCREENSHOT_WIDTH=1600 \ + SCREENSHOT_HEIGHT=1000 \ + node "$REPO_DIR/bin/take_screenshot.js" "$SNAPSHOT_VIEW_URL" "$CAPTURE_ROOT/snapshot-output-discovery.png" >"$SNAPSHOT_DISCOVERY_REPORT" + SNAPSHOT_OUTPUT_PLUGINS="$(UI_SCREENSHOT_DISCOVERY_REPORT="$SNAPSHOT_DISCOVERY_REPORT" uv run --project "$REPO_DIR" python -c \ + 'import json, os; from urllib.parse import urlsplit; report=json.load(open(os.environ["UI_SCREENSHOT_DISCOVERY_REPORT"])); print("\n".join("{}\t{}".format(output["plugin"], "wait-replay:Nick Sweeting" if urlsplit(output["previewUrl"]).path.endswith(".wacz") else "") for output in report["checks"]["snapshotOutputs"]))')" + if [[ -z "$SNAPSHOT_OUTPUT_PLUGINS" ]]; then + echo "[!] The Sweeting.me snapshot detail page exposed no selectable outputs" >&2 + exit 1 + fi + while IFS=$'\t' read -r plugin_name output_capture_mode; do + [[ -z "$plugin_name" ]] && continue + VIEWS+=("Snapshot View ($plugin_name)|$SNAPSHOT_VIEW_URL#$plugin_name|/|archivebox/templates/core/snapshot.html|$output_capture_mode") + done <<<"$SNAPSHOT_OUTPUT_PLUGINS" + VIEWS+=("Snapshot View (header collapsed)|$SNAPSHOT_VIEW_URL|/|archivebox/templates/core/snapshot.html|snapshot-collapsed") + fi + if [[ "$MAX_VIEWS" != "0" && "$capture_index" -ge "$MAX_VIEWS" ]]; then + break + fi +done + +[[ "$MAX_VIEWS" != "0" ]] && export UI_SCREENSHOT_ALLOW_PARTIAL=1 +uv run --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" build \ + "$MANIFEST_FILE" "$REPO_DIR/docs/Screenshots.md" "$PUBLIC_OUTPUT_DIR/index.html" + +echo "[+] Captured $capture_index views at desktop, tablet, and mobile sizes" +echo "[+] Documentation gallery: $REPO_DIR/docs/Screenshots.md" +echo "[+] GitHub Pages gallery: $PUBLIC_OUTPUT_DIR/index.html" diff --git a/bin/generate_ui_screenshot_gallery.py b/bin/generate_ui_screenshot_gallery.py new file mode 100755 index 00000000..6fcca979 --- /dev/null +++ b/bin/generate_ui_screenshot_gallery.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 + +import hashlib +import html +import json +import os +import struct +import sys +from pathlib import Path +from urllib.parse import urlparse + +SOURCE_BASE_URL = "https://github.com/ArchiveBox/ArchiveBox/blob/dev/" +CAPTURE_PROFILES = { + "desktop": (1600, 1000), + "tablet": (1024, 1366), + "mobile": (390, 844), +} + +REQUIRED_VIEW_NAMES = { + "Login", + "Public snapshot list", + "Add URLs", + "Admin dashboard", + "AI agent", + "Snapshots table", + "Snapshots grid", + "Snapshot admin detail", + "Snapshot View (capture in progress)", + "Snapshot View (header collapsed)", + "Snapshot files", + "Archive results", + "Archive result detail", + "Crawl detail", + "Crawl schedules", + "Crawl schedule detail", + "Persona detail", + "Machine detail", + "Network interface detail", + "Binary detail", + "Process detail", + "API tokens", + "API token detail", + "Webhooks", + "Webhook detail", + "Environment", + "Configuration", + "Configuration detail", + "Dependencies", + "Dependency detail", + "Plugins", + "Workers", + "Worker detail", + "Logs", + "Log detail", +} + + +def validate_navigation(metadata_path: Path, expected_path: str) -> None: + navigation = json.loads(metadata_path.read_text(encoding="utf-8")) + navigation = navigation.get("checks", navigation) + status = navigation.get("status") + final_url = navigation.get("finalUrl") or "" + actual_path = urlparse(final_url).path + if status != 200: + raise SystemExit(f"expected HTTP 200, got {status}: {final_url or metadata_path}") + if actual_path != expected_path: + raise SystemExit(f"expected final path {expected_path}, got {actual_path}: {final_url}") + + +def append_manifest(manifest_path: Path, screenshot_path: Path) -> None: + data = screenshot_path.read_bytes() + if data[:8] != b"\x89PNG\r\n\x1a\n" or len(data) < 24: + raise SystemExit(f"not a valid PNG: {screenshot_path}") + dimensions = struct.unpack(">II", data[16:24]) + profile = os.environ["UI_SCREENSHOT_PROFILE"] + if profile not in CAPTURE_PROFILES: + raise SystemExit(f"unknown screenshot profile: {profile}") + expected_dimensions = CAPTURE_PROFILES[profile] + if dimensions != expected_dimensions: + raise SystemExit( + f"expected {expected_dimensions[0]}x{expected_dimensions[1]} for {profile}, " + f"got {dimensions[0]}x{dimensions[1]}: {screenshot_path}", + ) + + item = { + "name": os.environ["UI_SCREENSHOT_NAME"], + "url": os.environ["UI_SCREENSHOT_URL"], + "source": os.environ["UI_SCREENSHOT_SOURCE"], + "filename": os.environ["UI_SCREENSHOT_FILENAME"], + "profile": profile, + "width": dimensions[0], + "height": dimensions[1], + } + with manifest_path.open("a", encoding="utf-8") as manifest: + manifest.write(json.dumps(item) + "\n") + + +def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -> None: + captures = [json.loads(line) for line in manifest_path.read_text(encoding="utf-8").splitlines() if line] + allow_partial = os.environ.get("UI_SCREENSHOT_ALLOW_PARTIAL") == "1" + grouped_captures: list[dict[str, object]] = [] + captures_by_name: dict[str, dict[str, object]] = {} + for capture in captures: + group = captures_by_name.get(capture["name"]) + if group is None: + group = { + "name": capture["name"], + "url": capture["url"], + "source": capture["source"], + "variants": {}, + } + captures_by_name[capture["name"]] = group + grouped_captures.append(group) + elif group["url"] != capture["url"] or group["source"] != capture["source"]: + raise SystemExit(f"inconsistent manifest metadata for {capture['name']}") + variants = group["variants"] + assert isinstance(variants, dict) + if capture["profile"] in variants: + raise SystemExit(f"duplicate {capture['profile']} capture for {capture['name']}") + variants[capture["profile"]] = capture + + complete_groups = [] + for group in grouped_captures: + variants = group["variants"] + assert isinstance(variants, dict) + missing_profiles = [profile for profile in CAPTURE_PROFILES if profile not in variants] + if missing_profiles: + if allow_partial: + continue + raise SystemExit(f"missing {', '.join(missing_profiles)} capture for {group['name']}") + complete_groups.append(group) + grouped_captures = complete_groups + + if not allow_partial: + captured_names = set(captures_by_name) + missing = sorted(REQUIRED_VIEW_NAMES - captured_names) + if missing: + raise SystemExit(f"required UI screenshot coverage is missing: {', '.join(missing)}") + snapshot_output_views = [ + name + for name in captured_names + if name.startswith("Snapshot View (") + and name not in {"Snapshot View (capture in progress)", "Snapshot View (header collapsed)"} + ] + if len(snapshot_output_views) < 20: + raise SystemExit( + f"expected at least 20 rendered Sweeting.me snapshot outputs, got {len(snapshot_output_views)}", + ) + markdown_sections = [] + html_sections = [] + for capture in grouped_captures: + variants = capture["variants"] + assert isinstance(variants, dict) + parsed_url = urlparse(str(capture["url"])) + route = parsed_url.path or "/" + if parsed_url.fragment: + route = f"{route}#{parsed_url.fragment}" + source_url = f"{SOURCE_BASE_URL}{capture['source']}" + markdown_cells = [] + html_figures = [] + for profile, (width, height) in CAPTURE_PROFILES.items(): + variant = variants[profile] + label = f"{profile.title()} ({width}x{height})" + filename = str(variant["filename"]) + screenshot_path = markdown_path.parent / "screenshots" / filename + if not screenshot_path.is_file(): + raise SystemExit(f"missing screenshot for gallery: {screenshot_path}") + cache_version = hashlib.sha256(screenshot_path.read_bytes()).hexdigest()[:12] + versioned_filename = f"{html.escape(filename)}?v={cache_version}" + markdown_cells.append( + f'{label}
    ' + f'', + ) + html_figures.append( + f'
    {label}
    ' + f'' + f'
    ', + ) + markdown_sections.append( + "\n".join( + ( + f"## {capture['name']}", + "", + f"View: [`{route}`]({capture['url']}) · [View code]({source_url})", + "", + "", + "".join(f"" for profile in CAPTURE_PROFILES), + "", + "".join(markdown_cells), + "
    {profile.title()}
    ", + ), + ), + ) + html_sections.append( + f"', + ) + + markdown_path.write_text( + "\n".join( + ( + "# UI Screenshots", + "", + "", + "", + ( + "These desktop, tablet, and mobile screenshots cover ArchiveBox's major public and authenticated UI views. " + "Raw API endpoints, API documentation, health-check, and error routes are intentionally excluded." + ), + "", + "\n\n".join(markdown_sections), + "", + ), + ), + encoding="utf-8", + ) + + html_path.write_text( + '' + '' + "ArchiveBox UI Screenshots

    ← ArchiveBox

    ArchiveBox UI Screenshots

    ' + "

    Generated from the current dev UI at desktop, tablet, and mobile viewports.

    " + + "".join(html_sections) + + "
    \n", + encoding="utf-8", + ) + + if not allow_partial: + expected_filenames = {capture["filename"] for capture in captures} + for screenshot_dir in (markdown_path.parent / "screenshots", html_path.parent): + for screenshot_path in screenshot_dir.glob("*.png"): + if screenshot_path.name not in expected_filenames: + screenshot_path.unlink() + + +def main() -> None: + if len(sys.argv) == 4 and sys.argv[1] == "validate": + validate_navigation(Path(sys.argv[2]), sys.argv[3]) + return + if len(sys.argv) == 4 and sys.argv[1] == "append": + append_manifest(Path(sys.argv[2]), Path(sys.argv[3])) + return + if len(sys.argv) == 5 and sys.argv[1] == "build": + build_galleries(Path(sys.argv[2]), Path(sys.argv[3]), Path(sys.argv[4])) + return + raise SystemExit( + "usage: generate_ui_screenshot_gallery.py append MANIFEST SCREENSHOT | " + "validate SCREENSHOT_JSON EXPECTED_PATH | build MANIFEST MARKDOWN HTML", + ) + + +if __name__ == "__main__": + main() diff --git a/bin/setup_ui_screenshot_data.js b/bin/setup_ui_screenshot_data.js new file mode 100755 index 00000000..cf022389 --- /dev/null +++ b/bin/setup_ui_screenshot_data.js @@ -0,0 +1,120 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const path = require('path'); +const puppeteer = require('puppeteer'); + +const [adminBaseUrl, username] = process.argv.slice(2); +if (!adminBaseUrl || !username || !process.env.SCREENSHOT_USER_DATA_DIR) { + throw new Error('usage: setup_ui_screenshot_data.js ADMIN_BASE_URL USERNAME (with SCREENSHOT_USER_DATA_DIR)'); +} + +function chromePath() { + const configured = process.env.CHROME_BINARY || process.env.PUPPETEER_EXECUTABLE_PATH; + return configured && fs.existsSync(configured) ? configured : undefined; +} + +async function submitAdminAdd(page, url, configure) { + await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 }); + if (!page.url().includes('/add/')) return false; + await configure(); + const saveButton = 'button[name="_save"], input[name="_save"]'; + await page.waitForSelector(saveButton, { visible: true, timeout: 10000 }); + await Promise.all([ + page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 60000 }), + page.click(saveButton), + ]); + if (page.url().includes('/add/')) { + const errors = await page.$$eval('.errornote, .errorlist', (nodes) => nodes.map((node) => node.textContent.trim()).join(' ')); + throw new Error(`admin setup form failed at ${page.url()}: ${errors || 'no form error shown'}`); + } + return true; +} + +async function selectUser(page) { + const selector = '#id_created_by'; + if (!(await page.$(selector))) return; + const value = await page.$eval(selector, (select, wanted) => { + const option = [...select.options].find((candidate) => candidate.textContent.trim() === wanted); + return option ? option.value : ''; + }, username); + if (value) await page.select(selector, value); +} + +async function selectFirstRealOption(page, selector) { + if (!(await page.$(selector))) return; + const value = await page.$eval(selector, (select) => { + const option = [...select.options].find((candidate) => candidate.value); + return option ? option.value : ''; + }); + if (value) await page.select(selector, value); +} + +async function replaceText(page, selector, value) { + if (!(await page.$(selector))) return; + await page.click(selector); + const modifier = process.platform === 'darwin' ? 'Meta' : 'Control'; + await page.keyboard.down(modifier); + await page.keyboard.press('KeyA'); + await page.keyboard.up(modifier); + await page.type(selector, value); +} + +async function collapseAdminFilters(page, adminBaseUrl) { + await page.goto(`${adminBaseUrl}/admin/core/snapshot/`, { waitUntil: 'domcontentloaded', timeout: 60000 }); + const toggle = await page.$('#changelist-filter-toggle'); + if (!toggle) throw new Error('snapshot filter control is missing'); + const expanded = await page.$eval('#changelist-filter-toggle', (button) => button.getAttribute('aria-expanded') === 'true'); + if (expanded) { + await page.click('#changelist-filter-toggle'); + await page.waitForFunction(() => document.body.classList.contains('filters-collapsed')); + } +} + +async function main() { + const launchOptions = { + headless: true, + userDataDir: path.resolve(process.env.SCREENSHOT_USER_DATA_DIR), + defaultViewport: { + width: Number(process.env.SCREENSHOT_WIDTH || 1600), + height: Number(process.env.SCREENSHOT_HEIGHT || 1000), + }, + protocolTimeout: 300000, + }; + const executablePath = chromePath(); + if (executablePath) launchOptions.executablePath = executablePath; + + const browser = await puppeteer.launch(launchOptions); + try { + const page = await browser.newPage(); + await page.goto(`${adminBaseUrl}/admin/`, { waitUntil: 'domcontentloaded', timeout: 60000 }); + if (page.url().includes('/admin/login/')) throw new Error('screenshot persona is not logged in'); + + if (process.env.CREATE_API_TOKEN === '1') { + await submitAdminAdd(page, `${adminBaseUrl}/admin/api/apitoken/add/`, async () => { + await selectUser(page); + }); + } + + if (process.env.CREATE_WEBHOOK === '1') await submitAdminAdd(page, `${adminBaseUrl}/admin/api/outboundwebhook/add/`, async () => { + await replaceText(page, '#id_name', 'Snapshot completion notifications'); + await selectFirstRealOption(page, '#id_signal'); + await selectFirstRealOption(page, '#id_ref'); + await replaceText(page, '#id_endpoint', 'https://httpbin.org/post'); + if (await page.$('#id_enabled')) { + const enabled = await page.$eval('#id_enabled', (input) => input.checked); + if (!enabled) await page.click('#id_enabled'); + } + await selectUser(page); + }); + + await collapseAdminFilters(page, adminBaseUrl); + } finally { + await browser.close(); + } +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/bin/take_screenshot.js b/bin/take_screenshot.js index fba8bf4a..d9619b21 100755 --- a/bin/take_screenshot.js +++ b/bin/take_screenshot.js @@ -16,9 +16,13 @@ Environment: SESSIONID Django session cookie value for admin.archivebox.localhost SCREENSHOT_COOKIE_NAME Cookie name, defaults to sessionid SCREENSHOT_COOKIE_DOMAIN Cookie domain, defaults to admin.archivebox.localhost + SCREENSHOT_USER_DATA_DIR Chrome profile directory (e.g. a persona's chrome_profile) + SCREENSHOT_LOGIN_USERNAME Log in through #id_username before capture + SCREENSHOT_LOGIN_PASSWORD Password used with SCREENSHOT_LOGIN_USERNAME CHROME_BINARY Chromium/Chrome executable path SCREENSHOT_WIDTH Viewport width, defaults to 1600 SCREENSHOT_HEIGHT Viewport height, defaults to 1400 + SCREENSHOT_VARIANTS_JSON JSON array of {path,width,height} captures from one loaded page SCREENSHOT_FULL_PAGE Set to 1 to capture the full page, defaults to viewport only SCREENSHOT_SCROLL_SELECTOR Scroll this selector into view before capture SCREENSHOT_WAIT_SELECTOR Wait for this selector before capture @@ -26,6 +30,10 @@ Environment: SCREENSHOT_AFTER_CLICK_WAIT_SELECTOR Wait for this selector after clicking SCREENSHOT_HOST_RESOLVER_RULES Chrome host resolver rules SCREENSHOT_SNAPSHOT_VIEW Set to list or grid before loading the page + SCREENSHOT_SNAPSHOT_HEADER Set to expanded or collapsed before loading a snapshot detail page + SCREENSHOT_EXPECT_PLUGIN Require this snapshot output plugin to be selected + SCREENSHOT_EXPECT_LIVE_PROGRESS Require real progress bars and a loaded screencast frame + SCREENSHOT_COLLAPSE_FILTERS Set to 1 to keep admin filters out of screenshots SCREENSHOT_RESET_FILTERS Set to 1 to clear the admin filter collapsed preference `); } @@ -58,6 +66,18 @@ async function main() { const width = Number(process.env.SCREENSHOT_WIDTH || 1600); const height = Number(process.env.SCREENSHOT_HEIGHT || 1400); const fullPage = process.env.SCREENSHOT_FULL_PAGE === '1'; + const variants = process.env.SCREENSHOT_VARIANTS_JSON + ? JSON.parse(process.env.SCREENSHOT_VARIANTS_JSON) + : []; + + if (!Array.isArray(variants)) { + throw new Error('SCREENSHOT_VARIANTS_JSON must be a JSON array'); + } + for (const variant of variants) { + if (!variant.path || !Number.isInteger(variant.width) || !Number.isInteger(variant.height)) { + throw new Error('Each screenshot variant requires path, integer width, and integer height'); + } + } fs.mkdirSync(path.dirname(output), { recursive: true }); @@ -66,19 +86,30 @@ async function main() { defaultViewport: { width, height }, executablePath: chromePath(), }; + if (process.env.SCREENSHOT_USER_DATA_DIR) { + launchOptions.userDataDir = path.resolve(process.env.SCREENSHOT_USER_DATA_DIR); + } if (process.env.SCREENSHOT_HOST_RESOLVER_RULES) { launchOptions.args = [`--host-resolver-rules=${process.env.SCREENSHOT_HOST_RESOLVER_RULES}`]; } const browser = await puppeteer.launch(launchOptions); try { + // Persistent persona profiles can restore tabs left by earlier launches. + // Close them before navigating so repeated captures do not multiply live + // admin requests and overwhelm the server being documented. + const restoredPages = await browser.pages(); const page = await browser.newPage(); + await Promise.all(restoredPages.map((restoredPage) => restoredPage.close())); page.setDefaultTimeout(45000); - if (process.env.SCREENSHOT_SNAPSHOT_VIEW || process.env.SCREENSHOT_RESET_FILTERS === '1') { - await page.evaluateOnNewDocument((snapshotView, resetFilters) => { + if (process.env.SCREENSHOT_SNAPSHOT_VIEW || process.env.SCREENSHOT_SNAPSHOT_HEADER || process.env.SCREENSHOT_COLLAPSE_FILTERS === '1' || process.env.SCREENSHOT_RESET_FILTERS === '1') { + await page.evaluateOnNewDocument((snapshotView, snapshotHeader, collapseFilters, resetFilters) => { if (snapshotView) localStorage.setItem('preferred_snapshot_view_mode', snapshotView); - if (resetFilters) localStorage.removeItem('admin-filters-collapsed'); - }, process.env.SCREENSHOT_SNAPSHOT_VIEW || '', process.env.SCREENSHOT_RESET_FILTERS === '1'); + if (snapshotHeader === 'expanded') localStorage.setItem('archivebox-snapshot-header-visible', 'true'); + if (snapshotHeader === 'collapsed') localStorage.setItem('archivebox-snapshot-header-visible', 'false'); + if (collapseFilters) localStorage.setItem('admin-filters-collapsed', 'true'); + else if (resetFilters) localStorage.removeItem('admin-filters-collapsed'); + }, process.env.SCREENSHOT_SNAPSHOT_VIEW || '', process.env.SCREENSHOT_SNAPSHOT_HEADER || '', process.env.SCREENSHOT_COLLAPSE_FILTERS === '1', process.env.SCREENSHOT_RESET_FILTERS === '1'); } if (process.env.SESSIONID) { @@ -95,7 +126,26 @@ async function main() { await page.setCookie(cookie); } - await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 }); + let navigationResponse = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 }); + + if (process.env.SCREENSHOT_LOGIN_USERNAME || process.env.SCREENSHOT_LOGIN_PASSWORD) { + if (!process.env.SCREENSHOT_LOGIN_USERNAME || !process.env.SCREENSHOT_LOGIN_PASSWORD) { + throw new Error('SCREENSHOT_LOGIN_USERNAME and SCREENSHOT_LOGIN_PASSWORD must be set together'); + } + await page.waitForSelector('#id_username'); + await page.type('#id_username', process.env.SCREENSHOT_LOGIN_USERNAME); + await page.type('#id_password', process.env.SCREENSHOT_LOGIN_PASSWORD); + const [loginResponse] = await Promise.all([ + page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 60000 }), + page.click('button[type="submit"], input[type="submit"]'), + ]); + navigationResponse = loginResponse || navigationResponse; + if (new URL(page.url()).pathname.includes('/admin/login/')) { + const loginError = await page.evaluate(() => Array.from(document.querySelectorAll('.errornote, .errorlist')).map((element) => element.textContent.trim()).filter(Boolean).join(' ')); + const cookieDetails = (await page.cookies()).map((cookie) => `${cookie.name}@${cookie.domain}${cookie.path}${cookie.secure ? ';Secure' : ''}`).join(', ') || 'none'; + throw new Error(`ArchiveBox persona login failed at ${page.url()}: ${loginError || 'no form error shown'} (cookies: ${cookieDetails})`); + } + } await page.waitForSelector('body'); await page.waitForSelector('#progress-monitor, #add-form', { timeout: 5000 }).catch(() => {}); @@ -116,6 +166,39 @@ async function main() { }, process.env.SCREENSHOT_SCROLL_SELECTOR); } + if (process.env.SCREENSHOT_EXPECT_PLUGIN) { + const expectedPlugin = process.env.SCREENSHOT_EXPECT_PLUGIN.toLowerCase(); + await page.waitForFunction((pluginName) => { + const selectedCard = document.querySelector('.thumb-card.selected-card[data-plugin-name]'); + const frame = document.querySelector('#main-frame'); + return selectedCard?.dataset.pluginName?.toLowerCase() === pluginName + && frame + && frame.getAttribute('src') + && frame.getAttribute('src') !== 'about:blank'; + }, { timeout: 45000 }, expectedPlugin); + } + + if (process.env.SCREENSHOT_EXPECT_LIVE_PROGRESS === '1') { + await page.waitForFunction(() => { + const monitor = document.querySelector('#progress-monitor'); + const bars = [...document.querySelectorAll('#progress-monitor .progress-bar')] + .filter((bar) => bar.getClientRects().length > 0 && bar.offsetWidth > 0 && bar.offsetHeight > 0); + const panel = document.querySelector('#progress-monitor .screencast-panel.visible'); + const image = panel?.querySelector('img'); + return monitor + && getComputedStyle(monitor).display !== 'none' + && !monitor.classList.contains('collapsed') + && monitor.querySelector('.progress-content')?.getClientRects().length > 0 + && bars.length >= 2 + && panel?.getClientRects().length > 0 + && panel.offsetWidth > 0 + && panel.offsetHeight > 0 + && image?.complete + && image.naturalWidth > 0 + && image.naturalHeight > 0; + }, { timeout: 120000, polling: 250 }); + } + const frameHandle = await page.$('.crawl-snapshots-embed iframe'); if (frameHandle) { const frame = await frameHandle.contentFrame(); @@ -147,7 +230,21 @@ async function main() { snapshotEmbed: Boolean(document.querySelector('.crawl-snapshots-embed iframe')), addForm: Boolean(document.querySelector('#add-form')), limitFields: Array.from(document.querySelectorAll('.crawl-limit-field label')).map((el) => el.textContent.trim()), + snapshotOutputPlugins: [...new Set( + [...document.querySelectorAll('.thumb-card[data-plugin-name] a[target="preview"]')] + .map((link) => link.closest('.thumb-card')?.dataset.pluginName) + .filter(Boolean), + )], + snapshotOutputs: [...document.querySelectorAll('.thumb-card[data-plugin-name]')] + .map((card) => ({ + plugin: card.dataset.pluginName || '', + previewUrl: card.dataset.previewUrl || card.querySelector('a[target="preview"]')?.getAttribute('href') || '', + })) + .filter((output) => output.plugin && output.previewUrl), + selectedSnapshotOutputPlugin: document.querySelector('.thumb-card.selected-card[data-plugin-name]')?.dataset.pluginName || '', })); + checks.status = navigationResponse ? navigationResponse.status() : null; + checks.finalUrl = page.url(); let frameChecks = null; const embeddedFrameHandle = await page.$('.crawl-snapshots-embed iframe'); @@ -165,8 +262,21 @@ async function main() { } } - await page.screenshot({ path: output, fullPage }); - console.log(JSON.stringify({ screenshotPath: output, checks, frameChecks }, null, 2)); + const screenshotPaths = []; + if (variants.length) { + for (const variant of variants) { + const variantPath = path.resolve(variant.path); + fs.mkdirSync(path.dirname(variantPath), { recursive: true }); + await page.setViewport({width: variant.width, height: variant.height, deviceScaleFactor: 1}); + await new Promise((resolve) => setTimeout(resolve, 300)); + await page.screenshot({ path: variantPath, fullPage }); + screenshotPaths.push(variantPath); + } + } else { + await page.screenshot({ path: output, fullPage }); + screenshotPaths.push(output); + } + console.log(JSON.stringify({ screenshotPath: screenshotPaths[0], screenshotPaths, checks, frameChecks }, null, 2)); } finally { await browser.close(); } diff --git a/docker-compose.yml b/docker-compose.yml index e9440a66..62d91264 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,8 +20,8 @@ services: environment: # - ADMIN_USERNAME=admin # creates an admin user on first run with the given user/pass combo # - ADMIN_PASSWORD=SomeSecretPassword - - BASE_URL=${BASE_URL:-http://archivebox.localhost:8000} # public URL used to build admin/web/api/snapshot links - - SERVER_SECURITY_MODE=${SERVER_SECURITY_MODE:-safe-subdomains-fullreplay} # safe-onedomain-nojsreplay if you can't do wildcard DNS *.your.domain + - BASE_URL # optional canonical URL; the first-run admin wizard fills it from the browser + - SERVER_SECURITY_MODE=${SERVER_SECURITY_MODE:-auto} # *.localhost uses isolated subdomains; other hosts use one-domain/no-JS replay - PUBLIC_ADD_VIEW=False # set to True to allow anonymous users to submit new URLs to archive # For all other options, it's better to use data/ArchiveBox.conf or the new Personas config feature in the admin UI... # - TIMEOUT=60 diff --git a/docs/Configuration.md b/docs/Configuration.md index 0478cb0f..0817a51f 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -388,14 +388,14 @@ ArchiveBox automatically derives the underlying Django `ALLOWED_HOSTS` and `CSRF > **Pin `BASE_URL` explicitly on any deployment using `safe-subdomains-fullreplay` mode.** A misconfig banner will surface in the rendered UI until you do. > [!NOTE] -> **Legacy upgrade path (0.7.3 → 0.9):** older deployments that set `CSRF_TRUSTED_ORIGINS=https://archive.example.com` for their reverse-proxy login but never set `BASE_URL` still work — when exactly one CSRF origin is present and `BASE_URL` is empty, ArchiveBox uses that origin as the implicit base URL. New installs should set `BASE_URL` directly; `CSRF_TRUSTED_ORIGINS` is no longer a user-settable knob. +> **Legacy upgrade path (0.7.3 → 0.9):** `archivebox init` preserves the complete legacy config and migrates the old `ARCHIVE_BASE_URL`, `ADMIN_BASE_URL`, or documented `LISTEN_HOST` hostname to `BASE_URL`. A single non-default `CSRF_TRUSTED_ORIGINS` or `ALLOWED_HOSTS` entry is used as a fallback when those settings are absent. New installs should set `BASE_URL` directly; the legacy hostname settings are no longer user-settable knobs. *Related options:* [`SERVER_SECURITY_MODE`](#server_security_mode), [`BIND_ADDR`](#bind_addr) --- #### `SERVER_SECURITY_MODE` -**Possible Values:** [`safe-subdomains-fullreplay`]/`safe-onedomain-nojsreplay`/`unsafe-onedomain-noadmin`/`danger-onedomain-fullreplay` +**Possible Values:** [`auto`]/`safe-subdomains-fullreplay`/`safe-onedomain-nojsreplay`/`unsafe-onedomain-noadmin`/`danger-onedomain-fullreplay` The top-level security posture of the server. Controls how archived content is served, whether the admin/API control plane is reachable, and which host(s) the UI is split across. **This is the most important security knob** — pick the most restrictive mode that still works for your use case. @@ -403,16 +403,22 @@ ArchiveBox splits its surfaces across three logical hosts: `admin.*` (Django adm | Mode | Host layout | JS replay | Control plane | Use when | |---|---|---|---|---| -| **`safe-subdomains-fullreplay`** *(default, recommended)* | admin/web/api/snap-* on separate subdomains | Full JS replay enabled | Enabled on `admin.*` only | You have wildcard DNS (`*.archive.example.com`) and a TLS cert that covers it. Archived JS runs sandboxed away from the admin origin. | -| **`safe-onedomain-nojsreplay`** | Everything on one host | JS in replays is neutered (served as `text/plain` or stripped) | Enabled | You can't get wildcard DNS. Trades replay fidelity for same-origin safety — archived pages won't execute scripts. | +| **`auto`** *(default, recommended)* | Subdomains for any `*.localhost` request; otherwise one host | Full raw replay on `*.localhost`; otherwise raw archived HTML uses no-JS replay. Plugins can opt into a trusted viewer with an explicit `full.html` preview template. | Enabled | The zero-configuration default. Localhost gets the highest-fidelity isolated setup; ordinary public/LAN hostnames do not require wildcard DNS or TLS. | +| **`safe-subdomains-fullreplay`** | admin/web/api/snap-* on separate subdomains | Full JS replay enabled | Enabled on `admin.*` only | You have wildcard DNS (`*.archive.example.com`) and a TLS cert that covers it. Archived JS runs sandboxed away from the admin origin. | +| **`safe-onedomain-nojsreplay`** | Everything on one host | Raw archived HTML cannot run JS. Explicit trusted plugin preview templates can run the JS needed by their viewer. | Enabled | You can't get wildcard DNS. Trades raw replay fidelity for same-origin safety while retaining trusted viewer formats. | | **`unsafe-onedomain-noadmin`** | Everything on one host | Full JS replay enabled | **Disabled** — `/admin`, `/accounts`, `/api`, `/add`, `/web` return 403; only GET/HEAD/OPTIONS allowed | Read-only public archive on a single host. Operate the instance via CLI only; the web admin is unreachable. | | **`danger-onedomain-fullreplay`** | Everything on one host | Full JS replay enabled | Enabled | Local dev / trusted-network only. Archived JS runs on the **same origin as the admin UI** — a malicious archived page can call admin endpoints with your session. **Do not expose this mode to the internet.** | +SingleFile output is served as ordinary HTML in every mode; it remains usable in no-JS modes because SingleFile removes the page's scripts during capture. ArchiveWeb.page/ReplayWeb.page and MHTML use their existing trusted preview templates. ArchiveBox always attempts to load these viewers, including when the incoming request is plain HTTP, because HTTPS may be terminated by an upstream proxy. Browser service-worker rules still require ReplayWeb.page to be reached through HTTPS or localhost for replay to initialize. + > [!WARNING] > Switching to any mode whose name starts with `unsafe-` or `danger-` is logged at startup and surfaces a banner in the UI. **Don't use these modes on a public hostname** — archived JavaScript will run on the same origin as your admin session. > [!NOTE] -> Subdomain mode requires both wildcard DNS (`*.archive.example.com`) and (if using TLS) a wildcard certificate. Without those, fall back to `safe-onedomain-nojsreplay`. +> Explicit subdomain mode requires both wildcard DNS (`*.archive.example.com`) and (if using TLS) a wildcard certificate. The default `auto` mode only selects subdomain routing for `*.localhost`, which browsers resolve locally without custom DNS or TLS setup. + +> [!NOTE] +> In one-domain `auto` mode, read-only requests are accepted through any valid ingress hostname, but POST/PUT/PATCH/DELETE requests are accepted only on the canonical `BASE_URL` host. Canonical links always use `BASE_URL`. Explicit subdomain mode continues to allow state-changing requests on its derived admin/API hosts. *Related options:* [`BASE_URL`](#base_url), [`PERMISSIONS`](#permissions) diff --git a/docs/Screenshots.md b/docs/Screenshots.md new file mode 100644 index 00000000..62aff0b2 --- /dev/null +++ b/docs/Screenshots.md @@ -0,0 +1,725 @@ +# UI Screenshots + + + +These desktop, tablet, and mobile screenshots cover ArchiveBox's major public and authenticated UI views. Raw API endpoints, API documentation, health-check, and error routes are intentionally excluded. + +## Login + +View: [`/admin/login/`](http://admin.archivebox.localhost:9292/admin/login/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/admin/login.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Login — desktop    		Tablet (1024x1366)
    Login — tablet    		Mobile (390x844)
    Login — mobile
    + +## Public snapshot list + +View: [`/public/`](http://web.archivebox.localhost:9292/public/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Public snapshot list — desktop    		Tablet (1024x1366)
    Public snapshot list — tablet    		Mobile (390x844)
    Public snapshot list — mobile
    + +## Snapshot View (capture in progress) + +View: [`/`](http://snap-8adbc4d6be2e.archivebox.localhost:9292) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (capture in progress) — desktop    		Tablet (1024x1366)
    Snapshot View (capture in progress) — tablet    		Mobile (390x844)
    Snapshot View (capture in progress) — mobile
    + +## Add URLs + +View: [`/add/`](http://admin.archivebox.localhost:9292/add/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Add URLs — desktop    		Tablet (1024x1366)
    Add URLs — tablet    		Mobile (390x844)
    Add URLs — mobile
    + +## Admin dashboard + +View: [`/admin/`](http://admin.archivebox.localhost:9292/admin/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/admin_site.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Admin dashboard — desktop    		Tablet (1024x1366)
    Admin dashboard — tablet    		Mobile (390x844)
    Admin dashboard — mobile
    + +## AI agent + +View: [`/admin/agent/`](http://admin.archivebox.localhost:9292/admin/agent/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/abx_plugins/plugins/opencode/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    AI agent — desktop    		Tablet (1024x1366)
    AI agent — tablet    		Mobile (390x844)
    AI agent — mobile
    + +## Snapshots table + +View: [`/admin/core/snapshot/`](http://admin.archivebox.localhost:9292/admin/core/snapshot/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/admin_snapshots.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshots table — desktop    		Tablet (1024x1366)
    Snapshots table — tablet    		Mobile (390x844)
    Snapshots table — mobile
    + +## Snapshots grid + +View: [`/admin/core/snapshot/grid/`](http://admin.archivebox.localhost:9292/admin/core/snapshot/grid/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/admin/snapshots_grid.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshots grid — desktop    		Tablet (1024x1366)
    Snapshots grid — tablet    		Mobile (390x844)
    Snapshots grid — mobile
    + +## Snapshot admin detail + +View: [`/admin/core/snapshot/06a628b5939c74da8000f9864c2b39b4/change/`](http://admin.archivebox.localhost:9292/admin/core/snapshot/06a628b5939c74da8000f9864c2b39b4/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/admin_snapshots.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot admin detail — desktop    		Tablet (1024x1366)
    Snapshot admin detail — tablet    		Mobile (390x844)
    Snapshot admin detail — mobile
    + +## Snapshot files + +View: [`/`](http://snap-f9864c2b39b4.archivebox.localhost:9292/?files=1) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/static_index.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot files — desktop    		Tablet (1024x1366)
    Snapshot files — tablet    		Mobile (390x844)
    Snapshot files — mobile
    + +## Archive results + +View: [`/admin/core/archiveresult/`](http://admin.archivebox.localhost:9292/admin/core/archiveresult/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/admin_archiveresults.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Archive results — desktop    		Tablet (1024x1366)
    Archive results — tablet    		Mobile (390x844)
    Archive results — mobile
    + +## Archive result detail + +View: [`/admin/core/archiveresult/06a628b7a41d7052800070cd8ff47301/change/`](http://admin.archivebox.localhost:9292/admin/core/archiveresult/06a628b7a41d7052800070cd8ff47301/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/admin_archiveresults.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Archive result detail — desktop    		Tablet (1024x1366)
    Archive result detail — tablet    		Mobile (390x844)
    Archive result detail — mobile
    + +## Tags + +View: [`/admin/core/tag/`](http://admin.archivebox.localhost:9292/admin/core/tag/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/admin_tags.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Tags — desktop    		Tablet (1024x1366)
    Tags — tablet    		Mobile (390x844)
    Tags — mobile
    + +## Tag detail + +View: [`/admin/core/tag/10129/change/`](http://admin.archivebox.localhost:9292/admin/core/tag/10129/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/admin_tags.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Tag detail — desktop    		Tablet (1024x1366)
    Tag detail — tablet    		Mobile (390x844)
    Tag detail — mobile
    + +## Users + +View: [`/admin/auth/user/`](http://admin.archivebox.localhost:9292/admin/auth/user/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/admin_users.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Users — desktop    		Tablet (1024x1366)
    Users — tablet    		Mobile (390x844)
    Users — mobile
    + +## User detail + +View: [`/admin/auth/user/106/change/`](http://admin.archivebox.localhost:9292/admin/auth/user/106/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/admin_users.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    User detail — desktop    		Tablet (1024x1366)
    User detail — tablet    		Mobile (390x844)
    User detail — mobile
    + +## Crawls + +View: [`/admin/crawls/crawl/`](http://admin.archivebox.localhost:9292/admin/crawls/crawl/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/crawls/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Crawls — desktop    		Tablet (1024x1366)
    Crawls — tablet    		Mobile (390x844)
    Crawls — mobile
    + +## Crawl detail + +View: [`/admin/crawls/crawl/06a628b548e575088000fd924686ae24/change/`](http://admin.archivebox.localhost:9292/admin/crawls/crawl/06a628b548e575088000fd924686ae24/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/crawls/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Crawl detail — desktop    		Tablet (1024x1366)
    Crawl detail — tablet    		Mobile (390x844)
    Crawl detail — mobile
    + +## Crawl schedules + +View: [`/admin/crawls/crawlschedule/`](http://admin.archivebox.localhost:9292/admin/crawls/crawlschedule/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/crawls/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Crawl schedules — desktop    		Tablet (1024x1366)
    Crawl schedules — tablet    		Mobile (390x844)
    Crawl schedules — mobile
    + +## Crawl schedule detail + +View: [`/admin/crawls/crawlschedule/06a5fc0c9ca870e080005b6823701cca/change/`](http://admin.archivebox.localhost:9292/admin/crawls/crawlschedule/06a5fc0c9ca870e080005b6823701cca/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/crawls/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Crawl schedule detail — desktop    		Tablet (1024x1366)
    Crawl schedule detail — tablet    		Mobile (390x844)
    Crawl schedule detail — mobile
    + +## Personas + +View: [`/admin/personas/persona/`](http://admin.archivebox.localhost:9292/admin/personas/persona/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/personas/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Personas — desktop    		Tablet (1024x1366)
    Personas — tablet    		Mobile (390x844)
    Personas — mobile
    + +## Persona detail + +View: [`/admin/personas/persona/019e84a0ffd376d98b410dfe1a409567/change/`](http://admin.archivebox.localhost:9292/admin/personas/persona/019e84a0ffd376d98b410dfe1a409567/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/personas/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Persona detail — desktop    		Tablet (1024x1366)
    Persona detail — tablet    		Mobile (390x844)
    Persona detail — mobile
    + +## Machines + +View: [`/admin/machine/machine/`](http://admin.archivebox.localhost:9292/admin/machine/machine/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/machine/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Machines — desktop    		Tablet (1024x1366)
    Machines — tablet    		Mobile (390x844)
    Machines — mobile
    + +## Machine detail + +View: [`/admin/machine/machine/019bd48b941f77fcafb2de79f2d2851e/change/`](http://admin.archivebox.localhost:9292/admin/machine/machine/019bd48b941f77fcafb2de79f2d2851e/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/machine/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Machine detail — desktop    		Tablet (1024x1366)
    Machine detail — tablet    		Mobile (390x844)
    Machine detail — mobile
    + +## Network interfaces + +View: [`/admin/machine/networkinterface/`](http://admin.archivebox.localhost:9292/admin/machine/networkinterface/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/machine/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Network interfaces — desktop    		Tablet (1024x1366)
    Network interfaces — tablet    		Mobile (390x844)
    Network interfaces — mobile
    + +## Network interface detail + +View: [`/admin/machine/networkinterface/06a2ee559ef570b080003f12b381ab42/change/`](http://admin.archivebox.localhost:9292/admin/machine/networkinterface/06a2ee559ef570b080003f12b381ab42/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/machine/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Network interface detail — desktop    		Tablet (1024x1366)
    Network interface detail — tablet    		Mobile (390x844)
    Network interface detail — mobile
    + +## Binaries + +View: [`/admin/machine/binary/`](http://admin.archivebox.localhost:9292/admin/machine/binary/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/machine/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Binaries — desktop    		Tablet (1024x1366)
    Binaries — tablet    		Mobile (390x844)
    Binaries — mobile
    + +## Binary detail + +View: [`/admin/machine/binary/06a14cf28b4a7b1d800008b92ca5b94a/change/`](http://admin.archivebox.localhost:9292/admin/machine/binary/06a14cf28b4a7b1d800008b92ca5b94a/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/machine/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Binary detail — desktop    		Tablet (1024x1366)
    Binary detail — tablet    		Mobile (390x844)
    Binary detail — mobile
    + +## Processes + +View: [`/admin/machine/process/`](http://admin.archivebox.localhost:9292/admin/machine/process/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/machine/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Processes — desktop    		Tablet (1024x1366)
    Processes — tablet    		Mobile (390x844)
    Processes — mobile
    + +## Process detail + +View: [`/admin/machine/process/06a628ec064b72d980002aac0a1bedd1/change/`](http://admin.archivebox.localhost:9292/admin/machine/process/06a628ec064b72d980002aac0a1bedd1/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/machine/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Process detail — desktop    		Tablet (1024x1366)
    Process detail — tablet    		Mobile (390x844)
    Process detail — mobile
    + +## API tokens + +View: [`/admin/api/apitoken/`](http://admin.archivebox.localhost:9292/admin/api/apitoken/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    API tokens — desktop    		Tablet (1024x1366)
    API tokens — tablet    		Mobile (390x844)
    API tokens — mobile
    + +## API token detail + +View: [`/admin/api/apitoken/06a628eae1477aae80008621afbbc237/change/`](http://admin.archivebox.localhost:9292/admin/api/apitoken/06a628eae1477aae80008621afbbc237/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    API token detail — desktop    		Tablet (1024x1366)
    API token detail — tablet    		Mobile (390x844)
    API token detail — mobile
    + +## Webhooks + +View: [`/admin/api/outboundwebhook/`](http://admin.archivebox.localhost:9292/admin/api/outboundwebhook/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Webhooks — desktop    		Tablet (1024x1366)
    Webhooks — tablet    		Mobile (390x844)
    Webhooks — mobile
    + +## Webhook detail + +View: [`/admin/api/outboundwebhook/06a628eb2801724880009c7991a4a522/change/`](http://admin.archivebox.localhost:9292/admin/api/outboundwebhook/06a628eb2801724880009c7991a4a522/change/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api/admin.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Webhook detail — desktop    		Tablet (1024x1366)
    Webhook detail — tablet    		Mobile (390x844)
    Webhook detail — mobile
    + +## Environment + +View: [`/admin/environment/`](http://admin.archivebox.localhost:9292/admin/environment/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/settings.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Environment — desktop    		Tablet (1024x1366)
    Environment — tablet    		Mobile (390x844)
    Environment — mobile
    + +## Configuration + +View: [`/admin/environment/config/`](http://admin.archivebox.localhost:9292/admin/environment/config/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Configuration — desktop    		Tablet (1024x1366)
    Configuration — tablet    		Mobile (390x844)
    Configuration — mobile
    + +## Configuration detail + +View: [`/admin/environment/config/BASE_URL/`](http://admin.archivebox.localhost:9292/admin/environment/config/BASE_URL/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Configuration detail — desktop    		Tablet (1024x1366)
    Configuration detail — tablet    		Mobile (390x844)
    Configuration detail — mobile
    + +## Dependencies + +View: [`/admin/environment/binaries/`](http://admin.archivebox.localhost:9292/admin/environment/binaries/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Dependencies — desktop    		Tablet (1024x1366)
    Dependencies — tablet    		Mobile (390x844)
    Dependencies — mobile
    + +## Dependency detail + +View: [`/admin/environment/binaries/abxbus/`](http://admin.archivebox.localhost:9292/admin/environment/binaries/abxbus/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Dependency detail — desktop    		Tablet (1024x1366)
    Dependency detail — tablet    		Mobile (390x844)
    Dependency detail — mobile
    + +## Plugins + +View: [`/admin/environment/plugins/`](http://admin.archivebox.localhost:9292/admin/environment/plugins/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/plugins/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Plugins — desktop    		Tablet (1024x1366)
    Plugins — tablet    		Mobile (390x844)
    Plugins — mobile
    + +## Workers + +View: [`/admin/environment/workers/`](http://admin.archivebox.localhost:9292/admin/environment/workers/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Workers — desktop    		Tablet (1024x1366)
    Workers — tablet    		Mobile (390x844)
    Workers — mobile
    + +## Worker detail + +View: [`/admin/environment/workers/supervisord/`](http://admin.archivebox.localhost:9292/admin/environment/workers/supervisord/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Worker detail — desktop    		Tablet (1024x1366)
    Worker detail — tablet    		Mobile (390x844)
    Worker detail — mobile
    + +## Logs + +View: [`/admin/environment/logs/`](http://admin.archivebox.localhost:9292/admin/environment/logs/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Logs — desktop    		Tablet (1024x1366)
    Logs — tablet    		Mobile (390x844)
    Logs — mobile
    + +## Log detail + +View: [`/admin/environment/logs/supervisord/`](http://admin.archivebox.localhost:9292/admin/environment/logs/supervisord/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/views.py) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Log detail — desktop    		Tablet (1024x1366)
    Log detail — tablet    		Mobile (390x844)
    Log detail — mobile
    + +## Snapshot View (singlefile) + +View: [`/#singlefile`](http://snap-f9864c2b39b4.archivebox.localhost:9292#singlefile) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (singlefile) — desktop    		Tablet (1024x1366)
    Snapshot View (singlefile) — tablet    		Mobile (390x844)
    Snapshot View (singlefile) — mobile
    + +## Snapshot View (screenshot) + +View: [`/#screenshot`](http://snap-f9864c2b39b4.archivebox.localhost:9292#screenshot) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (screenshot) — desktop    		Tablet (1024x1366)
    Snapshot View (screenshot) — tablet    		Mobile (390x844)
    Snapshot View (screenshot) — mobile
    + +## Snapshot View (wget) + +View: [`/#wget`](http://snap-f9864c2b39b4.archivebox.localhost:9292#wget) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (wget) — desktop    		Tablet (1024x1366)
    Snapshot View (wget) — tablet    		Mobile (390x844)
    Snapshot View (wget) — mobile
    + +## Snapshot View (dom) + +View: [`/#dom`](http://snap-f9864c2b39b4.archivebox.localhost:9292#dom) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (dom) — desktop    		Tablet (1024x1366)
    Snapshot View (dom) — tablet    		Mobile (390x844)
    Snapshot View (dom) — mobile
    + +## Snapshot View (pdf) + +View: [`/#pdf`](http://snap-f9864c2b39b4.archivebox.localhost:9292#pdf) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (pdf) — desktop    		Tablet (1024x1366)
    Snapshot View (pdf) — tablet    		Mobile (390x844)
    Snapshot View (pdf) — mobile
    + +## Snapshot View (readability) + +View: [`/#readability`](http://snap-f9864c2b39b4.archivebox.localhost:9292#readability) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (readability) — desktop    		Tablet (1024x1366)
    Snapshot View (readability) — tablet    		Mobile (390x844)
    Snapshot View (readability) — mobile
    + +## Snapshot View (ytdlp) + +View: [`/#ytdlp`](http://snap-f9864c2b39b4.archivebox.localhost:9292#ytdlp) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (ytdlp) — desktop    		Tablet (1024x1366)
    Snapshot View (ytdlp) — tablet    		Mobile (390x844)
    Snapshot View (ytdlp) — mobile
    + +## Snapshot View (responses) + +View: [`/#responses`](http://snap-f9864c2b39b4.archivebox.localhost:9292#responses) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (responses) — desktop    		Tablet (1024x1366)
    Snapshot View (responses) — tablet    		Mobile (390x844)
    Snapshot View (responses) — mobile
    + +## Snapshot View (chrome_mhtml) + +View: [`/#chrome_mhtml`](http://snap-f9864c2b39b4.archivebox.localhost:9292#chrome_mhtml) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (chrome_mhtml) — desktop    		Tablet (1024x1366)
    Snapshot View (chrome_mhtml) — tablet    		Mobile (390x844)
    Snapshot View (chrome_mhtml) — mobile
    + +## Snapshot View (defuddle) + +View: [`/#defuddle`](http://snap-f9864c2b39b4.archivebox.localhost:9292#defuddle) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (defuddle) — desktop    		Tablet (1024x1366)
    Snapshot View (defuddle) — tablet    		Mobile (390x844)
    Snapshot View (defuddle) — mobile
    + +## Snapshot View (mercury) + +View: [`/#mercury`](http://snap-f9864c2b39b4.archivebox.localhost:9292#mercury) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (mercury) — desktop    		Tablet (1024x1366)
    Snapshot View (mercury) — tablet    		Mobile (390x844)
    Snapshot View (mercury) — mobile
    + +## Snapshot View (archivewebpage) + +View: [`/#archivewebpage`](http://snap-f9864c2b39b4.archivebox.localhost:9292#archivewebpage) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (archivewebpage) — desktop    		Tablet (1024x1366)
    Snapshot View (archivewebpage) — tablet    		Mobile (390x844)
    Snapshot View (archivewebpage) — mobile
    + +## Snapshot View (chrome) + +View: [`/#chrome`](http://snap-f9864c2b39b4.archivebox.localhost:9292#chrome) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (chrome) — desktop    		Tablet (1024x1366)
    Snapshot View (chrome) — tablet    		Mobile (390x844)
    Snapshot View (chrome) — mobile
    + +## Snapshot View (consolelog) + +View: [`/#consolelog`](http://snap-f9864c2b39b4.archivebox.localhost:9292#consolelog) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (consolelog) — desktop    		Tablet (1024x1366)
    Snapshot View (consolelog) — tablet    		Mobile (390x844)
    Snapshot View (consolelog) — mobile
    + +## Snapshot View (dns) + +View: [`/#dns`](http://snap-f9864c2b39b4.archivebox.localhost:9292#dns) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (dns) — desktop    		Tablet (1024x1366)
    Snapshot View (dns) — tablet    		Mobile (390x844)
    Snapshot View (dns) — mobile
    + +## Snapshot View (sslcerts) + +View: [`/#sslcerts`](http://snap-f9864c2b39b4.archivebox.localhost:9292#sslcerts) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (sslcerts) — desktop    		Tablet (1024x1366)
    Snapshot View (sslcerts) — tablet    		Mobile (390x844)
    Snapshot View (sslcerts) — mobile
    + +## Snapshot View (redirects) + +View: [`/#redirects`](http://snap-f9864c2b39b4.archivebox.localhost:9292#redirects) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (redirects) — desktop    		Tablet (1024x1366)
    Snapshot View (redirects) — tablet    		Mobile (390x844)
    Snapshot View (redirects) — mobile
    + +## Snapshot View (headers) + +View: [`/#headers`](http://snap-f9864c2b39b4.archivebox.localhost:9292#headers) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (headers) — desktop    		Tablet (1024x1366)
    Snapshot View (headers) — tablet    		Mobile (390x844)
    Snapshot View (headers) — mobile
    + +## Snapshot View (seo) + +View: [`/#seo`](http://snap-f9864c2b39b4.archivebox.localhost:9292#seo) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (seo) — desktop    		Tablet (1024x1366)
    Snapshot View (seo) — tablet    		Mobile (390x844)
    Snapshot View (seo) — mobile
    + +## Snapshot View (accessibility) + +View: [`/#accessibility`](http://snap-f9864c2b39b4.archivebox.localhost:9292#accessibility) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (accessibility) — desktop    		Tablet (1024x1366)
    Snapshot View (accessibility) — tablet    		Mobile (390x844)
    Snapshot View (accessibility) — mobile
    + +## Snapshot View (htmltotext) + +View: [`/#htmltotext`](http://snap-f9864c2b39b4.archivebox.localhost:9292#htmltotext) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (htmltotext) — desktop    		Tablet (1024x1366)
    Snapshot View (htmltotext) — tablet    		Mobile (390x844)
    Snapshot View (htmltotext) — mobile
    + +## Snapshot View (trafilatura) + +View: [`/#trafilatura`](http://snap-f9864c2b39b4.archivebox.localhost:9292#trafilatura) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (trafilatura) — desktop    		Tablet (1024x1366)
    Snapshot View (trafilatura) — tablet    		Mobile (390x844)
    Snapshot View (trafilatura) — mobile
    + +## Snapshot View (liteparse) + +View: [`/#liteparse`](http://snap-f9864c2b39b4.archivebox.localhost:9292#liteparse) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (liteparse) — desktop    		Tablet (1024x1366)
    Snapshot View (liteparse) — tablet    		Mobile (390x844)
    Snapshot View (liteparse) — mobile
    + +## Snapshot View (parse_html_urls) + +View: [`/#parse_html_urls`](http://snap-f9864c2b39b4.archivebox.localhost:9292#parse_html_urls) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (parse_html_urls) — desktop    		Tablet (1024x1366)
    Snapshot View (parse_html_urls) — tablet    		Mobile (390x844)
    Snapshot View (parse_html_urls) — mobile
    + +## Snapshot View (parse_txt_urls) + +View: [`/#parse_txt_urls`](http://snap-f9864c2b39b4.archivebox.localhost:9292#parse_txt_urls) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (parse_txt_urls) — desktop    		Tablet (1024x1366)
    Snapshot View (parse_txt_urls) — tablet    		Mobile (390x844)
    Snapshot View (parse_txt_urls) — mobile
    + +## Snapshot View (parse_dom_outlinks) + +View: [`/#parse_dom_outlinks`](http://snap-f9864c2b39b4.archivebox.localhost:9292#parse_dom_outlinks) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (parse_dom_outlinks) — desktop    		Tablet (1024x1366)
    Snapshot View (parse_dom_outlinks) — tablet    		Mobile (390x844)
    Snapshot View (parse_dom_outlinks) — mobile
    + +## Snapshot View (hashes) + +View: [`/#hashes`](http://snap-f9864c2b39b4.archivebox.localhost:9292#hashes) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (hashes) — desktop    		Tablet (1024x1366)
    Snapshot View (hashes) — tablet    		Mobile (390x844)
    Snapshot View (hashes) — mobile
    + +## Snapshot View (header collapsed) + +View: [`/`](http://snap-f9864c2b39b4.archivebox.localhost:9292) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) + + + + + +
    DesktopTabletMobile
    Desktop (1600x1000)
    Snapshot View (header collapsed) — desktop    		Tablet (1024x1366)
    Snapshot View (header collapsed) — tablet    		Mobile (390x844)
    Snapshot View (header collapsed) — mobile
    diff --git a/docs/Security-Overview.md b/docs/Security-Overview.md index c11ad181..014ec8c8 100644 --- a/docs/Security-Overview.md +++ b/docs/Security-Overview.md @@ -60,7 +60,7 @@ If you're importing private links or authenticated content, you probably don't w - any cookies / secret state present in a Chrome user profile or `cookies.txt` file may be reflected in server responses and saved in the Snapshot output (e.g. in [`headers`](https://archivebox.github.io/abx-plugins/#headers) extractor output) — visible in cleartext to anyone viewing the Snapshot. **Don't use your personal Chrome profile for archiving** or people viewing your archive can then authenticate as you. - any secret tokens embedded in URLs (e.g. secret invite links, Google Doc URLs, etc.) will be visible on `archive.org` as the URLs are not filtered when saving to it. Disable submitting to Archive.org entirely with [`ARCHIVEDOTORG_ENABLED=False`](https://archivebox.github.io/abx-plugins/#archivedotorg). - the domain portion in archived URLs is sent to a favicon service in order to retrieve an icon more reliably than a janky internal implementation would be able to (if leaking domains is a concern, you can change the [`FAVICON_PROVIDER`](https://archivebox.github.io/abx-plugins/#favicon) or disable favicon fetching entirely with [`FAVICON_ENABLED=False`](https://archivebox.github.io/abx-plugins/#favicon)). -- [viewing malicious archived JS could allow an attacker to access your other archive items + the admin interface](https://github.com/ArchiveBox/ArchiveBox/issues/239) — use the default [`SERVER_SECURITY_MODE=safe-subdomains-fullreplay`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode) (which scopes admin cookies away from snapshot replay subdomains), or disable risky extractors entirely with [`WGET_ENABLED=False`](https://archivebox.github.io/abx-plugins/#wget) and [`DOM_ENABLED=False`](https://archivebox.github.io/abx-plugins/#dom). +- [viewing malicious archived JS could allow an attacker to access your other archive items + the admin interface](https://github.com/ArchiveBox/ArchiveBox/issues/239) — use the default [`SERVER_SECURITY_MODE=auto`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode) (`*.localhost` uses isolated replay subdomains and other hosts use one-domain/no-JS raw replay), or disable risky extractors entirely with [`WGET_ENABLED=False`](https://archivebox.github.io/abx-plugins/#wget) and [`DOM_ENABLED=False`](https://archivebox.github.io/abx-plugins/#dom). Plugins that need their own trusted viewer can opt in with an explicit `full.html` preview template; ArchiveBox applies the no-JS policy to all other raw archived HTML without maintaining a plugin whitelist.
    diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index 288d845d..08d6e278 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -14,6 +14,7 @@ - 🌐 [Web UI](Usage#ui-usage) - 🧩 [Browser Extension](Usage#browser-extension-usage) - 👾 [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496#issuecomment-2080174235) / [Webhooks](https://github.com/ArchiveBox/ArchiveBox/pull/1418) + - 📸 [[UI Screenshots|Screenshots]] - 📜 [Python API](https://docs.archivebox.io/dev/apidocs/index.html) / [REPL](Usage#python-shell-usage) / [SQL API](Usage#sql-shell-usage) # Reference diff --git a/publicsite/index.html b/publicsite/index.html index 3a02c25f..dad3fbe6 100644 --- a/publicsite/index.html +++ b/publicsite/index.html @@ -70,7 +70,7 @@

    ArchiveBox saves websites, bookmarks, RSS feeds, social posts, media, source code, and research material in durable files like HTML, PDF, PNG, TXT, JSON, WARC, MP4, and SQLite.

    Run with Docker Compose - View demo + View screenshots