diff --git a/archivebox/core/middleware.py b/archivebox/core/middleware.py index 2ed6260f..ab83df38 100644 --- a/archivebox/core/middleware.py +++ b/archivebox/core/middleware.py @@ -3,25 +3,26 @@ __package__ = "archivebox.core" import ipaddress import re from pathlib import Path + from django.conf import settings -from django.utils import timezone from django.contrib.auth.middleware import RemoteUserMiddleware from django.contrib.auth.models import AnonymousUser -from django.core.exceptions import ImproperlyConfigured -from django.shortcuts import redirect from django.contrib.staticfiles import finders -from django.utils.http import http_date +from django.core.exceptions import ImproperlyConfigured from django.http import HttpResponseForbidden, HttpResponseNotModified +from django.shortcuts import redirect +from django.utils import timezone +from django.utils.http import http_date -from archivebox.config.common import get_config from archivebox.config import VERSION +from archivebox.config.common import get_config from archivebox.config.version import get_COMMIT_HASH from archivebox.core.routes_util import ( - build_snapshot_url, build_admin_url, + build_snapshot_url, build_web_url, - get_api_host, get_admin_host, + get_api_host, get_base_host, get_listen_host, get_listen_subdomain, @@ -30,8 +31,7 @@ from archivebox.core.routes_util import ( is_snapshot_subdomain, split_host_port, ) -from archivebox.core.views import SnapshotHostView, OriginalDomainHostView - +from archivebox.core.views import OriginalDomainHostView, SnapshotHostView ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in" @@ -95,11 +95,15 @@ def AdminCookieIsolationMiddleware(get_response): if not config.USES_SUBDOMAIN_ROUTING: return response - request_host = (request.get_host() or "").lower() - if host_matches(request_host, get_admin_host(config=config)): + if not config.BASE_URL and request.path.startswith("/admin/"): return response - if host_matches(request_host, get_web_host(config=config)): + request_host = (request.get_host() or "").lower() + request_hostname, _request_port = split_host_port(request_host) + if host_matches(request_hostname, get_admin_host(config=config)): + return response + + if host_matches(request_hostname, get_web_host(config=config)): for cookie_name in tuple(response.cookies.keys()): if cookie_name != ADMIN_LOGIN_HINT_COOKIE: response.cookies.pop(cookie_name, None) @@ -144,14 +148,15 @@ def CacheControlMiddleware(get_response): response.headers["Last-Modified"] = http_date(mtime) return response - if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path): - if not response.get("Cache-Control"): - config = request.__dict__.get("archivebox_config") - if config is None: - config = get_config(resolve_plugins=False) - request.archivebox_config = config - policy = "private" if config.PERMISSIONS == "private" else "public" - response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300" + if ("/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path)) and not response.get( + "Cache-Control", + ): + config = request.__dict__.get("archivebox_config") + if config is None: + config = get_config(resolve_plugins=False) + request.archivebox_config = config + policy = "private" if config.PERMISSIONS == "private" else "public" + response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300" return response return middleware @@ -244,12 +249,11 @@ def HostRoutingMiddleware(get_response): req_host, req_port = split_host_port(request_host) listen_host_only, listen_port = split_host_port(listen_host) - if req_host.endswith(f".{listen_host_only}"): - if not listen_port or not req_port or listen_port == req_port: - target = build_web_url(request.path, request=request) - if request.META.get("QUERY_STRING"): - target = f"{target}?{request.META['QUERY_STRING']}" - return redirect(target) + if req_host.endswith(f".{listen_host_only}") and (not listen_port or not req_port or listen_port == req_port): + target = build_web_url(request.path, request=request) + if request.META.get("QUERY_STRING"): + target = f"{target}?{request.META['QUERY_STRING']}" + return redirect(target) return get_response(request) diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index f40c280f..d60d9843 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -1,25 +1,24 @@ __package__ = "archivebox.core" +import importlib +import inspect +import logging import os import sys -import inspect -import importlib - from pathlib import Path from django.conf.locale.en import formats as en_formats # type: ignore import archivebox - -from archivebox.config.constants import CONSTANTS from archivebox.config.common import get_config -from archivebox.core.routes_util import get_api_base_url, get_admin_base_url, get_base_url, normalize_base_url +from archivebox.config.constants import CONSTANTS +from archivebox.core.routes_util import get_admin_base_url, get_api_base_url, get_base_url, normalize_base_url # DATABASE_ENGINE config selects the backend (sqlite by default); the # sqlite-vs-postgres helpers live in archivebox.misc.db. from archivebox.misc.db import is_postgres, postgres_db_params -from .settings_logging import SETTINGS_LOGGING +from .settings_logging import SETTINGS_LOGGING IS_MIGRATING = "makemigrations" in sys.argv[:3] or "migrate" in sys.argv[:3] IS_TESTING = "test" in sys.argv[:3] @@ -27,6 +26,7 @@ IS_SHELL = "shell" in sys.argv[:3] or "shell_plus" in sys.argv[:3] IS_GETTING_VERSION_OR_HELP = "version" in sys.argv or "help" in sys.argv or "--version" in sys.argv or "--help" in sys.argv CONFIG = get_config() PACKAGE_DIR = CONSTANTS.PACKAGE_DIR +logger = logging.getLogger(__name__) ################################################################################ ### ArchiveBox Plugin Settings @@ -334,18 +334,18 @@ try: _persisted_keys = _BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE) _secret_persisted = bool((_persisted_keys.get("SECRET_KEY") or "").strip()) -except Exception: +except (OSError, RuntimeError, ValueError): _secret_persisted = True # err on the side of NOT touching disk if not _secret_persisted: try: from archivebox.config.collection import write_config_file write_config_file({"SECRET_KEY": SECRET_KEY}) - except Exception: + except (OSError, RuntimeError, ValueError): # Read-only mount, missing data dir, mid-init race — fall back to the # in-memory random key. The user will get logged out on the next boot # but the server still comes up. - pass + logger.debug("Unable to persist generated SECRET_KEY", exc_info=True) ALLOWED_HOSTS = [host.strip() for host in CONFIG.ALLOWED_HOSTS.split(",") if host.strip()] CSRF_TRUSTED_ORIGINS = list({origin.strip() for origin in CONFIG.CSRF_TRUSTED_ORIGINS.split(",") if origin.strip()}) @@ -379,15 +379,15 @@ SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin" # behind a TLS-terminating proxy/tunnel (the bundled traefik/cloudflared profiles, # or your own caddy/traefik/nginx) where the proxy -> archivebox hop is plain HTTP, so # request.is_secure() / request.scheme would otherwise report http. Honour the -# proxy's X-Forwarded-Proto so request-derived schemes are correct, and mark the -# admin session + CSRF cookies Secure so auth cookies are never sent in cleartext. +# proxy's X-Forwarded-Proto so first-run URL detection and CSRF origin checks are +# correct. Mark auth cookies Secure once the saved BASE_URL confirms HTTPS. # Derived from the RESOLVED base URL's scheme — no separate flag to keep in sync. # get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the # implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening -# isn't lost until BASE_URL is migrated. A plain-http base (e.g. local -# http://archivebox.localhost:8000) keeps the defaults below. +# isn't lost until BASE_URL is migrated. An explicit plain-http base (e.g. local +# http://archivebox.localhost:8000) disables proxy HTTPS handling. BASE_URL_IS_HTTPS = get_base_url(config=CONFIG).strip().lower().startswith("https://") -if BASE_URL_IS_HTTPS: +if BASE_URL_IS_HTTPS or not CONFIG.BASE_URL: SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") CSRF_COOKIE_SECURE = BASE_URL_IS_HTTPS diff --git a/archivebox/tests/test_cli_server.py b/archivebox/tests/test_cli_server.py index e1db5711..d2b57fa5 100644 --- a/archivebox/tests/test_cli_server.py +++ b/archivebox/tests/test_cli_server.py @@ -1,14 +1,13 @@ -#!/usr/bin/env python3 """ Tests for archivebox server command. Verify server can start (basic smoke tests only, no full server testing). """ -import os import asyncio import json -import signal +import os import shlex +import signal import socket import subprocess import sys @@ -22,17 +21,17 @@ from archivebox.tests.conftest import ( _wait_for_archivebox_workers, assert_no_processes_for_data_dir, assert_port_open, + cli_env, find_process, get_free_port, kill_processes_for_data_dir, - cli_env, pid_is_alive, + resolve_abxpkg_binary_env, + run_archivebox_cmd, start_archivebox_server, stop_archivebox_process, wait_for_log_count, wait_for_pid_to_disappear, - run_archivebox_cmd, - resolve_abxpkg_binary_env, ) @@ -162,6 +161,43 @@ def test_https_base_url_enables_proxy_ssl_header_and_secure_cookies(tmp_path): } +def test_unconfigured_base_url_enables_proxy_ssl_header_without_secure_cookies(tmp_path): + (tmp_path / ".archivebox_id").write_text("testcoll") + env = os.environ.copy() + env["BASE_URL"] = "" + env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings" + repo_root = Path(__file__).resolve().parents[2] + env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{env.get('PYTHONPATH', '')}" + + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import django, json;" + "django.setup();" + "from django.conf import settings;" + "print(json.dumps({" + "'csrf_secure': settings.CSRF_COOKIE_SECURE," + "'session_secure': settings.SESSION_COOKIE_SECURE," + "'proxy_ssl_header': settings.SECURE_PROXY_SSL_HEADER," + "}))" + ), + ], + capture_output=True, + text=True, + check=True, + env=env, + cwd=tmp_path, + ) + + assert json.loads(result.stdout) == { + "csrf_secure": False, + "session_secure": False, + "proxy_ssl_header": ["HTTP_X_FORWARDED_PROTO", "https"], + } + + def test_sqlite_connections_use_explicit_busy_timeout(): from archivebox.core.settings import SQLITE_CONNECTION_OPTIONS @@ -336,6 +372,7 @@ def test_live_server_machine_search_engine_update_reaches_subsequent_snapshot_ru capture_output=True, text=True, timeout=30, + check=False, ) assert setup_result.returncode == 0, setup_result.stderr or setup_result.stdout snapshot_id = setup_result.stdout.strip().splitlines()[-1] @@ -366,6 +403,7 @@ def test_live_server_machine_search_engine_update_reaches_subsequent_snapshot_ru capture_output=True, text=True, timeout=30, + check=False, ) assert result.returncode == 0, result.stderr or result.stdout resolved = json.loads(result.stdout.strip().splitlines()[-1]) @@ -392,9 +430,10 @@ def test_sonic_worker_is_disabled_when_sonic_disabled(tmp_path): def test_sonic_daemon_event_handler_accepts_real_running_worker(initialized_archive, archivebox_daemon_server): from abx_dl.events import ProcessStdoutEvent from abx_dl.orchestrator import create_bus - from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler from abx_plugins.plugins.search_backend_sonic.daemon import prepare_sonic_daemon + from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler + sonic_port = get_free_port() sonic_env = _resolve_sonic_env(initialized_archive) server = archivebox_daemon_server( @@ -434,6 +473,7 @@ def test_sonic_daemon_event_handler_accepts_real_running_worker(initialized_arch def test_supervisord_sync_does_not_start_duplicate_sonic_listener(initialized_archive, db): from abx_plugins.plugins.search_backend_sonic.daemon import get_sonic_supervisord_worker + from archivebox.tests.test_orm_helpers import use_archivebox_db from archivebox.workers.supervisord_util import ( get_or_create_supervisord_process, diff --git a/archivebox/tests/test_ui_admin_config_widget.py b/archivebox/tests/test_ui_admin_config_widget.py index 34cbd54a..1f28eaef 100644 --- a/archivebox/tests/test_ui_admin_config_widget.py +++ b/archivebox/tests/test_ui_admin_config_widget.py @@ -1,12 +1,15 @@ from pathlib import Path import pytest +from django.conf import settings from django.contrib.auth.models import AnonymousUser +from django.http import HttpResponse from django.template.loader import render_to_string from django.test import RequestFactory from archivebox.base_models.admin import KeyValueWidget from archivebox.config.common import get_config +from archivebox.core.middleware import AdminCookieIsolationMiddleware from archivebox.core.setup_wizard import get_base_url_mismatch_context, get_setup_wizard_context from archivebox.core.templatetags.core_tags import system_warnings_banner @@ -224,6 +227,27 @@ def test_unconfigured_banner_honors_forwarded_https_from_ingress(): assert context["suggested_base_url"] == "https://archivebox.example.test" +@pytest.mark.parametrize( + ("base_url", "request_host"), + ( + ("", "archivebox.example.test:18443"), + ("http://archivebox.localhost:8000", "admin.archivebox.localhost:18010"), + ), +) +def test_admin_cookie_isolation_accepts_first_run_and_external_port_mapping(base_url, request_host): + config = get_config(include_machine=False).model_copy( + update={"BASE_URL": base_url, "SERVER_SECURITY_MODE": "auto"}, + ) + request = RequestFactory().get("/admin/login/", HTTP_HOST=request_host) + request.archivebox_config = config + response = HttpResponse() + response.set_cookie(settings.CSRF_COOKIE_NAME, "test-token") + + actual_response = AdminCookieIsolationMiddleware(lambda _request: response)(request) + + assert settings.CSRF_COOKIE_NAME in actual_response.cookies + + def test_unconfigured_banner_does_not_show_setup_wizard_to_non_superusers(): html = render_to_string( "core/system_warnings_banner.html", diff --git a/bin/collect_ui_screenshots.sh b/bin/collect_ui_screenshots.sh index 6360574f..e5c8d260 100755 --- a/bin/collect_ui_screenshots.sh +++ b/bin/collect_ui_screenshots.sh @@ -389,7 +389,7 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do RECORD_CONFIG="$( ( cd "$DATA_DIR" uv run --project "$REPO_DIR" archivebox manage shell --no-imports -c \ - 'from django.db.models import Count; from archivebox.core.models import Snapshot, ArchiveResult, Tag; from archivebox.crawls.models import Crawl, CrawlSchedule; from archivebox.personas.models import Persona; from archivebox.machine.models import Machine, NetworkInterface, Binary, Process; from archivebox.api.models import APIToken; from django.contrib.auth import get_user_model; from signal_webhooks.utils import get_webhook_model; from archivebox.core.routes_util import build_snapshot_url; recent=list(Snapshot.objects.filter(url__startswith="https://sweeting.me",status=Snapshot.StatusChoices.SEALED).order_by("-bookmarked_at").values_list("id", flat=True)[:1000]); counts=dict(ArchiveResult.objects.filter(snapshot_id__in=recent,status="succeeded").values_list("snapshot_id").annotate(Count("id"))); snapshot_id=str(max(recent,key=lambda item: counts.get(item,0))); snapshot=Snapshot.objects.get(id=snapshot_id); result=ArchiveResult.objects.filter(snapshot_id=snapshot_id,status="succeeded").order_by("-output_size").first() or ArchiveResult.objects.filter(snapshot_id=snapshot_id).first(); tag=snapshot.tags.first() or Tag.objects.first(); crawl=snapshot.crawl or Crawl.objects.order_by("-created_at").first(); schedule=CrawlSchedule.objects.order_by("-created_at").first(); persona=Persona.objects.exclude(name="Default").order_by("-created_at").first() or Persona.objects.first(); machine=Machine.objects.order_by("-modified_at").first(); interface=NetworkInterface.objects.order_by("-modified_at").first(); binary=Binary.objects.order_by("-modified_at").first(); process=Process.objects.order_by("-created_at").first(); token=APIToken.objects.order_by("-created_at").first(); webhook=get_webhook_model().objects.order_by("-created_at").first(); user=get_user_model().objects.get(username="'"$USERNAME"'"); values={"SNAPSHOT_ID":snapshot_id,"SNAPSHOT_VIEW_URL":build_snapshot_url(snapshot_id,""),"SNAPSHOT_FILES_URL":build_snapshot_url(snapshot_id,"/?files=1"),"ARCHIVERESULT_ID":str(result.id),"TAG_ID":str(tag.id),"USER_ID":str(user.id),"CRAWL_ID":str(crawl.id),"SCHEDULE_ID":str(schedule.id),"PERSONA_ID":str(persona.id),"MACHINE_ID":str(machine.id),"INTERFACE_ID":str(interface.id),"BINARY_ID":str(binary.id),"PROCESS_ID":str(process.id),"TOKEN_ID":str(token.id),"WEBHOOK_ID":str(webhook.id)}; [print(f"{key}={value}") for key,value in values.items()]' + 'from django.db.models import Count; from archivebox.core.models import Snapshot, ArchiveResult, Tag; from archivebox.crawls.models import Crawl, CrawlSchedule; from archivebox.personas.models import Persona; from archivebox.machine.models import Machine, NetworkInterface, Binary, Process; from archivebox.api.models import APIToken; from django.contrib.auth import get_user_model; from signal_webhooks.utils import get_webhook_model; from archivebox.core.routes_util import build_snapshot_url; recent=list(Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED).order_by("-bookmarked_at").values_list("id", flat=True)[:1000]); counts=dict(ArchiveResult.objects.filter(snapshot_id__in=recent,status="succeeded").values_list("snapshot_id").annotate(Count("id"))); snapshot_id=str(max(recent,key=lambda item: counts.get(item,0))); snapshot=Snapshot.objects.get(id=snapshot_id); result=ArchiveResult.objects.filter(snapshot_id=snapshot_id,status="succeeded").order_by("-output_size").first() or ArchiveResult.objects.filter(snapshot_id=snapshot_id).first(); tag=snapshot.tags.first() or Tag.objects.first(); crawl=snapshot.crawl or Crawl.objects.order_by("-created_at").first(); schedule=CrawlSchedule.objects.order_by("-created_at").first(); persona=Persona.objects.exclude(name="Default").order_by("-created_at").first() or Persona.objects.first(); machine=Machine.objects.order_by("-modified_at").first(); interface=NetworkInterface.objects.order_by("-modified_at").first(); binary=Binary.objects.order_by("-modified_at").first(); process=Process.objects.order_by("-created_at").first(); token=APIToken.objects.order_by("-created_at").first(); webhook=get_webhook_model().objects.order_by("-created_at").first(); user=get_user_model().objects.get(username="'"$USERNAME"'"); values={"SNAPSHOT_ID":snapshot_id,"SNAPSHOT_VIEW_URL":build_snapshot_url(snapshot_id,""),"SNAPSHOT_FILES_URL":build_snapshot_url(snapshot_id,"/?files=1"),"ARCHIVERESULT_ID":str(result.id),"TAG_ID":str(tag.id),"USER_ID":str(user.id),"CRAWL_ID":str(crawl.id),"SCHEDULE_ID":str(schedule.id),"PERSONA_ID":str(persona.id),"MACHINE_ID":str(machine.id),"INTERFACE_ID":str(interface.id),"BINARY_ID":str(binary.id),"PROCESS_ID":str(process.id),"TOKEN_ID":str(token.id),"WEBHOOK_ID":str(webhook.id)}; [print(f"{key}={value}") for key,value in values.items()]' ) | tail -15)" eval "$RECORD_CONFIG" @@ -458,7 +458,7 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do SCREENSHOT_SNAPSHOT_HEADER=expanded \ SCREENSHOT_WIDTH=1600 \ SCREENSHOT_HEIGHT=1000 \ - node "$REPO_DIR/bin/take_screenshot.js" "$SNAPSHOT_VIEW_URL" "$CAPTURE_ROOT/snapshot-output-discovery.png" >"$SNAPSHOT_DISCOVERY_REPORT" + node "$REPO_DIR/bin/take_screenshot.js" "$LIVE_SNAPSHOT_VIEW_URL" "$CAPTURE_ROOT/snapshot-output-discovery.png" >"$SNAPSHOT_DISCOVERY_REPORT" SNAPSHOT_OUTPUT_PLUGINS="$(UI_SCREENSHOT_DISCOVERY_REPORT="$SNAPSHOT_DISCOVERY_REPORT" uv run --project "$REPO_DIR" python -c \ 'import json, os; from urllib.parse import urlsplit; report=json.load(open(os.environ["UI_SCREENSHOT_DISCOVERY_REPORT"])); print("\n".join("{}\t{}".format(output["plugin"], "wait-replay:Nick Sweeting" if urlsplit(output["previewUrl"]).path.endswith(".wacz") else "") for output in report["checks"]["snapshotOutputs"]))')" if [[ -z "$SNAPSHOT_OUTPUT_PLUGINS" ]]; then @@ -467,9 +467,9 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do fi while IFS=$'\t' read -r plugin_name output_capture_mode; do [[ -z "$plugin_name" ]] && continue - VIEWS+=("Snapshot View ($plugin_name)|$SNAPSHOT_VIEW_URL#$plugin_name|/|archivebox/templates/core/snapshot.html|$output_capture_mode") + VIEWS+=("Snapshot View ($plugin_name)|$LIVE_SNAPSHOT_VIEW_URL#$plugin_name|/|archivebox/templates/core/snapshot.html|$output_capture_mode") done <<<"$SNAPSHOT_OUTPUT_PLUGINS" - VIEWS+=("Snapshot View (header collapsed)|$SNAPSHOT_VIEW_URL|/|archivebox/templates/core/snapshot.html|snapshot-collapsed") + VIEWS+=("Snapshot View (header collapsed)|$LIVE_SNAPSHOT_VIEW_URL|/|archivebox/templates/core/snapshot.html|snapshot-collapsed") fi if [[ "$MAX_VIEWS" != "0" && "$capture_index" -ge "$MAX_VIEWS" ]]; then break