The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Fix screenshot deploy sequencing

Browse Source
This commit is contained in:
Nick Sweeting 2026-07-25 15:26:08 -07:00
parent 0fea07064d
commit 28d3392291
No known key found for this signature in database
5 changed files with 120 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
archivebox/core/middleware.py
View File

@ -3,25 +3,26 @@ __package__ = "archivebox.core"
import ipaddress
import re
from pathlib import Path
from django.conf import settings
from django.utils import timezone
from django.contrib.auth.middleware import RemoteUserMiddleware
from django.contrib.auth.models import AnonymousUser
from django.core.exceptions import ImproperlyConfigured
from django.shortcuts import redirect
from django.contrib.staticfiles import finders
from django.utils.http import http_date
from django.core.exceptions import ImproperlyConfigured
from django.http import HttpResponseForbidden, HttpResponseNotModified
from django.shortcuts import redirect
from django.utils import timezone
from django.utils.http import http_date
from archivebox.config.common import get_config
from archivebox.config import VERSION
from archivebox.config.common import get_config
from archivebox.config.version import get_COMMIT_HASH
from archivebox.core.routes_util import (
build_snapshot_url,
build_admin_url,
build_snapshot_url,
build_web_url,
get_api_host,
get_admin_host,
get_api_host,
get_base_host,
get_listen_host,
get_listen_subdomain,
@ -30,8 +31,7 @@ from archivebox.core.routes_util import (
is_snapshot_subdomain,
split_host_port,
)
from archivebox.core.views import SnapshotHostView, OriginalDomainHostView
from archivebox.core.views import OriginalDomainHostView, SnapshotHostView
ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
@ -95,11 +95,15 @@ def AdminCookieIsolationMiddleware(get_response):
if not config.USES_SUBDOMAIN_ROUTING:
return response
request_host = (request.get_host() or "").lower()
if host_matches(request_host, get_admin_host(config=config)):
if not config.BASE_URL and request.path.startswith("/admin/"):
return response
if host_matches(request_host, get_web_host(config=config)):
request_host = (request.get_host() or "").lower()
request_hostname, _request_port = split_host_port(request_host)
if host_matches(request_hostname, get_admin_host(config=config)):
return response
if host_matches(request_hostname, get_web_host(config=config)):
for cookie_name in tuple(response.cookies.keys()):
if cookie_name != ADMIN_LOGIN_HINT_COOKIE:
response.cookies.pop(cookie_name, None)
@ -144,14 +148,15 @@ def CacheControlMiddleware(get_response):
response.headers["Last-Modified"] = http_date(mtime)
return response
if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
if not response.get("Cache-Control"):
config = request.__dict__.get("archivebox_config")
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config
policy = "private" if config.PERMISSIONS == "private" else "public"
response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
if ("/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path)) and not response.get(
"Cache-Control",
):
config = request.__dict__.get("archivebox_config")
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config
policy = "private" if config.PERMISSIONS == "private" else "public"
response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
return response
return middleware
@ -244,12 +249,11 @@ def HostRoutingMiddleware(get_response):
req_host, req_port = split_host_port(request_host)
listen_host_only, listen_port = split_host_port(listen_host)
if req_host.endswith(f".{listen_host_only}"):
if not listen_port or not req_port or listen_port == req_port:
target = build_web_url(request.path, request=request)
if request.META.get("QUERY_STRING"):
target = f"{target}?{request.META['QUERY_STRING']}"
return redirect(target)
if req_host.endswith(f".{listen_host_only}") and (not listen_port or not req_port or listen_port == req_port):
target = build_web_url(request.path, request=request)
if request.META.get("QUERY_STRING"):
target = f"{target}?{request.META['QUERY_STRING']}"
return redirect(target)
return get_response(request)

30
archivebox/core/settings.py
View File

@ -1,25 +1,24 @@
__package__ = "archivebox.core"
import importlib
import inspect
import logging
import os
import sys
import inspect
import importlib
from pathlib import Path
from django.conf.locale.en import formats as en_formats # type: ignore
import archivebox
from archivebox.config.constants import CONSTANTS
from archivebox.config.common import get_config
from archivebox.core.routes_util import get_api_base_url, get_admin_base_url, get_base_url, normalize_base_url
from archivebox.config.constants import CONSTANTS
from archivebox.core.routes_util import get_admin_base_url, get_api_base_url, get_base_url, normalize_base_url
# DATABASE_ENGINE config selects the backend (sqlite by default); the
# sqlite-vs-postgres helpers live in archivebox.misc.db.
from archivebox.misc.db import is_postgres, postgres_db_params
from .settings_logging import SETTINGS_LOGGING
from .settings_logging import SETTINGS_LOGGING
IS_MIGRATING = "makemigrations" in sys.argv[:3] or "migrate" in sys.argv[:3]
IS_TESTING = "test" in sys.argv[:3]
@ -27,6 +26,7 @@ IS_SHELL = "shell" in sys.argv[:3] or "shell_plus" in sys.argv[:3]
IS_GETTING_VERSION_OR_HELP = "version" in sys.argv or "help" in sys.argv or "--version" in sys.argv or "--help" in sys.argv
CONFIG = get_config()
PACKAGE_DIR = CONSTANTS.PACKAGE_DIR
logger = logging.getLogger(__name__)
################################################################################
### ArchiveBox Plugin Settings
@ -334,18 +334,18 @@ try:
_persisted_keys = _BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
_secret_persisted = bool((_persisted_keys.get("SECRET_KEY") or "").strip())
except Exception:
except (OSError, RuntimeError, ValueError):
_secret_persisted = True # err on the side of NOT touching disk
if not _secret_persisted:
try:
from archivebox.config.collection import write_config_file
write_config_file({"SECRET_KEY": SECRET_KEY})
except Exception:
except (OSError, RuntimeError, ValueError):
# Read-only mount, missing data dir, mid-init race — fall back to the
# in-memory random key. The user will get logged out on the next boot
# but the server still comes up.
pass
logger.debug("Unable to persist generated SECRET_KEY", exc_info=True)
ALLOWED_HOSTS = [host.strip() for host in CONFIG.ALLOWED_HOSTS.split(",") if host.strip()]
CSRF_TRUSTED_ORIGINS = list({origin.strip() for origin in CONFIG.CSRF_TRUSTED_ORIGINS.split(",") if origin.strip()})
@ -379,15 +379,15 @@ SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
# behind a TLS-terminating proxy/tunnel (the bundled traefik/cloudflared profiles,
# or your own caddy/traefik/nginx) where the proxy -> archivebox hop is plain HTTP, so
# request.is_secure() / request.scheme would otherwise report http. Honour the
# proxy's X-Forwarded-Proto so request-derived schemes are correct, and mark the
# admin session + CSRF cookies Secure so auth cookies are never sent in cleartext.
# proxy's X-Forwarded-Proto so first-run URL detection and CSRF origin checks are
# correct. Mark auth cookies Secure once the saved BASE_URL confirms HTTPS.
# Derived from the RESOLVED base URL's scheme — no separate flag to keep in sync.
# get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the
# implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening
# isn't lost until BASE_URL is migrated. A plain-http base (e.g. local
# http://archivebox.localhost:8000) keeps the defaults below.
# isn't lost until BASE_URL is migrated. An explicit plain-http base (e.g. local
# http://archivebox.localhost:8000) disables proxy HTTPS handling.
BASE_URL_IS_HTTPS = get_base_url(config=CONFIG).strip().lower().startswith("https://")
if BASE_URL_IS_HTTPS:
if BASE_URL_IS_HTTPS or not CONFIG.BASE_URL:
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
CSRF_COOKIE_SECURE = BASE_URL_IS_HTTPS

54
archivebox/tests/test_cli_server.py
View File

@ -1,14 +1,13 @@
#!/usr/bin/env python3
"""
Tests for archivebox server command.
Verify server can start (basic smoke tests only, no full server testing).
"""
import os
import asyncio
import json
import signal
import os
import shlex
import signal
import socket
import subprocess
import sys
@ -22,17 +21,17 @@ from archivebox.tests.conftest import (
_wait_for_archivebox_workers,
assert_no_processes_for_data_dir,
assert_port_open,
cli_env,
find_process,
get_free_port,
kill_processes_for_data_dir,
cli_env,
pid_is_alive,
resolve_abxpkg_binary_env,
run_archivebox_cmd,
start_archivebox_server,
stop_archivebox_process,
wait_for_log_count,
wait_for_pid_to_disappear,
run_archivebox_cmd,
resolve_abxpkg_binary_env,
)
@ -162,6 +161,43 @@ def test_https_base_url_enables_proxy_ssl_header_and_secure_cookies(tmp_path):
}
def test_unconfigured_base_url_enables_proxy_ssl_header_without_secure_cookies(tmp_path):
(tmp_path / ".archivebox_id").write_text("testcoll")
env = os.environ.copy()
env["BASE_URL"] = ""
env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
repo_root = Path(__file__).resolve().parents[2]
env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{env.get('PYTHONPATH', '')}"
result = subprocess.run(
[
sys.executable,
"-c",
(
"import django, json;"
"django.setup();"
"from django.conf import settings;"
"print(json.dumps({"
"'csrf_secure': settings.CSRF_COOKIE_SECURE,"
"'session_secure': settings.SESSION_COOKIE_SECURE,"
"'proxy_ssl_header': settings.SECURE_PROXY_SSL_HEADER,"
"}))"
),
],
capture_output=True,
text=True,
check=True,
env=env,
cwd=tmp_path,
)
assert json.loads(result.stdout) == {
"csrf_secure": False,
"session_secure": False,
"proxy_ssl_header": ["HTTP_X_FORWARDED_PROTO", "https"],
}
def test_sqlite_connections_use_explicit_busy_timeout():
from archivebox.core.settings import SQLITE_CONNECTION_OPTIONS
@ -336,6 +372,7 @@ def test_live_server_machine_search_engine_update_reaches_subsequent_snapshot_ru
capture_output=True,
text=True,
timeout=30,
check=False,
)
assert setup_result.returncode == 0, setup_result.stderr or setup_result.stdout
snapshot_id = setup_result.stdout.strip().splitlines()[-1]
@ -366,6 +403,7 @@ def test_live_server_machine_search_engine_update_reaches_subsequent_snapshot_ru
capture_output=True,
text=True,
timeout=30,
check=False,
)
assert result.returncode == 0, result.stderr or result.stdout
resolved = json.loads(result.stdout.strip().splitlines()[-1])
@ -392,9 +430,10 @@ def test_sonic_worker_is_disabled_when_sonic_disabled(tmp_path):
def test_sonic_daemon_event_handler_accepts_real_running_worker(initialized_archive, archivebox_daemon_server):
from abx_dl.events import ProcessStdoutEvent
from abx_dl.orchestrator import create_bus
from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
from abx_plugins.plugins.search_backend_sonic.daemon import prepare_sonic_daemon
from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
sonic_port = get_free_port()
sonic_env = _resolve_sonic_env(initialized_archive)
server = archivebox_daemon_server(
@ -434,6 +473,7 @@ def test_sonic_daemon_event_handler_accepts_real_running_worker(initialized_arch
def test_supervisord_sync_does_not_start_duplicate_sonic_listener(initialized_archive, db):
from abx_plugins.plugins.search_backend_sonic.daemon import get_sonic_supervisord_worker
from archivebox.tests.test_orm_helpers import use_archivebox_db
from archivebox.workers.supervisord_util import (
get_or_create_supervisord_process,

24
archivebox/tests/test_ui_admin_config_widget.py
View File

@ -1,12 +1,15 @@
from pathlib import Path
import pytest
from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.http import HttpResponse
from django.template.loader import render_to_string
from django.test import RequestFactory
from archivebox.base_models.admin import KeyValueWidget
from archivebox.config.common import get_config
from archivebox.core.middleware import AdminCookieIsolationMiddleware
from archivebox.core.setup_wizard import get_base_url_mismatch_context, get_setup_wizard_context
from archivebox.core.templatetags.core_tags import system_warnings_banner
@ -224,6 +227,27 @@ def test_unconfigured_banner_honors_forwarded_https_from_ingress():
assert context["suggested_base_url"] == "https://archivebox.example.test"
@pytest.mark.parametrize(
("base_url", "request_host"),
(
("", "archivebox.example.test:18443"),
("http://archivebox.localhost:8000", "admin.archivebox.localhost:18010"),
),
)
def test_admin_cookie_isolation_accepts_first_run_and_external_port_mapping(base_url, request_host):
config = get_config(include_machine=False).model_copy(
update={"BASE_URL": base_url, "SERVER_SECURITY_MODE": "auto"},
)
request = RequestFactory().get("/admin/login/", HTTP_HOST=request_host)
request.archivebox_config = config
response = HttpResponse()
response.set_cookie(settings.CSRF_COOKIE_NAME, "test-token")
actual_response = AdminCookieIsolationMiddleware(lambda _request: response)(request)
assert settings.CSRF_COOKIE_NAME in actual_response.cookies
def test_unconfigured_banner_does_not_show_setup_wizard_to_non_superusers():
html = render_to_string(
"core/system_warnings_banner.html",

8
bin/collect_ui_screenshots.sh
View File

@ -389,7 +389,7 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
RECORD_CONFIG="$( (
cd "$DATA_DIR"
uv run --project "$REPO_DIR" archivebox manage shell --no-imports -c \
'from django.db.models import Count; from archivebox.core.models import Snapshot, ArchiveResult, Tag; from archivebox.crawls.models import Crawl, CrawlSchedule; from archivebox.personas.models import Persona; from archivebox.machine.models import Machine, NetworkInterface, Binary, Process; from archivebox.api.models import APIToken; from django.contrib.auth import get_user_model; from signal_webhooks.utils import get_webhook_model; from archivebox.core.routes_util import build_snapshot_url; recent=list(Snapshot.objects.filter(url__startswith="https://sweeting.me",status=Snapshot.StatusChoices.SEALED).order_by("-bookmarked_at").values_list("id", flat=True)[:1000]); counts=dict(ArchiveResult.objects.filter(snapshot_id__in=recent,status="succeeded").values_list("snapshot_id").annotate(Count("id"))); snapshot_id=str(max(recent,key=lambda item: counts.get(item,0))); snapshot=Snapshot.objects.get(id=snapshot_id); result=ArchiveResult.objects.filter(snapshot_id=snapshot_id,status="succeeded").order_by("-output_size").first() or ArchiveResult.objects.filter(snapshot_id=snapshot_id).first(); tag=snapshot.tags.first() or Tag.objects.first(); crawl=snapshot.crawl or Crawl.objects.order_by("-created_at").first(); schedule=CrawlSchedule.objects.order_by("-created_at").first(); persona=Persona.objects.exclude(name="Default").order_by("-created_at").first() or Persona.objects.first(); machine=Machine.objects.order_by("-modified_at").first(); interface=NetworkInterface.objects.order_by("-modified_at").first(); binary=Binary.objects.order_by("-modified_at").first(); process=Process.objects.order_by("-created_at").first(); token=APIToken.objects.order_by("-created_at").first(); webhook=get_webhook_model().objects.order_by("-created_at").first(); user=get_user_model().objects.get(username="'"$USERNAME"'"); values={"SNAPSHOT_ID":snapshot_id,"SNAPSHOT_VIEW_URL":build_snapshot_url(snapshot_id,""),"SNAPSHOT_FILES_URL":build_snapshot_url(snapshot_id,"/?files=1"),"ARCHIVERESULT_ID":str(result.id),"TAG_ID":str(tag.id),"USER_ID":str(user.id),"CRAWL_ID":str(crawl.id),"SCHEDULE_ID":str(schedule.id),"PERSONA_ID":str(persona.id),"MACHINE_ID":str(machine.id),"INTERFACE_ID":str(interface.id),"BINARY_ID":str(binary.id),"PROCESS_ID":str(process.id),"TOKEN_ID":str(token.id),"WEBHOOK_ID":str(webhook.id)}; [print(f"{key}={value}") for key,value in values.items()]'
'from django.db.models import Count; from archivebox.core.models import Snapshot, ArchiveResult, Tag; from archivebox.crawls.models import Crawl, CrawlSchedule; from archivebox.personas.models import Persona; from archivebox.machine.models import Machine, NetworkInterface, Binary, Process; from archivebox.api.models import APIToken; from django.contrib.auth import get_user_model; from signal_webhooks.utils import get_webhook_model; from archivebox.core.routes_util import build_snapshot_url; recent=list(Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED).order_by("-bookmarked_at").values_list("id", flat=True)[:1000]); counts=dict(ArchiveResult.objects.filter(snapshot_id__in=recent,status="succeeded").values_list("snapshot_id").annotate(Count("id"))); snapshot_id=str(max(recent,key=lambda item: counts.get(item,0))); snapshot=Snapshot.objects.get(id=snapshot_id); result=ArchiveResult.objects.filter(snapshot_id=snapshot_id,status="succeeded").order_by("-output_size").first() or ArchiveResult.objects.filter(snapshot_id=snapshot_id).first(); tag=snapshot.tags.first() or Tag.objects.first(); crawl=snapshot.crawl or Crawl.objects.order_by("-created_at").first(); schedule=CrawlSchedule.objects.order_by("-created_at").first(); persona=Persona.objects.exclude(name="Default").order_by("-created_at").first() or Persona.objects.first(); machine=Machine.objects.order_by("-modified_at").first(); interface=NetworkInterface.objects.order_by("-modified_at").first(); binary=Binary.objects.order_by("-modified_at").first(); process=Process.objects.order_by("-created_at").first(); token=APIToken.objects.order_by("-created_at").first(); webhook=get_webhook_model().objects.order_by("-created_at").first(); user=get_user_model().objects.get(username="'"$USERNAME"'"); values={"SNAPSHOT_ID":snapshot_id,"SNAPSHOT_VIEW_URL":build_snapshot_url(snapshot_id,""),"SNAPSHOT_FILES_URL":build_snapshot_url(snapshot_id,"/?files=1"),"ARCHIVERESULT_ID":str(result.id),"TAG_ID":str(tag.id),"USER_ID":str(user.id),"CRAWL_ID":str(crawl.id),"SCHEDULE_ID":str(schedule.id),"PERSONA_ID":str(persona.id),"MACHINE_ID":str(machine.id),"INTERFACE_ID":str(interface.id),"BINARY_ID":str(binary.id),"PROCESS_ID":str(process.id),"TOKEN_ID":str(token.id),"WEBHOOK_ID":str(webhook.id)}; [print(f"{key}={value}") for key,value in values.items()]'
) | tail -15)"
eval "$RECORD_CONFIG"
@ -458,7 +458,7 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
SCREENSHOT_SNAPSHOT_HEADER=expanded \
SCREENSHOT_WIDTH=1600 \
SCREENSHOT_HEIGHT=1000 \
node "$REPO_DIR/bin/take_screenshot.js" "$SNAPSHOT_VIEW_URL" "$CAPTURE_ROOT/snapshot-output-discovery.png" >"$SNAPSHOT_DISCOVERY_REPORT"
node "$REPO_DIR/bin/take_screenshot.js" "$LIVE_SNAPSHOT_VIEW_URL" "$CAPTURE_ROOT/snapshot-output-discovery.png" >"$SNAPSHOT_DISCOVERY_REPORT"
SNAPSHOT_OUTPUT_PLUGINS="$(UI_SCREENSHOT_DISCOVERY_REPORT="$SNAPSHOT_DISCOVERY_REPORT" uv run --project "$REPO_DIR" python -c \
'import json, os; from urllib.parse import urlsplit; report=json.load(open(os.environ["UI_SCREENSHOT_DISCOVERY_REPORT"])); print("\n".join("{}\t{}".format(output["plugin"], "wait-replay:Nick Sweeting" if urlsplit(output["previewUrl"]).path.endswith(".wacz") else "") for output in report["checks"]["snapshotOutputs"]))')"
if [[ -z "$SNAPSHOT_OUTPUT_PLUGINS" ]]; then
@ -467,9 +467,9 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
fi
while IFS=$'\t' read -r plugin_name output_capture_mode; do
[[ -z "$plugin_name" ]] && continue
VIEWS+=("Snapshot View ($plugin_name)|$SNAPSHOT_VIEW_URL#$plugin_name|/|archivebox/templates/core/snapshot.html|$output_capture_mode")
VIEWS+=("Snapshot View ($plugin_name)|$LIVE_SNAPSHOT_VIEW_URL#$plugin_name|/|archivebox/templates/core/snapshot.html|$output_capture_mode")
done <<<"$SNAPSHOT_OUTPUT_PLUGINS"
VIEWS+=("Snapshot View (header collapsed)|$SNAPSHOT_VIEW_URL|/|archivebox/templates/core/snapshot.html|snapshot-collapsed")
VIEWS+=("Snapshot View (header collapsed)|$LIVE_SNAPSHOT_VIEW_URL|/|archivebox/templates/core/snapshot.html|snapshot-collapsed")
fi
if [[ "$MAX_VIEWS" != "0" && "$capture_index" -ge "$MAX_VIEWS" ]]; then
break