diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index f902faf9..ba971fc9 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -13,7 +13,7 @@ import archivebox from archivebox.config.constants import CONSTANTS from archivebox.config.common import get_config -from archivebox.core.routes_util import normalize_base_url, get_admin_base_url, get_api_base_url +from archivebox.core.routes_util import get_api_base_url, get_admin_base_url, get_base_url, normalize_base_url from .settings_logging import SETTINGS_LOGGING @@ -419,8 +419,23 @@ SECURE_BROWSER_XSS_FILTER = True SECURE_CONTENT_TYPE_NOSNIFF = True SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin" -CSRF_COOKIE_SECURE = False -SESSION_COOKIE_SECURE = False +# When BASE_URL is an https:// URL the deployment is HTTPS end-to-end, typically +# behind a TLS-terminating proxy/tunnel (the bundled traefik/cloudflared profiles, +# or your own caddy/traefik/nginx) where the proxy -> archivebox hop is plain HTTP, so +# request.is_secure() / request.scheme would otherwise report http. Honour the +# proxy's X-Forwarded-Proto so request-derived schemes are correct, and mark the +# admin session + CSRF cookies Secure so auth cookies are never sent in cleartext. +# Derived from the RESOLVED base URL's scheme — no separate flag to keep in sync. +# get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the +# implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening +# isn't lost until BASE_URL is migrated. A plain-http base (e.g. local +# http://archivebox.localhost:8000) keeps the defaults below. +BASE_URL_IS_HTTPS = get_base_url(config=CONFIG).strip().lower().startswith("https://") +if BASE_URL_IS_HTTPS: + SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") + +CSRF_COOKIE_SECURE = BASE_URL_IS_HTTPS +SESSION_COOKIE_SECURE = BASE_URL_IS_HTTPS SESSION_COOKIE_HTTPONLY = True SESSION_COOKIE_NAME = f"archivebox_sessionid_{CONSTANTS.COLLECTION_ID}" CSRF_COOKIE_NAME = f"archivebox_csrftoken_{CONSTANTS.COLLECTION_ID}" diff --git a/archivebox/tests/test_cli_server.py b/archivebox/tests/test_cli_server.py index ce02d2b1..38989d05 100644 --- a/archivebox/tests/test_cli_server.py +++ b/archivebox/tests/test_cli_server.py @@ -14,6 +14,7 @@ import subprocess import sys import time from datetime import datetime +from pathlib import Path from types import SimpleNamespace import pytest @@ -98,6 +99,43 @@ def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path, mon assert first_lines[3:] == ["None", "False", "False"] +def test_https_base_url_enables_proxy_ssl_header_and_secure_cookies(tmp_path): + (tmp_path / ".archivebox_id").write_text("testcoll") + env = os.environ.copy() + env["BASE_URL"] = "https://archive.example.com" + env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings" + repo_root = Path(__file__).resolve().parents[2] + env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{env.get('PYTHONPATH', '')}" + + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import django, json;" + "django.setup();" + "from django.conf import settings;" + "print(json.dumps({" + "'csrf_secure': settings.CSRF_COOKIE_SECURE," + "'session_secure': settings.SESSION_COOKIE_SECURE," + "'proxy_ssl_header': settings.SECURE_PROXY_SSL_HEADER," + "}))" + ), + ], + capture_output=True, + text=True, + check=True, + env=env, + cwd=tmp_path, + ) + + assert json.loads(result.stdout) == { + "csrf_secure": True, + "session_secure": True, + "proxy_ssl_header": ["HTTP_X_FORWARDED_PROTO", "https"], + } + + def test_sqlite_connections_use_explicit_busy_timeout(): from archivebox.core.settings import SQLITE_CONNECTION_OPTIONS diff --git a/docker-compose.yml b/docker-compose.yml index d8374217..13c7d663 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,8 +20,8 @@ services: environment: # - ADMIN_USERNAME=admin # creates an admin user on first run with the given user/pass combo # - ADMIN_PASSWORD=SomeSecretPassword - # - SERVER_SECURITY_MODE=safe-subdomains-fullreplay # set to safe-onedomain-nojsreplay if you can't do wildcard DNS *.your.archivebox.domain - - BASE_URL=http://archivebox.localhost:8000 # public URL used to build admin/web/api/snapshot links + - BASE_URL=${BASE_URL:-http://archivebox.localhost:8000} # public URL used to build admin/web/api/snapshot links + - SERVER_SECURITY_MODE=${SERVER_SECURITY_MODE:-safe-subdomains-fullreplay} # safe-onedomain-nojsreplay if you can't do wildcard DNS *.your.domain - PUBLIC_ADD_VIEW=False # set to True to allow anonymous users to submit new URLs to archive # - PUID=911 # set to your host user's UID & GID if you encounter permissions issues # - PGID=911 # UID/GIDs lower than 500 may clash with system uids and are not recommended @@ -72,18 +72,159 @@ services: # - 127.0.0.1:8080:8080 - ### Example: Put Nginx in front of the ArchiveBox server for SSL termination and static file serving. - # You can also any other ingress provider for SSL like Apache, Caddy, Traefik, Cloudflare Tunnels, etc. - # Note you must set up wildcard DNS and TLS *.your.archivebox.domain because snapshots are served from unique subdomains for security. + ### TLS / HTTPS ingress (opt-in, everything below is driven by env vars only). + # + # ArchiveBox serves the admin/web/api/public control plane AND every archived + # snapshot on its own subdomain for security isolation, so a public deployment + # needs wildcard DNS + TLS for *.your.domain. Pick ONE of the two ingress options + # below by activating its profile (e.g. put COMPOSE_PROFILES=https or =tunnel in a + # .env file next to this one, then `docker compose up -d`). Both want: + # BASE_URL=https://archive.example.com + # SERVER_SECURITY_MODE=safe-subdomains-fullreplay - # nginx: - # image: nginx:alpine - # ports: - # - 443:443 - # - 80:80 - # volumes: - # - ./etc/nginx.conf:/etc/nginx/nginx.conf - # - ./data:/var/www + ### Option A — Cloudflare Tunnel (no public IP / behind NAT, e.g. home/NAS). + # Cloudflare's edge terminates TLS and resolves *.your.domain to a SINGLE tunnel; + # every snapshot/control subdomain rides one connection to archivebox:8000, which + # routes by Host header — so the tunnel itself needs no wildcard cert or per-host + # config. ZERO manual setup: the one-shot tunnel-init below uses your + # CLOUDFLARE_API_KEY (give it Account:Cloudflare Tunnel:Edit + Zone:DNS:Edit + # + Zone:Read) to create/reuse the tunnel, point *.your.domain and your.domain at + # it, and write its connector token — then cloudflared just runs it. + tunnel-init: + image: python:3-alpine # tiny stdlib-only provisioner; runs as root so it can chown the token + profiles: ["tunnel"] + restart: "no" + environment: + - BASE_URL=${BASE_URL:-https://archive.example.com} + - CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY:-} # a Cloudflare API *Token* (used as a Bearer token), NOT the legacy global API key + - CLOUDFLARE_ACCOUNT_ID=${CLOUDFLARE_ACCOUNT_ID:-} # optional; first account used if unset + - TUNNEL_SERVICE=http://archivebox:8000 + - TUNNEL_TOKEN_OUT=/shared/token + volumes: + - ./data/proxy/tunnel:/shared + entrypoint: + - python3 + - -c + - | + import os, json, base64, secrets, urllib.request, urllib.error + API = "https://api.cloudflare.com/client/v4" + TOKEN = os.environ["CLOUDFLARE_API_KEY"] + DOMAIN = os.environ.get("BASE_URL", "").split("://")[-1].split("/")[0].split(":")[0] + SERVICE = os.environ.get("TUNNEL_SERVICE", "http://archivebox:8000") + OUT = os.environ.get("TUNNEL_TOKEN_OUT", "/shared/token") + H = {"Authorization": "Bearer " + TOKEN, "Content-Type": "application/json"} + def call(method, path, data=None): + body = json.dumps(data).encode() if data is not None else None + req = urllib.request.Request(API + path, data=body, headers=H, method=method) + try: + with urllib.request.urlopen(req, timeout=30) as r: return json.load(r) + except urllib.error.HTTPError as e: return json.load(e) + assert DOMAIN and TOKEN, "set BASE_URL (https://archive.example.com) + CLOUDFLARE_API_KEY" + acct = os.environ.get("CLOUDFLARE_ACCOUNT_ID", "").strip() or call("GET", "/accounts")["result"][0]["id"] + labels = DOMAIN.split("."); zone = None # DOMAIN may be a subdomain; find its registrable zone + for i in range(len(labels) - 1): + res = call("GET", f"/zones?name={'.'.join(labels[i:])}")["result"] + if res: zone = res[0]["id"]; break + assert zone, f"no Cloudflare zone found for {DOMAIN}" + NAME = "archivebox-" + DOMAIN.replace(".", "-") + ts = call("GET", f"/accounts/{acct}/cfd_tunnel?name={NAME}&is_deleted=false")["result"] + tid = ts[0]["id"] if ts else call("POST", f"/accounts/{acct}/cfd_tunnel", {"name": NAME, + "tunnel_secret": base64.b64encode(secrets.token_bytes(32)).decode(), "config_src": "cloudflare"})["result"]["id"] + target = f"{tid}.cfargotunnel.com" + call("PUT", f"/accounts/{acct}/cfd_tunnel/{tid}/configurations", {"config": {"ingress": [ + {"hostname": f"*.{DOMAIN}", "service": SERVICE}, {"hostname": DOMAIN, "service": SERVICE}, + {"service": "http_status:404"}]}}) + for name in (DOMAIN, f"*.{DOMAIN}"): + recs = call("GET", f"/zones/{zone}/dns_records?name={name}")["result"] + cname = [r for r in recs if r["type"] == "CNAME"] + for r in [r for r in recs if r["type"] in ("A", "AAAA")] + cname[1:]: + call("DELETE", f"/zones/{zone}/dns_records/{r['id']}") + desired = {"type": "CNAME", "name": name, "content": target, "proxied": True, "ttl": 1} + call("PUT", f"/zones/{zone}/dns_records/{cname[0]['id']}", desired) if cname else call("POST", f"/zones/{zone}/dns_records", desired) + tok = call("GET", f"/accounts/{acct}/cfd_tunnel/{tid}/token")["result"] + os.makedirs(os.path.dirname(OUT) or ".", exist_ok=True) + with open(OUT, "w") as f: f.write(tok) + os.chmod(OUT, 0o600) # private: never world-readable on the host bind-mount + try: os.chown(OUT, 65532, 65532) # best-effort: own it by the cloudflared (uid 65532) connector that reads it + except OSError as e: print(f"[tunnel-init] warning: could not chown {OUT} to uid 65532 ({e}); ensure the cloudflared container can read it") + print(f"[tunnel-init] {NAME} ({tid}): *.{DOMAIN} + {DOMAIN} -> {SERVICE}; connector token -> {OUT}") + + cloudflared: + image: cloudflare/cloudflared + profiles: ["tunnel"] + restart: unless-stopped + depends_on: + archivebox: + condition: service_started + tunnel-init: + condition: service_completed_successfully + command: tunnel --no-autoupdate --protocol http2 run --token-file /shared/token + volumes: + - ./data/proxy/tunnel:/shared:ro + + ### Option B — Traefik reverse proxy + automatic wildcard TLS (you have a public IP). + # ONE container terminates TLS for the apex + every snapshot subdomain and proxies + # to archivebox:8000. Traefik is also an ACME client (it embeds go-acme/lego), so it + # fetches a single *.your.domain WILDCARD cert via DNS-01 and auto-renews it — no + # separate cert sidecar. All config is generated inline; no extra files. + # + # WILDCARD DNS — you must do this ONE manual step first (no proxy can do it for you): + # point a wildcard record at this server's public IP, e.g. at your DNS host add + # A *.archive.example.com -> + # A archive.example.com -> + # (AAAA too if you have IPv6). That's what makes snap-*.archive.example.com reach + # this box. Traefik then only needs the DNS *API* to solve the ACME DNS-01 challenge: + # + # set ARCHIVEBOX_ACME_DNS to your provider and put its credentials in a .env next to + # this file (passed straight through to Traefik/lego) — any of ~100 providers: + # cloudflare -> ARCHIVEBOX_ACME_DNS=cloudflare + CLOUDFLARE_DNS_API_TOKEN=... + # route53 -> ARCHIVEBOX_ACME_DNS=route53 + AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_REGION + # digitalocean -> ARCHIVEBOX_ACME_DNS=digitalocean + DO_AUTH_TOKEN + # ... full list + exact var names: https://doc.traefik.io/traefik/https/acme/#providers + # Leave ARCHIVEBOX_ACME_DNS unset to skip ACME — Traefik then serves its built-in + # self-signed cert (browser warning), handy for local/testing. + traefik: + image: traefik:v3 + profiles: ["https"] + restart: unless-stopped + depends_on: [archivebox] + ports: + - "80:80" + - "443:443" + environment: + - BASE_URL=${BASE_URL:-https://archive.example.com} + - ARCHIVEBOX_ACME_EMAIL=${ARCHIVEBOX_ACME_EMAIL:-admin@example.com} + - ARCHIVEBOX_ACME_DNS=${ARCHIVEBOX_ACME_DNS:-} + env_file: + - path: .env # passes your DNS provider's creds (CLOUDFLARE_DNS_API_TOKEN, AWS_*, DO_AUTH_TOKEN, ...) to Traefik + required: false + volumes: + - ./data/proxy/traefik:/certs # Traefik stores acme.json (the wildcard cert) here + entrypoint: + - sh + - -c + - | + set -eu + DOMAIN=$$(printf '%s' "$$BASE_URL" | sed -E 's#^[a-z]+://##; s#[:/].*##') + # catch-all router -> archivebox (Host-routed); domain-free, so no docker socket needed + printf 'http:\n routers:\n archivebox:\n rule: "HostRegexp(`^.+$$`)"\n service: archivebox\n services:\n archivebox:\n loadBalancer:\n servers:\n - url: "http://archivebox:8000"\n' > /etc/traefik/dynamic.yml + set -- --entrypoints.web.address=:80 --entrypoints.websecure.address=:443 \ + --entrypoints.web.http.redirections.entrypoint.to=websecure \ + --entrypoints.web.http.redirections.entrypoint.scheme=https \ + --providers.file.filename=/etc/traefik/dynamic.yml + if [ -n "$${ARCHIVEBOX_ACME_DNS:-}" ]; then + echo "[traefik] wildcard cert for *.$$DOMAIN via $$ARCHIVEBOX_ACME_DNS DNS-01" + set -- "$$@" --entrypoints.websecure.http.tls.certresolver=le \ + --entrypoints.websecure.http.tls.domains[0].main="$$DOMAIN" \ + --entrypoints.websecure.http.tls.domains[0].sans="*.$$DOMAIN" \ + --certificatesresolvers.le.acme.email="$$ARCHIVEBOX_ACME_EMAIL" \ + --certificatesresolvers.le.acme.storage=/certs/acme.json \ + --certificatesresolvers.le.acme.dnschallenge=true \ + --certificatesresolvers.le.acme.dnschallenge.provider="$$ARCHIVEBOX_ACME_DNS" + else + echo "[traefik] no ARCHIVEBOX_ACME_DNS set -> serving Traefik's default self-signed cert (set a DNS provider for real wildcard TLS)" + fi + exec traefik "$$@" ### Example: run all your ArchiveBox traffic through a WireGuard VPN tunnel to avoid IP blocks. # You can also use any other VPN that works at the docker/IP level, e.g. Tailscale, OpenVPN, etc. diff --git a/docs/Docker.md b/docs/Docker.md index dc5931e9..d701c75f 100644 --- a/docs/Docker.md +++ b/docs/Docker.md @@ -179,7 +179,12 @@ services: You can also specify an env file via CLI when running compose using `docker compose --env-file=/path/to/config.env ...` although you must specify the variables in the `environment:` section that you want to have passed down to the ArchiveBox container from the passed env file. -If you want to access your archive server with HTTPS, put a reverse proxy like Nginx or Caddy in front of `http://127.0.0.1:8000` to do SSL termination. Here is an example [ArchiveBox nginx container](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docker-compose.yml#:~:text=nginx) + [`nginx.conf`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/etc/nginx.conf) that you can modify to add your preferred TLS settings. +If you want to access your archive server with HTTPS, the bundled `docker-compose.yml` includes two opt-in ingress profiles: + +- `COMPOSE_PROFILES=https` runs Traefik in front of ArchiveBox for HTTPS/TLS, with optional wildcard certificates via DNS-01. +- `COMPOSE_PROFILES=tunnel` runs a Cloudflare Tunnel for deployments without a public IP. + +Set `BASE_URL=https://archive.example.com` in the `.env` file next to `docker-compose.yml`, then follow the inline comments in the compose file for the profile you choose. You can still bring your own reverse proxy such as Nginx or Caddy in front of `http://127.0.0.1:8000`; [`etc/nginx.conf`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/etc/nginx.conf) remains a standalone example.
diff --git a/docs/Publishing-Your-Archive.md b/docs/Publishing-Your-Archive.md index 62c6ab51..93c2e177 100644 --- a/docs/Publishing-Your-Archive.md +++ b/docs/Publishing-Your-Archive.md @@ -20,8 +20,8 @@ archivebox server 0.0.0.0:8000 open http://127.0.0.1:8000 ``` -This server is enabled out-of-the-box if you're using `docker-compose` to run ArchiveBox, -and there is a commented-out example nginx config with SSL set up as well. If hosting publicly, it's essential to place an SSL termination server in front of ArchiveBox (e.g. [`traefik`](https://github.com/traefik/traefik), [`caddy`](https://caddyserver.com/docs/automatic-https#activation), or [`cloudflared`](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/)), +This server is enabled out-of-the-box if you're using `docker-compose` to run ArchiveBox. +If hosting publicly, it's essential to place an SSL termination server in front of ArchiveBox. The bundled compose file includes opt-in `https` (Traefik) and `tunnel` (Cloudflare Tunnel) profiles, or you can bring your own reverse proxy such as [`traefik`](https://github.com/traefik/traefik), [`caddy`](https://caddyserver.com/docs/automatic-https#activation), or [`cloudflared`](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/). > [!TIP] > Advanced: You can use nginx to serve the static `/archive/` dir directly from the filesystem to increase performance. diff --git a/etc/README.md b/etc/README.md index 1b7f0865..6244aa61 100644 --- a/etc/README.md +++ b/etc/README.md @@ -4,6 +4,19 @@ In this folder are some example config files you can use for setting up ArchiveB E.g. see `nginx.conf` for an example nginx config to serve your archive with SSL, or `fly.toml` for an example deployment to the Fly.io hosting platform. +For the recommended, batteries-included reverse proxy and TLS, you don't need a file +here at all — it's built into the main `../docker-compose.yml` as two opt-in, env-var +driven profiles (no extra files, Dockerfiles, or scripts) — set the documented env +vars in a `.env` next to `../docker-compose.yml`: + +- `https` — a single Traefik container terminates TLS and fetches/auto-renews one + `*.` wildcard cert via DNS-01 (covering unlimited `snap-*` subdomains, + ~100 DNS providers via its embedded lego, no per-provider code), serving Traefik's + default self-signed cert if no DNS provider is configured. +- `tunnel` — a Cloudflare Tunnel whose tunnel/DNS are auto-provisioned from your API + token, so Cloudflare's edge terminates TLS and routes `*.` through one + tunnel to ArchiveBox (Host-routed) — no public IP or wildcard cert needed locally. + Please contribute your etc files here! Example contributions - supervisord config