refactor(proxy): derive scheme from BASE_URL, provider-agnostic certs, slim tunnel-init

- Drop REVERSE_PROXY_TRUST_FORWARDED_PROTO; derive SECURE_PROXY_SSL_HEADER + Secure cookies from BASE_URL's https:// scheme (one knob, no overlapping config). - Caddy https profile is no longer Cloudflare-specific: wildcard cert via lego with ANY of ~150 DNS providers (ARCHIVEBOX_ACME_DNS + provider's native creds passed through .env), OR no DNS provider at all -> plain Let's Encrypt on-demand per-host (HTTP-01), OR self-signed internal locally. Caddyfiles validated. - tunnel-init: use python:3-alpine instead of the heavy archivebox image. - Rename CLOUDFLARE_DNS_API_TOKEN -> CLOUDFLARE_API_KEY for the tunnel's CF API use.
2026-09-12 19:50:57 +05:00 · 2026-06-05 02:29:49 +00:00 · 2026-06-05 02:29:49 +00:00 · f7117a40cd
commit f7117a40cd
parent f3af045974
3 changed files with 55 additions and 48 deletions
--- a/archivebox/config/common.py
+++ b/archivebox/config/common.py
@ -293,7 +293,6 @@ class ServerConfig(BaseConfigSet):

    REVERSE_PROXY_USER_HEADER: str = Field(default="Remote-User")
    REVERSE_PROXY_WHITELIST: str = Field(default="")
-    REVERSE_PROXY_TRUST_FORWARDED_PROTO: bool = Field(default=False)
    LOGOUT_REDIRECT_URL: str = Field(default="/")

    @field_validator("SERVER_SECURITY_MODE", mode="after")
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@ -419,24 +419,20 @@ SECURE_BROWSER_XSS_FILTER = True
 SECURE_CONTENT_TYPE_NOSNIFF = True
 SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"

-# When ArchiveBox sits behind a TLS-terminating reverse proxy (the bundled
-# Caddy profile, the user's own caddy/traefik/nginx, or an ingress like
-# Cloudflare), the hop from proxy → archivebox is plain HTTP, so
-# request.is_secure() / request.scheme would report http. That makes the
-# cross-subdomain login-hint cookie (core/middleware.py) lose its Secure flag
-# and request-derived schemes fall back to http. Honour the proxy's
-# X-Forwarded-Proto header so request.is_secure() reflects the real
-# client-facing scheme. Gated behind a config flag (default off) because
-# trusting a forwarded header is only safe when a proxy you control always
-# sets it — the bundled proxy profile turns it on via env.
-if CONFIG.REVERSE_PROXY_TRUST_FORWARDED_PROTO:
+# When BASE_URL is an https:// URL the deployment is HTTPS end-to-end — typically
+# behind a TLS-terminating proxy/tunnel (the bundled caddy/cloudflared profiles, or
+# your own caddy/traefik/nginx) where the proxy → archivebox hop is plain HTTP, so
+# request.is_secure() / request.scheme would otherwise report http. Honour the
+# proxy's X-Forwarded-Proto so request-derived schemes are correct, and mark the
+# admin session + CSRF cookies Secure so auth cookies are never sent in cleartext.
+# Derived straight from BASE_URL's scheme — no separate flag to keep in sync. A
+# plain-http BASE_URL (e.g. local http://archivebox.localhost:8000) keeps defaults.
+BASE_URL_IS_HTTPS = CONFIG.BASE_URL.strip().lower().startswith("https://")
+if BASE_URL_IS_HTTPS:
    SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")

-# When the proxy is trusted the deployment is HTTPS end-to-end, so also mark the
-# admin session + CSRF cookies Secure (never sent over plain HTTP). Scoped to the
-# same flag so the default http://localhost:8000 setup keeps working unchanged.
-CSRF_COOKIE_SECURE = CONFIG.REVERSE_PROXY_TRUST_FORWARDED_PROTO
-SESSION_COOKIE_SECURE = CONFIG.REVERSE_PROXY_TRUST_FORWARDED_PROTO
+CSRF_COOKIE_SECURE = BASE_URL_IS_HTTPS
+SESSION_COOKIE_SECURE = BASE_URL_IS_HTTPS
 SESSION_COOKIE_HTTPONLY = True
 SESSION_COOKIE_NAME = f"archivebox_sessionid_{CONSTANTS.COLLECTION_ID}"
 CSRF_COOKIE_NAME = f"archivebox_csrftoken_{CONSTANTS.COLLECTION_ID}"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -22,11 +22,9 @@ services:
            # - ADMIN_PASSWORD=SomeSecretPassword
            - BASE_URL=${BASE_URL:-http://archivebox.localhost:8000}                  # public URL used to build admin/web/api/snapshot links
            - SERVER_SECURITY_MODE=${SERVER_SECURITY_MODE:-safe-subdomains-fullreplay} # safe-onedomain-nojsreplay if you can't do wildcard DNS *.your.domain
-            # Trust the bundled ingress (caddy/cloudflared below) for the request scheme
-            # so https links + secure cookies work end-to-end. Leave False unless you put
-            # an ingress in front (and don't also expose :8000 publicly), or the scheme
-            # could be spoofed by a client hitting :8000 directly.
-            - REVERSE_PROXY_TRUST_FORWARDED_PROTO=${REVERSE_PROXY_TRUST_FORWARDED_PROTO:-False}
+            # An https:// BASE_URL means HTTPS end-to-end (behind the ingress below or
+            # your own proxy): ArchiveBox then trusts X-Forwarded-Proto and marks auth
+            # cookies Secure automatically — no extra flag.
            - PUBLIC_ADD_VIEW=False                              # set to True to allow anonymous users to submit new URLs to archive
            # - PUID=911                                         # set to your host user's UID & GID if you encounter permissions issues
            # - PGID=911                                         # UID/GIDs lower than 500 may clash with system uids and are not recommended
@ -84,25 +82,24 @@ services:
    # needs wildcard DNS + TLS for *.your.domain. Pick ONE of the two ingress options
    # below by activating its profile (e.g. put COMPOSE_PROFILES=https or =tunnel in a
    # .env file next to this one, then `docker compose up -d`). Both want:
-    #     BASE_URL=https://archive.example.com
-    #     SERVER_SECURITY_MODE=safe-subdomains-fullreplay
-    #     REVERSE_PROXY_TRUST_FORWARDED_PROTO=True
+    #     BASE_URL=https://archive.example.com   (the https:// scheme is what flips on
+    #     SERVER_SECURITY_MODE=safe-subdomains-fullreplay   proxy-scheme trust + Secure cookies)

    ### Option A — Cloudflare Tunnel (no public IP / behind NAT, e.g. home/NAS).
    # Cloudflare's edge terminates TLS and resolves *.your.domain to a SINGLE tunnel;
    # every snapshot/control subdomain rides one connection to archivebox:8000, which
    # routes by Host header — so the tunnel itself needs no wildcard cert or per-host
    # config. ZERO manual setup: the one-shot tunnel-init below uses your
-    # CLOUDFLARE_DNS_API_TOKEN (give it Account:Cloudflare Tunnel:Edit + Zone:DNS:Edit
+    # CLOUDFLARE_API_KEY (give it Account:Cloudflare Tunnel:Edit + Zone:DNS:Edit
    # + Zone:Read) to create/reuse the tunnel, point *.your.domain and your.domain at
    # it, and write its connector token — then cloudflared just runs it.
    tunnel-init:
-        image: ${ARCHIVEBOX_IMAGE:-archivebox/archivebox:dev}   # reuse the image (its python); no extra image
+        image: python:3-alpine   # tiny stdlib-only provisioner; runs as root so it can chown the token
        profiles: ["tunnel"]
        restart: "no"
        environment:
            - BASE_URL=${BASE_URL:-https://archive.example.com}
-            - CLOUDFLARE_DNS_API_TOKEN=${CLOUDFLARE_DNS_API_TOKEN:-}
+            - CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY:-}
            - CLOUDFLARE_ACCOUNT_ID=${CLOUDFLARE_ACCOUNT_ID:-}   # optional; first account used if unset
            - TUNNEL_SERVICE=http://archivebox:8000
            - TUNNEL_TOKEN_OUT=/shared/token
@ -114,7 +111,7 @@ services:
            - |
              import os, json, base64, secrets, urllib.request, urllib.error
              API = "https://api.cloudflare.com/client/v4"
-              TOKEN = os.environ["CLOUDFLARE_DNS_API_TOKEN"]
+              TOKEN = os.environ["CLOUDFLARE_API_KEY"]
              DOMAIN = os.environ.get("BASE_URL", "").split("://")[-1].split("/")[0].split(":")[0]
              SERVICE = os.environ.get("TUNNEL_SERVICE", "http://archivebox:8000")
              OUT = os.environ.get("TUNNEL_TOKEN_OUT", "/shared/token")
@ -125,7 +122,7 @@ services:
                  try:
                      with urllib.request.urlopen(req, timeout=30) as r: return json.load(r)
                  except urllib.error.HTTPError as e: return json.load(e)
-              assert DOMAIN and TOKEN, "set BASE_URL (https://archive.example.com) + CLOUDFLARE_DNS_API_TOKEN"
+              assert DOMAIN and TOKEN, "set BASE_URL (https://archive.example.com) + CLOUDFLARE_API_KEY"
              acct = os.environ.get("CLOUDFLARE_ACCOUNT_ID", "").strip() or call("GET", "/accounts")["result"][0]["id"]
              labels = DOMAIN.split("."); zone = None   # DOMAIN may be a subdomain; find its registrable zone
              for i in range(len(labels) - 1):
@ -168,12 +165,16 @@ services:
        volumes:
            - ./data/proxy/tunnel:/shared:ro

-    ### Option B — Caddy + automatic Let's Encrypt wildcard cert (you have a public IP).
-    # The lego sidecar fetches ONE *.your.domain wildcard cert via DNS-01 (≈150
-    # providers; cloudflare shown) using CLOUDFLARE_DNS_API_TOKEN, and Caddy serves it
-    # for unlimited snap-* subdomains. If no cert is available (no token / DNS-01
-    # failed / first boot) Caddy GRACEFULLY DEGRADES to a self-signed internal cert so
-    # the site always comes up on https. All config is inlined here — no extra files.
+    ### Option B — Caddy reverse proxy + automatic TLS (you have a public IP, ports 80/443).
+    # Caddy terminates TLS for the apex + every snapshot subdomain, picking the best
+    # source automatically:
+    #   1. *.your.domain WILDCARD cert from the lego sidecar (any of ~150 DNS providers,
+    #      see ARCHIVEBOX_ACME_DNS below) — one cert, unlimited snap-* subdomains.
+    #   2. else, if BASE_URL is https://, plain Let's Encrypt with NO DNS provider:
+    #      on-demand per-host certs over HTTP-01 (needs no credentials; subject to LE
+    #      rate limits if you have very many distinct subdomains).
+    #   3. else (local http:// BASE_URL) a self-signed internal cert.
+    # All config is generated inline — no extra files.
    caddy:
        image: caddy:2-alpine
        profiles: ["https"]
@ -196,20 +197,28 @@ services:
              set -eu
              DOMAIN=$$(printf '%s' "$$BASE_URL" | sed -E 's#^[a-z]+://##; s#[:/].*##')
              CRT="/certs/lego/certificates/_.$$DOMAIN.crt"; KEY="/certs/lego/certificates/_.$$DOMAIN.key"
-              # wait briefly for the lego sidecar to mint the wildcard cert on first boot
+              # give a DNS-01 sidecar (if configured) a moment to mint the wildcard cert on first boot
              i=0; while [ ! -s "$$CRT" ] && [ $$i -lt 20 ]; do sleep 3; i=$$((i+1)); done
              if [ -s "$$CRT" ] && [ -s "$$KEY" ]; then
-                TLS="$$CRT $$KEY"; echo "[caddy] serving real wildcard cert for *.$$DOMAIN"
+                TLS="tls $$CRT $$KEY"; echo "[caddy] wildcard cert for *.$$DOMAIN"
+              elif printf '%s' "$$BASE_URL" | grep -q '^https://'; then
+                TLS=$$(printf 'tls {\n\t\ton_demand\n\t}'); echo "[caddy] on-demand Let's Encrypt per host for *.$$DOMAIN"
              else
-                TLS=internal; echo "[caddy] no wildcard cert yet -> self-signed internal CA for *.$$DOMAIN"
+                TLS="tls internal"; echo "[caddy] self-signed internal CA for *.$$DOMAIN"
              fi
-              printf '{\n\temail %s\n\tauto_https disable_redirects\n}\n*.%s, %s {\n\ttls %s\n\tencode zstd gzip\n\treverse_proxy archivebox:8000\n}\n' \
+              printf '{\n\temail %s\n\tauto_https disable_redirects\n\ton_demand_tls {\n\t\task http://localhost:7000/check\n\t}\n}\n:7000 {\n\trespond /check 200\n}\n*.%s, %s {\n\t%s\n\tencode zstd gzip\n\treverse_proxy archivebox:8000\n}\n' \
                "$$ARCHIVEBOX_ACME_EMAIL" "$$DOMAIN" "$$DOMAIN" "$$TLS" > /tmp/Caddyfile
              exec caddy run --config /tmp/Caddyfile --adapter caddyfile

-    # DNS-01 wildcard cert sidecar for the `https` profile (go-acme/lego). Issues then
-    # auto-renews ONE *.your.domain cert. Swap --dns / creds for any lego provider:
-    #   route53 -> AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY; digitalocean -> DO_AUTH_TOKEN; ...
+    # Optional DNS-01 WILDCARD cert sidecar for the `https` profile (go-acme/lego,
+    # ~150 DNS providers). Pick one with ARCHIVEBOX_ACME_DNS and put that provider's
+    # native credentials in a .env next to this file — they're passed straight to lego:
+    #   cloudflare   -> ARCHIVEBOX_ACME_DNS=cloudflare   + CLOUDFLARE_DNS_API_TOKEN=...
+    #   route53      -> ARCHIVEBOX_ACME_DNS=route53      + AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_REGION
+    #   digitalocean -> ARCHIVEBOX_ACME_DNS=digitalocean + DO_AUTH_TOKEN
+    #   ... full list + var names: https://go-acme.github.io/lego/dns/
+    # Leave ARCHIVEBOX_ACME_DNS unset to skip DNS-01 entirely — Caddy then does plain
+    # Let's Encrypt on-demand (HTTP-01), so no DNS provider/credentials are required.
    lego:
        image: goacme/lego
        profiles: ["https"]
@ -217,7 +226,10 @@ services:
        environment:
            - BASE_URL=${BASE_URL:-https://archive.example.com}
            - ARCHIVEBOX_ACME_EMAIL=${ARCHIVEBOX_ACME_EMAIL:-admin@example.com}
-            - CLOUDFLARE_DNS_API_TOKEN=${CLOUDFLARE_DNS_API_TOKEN:-}
+            - ARCHIVEBOX_ACME_DNS=${ARCHIVEBOX_ACME_DNS:-}
+        env_file:
+            - path: .env       # passes your provider's native creds (CLOUDFLARE_DNS_API_TOKEN, AWS_*, DO_AUTH_TOKEN, ...) to lego
+              required: false
        volumes:
            - ./data/proxy/certs:/certs
        entrypoint:
@ -226,14 +238,14 @@ services:
            - |
              set -u
              DOMAIN=$$(printf '%s' "$$BASE_URL" | sed -E 's#^[a-z]+://##; s#[:/].*##')
-              if [ -z "$${CLOUDFLARE_DNS_API_TOKEN:-}" ]; then
-                echo "[lego] no CLOUDFLARE_DNS_API_TOKEN -> skipping; caddy will self-sign for $$DOMAIN"
+              if [ -z "$${ARCHIVEBOX_ACME_DNS:-}" ]; then
+                echo "[lego] no ARCHIVEBOX_ACME_DNS set -> skipping DNS-01; caddy will use on-demand Let's Encrypt for $$DOMAIN"
                exec sleep infinity
              fi
              ACTION=run; EXTRA=""
              while :; do
-                echo "[lego] $$ACTION *.$$DOMAIN + $$DOMAIN via cloudflare DNS-01"
-                /lego $$ACTION --accept-tos --email "$$ARCHIVEBOX_ACME_EMAIL" --dns cloudflare \
+                echo "[lego] $$ACTION *.$$DOMAIN + $$DOMAIN via $$ARCHIVEBOX_ACME_DNS DNS-01"
+                /lego $$ACTION --accept-tos --email "$$ARCHIVEBOX_ACME_EMAIL" --dns "$$ARCHIVEBOX_ACME_DNS" \
                  -d "*.$$DOMAIN" -d "$$DOMAIN" --path /certs/lego $$EXTRA \
                  || echo "[lego] $$ACTION failed; will retry next cycle"
                ACTION=renew; EXTRA="--days 30"