From 2fea001d368cdadbd0130b183cc2f47ecd70bf96 Mon Sep 17 00:00:00 2001 From: archivebox Date: Fri, 5 Jun 2026 04:18:41 +0000 Subject: [PATCH] refactor(proxy): replace caddy+lego with a single Traefik container The https profile is now ONE container (was 3): Traefik terminates TLS and, via its embedded go-acme/lego (~100 DNS providers), fetches + auto-renews a single *. wildcard cert over DNS-01; no DNS provider -> Traefik's default self-signed cert. Catch-all router -> archivebox:8000 via an inline file-provider config (no docker socket, no extra files). Config validated against the real traefik v3.7 binary. Comments document the one manual step (wildcard DNS A-record). Also address review: derive BASE_URL_IS_HTTPS from the RESOLVED base url (get_base_url(), so CSRF_TRUSTED_ORIGINS-only upgrades keep HTTPS hardening); clarify CLOUDFLARE_API_KEY is an API Token (Bearer), not the legacy global key. --- archivebox/core/settings.py | 11 ++-- docker-compose.yml | 115 +++++++++++++++--------------------- etc/README.md | 8 +-- 3 files changed, 57 insertions(+), 77 deletions(-) diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 2cebd534..1f95bb58 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -13,7 +13,7 @@ import archivebox from archivebox.config.constants import CONSTANTS from archivebox.config.common import get_config -from archivebox.core.routes_util import normalize_base_url, get_admin_base_url, get_api_base_url +from archivebox.core.routes_util import normalize_base_url, get_admin_base_url, get_api_base_url, get_base_url from .settings_logging import SETTINGS_LOGGING @@ -425,9 +425,12 @@ SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin" # request.is_secure() / request.scheme would otherwise report http. Honour the # proxy's X-Forwarded-Proto so request-derived schemes are correct, and mark the # admin session + CSRF cookies Secure so auth cookies are never sent in cleartext. -# Derived straight from BASE_URL's scheme — no separate flag to keep in sync. A -# plain-http BASE_URL (e.g. local http://archivebox.localhost:8000) keeps defaults. -BASE_URL_IS_HTTPS = CONFIG.BASE_URL.strip().lower().startswith("https://") +# Derived from the RESOLVED base URL's scheme — no separate flag to keep in sync. +# get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the +# implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening +# isn't lost until BASE_URL is migrated. A plain-http base (e.g. local +# http://archivebox.localhost:8000) keeps the defaults below. +BASE_URL_IS_HTTPS = get_base_url().strip().lower().startswith("https://") if BASE_URL_IS_HTTPS: SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") diff --git a/docker-compose.yml b/docker-compose.yml index 8c70347c..fa9cc6e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,7 @@ services: restart: "no" environment: - BASE_URL=${BASE_URL:-https://archive.example.com} - - CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY:-} + - CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY:-} # a Cloudflare API *Token* (used as a Bearer token), NOT the legacy global API key - CLOUDFLARE_ACCOUNT_ID=${CLOUDFLARE_ACCOUNT_ID:-} # optional; first account used if unset - TUNNEL_SERVICE=http://archivebox:8000 - TUNNEL_TOKEN_OUT=/shared/token @@ -165,92 +165,69 @@ services: volumes: - ./data/proxy/tunnel:/shared:ro - ### Option B — Caddy reverse proxy + automatic TLS (you have a public IP, ports 80/443). - # Caddy terminates TLS for the apex + every snapshot subdomain, picking the best - # source automatically: - # 1. *.your.domain WILDCARD cert from the lego sidecar (any of ~150 DNS providers, - # see ARCHIVEBOX_ACME_DNS below) — one cert, unlimited snap-* subdomains. - # 2. else, if BASE_URL is https://, plain Let's Encrypt with NO DNS provider: - # on-demand per-host certs over HTTP-01 (needs no credentials; subject to LE - # rate limits if you have very many distinct subdomains). - # 3. else (local http:// BASE_URL) a self-signed internal cert. - # All config is generated inline — no extra files. - caddy: - image: caddy:2-alpine + ### Option B — Traefik reverse proxy + automatic wildcard TLS (you have a public IP). + # ONE container terminates TLS for the apex + every snapshot subdomain and proxies + # to archivebox:8000. Traefik is also an ACME client (it embeds go-acme/lego), so it + # fetches a single *.your.domain WILDCARD cert via DNS-01 and auto-renews it — no + # separate cert sidecar. All config is generated inline; no extra files. + # + # WILDCARD DNS — you must do this ONE manual step first (no proxy can do it for you): + # point a wildcard record at this server's public IP, e.g. at your DNS host add + # A *.archive.example.com -> + # A archive.example.com -> + # (AAAA too if you have IPv6). That's what makes snap-*.archive.example.com reach + # this box. Traefik then only needs the DNS *API* to solve the ACME DNS-01 challenge: + # + # set ARCHIVEBOX_ACME_DNS to your provider and put its credentials in a .env next to + # this file (passed straight through to Traefik/lego) — any of ~100 providers: + # cloudflare -> ARCHIVEBOX_ACME_DNS=cloudflare + CLOUDFLARE_DNS_API_TOKEN=... + # route53 -> ARCHIVEBOX_ACME_DNS=route53 + AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_REGION + # digitalocean -> ARCHIVEBOX_ACME_DNS=digitalocean + DO_AUTH_TOKEN + # ... full list + exact var names: https://doc.traefik.io/traefik/https/acme/#providers + # Leave ARCHIVEBOX_ACME_DNS unset to skip ACME — Traefik then serves its built-in + # self-signed cert (browser warning), handy for local/testing. + traefik: + image: traefik:v3 profiles: ["https"] restart: unless-stopped - depends_on: [archivebox, lego] + depends_on: [archivebox] ports: - "80:80" - "443:443" - - "443:443/udp" # HTTP/3 - environment: - - BASE_URL=${BASE_URL:-https://archive.example.com} - - ARCHIVEBOX_ACME_EMAIL=${ARCHIVEBOX_ACME_EMAIL:-admin@example.com} - volumes: - - ./data/proxy/caddy:/data - - ./data/proxy/certs:/certs:ro - command: - - sh - - -c - - | - set -eu - DOMAIN=$$(printf '%s' "$$BASE_URL" | sed -E 's#^[a-z]+://##; s#[:/].*##') - CRT="/certs/lego/certificates/_.$$DOMAIN.crt"; KEY="/certs/lego/certificates/_.$$DOMAIN.key" - # give a DNS-01 sidecar (if configured) a moment to mint the wildcard cert on first boot - i=0; while [ ! -s "$$CRT" ] && [ $$i -lt 20 ]; do sleep 3; i=$$((i+1)); done - if [ -s "$$CRT" ] && [ -s "$$KEY" ]; then - TLS="tls $$CRT $$KEY"; echo "[caddy] wildcard cert for *.$$DOMAIN" - elif printf '%s' "$$BASE_URL" | grep -q '^https://'; then - TLS=$$(printf 'tls {\n\t\ton_demand\n\t}'); echo "[caddy] on-demand Let's Encrypt per host for *.$$DOMAIN" - else - TLS="tls internal"; echo "[caddy] self-signed internal CA for *.$$DOMAIN" - fi - printf '{\n\temail %s\n\tauto_https disable_redirects\n\ton_demand_tls {\n\t\task http://localhost:7000/check\n\t}\n}\n:7000 {\n\trespond /check 200\n}\n*.%s, %s {\n\t%s\n\tencode zstd gzip\n\treverse_proxy archivebox:8000\n}\n' \ - "$$ARCHIVEBOX_ACME_EMAIL" "$$DOMAIN" "$$DOMAIN" "$$TLS" > /tmp/Caddyfile - exec caddy run --config /tmp/Caddyfile --adapter caddyfile - - # Optional DNS-01 WILDCARD cert sidecar for the `https` profile (go-acme/lego, - # ~150 DNS providers). Pick one with ARCHIVEBOX_ACME_DNS and put that provider's - # native credentials in a .env next to this file — they're passed straight to lego: - # cloudflare -> ARCHIVEBOX_ACME_DNS=cloudflare + CLOUDFLARE_DNS_API_TOKEN=... - # route53 -> ARCHIVEBOX_ACME_DNS=route53 + AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_REGION - # digitalocean -> ARCHIVEBOX_ACME_DNS=digitalocean + DO_AUTH_TOKEN - # ... full list + var names: https://go-acme.github.io/lego/dns/ - # Leave ARCHIVEBOX_ACME_DNS unset to skip DNS-01 entirely — Caddy then does plain - # Let's Encrypt on-demand (HTTP-01), so no DNS provider/credentials are required. - lego: - image: goacme/lego - profiles: ["https"] - restart: unless-stopped environment: - BASE_URL=${BASE_URL:-https://archive.example.com} - ARCHIVEBOX_ACME_EMAIL=${ARCHIVEBOX_ACME_EMAIL:-admin@example.com} - ARCHIVEBOX_ACME_DNS=${ARCHIVEBOX_ACME_DNS:-} env_file: - - path: .env # passes your provider's native creds (CLOUDFLARE_DNS_API_TOKEN, AWS_*, DO_AUTH_TOKEN, ...) to lego + - path: .env # passes your DNS provider's creds (CLOUDFLARE_DNS_API_TOKEN, AWS_*, DO_AUTH_TOKEN, ...) to Traefik required: false volumes: - - ./data/proxy/certs:/certs + - ./data/proxy/traefik:/certs # Traefik stores acme.json (the wildcard cert) here entrypoint: - sh - -c - | - set -u + set -eu DOMAIN=$$(printf '%s' "$$BASE_URL" | sed -E 's#^[a-z]+://##; s#[:/].*##') - if [ -z "$${ARCHIVEBOX_ACME_DNS:-}" ]; then - echo "[lego] no ARCHIVEBOX_ACME_DNS set -> skipping DNS-01; caddy will use on-demand Let's Encrypt for $$DOMAIN" - exec sleep infinity + # catch-all router -> archivebox (Host-routed); domain-free, so no docker socket needed + printf 'http:\n routers:\n archivebox:\n rule: "HostRegexp(`^.+$$`)"\n service: archivebox\n services:\n archivebox:\n loadBalancer:\n servers:\n - url: "http://archivebox:8000"\n' > /etc/traefik/dynamic.yml + set -- --entrypoints.web.address=:80 --entrypoints.websecure.address=:443 \ + --entrypoints.web.http.redirections.entrypoint.to=websecure \ + --entrypoints.web.http.redirections.entrypoint.scheme=https \ + --providers.file.filename=/etc/traefik/dynamic.yml + if [ -n "$${ARCHIVEBOX_ACME_DNS:-}" ]; then + echo "[traefik] wildcard cert for *.$$DOMAIN via $$ARCHIVEBOX_ACME_DNS DNS-01" + set -- "$$@" --entrypoints.websecure.http.tls.certresolver=le \ + --entrypoints.websecure.http.tls.domains[0].main="$$DOMAIN" \ + --entrypoints.websecure.http.tls.domains[0].sans="*.$$DOMAIN" \ + --certificatesresolvers.le.acme.email="$$ARCHIVEBOX_ACME_EMAIL" \ + --certificatesresolvers.le.acme.storage=/certs/acme.json \ + --certificatesresolvers.le.acme.dnschallenge=true \ + --certificatesresolvers.le.acme.dnschallenge.provider="$$ARCHIVEBOX_ACME_DNS" + else + echo "[traefik] no ARCHIVEBOX_ACME_DNS set -> serving Traefik's default self-signed cert (set a DNS provider for real wildcard TLS)" fi - ACTION=run; EXTRA="" - while :; do - echo "[lego] $$ACTION *.$$DOMAIN + $$DOMAIN via $$ARCHIVEBOX_ACME_DNS DNS-01" - /lego $$ACTION --accept-tos --email "$$ARCHIVEBOX_ACME_EMAIL" --dns "$$ARCHIVEBOX_ACME_DNS" \ - -d "*.$$DOMAIN" -d "$$DOMAIN" --path /certs/lego $$EXTRA \ - || echo "[lego] $$ACTION failed; will retry next cycle" - ACTION=renew; EXTRA="--days 30" - sleep 43200 # 12h; renew is a no-op until within 30 days of expiry - done + exec traefik "$$@" ### Example: run all your ArchiveBox traffic through a WireGuard VPN tunnel to avoid IP blocks. # You can also use any other VPN that works at the docker/IP level, e.g. Tailscale, OpenVPN, etc. diff --git a/etc/README.md b/etc/README.md index b45a4b60..6244aa61 100644 --- a/etc/README.md +++ b/etc/README.md @@ -9,10 +9,10 @@ here at all — it's built into the main `../docker-compose.yml` as two opt-in, driven profiles (no extra files, Dockerfiles, or scripts) — set the documented env vars in a `.env` next to `../docker-compose.yml`: -- `https` — Caddy + a `goacme/lego` DNS-01 sidecar fetch a single `*.` - wildcard cert (covering unlimited `snap-*` subdomains, ~150 DNS providers, no - per-provider code) and serve it, gracefully degrading to a self-signed cert if none - is available yet. +- `https` — a single Traefik container terminates TLS and fetches/auto-renews one + `*.` wildcard cert via DNS-01 (covering unlimited `snap-*` subdomains, + ~100 DNS providers via its embedded lego, no per-provider code), serving Traefik's + default self-signed cert if no DNS provider is configured. - `tunnel` — a Cloudflare Tunnel whose tunnel/DNS are auto-provisioned from your API token, so Cloudflare's edge terminates TLS and routes `*.` through one tunnel to ArchiveBox (Host-routed) — no public IP or wildcard cert needed locally.