From 6230d0297bd8dc6666d78ee6d3d2e3857e4cc047 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sun, 31 May 2026 02:20:11 -0700 Subject: [PATCH] wip: in-progress models tweaks + Dockerfile.multistage Co-Authored-By: Claude Opus 4.7 (1M context) --- Dockerfile.multistage | 218 +++++++++++++++++++++++++++++++++++ archivebox/core/models.py | 64 +++++----- archivebox/workers/models.py | 15 +-- 3 files changed, 245 insertions(+), 52 deletions(-) create mode 100644 Dockerfile.multistage diff --git a/Dockerfile.multistage b/Dockerfile.multistage new file mode 100644 index 00000000..fee94595 --- /dev/null +++ b/Dockerfile.multistage @@ -0,0 +1,218 @@ +# syntax=docker/dockerfile:1.7 + +# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image. +# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes. +# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime. +# Build abx-dl first, then point this file at it: +# docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \ +# --build-context abxbus=../abxbus \ +# --build-context abxpkg=../abxpkg \ +# --build-context abx-plugins=../abx-plugins \ +# -t archivebox/abx-dl:dev +# docker buildx build . -f Dockerfile.multistage \ +# --build-context abx-dl=docker-image://archivebox/abx-dl:dev \ +# --build-arg ABX_DL_IMAGE=abx-dl \ +# -t archivebox:multistage + +ARG TARGETPLATFORM=linux/amd64 +ARG TARGETOS=linux +ARG TARGETARCH=amd64 +ARG TARGETVARIANT= +ARG ABX_DL_IMAGE=abx-dl + +FROM ${ABX_DL_IMAGE} AS abx-dl +FROM archivebox/sonic:1.4.9 AS sonic +FROM ubuntu:24.04 AS archivebox-runtime-base + +ARG TARGETPLATFORM=linux/amd64 +ARG TARGETOS=linux +ARG TARGETARCH=amd64 +ARG TARGETVARIANT + +ENV TZ=UTC \ + LANGUAGE=en_US:en \ + LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 \ + DEBIAN_FRONTEND=noninteractive \ + APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \ + PYTHONIOENCODING=UTF-8 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + npm_config_loglevel=error + +ENV PYTHON_VERSION=3.13 \ + NODE_VERSION=22.22.3 + +ENV ARCHIVEBOX_USER=archivebox \ + DEFAULT_PUID=911 \ + DEFAULT_PGID=911 \ + IN_DOCKER=True + +ENV CODE_DIR=/app \ + DATA_DIR=/data \ + LIB_DIR=/opt/archivebox/lib \ + LIB_BIN_DIR=/opt/archivebox/lib/bin \ + ABXPKG_LIB_DIR=/opt/archivebox/lib \ + PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \ + PERSONAS_DIR=/data/personas \ + CHROME_EXTENSIONS_DIR=/opt/archivebox/lib/chrome_extensions \ + CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \ + CHROME_HEADLESS=true \ + CHROME_SANDBOX=false \ + CHROME_ISOLATION=crawl \ + CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]' + +ENV TMP_DIR=/tmp/archivebox \ + PIP_VENV_PYTHON=/venv/bin/python3 \ + GOOGLE_API_KEY=no \ + GOOGLE_DEFAULT_CLIENT_ID=no \ + GOOGLE_DEFAULT_CLIENT_SECRET=no + +ENV UV_COMPILE_BYTECODE=0 \ + UV_PYTHON_PREFERENCE=managed \ + UV_PYTHON_INSTALL_DIR=/opt/uv/python \ + UV_LINK_MODE=copy \ + UV_PROJECT_ENVIRONMENT=/venv \ + VIRTUAL_ENV=/venv \ + PATH="/opt/archivebox/lib/bin:/venv/bin:/opt/node/bin:$PATH" + +SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] +WORKDIR "$CODE_DIR" + +RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \ + && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \ + && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \ + && rm -f /etc/apt/apt.conf.d/docker-clean + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ + echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \ + && apt-get update -qq \ + && apt-get install -qq -y \ + apt-transport-https apt-utils ca-certificates curl wget gnupg2 \ + dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \ + cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \ + libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \ + libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \ + libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \ + libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \ + fonts-liberation fonts-noto-color-emoji xdg-utils \ + && rm -rf /var/lib/apt/lists/* + +# Runtime-owned layers copied from the abx-dl image. +COPY --from=abx-dl /bin/uv /bin/uv +COPY --from=abx-dl /opt/uv/python /opt/uv/python +COPY --from=abx-dl /opt/node /opt/node +COPY --from=abx-dl /venv /venv +COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt + +RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ + && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \ + && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \ + && uname -a \ + && sed -n '1,7p' /etc/os-release \ + && which python3 && python3 --version \ + && which node && node --version \ + && which uv && uv self version \ + ) | tee -a /VERSION.txt + +FROM archivebox-runtime-base AS archivebox-builder + +WORKDIR "$CODE_DIR" +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ + --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ + --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ + echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \ + && apt-get update -qq \ + && apt-get install -qq -y --no-install-recommends \ + build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ + && uv sync \ + --refresh \ + --no-dev \ + --inexact \ + --all-extras \ + --no-install-project \ + --no-install-workspace \ + --no-sources \ + && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ + && apt-get autoremove -y \ + && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \ + && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ + && rm -rf /var/lib/apt/lists/* + +COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" +RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ + echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \ + && uv pip install --no-deps "$CODE_DIR" \ + && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \ + && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ + && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete + +FROM archivebox-runtime-base + +LABEL name="archivebox" \ + maintainer="Nick Sweeting " \ + description="All-in-one self-hosted internet archiving solution" \ + homepage="https://github.com/ArchiveBox/ArchiveBox" \ + documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \ + org.opencontainers.image.title="ArchiveBox" \ + org.opencontainers.image.vendor="ArchiveBox" \ + org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \ + org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \ + com.docker.image.source.entrypoint="Dockerfile.multistage" + +COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic +COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg + +COPY --from=archivebox-builder /venv /venv +COPY --from=archivebox-builder /app /app +COPY --from=archivebox-builder /VERSION.txt /VERSION.txt +COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib + +RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ + && groupadd --system "$ARCHIVEBOX_USER" \ + && useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \ + && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ + && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ + && (which sonic && sonic --version) | tee -a /VERSION.txt \ + && mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$LIB_BIN_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && openssl rand -hex 16 > /etc/machine-id \ + && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \ + && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt + +WORKDIR "$DATA_DIR" +RUN echo "[+] Initializing image collection..." \ + && PUID=0 PGID=0 archivebox init \ + && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ + && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ + && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \ + "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \ + "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \ + "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \ + "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \ + 2>/dev/null || true) + +RUN chmod +x "$CODE_DIR"/bin/*.sh \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ + && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_BIN_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \ + && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \ + && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ + && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ + && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/* + +RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \ + && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \ + && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \ + ) | tee -a /VERSION.txt + +WORKDIR "$DATA_DIR" +VOLUME "$DATA_DIR" +EXPOSE 8000 + +HEALTHCHECK --interval=30s --timeout=20s --retries=15 \ + CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK' + +ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"] +CMD ["archivebox", "server", "--init", "0.0.0.0:8000"] diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 496e57cb..46ddf6f9 100755 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -14,7 +14,8 @@ from urllib.parse import urlparse from statemachine import State, registry from django.db import models, transaction -from django.db.models import Q, QuerySet, Sum +from django.db.models import Case, Q, QuerySet, Sum, Value, When +from django.db.models.functions import Concat from django.db.models.fields.json import KT from django.utils.functional import cached_property from django.utils.text import slugify @@ -756,44 +757,31 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW crawl = self.crawl if not crawl.url_passes_filters(self.url, snapshot=self): return - # TEMP INSTRUMENTATION: count retries to verify root cause - import logging as _logging - import os - - for attempt in range(16): - crawl.refresh_from_db() - existing_urls = {url for _raw_line, url in crawl._iter_url_lines() if url} - if self.url in existing_urls: - if attempt > 0: - _logging.getLogger("archivebox.crawls").warning( - "CAS_RETRY[%d] another_writer_won pid=%d snap=%s crawl=%s url=%s", - attempt, - os.getpid(), - self.id, - crawl.id, - self.url, - ) - return - urls = f"{crawl.urls}\n{self.url}" - if crawl.safe_update({"urls": urls, "modified_at": timezone.now()}, refresh=False): - if attempt > 0: - _logging.getLogger("archivebox.crawls").warning( - "CAS_RETRY[%d] succeeded pid=%d snap=%s crawl=%s url=%s", - attempt, - os.getpid(), - self.id, - crawl.id, - self.url, - ) - crawl.urls = urls - return - _logging.getLogger("archivebox.crawls").error( - "CAS_RETRY exhausted pid=%d snap=%s crawl=%s url=%s", - os.getpid(), - self.id, - crawl.id, - self.url, + # Best-effort skip if our URL is already recorded on the crawl; + # the atomic UPDATE below is what actually prevents clobbering. + crawl.refresh_from_db(fields=["urls"]) + if self.url in {url for _raw_line, url in crawl._iter_url_lines() if url}: + return + now = timezone.now() + # Atomic append: SQLite reads `urls` inside the UPDATE statement, + # so concurrent appends never clobber each other (no read-then-write + # window, no CAS retry needed). A rare duplicate URL on a race is + # harmless — downstream consumers dedupe via Snapshot uniqueness. + text = models.TextField() + type(crawl).objects.filter(pk=crawl.pk).update( + urls=Case( + When(Q(urls="") | Q(urls__isnull=True), then=Value(self.url, output_field=text)), + default=Concat( + "urls", + Value("\n", output_field=text), + Value(self.url, output_field=text), + output_field=text, + ), + output_field=text, + ), + modified_at=now, ) + crawl.modified_at = now # get_or_create/update_or_create wrap save() in atomic(); defer filesystem # work and crawl maintenance so SQLite commits before touching the disk. diff --git a/archivebox/workers/models.py b/archivebox/workers/models.py index 85f6cade..bc24cb43 100644 --- a/archivebox/workers/models.py +++ b/archivebox/workers/models.py @@ -228,21 +228,9 @@ class BaseModelWithStateMachine(models.Model, MachineMixin): current = type(self).objects.filter(pk=self.pk).values("modified_at", self.state_field_name).first() current_modified_at = current.get("modified_at") if current else "" current_status = current.get(self.state_field_name) if current else "" - # TEMP INSTRUMENTATION: capture caller stack so we can identify - # who's racing on the same row. - import traceback as _tb - - caller_frame = "" - try: - stack = _tb.extract_stack(limit=12)[:-1] - relevant = [f"{Path(s.filename).name}:{s.lineno}:{s.name}" for s in stack if "site-packages" not in s.filename][-6:] - caller_frame = " <- ".join(relevant) - except Exception: - pass logger.error( "\nSafeUpdateCASMiss: %s row %s was modified by another process while writing; " - "loaded modified_at=%s loaded %s=%s current modified_at=%s current %s=%s " - "update_fields=%s extra_filter=%s caller=%s\n", + "loaded modified_at=%s loaded %s=%s current modified_at=%s current %s=%s update_fields=%s extra_filter=%s\n", type(self).__name__, self.pk, self.modified_at, @@ -253,7 +241,6 @@ class BaseModelWithStateMachine(models.Model, MachineMixin): current_status, sorted(values), extra_filter or {}, - caller_frame, ) if refresh: try: