mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-12 19:50:57 +05:00
wip: in-progress models tweaks + Dockerfile.multistage
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Sweeting
parent
7ea387887c
commit
6230d0297b
218
Dockerfile.multistage
Normal file
218
Dockerfile.multistage
Normal file
@ -0,0 +1,218 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
|
||||
# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
|
||||
# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
|
||||
# Build abx-dl first, then point this file at it:
|
||||
# docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
|
||||
# --build-context abxbus=../abxbus \
|
||||
# --build-context abxpkg=../abxpkg \
|
||||
# --build-context abx-plugins=../abx-plugins \
|
||||
# -t archivebox/abx-dl:dev
|
||||
# docker buildx build . -f Dockerfile.multistage \
|
||||
# --build-context abx-dl=docker-image://archivebox/abx-dl:dev \
|
||||
# --build-arg ABX_DL_IMAGE=abx-dl \
|
||||
# -t archivebox:multistage
|
||||
|
||||
ARG TARGETPLATFORM=linux/amd64
|
||||
ARG TARGETOS=linux
|
||||
ARG TARGETARCH=amd64
|
||||
ARG TARGETVARIANT=
|
||||
ARG ABX_DL_IMAGE=abx-dl
|
||||
|
||||
FROM ${ABX_DL_IMAGE} AS abx-dl
|
||||
FROM archivebox/sonic:1.4.9 AS sonic
|
||||
FROM ubuntu:24.04 AS archivebox-runtime-base
|
||||
|
||||
ARG TARGETPLATFORM=linux/amd64
|
||||
ARG TARGETOS=linux
|
||||
ARG TARGETARCH=amd64
|
||||
ARG TARGETVARIANT
|
||||
|
||||
ENV TZ=UTC \
|
||||
LANGUAGE=en_US:en \
|
||||
LC_ALL=C.UTF-8 \
|
||||
LANG=C.UTF-8 \
|
||||
DEBIAN_FRONTEND=noninteractive \
|
||||
APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
|
||||
PYTHONIOENCODING=UTF-8 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
npm_config_loglevel=error
|
||||
|
||||
ENV PYTHON_VERSION=3.13 \
|
||||
NODE_VERSION=22.22.3
|
||||
|
||||
ENV ARCHIVEBOX_USER=archivebox \
|
||||
DEFAULT_PUID=911 \
|
||||
DEFAULT_PGID=911 \
|
||||
IN_DOCKER=True
|
||||
|
||||
ENV CODE_DIR=/app \
|
||||
DATA_DIR=/data \
|
||||
LIB_DIR=/opt/archivebox/lib \
|
||||
LIB_BIN_DIR=/opt/archivebox/lib/bin \
|
||||
ABXPKG_LIB_DIR=/opt/archivebox/lib \
|
||||
PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
|
||||
PERSONAS_DIR=/data/personas \
|
||||
CHROME_EXTENSIONS_DIR=/opt/archivebox/lib/chrome_extensions \
|
||||
CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
|
||||
CHROME_HEADLESS=true \
|
||||
CHROME_SANDBOX=false \
|
||||
CHROME_ISOLATION=crawl \
|
||||
CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
|
||||
|
||||
ENV TMP_DIR=/tmp/archivebox \
|
||||
PIP_VENV_PYTHON=/venv/bin/python3 \
|
||||
GOOGLE_API_KEY=no \
|
||||
GOOGLE_DEFAULT_CLIENT_ID=no \
|
||||
GOOGLE_DEFAULT_CLIENT_SECRET=no
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=0 \
|
||||
UV_PYTHON_PREFERENCE=managed \
|
||||
UV_PYTHON_INSTALL_DIR=/opt/uv/python \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PROJECT_ENVIRONMENT=/venv \
|
||||
VIRTUAL_ENV=/venv \
|
||||
PATH="/opt/archivebox/lib/bin:/venv/bin:/opt/node/bin:$PATH"
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
|
||||
WORKDIR "$CODE_DIR"
|
||||
|
||||
RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
|
||||
&& echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
|
||||
&& echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
|
||||
&& rm -f /etc/apt/apt.conf.d/docker-clean
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y \
|
||||
apt-transport-https apt-utils ca-certificates curl wget gnupg2 \
|
||||
dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
|
||||
cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \
|
||||
libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \
|
||||
libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
|
||||
libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \
|
||||
libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \
|
||||
fonts-liberation fonts-noto-color-emoji xdg-utils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Runtime-owned layers copied from the abx-dl image.
|
||||
COPY --from=abx-dl /bin/uv /bin/uv
|
||||
COPY --from=abx-dl /opt/uv/python /opt/uv/python
|
||||
COPY --from=abx-dl /opt/node /opt/node
|
||||
COPY --from=abx-dl /venv /venv
|
||||
COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
|
||||
|
||||
RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
|
||||
&& echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
|
||||
&& uname -a \
|
||||
&& sed -n '1,7p' /etc/os-release \
|
||||
&& which python3 && python3 --version \
|
||||
&& which node && node --version \
|
||||
&& which uv && uv self version \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
FROM archivebox-runtime-base AS archivebox-builder
|
||||
|
||||
WORKDIR "$CODE_DIR"
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
|
||||
echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y --no-install-recommends \
|
||||
build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
|
||||
&& uv sync \
|
||||
--refresh \
|
||||
--no-dev \
|
||||
--inexact \
|
||||
--all-extras \
|
||||
--no-install-project \
|
||||
--no-install-workspace \
|
||||
--no-sources \
|
||||
&& apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
|
||||
&& apt-get autoremove -y \
|
||||
&& find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
|
||||
RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
|
||||
&& uv pip install --no-deps "$CODE_DIR" \
|
||||
&& (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
|
||||
&& find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
|
||||
|
||||
FROM archivebox-runtime-base
|
||||
|
||||
LABEL name="archivebox" \
|
||||
maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
|
||||
description="All-in-one self-hosted internet archiving solution" \
|
||||
homepage="https://github.com/ArchiveBox/ArchiveBox" \
|
||||
documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
|
||||
org.opencontainers.image.title="ArchiveBox" \
|
||||
org.opencontainers.image.vendor="ArchiveBox" \
|
||||
org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
|
||||
org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
|
||||
com.docker.image.source.entrypoint="Dockerfile.multistage"
|
||||
|
||||
COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
|
||||
COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
|
||||
|
||||
COPY --from=archivebox-builder /venv /venv
|
||||
COPY --from=archivebox-builder /app /app
|
||||
COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
|
||||
COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib
|
||||
|
||||
RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
|
||||
&& groupadd --system "$ARCHIVEBOX_USER" \
|
||||
&& useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
|
||||
&& usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
|
||||
&& groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
|
||||
&& (which sonic && sonic --version) | tee -a /VERSION.txt \
|
||||
&& mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$LIB_BIN_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& openssl rand -hex 16 > /etc/machine-id \
|
||||
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
|
||||
&& echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
|
||||
|
||||
WORKDIR "$DATA_DIR"
|
||||
RUN echo "[+] Initializing image collection..." \
|
||||
&& PUID=0 PGID=0 archivebox init \
|
||||
&& find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
|
||||
&& (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
|
||||
"$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
|
||||
"$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
|
||||
"$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
|
||||
"$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
|
||||
2>/dev/null || true)
|
||||
|
||||
RUN chmod +x "$CODE_DIR"/bin/*.sh \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
|
||||
&& chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_BIN_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
|
||||
&& gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
|
||||
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
|
||||
|
||||
RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
|
||||
&& echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
WORKDIR "$DATA_DIR"
|
||||
VOLUME "$DATA_DIR"
|
||||
EXPOSE 8000
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
|
||||
CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
|
||||
|
||||
ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
|
||||
CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
|
||||
@ -14,7 +14,8 @@ from urllib.parse import urlparse
|
||||
from statemachine import State, registry
|
||||
|
||||
from django.db import models, transaction
|
||||
from django.db.models import Q, QuerySet, Sum
|
||||
from django.db.models import Case, Q, QuerySet, Sum, Value, When
|
||||
from django.db.models.functions import Concat
|
||||
from django.db.models.fields.json import KT
|
||||
from django.utils.functional import cached_property
|
||||
from django.utils.text import slugify
|
||||
@ -756,44 +757,31 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
|
||||
crawl = self.crawl
|
||||
if not crawl.url_passes_filters(self.url, snapshot=self):
|
||||
return
|
||||
# TEMP INSTRUMENTATION: count retries to verify root cause
|
||||
import logging as _logging
|
||||
import os
|
||||
|
||||
for attempt in range(16):
|
||||
crawl.refresh_from_db()
|
||||
existing_urls = {url for _raw_line, url in crawl._iter_url_lines() if url}
|
||||
if self.url in existing_urls:
|
||||
if attempt > 0:
|
||||
_logging.getLogger("archivebox.crawls").warning(
|
||||
"CAS_RETRY[%d] another_writer_won pid=%d snap=%s crawl=%s url=%s",
|
||||
attempt,
|
||||
os.getpid(),
|
||||
self.id,
|
||||
crawl.id,
|
||||
self.url,
|
||||
)
|
||||
return
|
||||
urls = f"{crawl.urls}\n{self.url}"
|
||||
if crawl.safe_update({"urls": urls, "modified_at": timezone.now()}, refresh=False):
|
||||
if attempt > 0:
|
||||
_logging.getLogger("archivebox.crawls").warning(
|
||||
"CAS_RETRY[%d] succeeded pid=%d snap=%s crawl=%s url=%s",
|
||||
attempt,
|
||||
os.getpid(),
|
||||
self.id,
|
||||
crawl.id,
|
||||
self.url,
|
||||
)
|
||||
crawl.urls = urls
|
||||
return
|
||||
_logging.getLogger("archivebox.crawls").error(
|
||||
"CAS_RETRY exhausted pid=%d snap=%s crawl=%s url=%s",
|
||||
os.getpid(),
|
||||
self.id,
|
||||
crawl.id,
|
||||
self.url,
|
||||
# Best-effort skip if our URL is already recorded on the crawl;
|
||||
# the atomic UPDATE below is what actually prevents clobbering.
|
||||
crawl.refresh_from_db(fields=["urls"])
|
||||
if self.url in {url for _raw_line, url in crawl._iter_url_lines() if url}:
|
||||
return
|
||||
now = timezone.now()
|
||||
# Atomic append: SQLite reads `urls` inside the UPDATE statement,
|
||||
# so concurrent appends never clobber each other (no read-then-write
|
||||
# window, no CAS retry needed). A rare duplicate URL on a race is
|
||||
# harmless — downstream consumers dedupe via Snapshot uniqueness.
|
||||
text = models.TextField()
|
||||
type(crawl).objects.filter(pk=crawl.pk).update(
|
||||
urls=Case(
|
||||
When(Q(urls="") | Q(urls__isnull=True), then=Value(self.url, output_field=text)),
|
||||
default=Concat(
|
||||
"urls",
|
||||
Value("\n", output_field=text),
|
||||
Value(self.url, output_field=text),
|
||||
output_field=text,
|
||||
),
|
||||
output_field=text,
|
||||
),
|
||||
modified_at=now,
|
||||
)
|
||||
crawl.modified_at = now
|
||||
|
||||
# get_or_create/update_or_create wrap save() in atomic(); defer filesystem
|
||||
# work and crawl maintenance so SQLite commits before touching the disk.
|
||||
|
||||
@ -228,21 +228,9 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
|
||||
current = type(self).objects.filter(pk=self.pk).values("modified_at", self.state_field_name).first()
|
||||
current_modified_at = current.get("modified_at") if current else "<deleted>"
|
||||
current_status = current.get(self.state_field_name) if current else "<deleted>"
|
||||
# TEMP INSTRUMENTATION: capture caller stack so we can identify
|
||||
# who's racing on the same row.
|
||||
import traceback as _tb
|
||||
|
||||
caller_frame = "<unknown>"
|
||||
try:
|
||||
stack = _tb.extract_stack(limit=12)[:-1]
|
||||
relevant = [f"{Path(s.filename).name}:{s.lineno}:{s.name}" for s in stack if "site-packages" not in s.filename][-6:]
|
||||
caller_frame = " <- ".join(relevant)
|
||||
except Exception:
|
||||
pass
|
||||
logger.error(
|
||||
"\nSafeUpdateCASMiss: %s row %s was modified by another process while writing; "
|
||||
"loaded modified_at=%s loaded %s=%s current modified_at=%s current %s=%s "
|
||||
"update_fields=%s extra_filter=%s caller=%s\n",
|
||||
"loaded modified_at=%s loaded %s=%s current modified_at=%s current %s=%s update_fields=%s extra_filter=%s\n",
|
||||
type(self).__name__,
|
||||
self.pk,
|
||||
self.modified_at,
|
||||
@ -253,7 +241,6 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
|
||||
current_status,
|
||||
sorted(values),
|
||||
extra_filter or {},
|
||||
caller_frame,
|
||||
)
|
||||
if refresh:
|
||||
try:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user