The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' into claude/upbeat-rubin-FQdEL
Some checks failed
Build Docker image / build ${{ matrix.platform }} (digest-linux-amd64, docker-amd64, linux/amd64, ubuntu-24.04) (push) Has been cancelled
Details
Build Docker image / build ${{ matrix.platform }} (digest-linux-arm64, docker-arm64, linux/arm64, ubuntu-24.04-arm) (push) Has been cancelled
Details
Run linters / lint (push) Has been cancelled
Details
Build Pip package / build (push) Has been cancelled
Details
Release State / release-state (push) Has been cancelled
Details
Run tests / python_tests (ubuntu-22.04, 3.13) (push) Has been cancelled
Details
Run tests / docker_tests (push) Has been cancelled
Details
Build Docker image / publish multiarch tags (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-06 02:38:43 -04:00 committed by GitHub
commit cea57de5f6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
32 changed files with 904 additions and 577 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.dockerignore
View File

@ -12,9 +12,14 @@ __pycache__/
.pdm-python
.eggs/
.git/
!.git/
.git/*
.vscode/
!.git/HEAD
!.git/refs/heads/*
!.git/packed-refs
!.git/refs/
!.git/refs/heads/
!.git/refs/heads/**
venv/
.venv/

72
.github/workflows/docker.yml vendored
View File

@ -5,7 +5,7 @@ on:
workflow_call:
push:
branches:
- dev
- "**"
tags:
- 'v*'
# pull_request:
@ -13,13 +13,14 @@ on:
env:
DOCKERHUB_IMAGE: archivebox/archivebox
GHCR_IMAGE: ghcr.io/archivebox/archivebox
ABX_DL_IMAGE: archivebox/abx-dl:latest
permissions:
contents: read
packages: write
concurrency:
group: docker-${{ github.ref }}
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
@ -96,6 +97,24 @@ jobs:
- name: Available platforms
run: echo ${{ steps.buildx.outputs.platforms }}
- name: Wait for published abx-dl image
id: abx_dl_image
shell: bash
run: |
set -Eeuo pipefail
deadline=$((SECONDS + 1800))
until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
if (( SECONDS >= deadline )); then
echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
exit 1
fi
echo "${ABX_DL_IMAGE} is not published yet; waiting..."
sleep 30
done
echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
docker buildx imagetools inspect "${ABX_DL_IMAGE}"
- name: Login to Docker Hub
uses: docker/login-action@v3
if: github.event_name != 'pull_request'
@ -116,13 +135,10 @@ jobs:
run: |
set -Eeuo pipefail
VERSION="$(python3 - <<'PY'
from pathlib import Path
import re
import tomllib
match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
if not match:
raise SystemExit("Failed to read version from pyproject.toml")
print(match.group(1))
with open("pyproject.toml", "rb") as f:
print(tomllib.load(f)["project"]["version"])
PY
)"
@ -131,11 +147,12 @@ jobs:
echo "org.opencontainers.image.version=${VERSION}"
echo "org.opencontainers.image.revision=${GITHUB_SHA}"
echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
echo 'EOF'
echo "version=${VERSION}"
} >> "$GITHUB_OUTPUT"
echo "[+] Building ${{ matrix.platform }} for ${VERSION}"
echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
- name: Build and push digest
id: docker_build
@ -149,6 +166,8 @@ jobs:
${{ env.DOCKERHUB_IMAGE }}
${{ env.GHCR_IMAGE }}
labels: ${{ steps.docker_meta.outputs.labels }}
build-args: |
ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
cache-from: type=gha,scope=${{ matrix.cache_scope }}
cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
platforms: ${{ matrix.platform }}
@ -215,36 +234,49 @@ jobs:
run: |
set -Eeuo pipefail
VERSION="$(python3 - <<'PY'
from pathlib import Path
import re
import tomllib
match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
if not match:
raise SystemExit("Failed to read version from pyproject.toml")
print(match.group(1))
with open("pyproject.toml", "rb") as f:
print(tomllib.load(f)["project"]["version"])
PY
)"
SHORT_SHA="${GITHUB_SHA::8}"
BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
SHORT_SHA="${GITHUB_SHA::12}"
test -n "$BRANCH_TAG"
test -n "$SHORT_SHA"
{
echo 'dockerhub_tags<<EOF'
echo "${DOCKERHUB_IMAGE}:dev"
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
echo "${DOCKERHUB_IMAGE}:latest"
fi
echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
echo "${DOCKERHUB_IMAGE}:${VERSION}"
echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
echo 'EOF'
echo 'ghcr_tags<<EOF'
echo "${GHCR_IMAGE}:dev"
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
echo "${GHCR_IMAGE}:latest"
fi
echo "${GHCR_IMAGE}:${BRANCH_TAG}"
echo "${GHCR_IMAGE}:${VERSION}"
echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
echo 'EOF'
echo "version=${VERSION}"
echo "branch_tag=${BRANCH_TAG}"
echo "short_sha=${SHORT_SHA}"
} >> "$GITHUB_OUTPUT"
echo "[+] Publishing Docker Hub tags:"
printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
fi
printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
echo "[+] Publishing GHCR tags:"
printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
printf '%s\n' "${GHCR_IMAGE}:latest"
fi
printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
- name: Create Docker Hub manifest
shell: bash

491
Dockerfile
View File

@ -1,66 +1,29 @@
# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
# python3.13, uv, python3-ldap
# curl, wget, git, dig, ping, tree, nano
# node, npm
# ArchiveBox and plugin runtime dependencies installed by archivebox init --install
# Usage:
# git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
# docker build . -t archivebox
# docker run -v "$PWD/data":/data archivebox init
# docker run -v "$PWD/data":/data archivebox add 'https://example.com'
# docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
# docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
# docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
# syntax=docker/dockerfile:1.7
# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
# Build abx-dl first, then point this file at it:
# docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
# --build-context abxbus=../abxbus \
# --build-context abxpkg=../abxpkg \
# --build-context abx-plugins=../abx-plugins \
# -t archivebox/abx-dl:dev
# docker buildx build . -f Dockerfile \
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
# -t archivebox:multistage
#########################################################################################
### Example: Using ArchiveBox in your own project's Dockerfile ########
# FROM python:3.13-slim
# WORKDIR /data
# RUN pip install archivebox>=0.9.0 # use latest release here
# RUN archivebox install
# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
#########################################################################################
ARG TARGETPLATFORM
ARG TARGETOS
ARG TARGETARCH
ARG TARGETVARIANT=
ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
FROM ${ABX_DL_IMAGE} AS abx-dl
FROM archivebox/sonic:1.4.9 AS sonic
FROM ubuntu:24.04
LABEL name="archivebox" \
maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
description="All-in-one self-hosted internet archiving solution" \
homepage="https://github.com/ArchiveBox/ArchiveBox" \
documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
org.opencontainers.image.title="ArchiveBox" \
org.opencontainers.image.vendor="ArchiveBox" \
org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
com.docker.image.source.entrypoint="Dockerfile" \
# TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
# https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
com.docker.desktop.extension.api.version=">= 1.4.7" \
com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
com.docker.extension.publisher-url="https://archivebox.io" \
com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
com.docker.extension.categories='database,utility-tools'
FROM ubuntu:24.04 AS archivebox-runtime-base
ARG TARGETPLATFORM
ARG TARGETOS
ARG TARGETARCH
ARG TARGETVARIANT
######### Environment Variables #################################
# Global build-time and runtime environment constants + default pkg manager config
ENV TZ=UTC \
LANGUAGE=en_US:en \
LC_ALL=C.UTF-8 \
@ -73,204 +36,93 @@ ENV TZ=UTC \
PIP_ONLY_BINARY=aiohttp \
npm_config_loglevel=error
# Language Version config
ENV PYTHON_VERSION=3.13 \
NODE_VERSION=22.22.3
# Non-root User config
ENV ARCHIVEBOX_USER="archivebox" \
ENV ARCHIVEBOX_USER=archivebox \
DEFAULT_PUID=911 \
DEFAULT_PGID=911 \
IN_DOCKER=True \
BIND_ADDR=0.0.0.0:8000
# Docker has to listen on all interfaces, not just localhost.
IN_DOCKER=True
# ArchiveBox Source Code + Lib + Data paths
ENV CODE_DIR=/app \
DATA_DIR=/data \
LIB_DIR=/opt/archivebox/lib \
ABXPKG_LIB_DIR=/opt/archivebox/lib \
SONIC_BINARY=/opt/archivebox/lib/env/bin/sonic \
PLAYWRIGHT_BROWSERS_PATH=/browsers
# Bash SHELL config
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
######### System Environment ####################################
# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
WORKDIR "$CODE_DIR"
# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
&& echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
&& echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
&& rm -f /etc/apt/apt.conf.d/docker-clean
# Print debug info about build and save it to disk, for human eyes only, not used by anything else
RUN (echo "[i] Docker build for ArchiveBox starting..." \
&& echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
&& echo \
&& echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
&& echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
&& echo \
&& uname -a \
&& sed -n '1,7p' /etc/os-release \
&& which bash && bash --version | sed -n '1p' \
&& which dpkg && dpkg --version | sed -n '1p' \
&& echo -e '\n\n' && env && echo -e '\n\n' \
) | tee -a /VERSION.txt
# Create non-privileged user for archivebox and chrome
RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
&& groupadd --system $ARCHIVEBOX_USER \
&& useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
&& usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
&& groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
| tee -a /VERSION.txt
# DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
# https://docs.linuxserver.io/general/understanding-puid-and-pgid
# Install system apt dependencies (adding backports to access more recent apt updates)
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
&& mkdir -p /etc/apt/keyrings \
&& apt-get update -qq \
&& apt-get install -qq -y \
# 1. packaging dependencies
apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
# 2. docker and init system dependencies
zlib1g-dev dumb-init gosu cron unzip grep dnsutils git ripgrep python3.12-venv default-jre-headless \
# 3. frivolous CLI helpers to make debugging failed archiving easier
tree nano iputils-ping \
# nano iputils-ping dnsutils htop procps jq yq
&& rm -rf /var/lib/apt/lists/*
# Install sonic search backend
COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
RUN (which sonic && sonic --version) | tee -a /VERSION.txt
######### Language Environments ####################################
# Set up Python environment
# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
# --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
# && apt-get update -qq \
# && apt-get install -qq -y --no-upgrade \
# python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
# && rm -rf /var/lib/apt/lists/* \
# tell PDM to allow using global system python site packages
# && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
# && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
# create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
# && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
# && python3 -m venv --system-site-packages $GLOBAL_VENV \
# && python3 -m venv $GLOBAL_VENV \
# install global dependencies / python build dependencies in GLOBAL_VENV
# && pip install --upgrade pip setuptools wheel \
# Save version info
# && ( \
# which python3 && python3 --version | grep " $PYTHON_VERSION" \
# && which pip && pip --version \
# # && which pdm && pdm --version \
# && echo -e '\n\n' \
# ) | tee -a /VERSION.txt
# Set up Node environment from the official platform tarball. This avoids
# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
# emulated Docker builds.
RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
case "$TARGETARCH" in \
amd64) NODE_DIST_ARCH="x64" ;; \
arm64) NODE_DIST_ARCH="arm64" ;; \
*) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
esac \
&& NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
&& echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
&& curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
&& curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
&& grep " ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
&& tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
&& rm "$NODE_TARBALL" SHASUMS256.txt \
# Save version info
&& ( \
which node && node --version \
&& which npm && npm --version \
&& echo -e '\n\n' \
) | tee -a /VERSION.txt
# Set up uv and main app /venv
RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
ENV UV_COMPILE_BYTECODE=0 \
UV_PYTHON_PREFERENCE=managed \
UV_PYTHON_INSTALL_DIR=/opt/uv/python \
UV_LINK_MODE=copy \
UV_PROJECT_ENVIRONMENT=/venv
WORKDIR "$CODE_DIR"
# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
&& uv venv /venv --python ${PYTHON_VERSION}
ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
RUN uv pip install setuptools pip \
&& ( \
which python3 && python3 --version \
&& which uv && uv self version \
&& uv python find --system && uv python find \
&& echo -e '\n\n' \
) | tee -a /VERSION.txt
######### ArchiveBox & Extractor Dependencies ##################################
# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
WORKDIR "$CODE_DIR"
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
--mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
#--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
&& apt-get update -qq \
&& apt-get install -qq -y --no-install-recommends \
build-essential gcc \
python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
&& uv pip install \
"python-ldap>=3.4.3" \
&& apt-get purge -y \
python3-dev build-essential gcc \
&& apt-get autoremove -y \
&& rm -rf /var/lib/apt/lists/*
# Runtime config used by plugin hooks. Plugin binaries and npm packages are
# installed into LIB_DIR below by archivebox init --install and resolved from
# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
ENV PERSONAS_DIR=/data/personas \
PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
PERSONAS_DIR=/data/personas \
CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
CHROME_HEADLESS=true \
CHROME_SANDBOX=false \
CHROME_ISOLATION=crawl \
CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
######### Build Dependencies ####################################
ENV TMP_DIR=/tmp/archivebox \
PIP_VENV_PYTHON=/venv/bin/python3 \
GOOGLE_API_KEY=no \
GOOGLE_DEFAULT_CLIENT_ID=no \
GOOGLE_DEFAULT_CLIENT_SECRET=no
ENV UV_COMPILE_BYTECODE=0 \
UV_PYTHON_PREFERENCE=managed \
UV_PYTHON_INSTALL_DIR=/opt/uv/python \
UV_LINK_MODE=copy \
UV_PROJECT_ENVIRONMENT=/venv \
VIRTUAL_ENV=/venv \
PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
# Install ArchiveBox Python venv dependencies from pyproject.toml.
RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
--mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
--mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
WORKDIR "$CODE_DIR"
RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
&& echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
&& echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
&& rm -f /etc/apt/apt.conf.d/docker-clean
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \
&& apt-get update -qq \
&& apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
&& uv --no-cache sync \
&& apt-get install -qq -y \
apt-transport-https apt-utils ca-certificates curl wget gnupg2 \
dumb-init util-linux unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \
libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \
libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \
libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \
fonts-liberation fonts-noto-color-emoji xdg-utils \
ffmpeg imagemagick tesseract-ocr tesseract-ocr-eng openjdk-21-jre-headless \
&& rm -rf /var/lib/apt/lists/*
# Runtime-owned layers copied from the abx-dl image.
COPY --from=abx-dl /bin/uv /bin/uv
COPY --from=abx-dl /opt/uv/python /opt/uv/python
COPY --from=abx-dl /opt/node /opt/node
COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
&& echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
&& uname -a \
&& sed -n '1,7p' /etc/os-release \
&& which node && node --version \
&& which uv && uv self version \
) | tee -a /VERSION.txt
ENV PYTHONDONTWRITEBYTECODE=1
FROM archivebox-runtime-base AS archivebox-builder
WORKDIR "$CODE_DIR"
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
--mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
--mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
&& apt-get update -qq \
&& apt-get install -qq -y --no-install-recommends \
build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
&& uv venv /venv --python "${PYTHON_VERSION}" \
&& uv pip install setuptools pip wheel \
&& uv sync \
--refresh \
--no-dev \
--inexact \
@ -278,110 +130,73 @@ RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
--no-install-project \
--no-install-workspace \
--no-sources \
&& apt-get purge -y python3-dev build-essential gcc \
&& apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
&& apt-get autoremove -y \
&& find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
&& find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
&& rm -rf /var/lib/apt/lists/*
# installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
# Setup ArchiveBox runtime config
ENV TMP_DIR=/tmp/archivebox \
PIP_VENV_PYTHON=/usr/bin/python3.12 \
GOOGLE_API_KEY=no \
GOOGLE_DEFAULT_CLIENT_ID=no \
GOOGLE_DEFAULT_CLIENT_SECRET=no
WORKDIR "$DATA_DIR"
RUN openssl rand -hex 16 > /etc/machine-id \
&& mkdir -p "$DATA_DIR" \
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
&& mkdir -p "$TMP_DIR" \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
&& mkdir -p "$LIB_DIR" \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
&& mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
&& echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
# path users run later, before copying ArchiveBox source so source-only edits do
# not invalidate the heavy browser/plugin dependency layer.
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
--mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
--mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
--mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
--mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
&& export PERSONAS_DIR="$LIB_DIR/personas" \
&& export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
&& export ABX_RUNTIME=archivebox ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \
&& mkdir -p "$LIB_DIR" \
&& apt-get update -qq \
&& apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \
&& abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" gallery-dl \
&& abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" --overrides='{"pip":{"install_args":["--no-deps","forum-dl","chardet==5.2.0","pydantic==2.12.3","pydantic-core==2.41.4","typing-extensions>=4.14.1","annotated-types>=0.6.0","typing-inspection>=0.4.2","beautifulsoup4","soupsieve","lxml","requests","urllib3","certifi","idna","charset-normalizer","tenacity","python-dateutil","six","html2text","warcio"]}}' forum-dl \
&& if [ "$TARGETARCH" = "arm64" ]; then \
abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
fi \
&& ABXPKG_INSTALL_TIMEOUT=600 TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
accessibility archivedotorg archivewebpage base chrome chrome_mhtml chrome_screencast \
claudechrome claudecode claudecodecleanup claudecodeextract consolelog defuddle dns dom \
favicon forumdl gallerydl git hashes headers htmltotext infiniscroll \
istilldontcareaboutcookies liteparse media mercury modalcloser opencode opendataloader papersdl \
parse_dom_outlinks parse_html_urls parse_jsonl_urls parse_netscape_urls parse_rss_urls \
parse_txt_urls pdf readability redirects responses screenshot search_backend_ripgrep \
search_backend_sonic search_backend_sqlite seo singlefile ssl sslcerts staticfile title \
trafilatura ublock wget ytdlp \
&& abxpkg install --no-cache --binproviders=chromewebstore --overrides='{"chromewebstore":{"install_args":["fpeoodllldobpkbkabpblcfaogecpndd","--name=archivewebpage"]}}' archivewebpage \
&& test -f "$LIB_DIR/chromewebstore/extensions/fpeoodllldobpkbkabpblcfaogecpndd__archivewebpage/manifest.json" \
&& mkdir -p "$LIB_DIR/env/bin" \
&& ln -sf "$(command -v node)" "$LIB_DIR/env/bin/node" \
&& ln -sf "$(command -v npm)" "$LIB_DIR/env/bin/npm" \
&& ln -sf "$(command -v java)" "$LIB_DIR/env/bin/java" \
&& ln -sf "$(command -v git)" "$LIB_DIR/env/bin/git" \
&& ln -sf "$(command -v rg)" "$LIB_DIR/env/bin/rg" \
&& ln -sf "$(command -v sonic)" "$LIB_DIR/env/bin/sonic" \
&& find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
&& find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
&& rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
&& mkdir -p /opt/archivebox/lib-layer \
&& cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
&& apt-get purge -y build-essential \
&& apt-get autoremove -y \
&& rm -rf /var/lib/apt/lists/* \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
RUN rm -rf "$LIB_DIR" \
&& mv /opt/archivebox/lib-layer "$LIB_DIR" \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
# Install ArchiveBox Python package from the checked-out source.
WORKDIR "$CODE_DIR"
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
&& pip install \
--no-deps \
"$CODE_DIR" \
&& ( \
pip show archivebox \
&& which archivebox \
&& echo -e '\n\n' \
) | tee -a /VERSION.txt \
&& COMMIT_HASH="$( \
if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
echo "$HEAD_REF"; \
elif [[ "$HEAD_REF" == ref:\ * ]]; then \
REF_PATH="${HEAD_REF#ref: }"; \
cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
fi; \
fi)" \
&& if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
&& uv pip install --no-deps "$CODE_DIR" \
&& (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
&& find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
&& find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
# installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
&& find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
&& rm -rf "$CODE_DIR/.git"
FROM archivebox-runtime-base
LABEL name="archivebox" \
maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
description="All-in-one self-hosted internet archiving solution" \
homepage="https://github.com/ArchiveBox/ArchiveBox" \
documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
org.opencontainers.image.title="ArchiveBox" \
org.opencontainers.image.vendor="ArchiveBox" \
org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
com.docker.image.source.entrypoint="Dockerfile"
COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
COPY --from=archivebox-builder /venv /venv
COPY --from=archivebox-builder /app /app
COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib
RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
&& groupadd --system "$ARCHIVEBOX_USER" \
&& useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
&& usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
&& groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
&& (which sonic && sonic --version) | tee -a /VERSION.txt \
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/uv" \
&& openssl rand -hex 16 > /etc/machine-id \
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
&& echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
# Initialize an empty image collection without rerunning dependency installs.
WORKDIR "$DATA_DIR"
RUN echo "[+] Initializing image collection..." \
&& find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
&& PUID=0 PGID=0 archivebox init \
&& find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
&& find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
&& (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
"$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
"$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
@ -389,25 +204,35 @@ RUN echo "[+] Initializing image collection..." \
"$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
2>/dev/null || true)
# Print version for nice docker finish summary
RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
&& echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
&& echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
) | tee -a /VERSION.txt
# Verify ArchiveBox is installed and write full version/dependency info.
RUN chmod +x "$CODE_DIR"/bin/*.sh \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
&& chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode papersdl parse_rss_urls readability search_backend_sonic opendataloader search_backend_ripgrep 2>&1 | tee -a /VERSION.txt \
&& gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
&& chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \
&& env -u PNPM_HOME PATH="/opt/node/bin:$PATH" /opt/node/bin/corepack pnpm add --loglevel=error --store-dir="$TMP_DIR/pnpm-store" --config.dangerouslyAllowAllBuilds=true --dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai 2>&1 | tee -a /VERSION.txt \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \
&& rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node \
&& ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode" \
&& ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
&& chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/env/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
&& "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
&& "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
&& /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
&& /usr/local/bin/sonic --version | tee -a /VERSION.txt \
&& /venv/bin/supervisord --version | tee -a /VERSION.txt \
&& ! command -v gcc \
&& ! command -v g++ \
&& ! command -v make \
&& HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
&& rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
####################################################
RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
&& echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
&& echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
) | tee -a /VERSION.txt
# Expose ArchiveBox's main interfaces to the outside world
WORKDIR "$DATA_DIR"
VOLUME "$DATA_DIR"
EXPOSE 8000

63
Dockerfile.multistage
View File

@ -9,24 +9,19 @@
# --build-context abxpkg=../abxpkg \
# --build-context abx-plugins=../abx-plugins \
# -t archivebox/abx-dl:dev
# docker buildx build . -f Dockerfile.multistage \
# --build-context abx-dl=docker-image://archivebox/abx-dl:dev \
# --build-arg ABX_DL_IMAGE=abx-dl \
# docker buildx build . -f Dockerfile \
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
# -t archivebox:multistage
ARG TARGETPLATFORM=linux/amd64
ARG TARGETOS=linux
ARG TARGETARCH=amd64
ARG TARGETVARIANT=
ARG ABX_DL_IMAGE=abx-dl
ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
FROM ${ABX_DL_IMAGE} AS abx-dl
FROM archivebox/sonic:1.4.9 AS sonic
FROM ubuntu:24.04 AS archivebox-runtime-base
ARG TARGETPLATFORM=linux/amd64
ARG TARGETOS=linux
ARG TARGETARCH=amd64
ARG TARGETPLATFORM
ARG TARGETOS
ARG TARGETARCH
ARG TARGETVARIANT
ENV TZ=UTC \
@ -73,7 +68,7 @@ ENV UV_COMPILE_BYTECODE=0 \
UV_LINK_MODE=copy \
UV_PROJECT_ENVIRONMENT=/venv \
VIRTUAL_ENV=/venv \
PATH="/venv/bin:/opt/node/bin:$PATH"
PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
WORKDIR "$CODE_DIR"
@ -88,7 +83,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
&& apt-get update -qq \
&& apt-get install -qq -y \
apt-transport-https apt-utils ca-certificates curl wget gnupg2 \
dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
dumb-init util-linux unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \
libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \
libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
@ -102,7 +97,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
COPY --from=abx-dl /bin/uv /bin/uv
COPY --from=abx-dl /opt/uv/python /opt/uv/python
COPY --from=abx-dl /opt/node /opt/node
COPY --from=abx-dl /venv /venv
COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
@ -110,7 +104,6 @@ RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
&& uname -a \
&& sed -n '1,7p' /etc/os-release \
&& which python3 && python3 --version \
&& which node && node --version \
&& which uv && uv self version \
) | tee -a /VERSION.txt
@ -127,6 +120,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
&& apt-get update -qq \
&& apt-get install -qq -y --no-install-recommends \
build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
&& uv venv /venv --python "${PYTHON_VERSION}" \
&& uv pip install setuptools pip wheel \
&& uv sync \
--refresh \
--no-dev \
@ -144,10 +139,22 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
&& COMMIT_HASH="$( \
if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
echo "$HEAD_REF"; \
elif [[ "$HEAD_REF" == ref:\ * ]]; then \
REF_PATH="${HEAD_REF#ref: }"; \
cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
fi; \
fi)" \
&& if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
&& uv pip install --no-deps "$CODE_DIR" \
&& (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
&& find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
&& find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
&& find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
&& rm -rf "$CODE_DIR/.git"
FROM archivebox-runtime-base
@ -160,11 +167,12 @@ LABEL name="archivebox" \
org.opencontainers.image.vendor="ArchiveBox" \
org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
com.docker.image.source.entrypoint="Dockerfile.multistage"
com.docker.image.source.entrypoint="Dockerfile"
COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
COPY --from=archivebox-builder /venv /venv
COPY --from=archivebox-builder /app /app
COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
@ -176,8 +184,9 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
&& usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
&& groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
&& (which sonic && sonic --version) | tee -a /VERSION.txt \
&& mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/uv" \
&& openssl rand -hex 16 > /etc/machine-id \
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
&& echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
@ -188,7 +197,6 @@ RUN echo "[+] Initializing image collection..." \
&& PUID=0 PGID=0 archivebox init \
&& find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
&& find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
&& (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
"$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
"$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
@ -197,18 +205,25 @@ RUN echo "[+] Initializing image collection..." \
2>/dev/null || true)
RUN chmod +x "$CODE_DIR"/bin/*.sh \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
&& chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \
&& env -u PNPM_HOME PATH="/opt/node/bin:$PATH" /opt/node/bin/corepack pnpm add --loglevel=error --store-dir="$TMP_DIR/pnpm-store" --config.dangerouslyAllowAllBuilds=true --dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai 2>&1 | tee -a /VERSION.txt \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \
&& rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node \
&& ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode" \
&& ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
&& chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/env/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
&& "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
&& "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
&& "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
&& /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
&& /usr/local/bin/sonic --version | tee -a /VERSION.txt \
&& /venv/bin/supervisord --version | tee -a /VERSION.txt \
&& ! command -v gcc \
&& ! command -v g++ \
&& ! command -v make \
&& gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
&& HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
&& rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*

2
archivebox/cli/archivebox_extract.py
View File

@ -211,7 +211,7 @@ def run_plugins(
if snapshot_id in existing_snapshot_ids
for plugin_name in plugin_names
)
plugins_by_name = discover_plugins()
plugins_by_name = discover_plugins(runtime="archivebox")
requested_rows: set[tuple[str, str, str]] = set()
for snapshot_id, plugin_name in requested_pairs:
plugin = plugins_by_name.get(plugin_name)

18
archivebox/cli/archivebox_run.py
View File

@ -243,6 +243,7 @@ def process_stdin_records() -> int:
crawl_id,
snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
selected_plugins_are_explicit=False,
)
return 0
@ -342,7 +343,8 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
@click.option("--snapshot-id", help="Run one snapshot through its crawl")
@click.option("--binary-id", help="Run one queued binary install directly on the bus")
@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool):
@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool, no_stdin: bool):
"""
Process queued work.
@ -394,7 +396,7 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
if maintenance_only:
sys.exit(run_runner(daemon=daemon, maintenance_only=True))
if not sys.stdin.isatty():
if not no_stdin and not sys.stdin.isatty():
sys.exit(process_stdin_records())
else:
sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
@ -409,10 +411,14 @@ def run_snapshot_worker(snapshot_id: str) -> int:
snapshot = None
try:
with foreground_shutdown_signals(), foreground_parent_watchdog():
snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
if snapshot.retry_at is None:
snapshot.update_and_requeue(retry_at=timezone.now())
run_due_snapshot(snapshot, lock_seconds=60)
for _ in range(10):
snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
if snapshot.retry_at is None:
snapshot.update_and_requeue(retry_at=timezone.now())
elif snapshot.retry_at > timezone.now():
break
if not run_due_snapshot(snapshot, lock_seconds=60):
break
return 0
except KeyboardInterrupt:
try:

32
archivebox/cli/archivebox_update.py
View File

@ -33,17 +33,19 @@ def _get_snapshot_crawl(snapshot: Snapshot) -> Crawl | None:
def _get_search_indexing_plugins() -> list[str]:
from abx_dl.models import discover_plugins
from archivebox.config.common import get_config
from archivebox.plugins.hooks import discover_hooks
from archivebox.plugins.discovery import get_search_backends
available_backends = set(get_search_backends())
plugins = discover_plugins()
return sorted(
plugin_name
for plugin_name, plugin in plugins.items()
if plugin_name.startswith("search_backend_")
and plugin_name.removeprefix("search_backend_") in available_backends
and any("Snapshot" in hook.name and "index" in hook.name.lower() for hook in plugin.hooks)
for plugin_name in {
hook.parent.name
for hook in discover_hooks("Snapshot", config=get_config())
if hook.parent.name.startswith("search_backend_") and "index" in hook.name.lower()
}
if plugin_name.startswith("search_backend_") and plugin_name.removeprefix("search_backend_") in available_backends
)
@ -79,7 +81,7 @@ def reindex_snapshots(
stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
records: list[dict[str, str]] = []
plugins_by_name = discover_plugins()
plugins_by_name = discover_plugins(runtime="archivebox")
required_hooks_by_plugin = {
plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
for plugin_name in search_plugins
@ -340,7 +342,7 @@ def update(
(
stats_combined["phase1"].get("queued", 0),
stats_combined["phase2"].get("queued", 0),
stats_combined["phase2"].get("crawls_queued", 0),
stats_combined["phase2"].get("crawls_sealed", 0),
),
)
runner_work_queued = runner_work_queued or maintenance_work_queued
@ -724,7 +726,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
"updated_db": 0,
"queued": 0,
"sealed": 0,
"crawls_queued": 0,
"crawls_sealed": 0,
}
current_fs_version = Snapshot._fs_current_version()
@ -831,7 +833,10 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
queue_stale_fs_batch()
now = timezone.now()
stats["crawls_queued"] = (
# Crawls with no open child snapshots are already finished. Seal them here
# instead of waking the foreground runner; otherwise migration/update can
# accidentally re-enter full crawl execution for historical rows.
stats["crawls_sealed"] = (
Crawl.objects.filter(
status__in=Crawl.RUNNABLE_STATES,
)
@ -839,11 +844,12 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
snapshot_set__status__in=Snapshot.OPEN_STATES,
)
.update(
retry_at=now,
status=Crawl.StatusChoices.SEALED,
retry_at=None,
modified_at=now,
)
)
stats["updated_db"] += stats["crawls_queued"]
stats["updated_db"] += stats["crawls_sealed"]
return stats
@ -981,7 +987,7 @@ Phase 2 (Process DB):
Updated JSON: {s2.get("updated_json", 0)}
Updated DB rows: {s2.get("updated_db", 0)}
Sealed snapshots: {s2.get("sealed", 0)}
Queued crawls: {s2.get("crawls_queued", 0)}
Sealed crawls: {s2.get("crawls_sealed", 0)}
""")

53
archivebox/config/version.py
View File

@ -6,6 +6,7 @@ import importlib.metadata
from pathlib import Path
from functools import cache
from datetime import datetime
import re
#############################################################################################
@ -41,11 +42,57 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):
@cache
def get_COMMIT_HASH() -> str | None:
for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"):
env_commit_hash = os.environ.get(env_var, "").strip()
if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
return env_commit_hash
if IN_DOCKER:
try:
version_txt = Path("/VERSION.txt").read_text()
docker_commit_hashes = re.findall(r"COMMIT_HASH=([0-9a-fA-F]{40})", version_txt)
if docker_commit_hashes:
return docker_commit_hashes[-1]
except Exception:
pass
def _read_git_file(git_dir: Path, ref: str) -> str | None:
try:
return git_dir.joinpath(ref).read_text().strip()
except Exception:
pass
try:
packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines()
except Exception:
return None
for line in packed_refs:
if line.startswith("#") or line.startswith("^") or not line.strip():
continue
commit_hash, packed_ref = line.split(" ", 1)
if packed_ref == ref:
return commit_hash.strip()
return None
try:
git_dir = PACKAGE_DIR.parent / ".git"
ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1]
commit_hash = git_dir.joinpath(ref).read_text().strip()
return commit_hash
if git_dir.is_file():
gitdir_line = git_dir.read_text().strip()
gitdir_path = gitdir_line.removeprefix("gitdir:").strip()
git_dir = Path(gitdir_path)
if not git_dir.is_absolute():
git_dir = PACKAGE_DIR.parent / git_dir
head = (git_dir / "HEAD").read_text().strip()
if re.fullmatch(r"[0-9a-fA-F]{40}", head):
return head
ref = head.removeprefix("ref:").strip()
commit_hash = _read_git_file(git_dir, ref)
if commit_hash:
return commit_hash
except Exception:
pass

19
archivebox/core/models.py
View File

@ -2737,7 +2737,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
return snapshot
def create_pending_archiveresults(self) -> list["ArchiveResult"]:
def create_pending_archiveresults(self, hooks: Iterable[tuple[str, str]] | None = None) -> list["ArchiveResult"]:
"""
Create ArchiveResult records for all enabled hooks.
@ -2748,18 +2748,17 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
Creates one ArchiveResult per hook (not per plugin), with hook_name set.
This enables step-based execution where all hooks in a step can run in parallel.
"""
from archivebox.plugins.hooks import discover_hooks
from archivebox.config.common import get_config
if hooks is None:
from archivebox.plugins.hooks import discover_hooks
from archivebox.config.common import get_config
# Get merged config with crawl-specific PLUGINS filter
config = get_config(crawl=self.crawl, snapshot=self)
hooks = discover_hooks("Snapshot", config=config)
# Compatibility path for direct model callers. The runner passes its
# abx-dl hook inventory explicitly so queued rows match execution.
config = get_config(crawl=self.crawl, snapshot=self)
hooks = ((hook_path.parent.name, hook_path.stem) for hook_path in discover_hooks("Snapshot", config=config))
archiveresults = []
for hook_path in hooks:
hook_name = hook_path.stem # e.g., 'on_Snapshot__50_wget'
plugin = hook_path.parent.name # e.g., 'wget'
for plugin, hook_name in hooks:
# ArchiveResult output is one filesystem directory per plugin hook, so
# retries must update this row in place instead of creating siblings.
archiveresult, _created = ArchiveResult.objects.get_or_create(

79
archivebox/core/recovery_util.py
View File

@ -1,13 +1,21 @@
from __future__ import annotations
from pathlib import Path
from django.utils import timezone
from rich.console import Console
def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
return exit_code is not None and (exit_code < 0 or exit_code >= 128)
def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]:
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.services.archive_result_service import _collect_output_metadata
from archivebox.machine.models import Process
from django.core.exceptions import ValidationError
from django.db.models import Exists, OuterRef, Q, Subquery, Value
from django.db.models.functions import Coalesce
@ -22,6 +30,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
"archiveresults_backoff": 0,
"snapshots_queued_plugin_rows_waiting_on_stale_lease": 0,
"archiveresults_started_without_running_process": 0,
"archiveresults_missing_for_orphaned_hook_processes": 0,
"snapshots_started_without_running_results": 0,
"crawls_started_with_due_snapshots": 0,
"crawls_started_waiting_on_future_snapshots": 0,
@ -100,6 +109,76 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
process=None,
modified_at=now,
)
orphaned_hook_processes = Process.objects.filter(
process_type=Process.TypeChoices.HOOK,
archiveresult__isnull=True,
).exclude(status=Process.StatusChoices.RUNNING)
for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status"):
hook_script_name = process.hook_script_name
if not hook_script_name or not process.pwd:
continue
plugin_dir = Path(process.pwd)
try:
# Old or synthetic hook Process rows can point at arbitrary paths.
# Only paths whose parent directory is a valid Snapshot id can be
# reconstructed into ArchiveResult rows.
snapshot = Snapshot.objects.filter(id=plugin_dir.parent.name).first()
except ValidationError:
continue
if snapshot is None:
continue
result, created = ArchiveResult.objects.get_or_create(
snapshot=snapshot,
plugin=plugin_dir.name,
hook_name=Path(hook_script_name).stem,
defaults={
"status": ArchiveResult.StatusChoices.QUEUED,
},
)
if result.status == ArchiveResult.StatusChoices.QUEUED:
requeue_snapshot = False
# A runner can die after the hook Process exits but before the
# ProcessCompletedEvent projector links/finalizes ArchiveResult.
# Reconstruct only that exact hook row from the durable Process row.
output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
result.process = process
if _is_signal_interrupted_exit(process.exit_code):
# The owning runner died or was asked to stop while the hook was
# still active. Keep the work item queued so takeover retries the
# same hook; treating an unknown signal exit as success would
# silently skip unfinished side effects.
result.output_files = {}
result.output_size = 0
result.output_mimetypes = ""
result.output_str = ""
result.status = ArchiveResult.StatusChoices.QUEUED
requeue_snapshot = True
else:
result.output_files = output_files
result.output_size = output_size
result.output_mimetypes = output_mimetypes
result.output_str = process.stderr if process.exit_code not in (0, None) else ""
result.status = (
ArchiveResult.StatusChoices.FAILED
if process.exit_code not in (0, None)
else (ArchiveResult.StatusChoices.SUCCEEDED if output_files else ArchiveResult.StatusChoices.NORESULTS)
)
result.save(
update_fields=[
"process",
"output_files",
"output_size",
"output_mimetypes",
"output_str",
"status",
"modified_at",
],
)
if requeue_snapshot:
Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
if created:
cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1
Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
Q(retry_at__isnull=True) | Q(retry_at__gt=now),
)

2
archivebox/core/views.py
View File

@ -328,7 +328,7 @@ class SnapshotView(View):
hidden_card_plugins = {"archivedotorg", "favicon", "title"}
outputs = [
out
for out in snapshot.discover_outputs(include_filesystem_fallback=False)
for out in snapshot.discover_outputs(include_filesystem_fallback=True)
if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins
]
archiveresults = {out["name"]: out for out in outputs}

23
archivebox/machine/models.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
__package__ = "archivebox.machine"
import os
import signal
import sys
import uuid
import socket
@ -55,6 +56,13 @@ PROCESS_TIMEOUT_GRACE = timedelta(seconds=30) # Extra margin before force-clean
START_TIME_TOLERANCE = 5.0 # Seconds tolerance for start time matching
def _default_exit_code_for_unowned_process(process_type: str) -> int:
# Hooks are externally visible work items. If their owning runner disappeared
# before recording the real exit code, retrying is safer than converting an
# unknown interrupted extraction into a durable success/no-result row.
return 128 + signal.SIGTERM if process_type == Process.TypeChoices.HOOK else 0
def _find_existing_binary_for_reference(machine: Machine, reference: str) -> Binary | None:
reference = str(reference or "").strip()
if not reference:
@ -1608,7 +1616,9 @@ class Process(ModelWithDeleteAfter, models.Model):
is_stale = True # Process no longer exists
if is_stale:
proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
proc.mark_exited(
exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
)
cleaned += 1
return cleaned
@ -2138,8 +2148,7 @@ class Process(ModelWithDeleteAfter, models.Model):
# TODO: Uncomment to cleanup (keeping for debugging for now)
# self.stderr_file.unlink(missing_ok=True)
# Try to get exit code from proc or default to unknown
self.exit_code = self.exit_code if self.exit_code is not None else 0
self.exit_code = self.exit_code if self.exit_code is not None else _default_exit_code_for_unowned_process(self.process_type)
if self.exit_code == -1:
self.exit_code = 137
self.ended_at = timezone.now()
@ -2503,7 +2512,9 @@ class Process(ModelWithDeleteAfter, models.Model):
# orphaned Process backlog cannot be materialized in memory at once.
for proc in running_children.iterator(chunk_size=100):
if not proc.is_running:
proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
proc.mark_exited(
exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
)
cleaned += 1
continue
@ -2527,7 +2538,9 @@ class Process(ModelWithDeleteAfter, models.Model):
):
continue
proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
proc.mark_exited(
exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),
)
cleaned += 1
if cleaned:

22
archivebox/services/archive_result_service.py
View File

@ -240,6 +240,10 @@ def _has_content_files(output_files: Any) -> bool:
return any(Path(path).suffix not in {".log", ".pid", ".sh"} for path in _normalize_output_files(output_files))
def _is_signal_interrupted_exit(exit_code: int) -> bool:
return exit_code < 0 or (exit_code >= 128 and exit_code != PROCESS_EXIT_SKIPPED)
def _iter_archiveresult_records(stdout: str) -> list[dict]:
records: list[dict] = []
for raw_line in stdout.splitlines():
@ -352,6 +356,13 @@ def _save_archiveresult_event_to_db(
with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_update"):
result.save(update_fields=[*update_fields, "modified_at"])
if result.status == ArchiveResult.StatusChoices.QUEUED:
# ArchiveResult has no retry_at column. If a shutdown/takeover projects
# a killed hook back to QUEUED, wake the parent Snapshot/Crawl so the
# next runner retries that exact hook instead of waiting on a stale
# active-state lease.
snapshot.update_and_requeue(retry_at=timezone.now())
if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.title_update"):
title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else ""
@ -435,14 +446,21 @@ class ArchiveResultService(BaseService):
# TODO: consider moving this fallback derivation into abx-dl itself.
# First try both patterns: if the whole abx-dl process crashes, restarting
# the snapshot may be enough, but don't guess before validating it.
process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED)
process_interrupted = _is_signal_interrupted_exit(event.exit_code)
process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED) and not process_interrupted
with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"):
await event.emit(
ArchiveResultEvent(
snapshot_id=snapshot_event.snapshot_id,
plugin=event.plugin_name,
hook_name=event.hook_name,
status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "noresult"),
status=(
"queued"
if process_interrupted
else "failed"
if process_failed
else ("succeeded" if _has_content_files(event.output_files) else "noresult")
),
output_str=event.stderr if process_failed else "",
output_files=event.output_files,
start_ts=event.start_ts,

8
archivebox/services/binary_service.py
View File

@ -193,6 +193,7 @@ class ArchiveBoxBinaryService(BaseService):
def __init__(self, bus: EventBus):
super().__init__(bus)
self.process_ids_by_request_id: dict[str, str] = {}
self._missing_finalize_tasks: set[asyncio.Task] = set()
self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__project_process)
self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__schedule_missing_finalize)
self.bus.on(BinaryEvent, self.on_BinaryEvent__finalize_process)
@ -353,7 +354,12 @@ class ArchiveBoxBinaryService(BaseService):
def _schedule_missing_finalize(self, request: BinaryRequestEvent) -> None:
task = asyncio.create_task(self._finalize_request_when_done(request))
task.add_done_callback(lambda done: None if done.cancelled() else done.exception())
self._missing_finalize_tasks.add(task)
task.add_done_callback(lambda done: self._missing_finalize_tasks.discard(done) or (None if done.cancelled() else done.exception()))
async def flush_missing_finalizers(self) -> None:
if self._missing_finalize_tasks:
await asyncio.gather(*tuple(self._missing_finalize_tasks), return_exceptions=False)
async def on_BinaryRequestEvent__schedule_missing_finalize(self, request: BinaryRequestEvent) -> None:
self._schedule_missing_finalize(request)

165
archivebox/services/runner.py
View File

@ -10,7 +10,6 @@ import threading
import time
from contextlib import nullcontext
from datetime import timedelta
from functools import lru_cache
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any
@ -214,6 +213,7 @@ class CrawlRunner:
show_progress: bool = True,
interactive_interrupts: bool = False,
config_overrides: dict[str, Any] | None = None,
selected_plugins_are_explicit: bool = True,
):
self.crawl = crawl
self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0)
@ -242,6 +242,7 @@ class CrawlRunner:
)
ArchiveResultService(self.bus)
self.selected_plugins = selected_plugins
self.selected_plugins_from_args = selected_plugins is not None and selected_plugins_are_explicit
self.initial_snapshot_ids = snapshot_ids
self.snapshot_tasks: dict[str, asyncio.Task[None]] = {}
self.snapshot_semaphore = asyncio.Semaphore(1)
@ -1015,6 +1016,7 @@ class CrawlRunner:
await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id)
return
snapshot_selected_plugins = self.selected_plugins
selected_hooks_by_plugin = None
if snapshot["status"] == "started":
_reset_count, running_count = await sync_to_async(snapshot["_snapshot"].reset_abandoned_results, thread_sensitive=True)()
if running_count:
@ -1025,10 +1027,36 @@ class CrawlRunner:
thread_sensitive=True,
)()
return
snapshot_selected_plugins = snapshot_selected_plugins or await sync_to_async(
queued_plugins_for_snapshot,
if await sync_to_async(snapshot["_snapshot"].is_finished_processing, thread_sensitive=True)():
await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
snapshot["id"],
output_dir=Path(snapshot["output_dir"]),
)
return
if not self.selected_plugins_from_args:
queued_plugins, selected_hooks_by_plugin = await sync_to_async(
queued_plugins_and_hooks_for_snapshot,
thread_sensitive=True,
)(snapshot["id"])
if queued_plugins:
if snapshot_selected_plugins:
queued_plugins = [plugin for plugin in queued_plugins if plugin in snapshot_selected_plugins]
selected_hooks_by_plugin = {
plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
}
snapshot_selected_plugins = queued_plugins
elif not self.selected_plugins_from_args:
queued_plugins, selected_hooks_by_plugin = await sync_to_async(
queued_plugins_and_hooks_for_snapshot,
thread_sensitive=True,
)(snapshot["id"])
if queued_plugins:
if snapshot_selected_plugins:
queued_plugins = [plugin for plugin in queued_plugins if plugin in snapshot_selected_plugins]
selected_hooks_by_plugin = {
plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
}
snapshot_selected_plugins = queued_plugins
if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() in (
"crawl_max_size",
"crawl_timeout",
@ -1043,6 +1071,23 @@ class CrawlRunner:
if snapshot_selected_plugins
else self.plugins
)
if selected_hooks_by_plugin is not None:
await sync_to_async(fail_unavailable_queued_hooks, thread_sensitive=True)(
snapshot["id"],
selected_hooks_by_plugin,
plugins,
)
filtered_plugins = {}
for plugin_name, plugin in plugins.items():
selected_hook_names = selected_hooks_by_plugin.get(plugin_name)
if selected_hook_names is None:
filtered_plugins[plugin_name] = plugin
continue
filtered_hooks = [
hook for hook in plugin.hooks if hook.name in selected_hook_names or Path(hook.name).stem in selected_hook_names
]
filtered_plugins[plugin_name] = plugin.model_copy(update={"hooks": filtered_hooks})
plugins = filtered_plugins
abx_snapshot = AbxSnapshot(
id=snapshot["id"],
url=snapshot["url"],
@ -1062,6 +1107,7 @@ class CrawlRunner:
snapshot_cleanup_enabled=True,
snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
abort_requested=self.crawl_is_cancelled,
selected_hooks_by_plugin=selected_hooks_by_plugin,
)
try:
snapshot_event = SnapshotEvent(
@ -1137,6 +1183,7 @@ def run_crawl(
show_progress: bool = True,
interactive_interrupts: bool = False,
config_overrides: dict[str, Any] | None = None,
selected_plugins_are_explicit: bool = True,
) -> None:
from archivebox.crawls.models import Crawl
from django.db import close_old_connections
@ -1154,6 +1201,7 @@ def run_crawl(
show_progress=show_progress,
interactive_interrupts=interactive_interrupts,
config_overrides=config_overrides,
selected_plugins_are_explicit=selected_plugins_are_explicit,
).run(),
)
finally:
@ -1191,7 +1239,7 @@ async def _run_binary(binary_id: str) -> None:
config = normalize_runtime_config(config)
bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0)
process_service = PersistedProcessService(bus)
ArchiveBoxBinaryService(bus)
binary_process_service = ArchiveBoxBinaryService(bus)
BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
BinaryService(bus)
TagService(bus)
@ -1233,6 +1281,7 @@ async def _run_binary(binary_id: str) -> None:
).now(first_result=True)
finally:
await bus.wait_until_idle()
await binary_process_service.flush_missing_finalizers()
await process_service.flush_completed()
@ -1240,14 +1289,7 @@ def run_binary(binary_id: str) -> None:
asyncio.run(_run_binary(binary_id))
@lru_cache(maxsize=1)
def _snapshot_hook_names_by_plugin() -> dict[str, frozenset[str]]:
return {
plugin.name: frozenset(hook.name for hook in plugin.filter_hooks("Snapshot")) for plugin in _discover_archivebox_plugins().values()
}
def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
def queued_plugins_and_hooks_for_snapshot(snapshot_id: str) -> tuple[list[str] | None, dict[str, set[str] | None] | None]:
from archivebox.core.models import ArchiveResult
queued_results = list(
@ -1258,30 +1300,72 @@ def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
.exclude(plugin="")
.only("id", "plugin", "hook_name"),
)
hooks_by_plugin = _snapshot_hook_names_by_plugin()
obsolete_result_ids = [
result.id
for result in queued_results
if result.hook_name and result.hook_name not in hooks_by_plugin.get(result.plugin, frozenset())
]
if obsolete_result_ids:
# Hook names are the scheduler identity for ArchiveResults. If an old
# queued row names a hook that the current plugin model cannot run, hard
# fail only that row so the scheduler drains without hiding stale/broken
# plugin state as an intentional skip.
selected_hooks_by_plugin: dict[str, set[str] | None] = {}
queued_plugins = sorted({result.plugin for result in queued_results})
for result in queued_results:
# hook_name is the modern scheduler identity. Empty hook_name rows are
# legacy plugin-level work and must keep running the whole plugin.
if not result.hook_name:
selected_hooks_by_plugin[result.plugin] = None
elif result.plugin not in selected_hooks_by_plugin:
selected_hooks_by_plugin[result.plugin] = {result.hook_name}
elif selected_hooks_by_plugin[result.plugin] is not None:
selected_hooks_by_plugin[result.plugin].add(result.hook_name)
if queued_plugins:
return queued_plugins, selected_hooks_by_plugin
return None, None
def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
queued_plugins, _selected_hooks_by_plugin = queued_plugins_and_hooks_for_snapshot(snapshot_id)
return queued_plugins
def fail_unavailable_queued_hooks(
snapshot_id: str,
selected_hooks_by_plugin: dict[str, set[str] | None],
plugins: dict[str, Plugin],
) -> None:
from archivebox.core.models import ArchiveResult
now = timezone.now()
for plugin_name, selected_hook_names in selected_hooks_by_plugin.items():
if selected_hook_names is None or plugin_name not in plugins:
continue
available_hook_names = {
name for hook in plugins[plugin_name].filter_hooks("Snapshot") for name in (hook.name, Path(hook.name).stem)
}
missing_hook_names = [hook_name for hook_name in selected_hook_names if hook_name not in available_hook_names]
if not missing_hook_names:
continue
# Hook-level resume rows are durable scheduler state. If a plugin is
# installed but no longer exposes a queued hook, mark that row failed so
# the snapshot is not retried forever with no hook left to execute.
ArchiveResult.objects.filter(
id__in=obsolete_result_ids,
snapshot_id=snapshot_id,
plugin=plugin_name,
hook_name__in=missing_hook_names,
status=ArchiveResult.StatusChoices.QUEUED,
).update(
status=ArchiveResult.StatusChoices.FAILED,
output_str="Hook no longer exists in the current plugin set.",
modified_at=timezone.now(),
start_ts=now,
end_ts=now,
output_str="Queued hook is no longer available in the installed plugin",
)
queued_plugins = sorted({result.plugin for result in queued_results if result.id not in obsolete_result_ids})
if queued_plugins:
return queued_plugins
return None
def snapshot_hooks_for_pending_archiveresults(snapshot) -> list[tuple[str, str]]:
from archivebox.config.common import get_config
config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
plugin_names = [name.strip() for name in str(config.PLUGINS or "").split(",") if name.strip()]
plugins = (
filter_plugins(_discover_archivebox_plugins(), plugin_names, include_providers=True)
if plugin_names
else _discover_archivebox_plugins()
)
return sorted((plugin.name, hook.name) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot"))
def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None) -> bool:
@ -1432,11 +1516,11 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
if not selected_plugins:
# No targeted plugin rows remain, so put paused snapshots back
# behind the indefinite retry_at marker. If queued plugin rows
# do remain, run_snapshot_maintenance kept retry_at due so the
# next tick can process them and the finally block below will
# restore the paused marker after that targeted work completes.
# remain, continue into the targeted plugin path below and let
# its finally block restore the paused marker after completion.
snapshot.restore_paused_scheduler_marker()
return True
return True
snapshot.refresh_from_db()
if not selected_plugins:
# Paused is a real lifecycle state; retry_at=MAX is only the
# orchestrator selection marker. If a direct maintenance/update
@ -1460,6 +1544,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
selected_plugins=selected_plugins,
process_discovered_snapshots_inline=True,
interactive_interrupts=interactive_interrupts,
selected_plugins_are_explicit=False,
)
finally:
# Targeted plugin rows can complete while the Snapshot remains
@ -1491,6 +1576,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
selected_plugins=selected_plugins,
process_discovered_snapshots_inline=True,
interactive_interrupts=interactive_interrupts,
selected_plugins_are_explicit=False,
)
if search_only_plugins:
from archivebox.core.models import ArchiveResult
@ -1532,7 +1618,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
# rows before ticking so maintenance-only final rows, e.g. search
# backfill on a paused snapshot, cannot make queued -> sealed skip the
# real extraction work after resume.
snapshot.create_pending_archiveresults()
snapshot.create_pending_archiveresults(hooks=snapshot_hooks_for_pending_archiveresults(snapshot))
snapshot.sm.tick()
snapshot.refresh_from_db()
if snapshot.status == Snapshot.StatusChoices.SEALED:
@ -1551,7 +1637,15 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
selected_plugins=queued_plugins_for_snapshot(str(snapshot.id)),
process_discovered_snapshots_inline=True,
interactive_interrupts=interactive_interrupts,
selected_plugins_are_explicit=False,
)
snapshot.refresh_from_db()
if queued_plugins_for_snapshot(str(snapshot.id)):
# Hook-level resume work is tracked by queued ArchiveResult rows, not by
# the Snapshot lease. If a partial pass returns with rows still queued,
# wake the Snapshot immediately so takeover does not wait out a stale
# active-state lock before running the remaining hooks.
snapshot.update_and_requeue(retry_at=timezone.now())
return True
@ -1825,6 +1919,7 @@ def _run_due_queued_plugin_result(
config_overrides={
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": QUEUED_PLUGIN_RESULT_BATCH_SIZE,
},
selected_plugins_are_explicit=False,
)
if all(plugin.startswith("search_backend_") for plugin in selected_plugins):
queued_results = ArchiveResult.objects.filter(

29
archivebox/tests/conftest.py
View File

@ -59,9 +59,10 @@ def _assert_safe_runtime_paths(*, cwd: Path | None = None, env: dict[str, str] |
def _test_source_pythonpath() -> str:
entries: list[str] = []
for repo_name in ("abxpkg", "abx-plugins", "abx-dl"):
repo_path = WORKSPACE_ROOT / repo_name
if repo_path.exists():
entries.append(str(repo_path.resolve(strict=False)))
for repo_path in (WORKSPACE_ROOT / repo_name, REPO_ROOT / repo_name):
if repo_path.exists():
entries.append(str(repo_path.resolve(strict=False)))
break
return os.pathsep.join(entries)
@ -592,9 +593,11 @@ def cli_env(
if disable_extractors or live or server:
env.update(
{
"PLUGINS": "__archivebox_test_no_plugins__",
"SAVE_ARCHIVEDOTORG": "False",
"SAVE_TITLE": "False",
"SAVE_FAVICON": "False",
"SAVE_WGET": "False",
"SAVE_WARC": "False",
"SAVE_PDF": "False",
"SAVE_SCREENSHOT": "False",
@ -1425,26 +1428,6 @@ def _find_system_browser() -> Path | None:
return None
def _ensure_puppeteer(shared_lib: Path) -> None:
pnpm_prefix = shared_lib / "pnpm" / "packages" / "chrome"
node_modules = pnpm_prefix / "node_modules"
puppeteer_dir = node_modules / "puppeteer"
if puppeteer_dir.exists():
return
pnpm_prefix.mkdir(parents=True, exist_ok=True)
env = os.environ.copy()
env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
subprocess.run(
["pnpm", "add", "--dir", str(pnpm_prefix), "puppeteer"],
cwd=str(pnpm_prefix),
env=env,
check=True,
capture_output=True,
text=True,
timeout=600,
)
@pytest.fixture(scope="class")
def real_archive_with_example(tmp_path_factory, request):
"""

10
archivebox/tests/test_api_v1_cli_update.py
View File

@ -72,6 +72,16 @@ def test_cli_update_api_supports_all_snapshot_list_filters_with_real_rows(tmp_pa
]
stdin = "\n".join(json.dumps(record) for record in records) + "\n"
run_archivebox_cmd(["snapshot", "create"], cwd=tmp_path, stdin=stdin, env=env, check=True)
list_result = run_archivebox_cmd(["snapshot", "list", "--sort", "timestamp"], cwd=tmp_path, env=env, check=True)
initial_snapshots = {record["url"]: record for record in parse_jsonl_output(list_result.stdout) if record.get("type") == "Snapshot"}
alpha = initial_snapshots["https://alpha.example.com/articles/needle"]
run_archivebox_cmd(
["snapshot", "update", "--status=paused"],
cwd=tmp_path,
stdin=json.dumps(alpha),
env=env,
check=True,
)
port = get_free_port()
env = {

21
archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py
View File

@ -130,6 +130,22 @@ def wait_for_crawl_wget_success_or_sealed(cwd, crawl_id, timeout=240):
raise AssertionError(f"timed out waiting for crawl resume completion for crawl {crawl_id}: {latest_state}")
def wait_for_sqlite_index_result(cwd, crawl_id, timeout=45):
deadline = time.time() + timeout
latest_state = None
while time.time() < deadline:
latest_state = get_crawl_runtime_state(cwd, crawl_id)
final_results = [
result
for result in latest_state["results"]
if result["plugin"] == "search_backend_sqlite" and result["status"] not in {"queued", "started", "paused"}
]
if final_results:
return latest_state
time.sleep(0.2)
raise AssertionError(f"timed out waiting for sqlite index result for crawl {crawl_id}: {latest_state}")
def make_snapshot(*, user, url: str, title: str, bookmarked_at: datetime):
crawl = Crawl.objects.create(urls=url, created_by=user)
snapshot = Snapshot.objects.create(
@ -452,7 +468,6 @@ def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(t
timeout=10,
)
assert pause_response.status_code == 200, pause_response.text
assert pause_response.json()["status"] == "paused"
paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id)
snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed"
finally:
@ -486,14 +501,14 @@ def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(t
)
assert update_process.returncode == 0, update_process.stderr
indexed_state = get_crawl_runtime_state(tmp_path, crawl_id)
indexed_state = wait_for_sqlite_index_result(tmp_path, crawl_id)
assert indexed_state["crawl_status"] == "paused"
assert indexed_state["crawl_retry_at"] == indexed_state["retry_at_max"]
assert indexed_state["snapshots"][0]["status"] == "paused"
assert indexed_state["snapshots"][0]["retry_at"] == indexed_state["retry_at_max"]
search_results = [result for result in indexed_state["results"] if result["plugin"] == "search_backend_sqlite"]
assert search_results
assert all(result["status"] not in {"queued", "started", "paused"} for result in search_results)
assert any(result["status"] not in {"queued", "started", "paused"} for result in search_results)
try:
start_archivebox_server(tmp_path, env=env, port=port)

2
archivebox/tests/test_cli_remove.py
View File

@ -219,7 +219,7 @@ def test_remove_reports_remaining_link_count_correctly(initialized_archive):
def test_remove_after_flag(initialized_archive):
"""Test remove --after flag removes snapshots after date."""
env = cli_env()
env = cli_env(disable_extractors=True)
run_archivebox_cmd(
["add", "--bg", "--depth=0", "https://example.com"],

46
archivebox/tests/test_cli_run.py
View File

@ -1390,13 +1390,14 @@ class TestRecoverOrchestratorState:
assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
assert snapshot.retry_at is None
def test_run_due_snapshot_runs_queued_plugin_after_fs_migration(self, monkeypatch):
@pytest.mark.django_db(transaction=True)
def test_run_due_snapshot_runs_queued_plugin_after_fs_migration(self):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.services import runner
from archivebox.services.runner import run_due_snapshot
crawl = Crawl.objects.create(
urls="https://example.com",
@ -1412,45 +1413,28 @@ class TestRecoverOrchestratorState:
)
Snapshot.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
snapshot.refresh_from_db()
snapshot.output_dir.mkdir(parents=True, exist_ok=True)
title_dir = snapshot.output_dir / "title"
title_dir.mkdir(parents=True, exist_ok=True)
(title_dir / "title.txt").write_text("Example Domain\n", encoding="utf-8")
result = ArchiveResult.objects.create(
snapshot=snapshot,
plugin="search_backend_sonic",
hook_name="on_Snapshot__91_index_sonic",
plugin="search_backend_sqlite",
hook_name="on_Snapshot__90_index_sqlite",
status=ArchiveResult.StatusChoices.QUEUED,
)
calls = []
def fake_run_crawl(crawl_id, *, snapshot_ids=None, selected_plugins=None, **kwargs):
calls.append((crawl_id, snapshot_ids, selected_plugins, kwargs))
ArchiveResult.objects.filter(pk=result.pk).update(
status=ArchiveResult.StatusChoices.NORESULTS,
start_ts=timezone.now(),
end_ts=timezone.now(),
output_str="No indexable content",
)
monkeypatch.setattr(
runner,
"_snapshot_hook_names_by_plugin",
lambda: {"search_backend_sonic": frozenset({"on_Snapshot__91_index_sonic"})},
)
monkeypatch.setattr(runner, "run_crawl", fake_run_crawl)
assert runner.run_due_snapshot(snapshot, lock_seconds=60) is True
assert run_due_snapshot(snapshot, lock_seconds=60) is True
snapshot.refresh_from_db()
result.refresh_from_db()
assert snapshot.fs_version == Snapshot._fs_current_version()
assert result.status == ArchiveResult.StatusChoices.NORESULTS
assert calls == [
(
str(crawl.id),
[str(snapshot.id)],
["search_backend_sonic"],
{"process_discovered_snapshots_inline": True, "interactive_interrupts": False},
),
]
assert result.status in ArchiveResult.FINAL_STATES
assert result.status != ArchiveResult.StatusChoices.QUEUED
assert result.start_ts is not None
assert result.end_ts is not None
@pytest.mark.django_db(transaction=True)
def test_run_due_snapshot_fails_obsolete_queued_hook_name(self):
from django.utils import timezone

6
archivebox/tests/test_cli_update.py
View File

@ -26,7 +26,7 @@ def test_update_runs_successfully_on_empty_archive(initialized_archive):
assert "Phase 1: Draining old archive/ directories" in output
assert "Phase 2: Processing all database snapshots" in output
assert "Updated DB rows: 0" in output
assert "Queued crawls: 0" in output
assert "Sealed crawls: 0" in output
with use_archivebox_db(initialized_archive):
assert Snapshot.objects.count() == 0
@ -86,7 +86,7 @@ def test_update_specific_snapshot_by_filter(initialized_archive):
output = result.stdout + result.stderr
assert result.returncode == 0, output
assert "Processing filtered snapshots from database" in output
assert "example.com" in output
assert "Found 1 matching snapshots" in output
with use_archivebox_db(initialized_archive):
assert Snapshot.objects.filter(url="https://example.com", status="sealed").count() == 1
@ -147,7 +147,7 @@ def test_update_seals_migrated_snapshots(initialized_archive):
output = result.stdout + result.stderr
assert result.returncode == 0, output
assert "Phase 3: Running" in output
assert "No queued/interrupted crawl work found" in output
# Check that snapshot remains archived instead of being queued for a full re-crawl.
with use_archivebox_db(initialized_archive):

95
archivebox/tests/test_dockerfiles.py
View File

@ -15,6 +15,8 @@ _REQUIRED_DOCKER_INSTALL_TARGETS = {
"istilldontcareaboutcookies",
"liteparse",
"mercury",
"opencode",
"opendataloader",
"papersdl",
"parse_rss_urls",
"readability",
@ -76,9 +78,10 @@ def test_dockerfiles_install_binaries_required_by_version_validation() -> None:
def test_dockerfile_prewarms_stable_plugin_dependencies_without_optional_captcha() -> None:
targets = _abx_dl_plugin_install_targets(REPO_ROOT / "Dockerfile")
assert _REQUIRED_DOCKER_PREINSTALL_TARGETS <= targets
assert not (_DOCKER_PREINSTALL_EXCLUDED_TARGETS & targets)
text = (REPO_ROOT / "Dockerfile").read_text()
assert "archivebox/abx-dl:latest" in text
assert "archivebox/abxdl" not in text
assert _abx_dl_plugin_install_targets(REPO_ROOT / "Dockerfile") == set()
def test_dockerfile_build_installs_disable_release_age_gate() -> None:
@ -105,3 +108,89 @@ def test_dockerfiles_pin_project_binary_paths_for_validation() -> None:
assert 'GIT_BINARY="$LIB_DIR/env/bin/git"' in text
assert 'GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl"' in text
assert 'FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl"' in text
assert 'OPENCODE_BINARY="$LIB_DIR/env/bin/opencode"' in text
def test_dockerfiles_preinstall_opencode_without_pnpm_home_override() -> None:
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
text = (REPO_ROOT / dockerfile_name).read_text()
assert 'PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"' in text
assert "env -u PNPM_HOME" in text
assert "/opt/node/bin/corepack pnpm add" in text
assert '--store-dir="$TMP_DIR/pnpm-store"' in text
assert '--dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai' in text
assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode"' in text
assert 'rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node' in text
assert 'ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode"' in text
assert 'ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode"' in text
assert 'chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"' in text
assert 'chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode"' in text
def test_dockerfiles_use_setpriv_instead_of_gosu() -> None:
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
text = (REPO_ROOT / dockerfile_name).read_text()
assert "gosu" not in text
assert "setpriv" in text
entrypoint = (REPO_ROOT / "bin/docker_entrypoint.sh").read_text()
assert "gosu" not in entrypoint
assert "setpriv" in entrypoint
def test_dockerfiles_create_archivebox_venv_in_archivebox_builder() -> None:
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
text = (REPO_ROOT / dockerfile_name).read_text()
assert "COPY --from=abx-dl /venv /venv" not in text
assert 'uv venv /venv --python "${PYTHON_VERSION}"' in text
assert "COPY --from=archivebox-builder /opt/uv/python /opt/uv/python" in text
assert 'COMMIT_HASH="$(' in text
assert 'HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"' in text
assert 'echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt' in text
assert 'rm -rf "$CODE_DIR/.git"' in text
def test_dockerfiles_do_not_duplicate_runtime_lib_with_recursive_chown() -> None:
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
text = (REPO_ROOT / dockerfile_name).read_text()
assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"' not in text
assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR"' not in text
assert 'chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"' in text
assert "COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib" in text
def test_dockerfiles_validate_current_papers_dl_path() -> None:
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
text = (REPO_ROOT / dockerfile_name).read_text()
assert '"$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version' in text
assert "$LIB_DIR/pip/packages/papers-dl" not in text
def test_dockerfiles_do_not_default_multiarch_builds_to_amd64() -> None:
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
text = (REPO_ROOT / dockerfile_name).read_text()
assert "ARG TARGETPLATFORM=linux/amd64" not in text
assert "ARG TARGETARCH=amd64" not in text
assert "ARG TARGETPLATFORM\n" in text
assert "FROM ${ABX_DL_IMAGE} AS abx-dl" in text
def test_dockerfiles_set_archivebox_home_for_setpriv_commands() -> None:
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
text = (REPO_ROOT / dockerfile_name).read_text()
assert (
'install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus"'
) in text
assert 'HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config"' in text
def test_dockerignore_keeps_minimal_git_metadata_for_image_version() -> None:
text = (REPO_ROOT / ".dockerignore").read_text()
assert ".git/" in text
assert "!.git/" in text
assert ".git/*" in text
assert "!.git/HEAD" in text
assert "!.git/packed-refs" in text
assert "!.git/refs/" in text
assert "!.git/refs/heads/" in text
assert "!.git/refs/heads/**" in text

6
archivebox/tests/test_hooks.py
View File

@ -106,9 +106,9 @@ def test_cli_env_does_not_emit_relative_pythonpath_entries():
os.environ["PYTHONPATH"] = old_pythonpath
pythonpath_entries = env["PYTHONPATH"].split(os.pathsep)
assert str((WORKSPACE_ROOT / "abxpkg").resolve(strict=False)) in pythonpath_entries
assert str((WORKSPACE_ROOT / "abx-plugins").resolve(strict=False)) in pythonpath_entries
assert str((WORKSPACE_ROOT / "abx-dl").resolve(strict=False)) in pythonpath_entries
for repo_name in ("abxpkg", "abx-plugins", "abx-dl"):
repo_path = next(path for path in (WORKSPACE_ROOT / repo_name, REPO_ROOT / repo_name) if path.exists())
assert str(repo_path.resolve(strict=False)) in pythonpath_entries
assert all(Path(entry).is_absolute() for entry in pythonpath_entries)
assert not any(entry.startswith("..") for entry in pythonpath_entries)

2
archivebox/tests/test_machine_models.py
View File

@ -929,7 +929,7 @@ class TestProcessClassMethods:
child.refresh_from_db()
assert child.status == Process.StatusChoices.EXITED
assert child.ended_at is not None
assert child.exit_code == 0
assert child.exit_code == 143
class TestProcessStateMachine:

8
archivebox/tests/test_opencode_agent.py
View File

@ -48,7 +48,6 @@ def _set_archivebox_config(data_dir: Path, *values: str, env: dict[str, str] | N
@pytest.fixture
def opencode_archive_config(initialized_archive):
port = _free_port()
lib_dir = initialized_archive / "lib"
state_dir = initialized_archive / "opencode"
env = os.environ.copy()
env.update(
@ -57,8 +56,6 @@ def opencode_archive_config(initialized_archive):
"ABXPKG_MIN_RELEASE_AGE": "0",
"ABX_RUNTIME": "archivebox",
"ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS": "true",
"LIB_DIR": str(lib_dir),
"ABXPKG_LIB_DIR": str(lib_dir),
"OPENCODE_ENABLED": "True",
"OPENCODE_HOST": "127.0.0.1",
"OPENCODE_PORT": str(port),
@ -75,10 +72,9 @@ def opencode_archive_config(initialized_archive):
f"OPENCODE_WORKDIR={initialized_archive}",
f"OPENCODE_STATE_DIR={state_dir}",
"OPENCODE_TIMEOUT=60",
f"LIB_DIR={lib_dir}",
env=env,
)
return SimpleNamespace(data_dir=initialized_archive, lib_dir=lib_dir, port=port, state_dir=state_dir, env=env)
return SimpleNamespace(data_dir=initialized_archive, port=port, state_dir=state_dir, env=env)
@pytest.fixture
@ -242,7 +238,7 @@ def test_opencode_starts_with_data_dir_cwd_and_isolated_state(live_opencode):
assert env["XDG_STATE_HOME"] == str(live_opencode.config.state_dir / "state")
assert env["XDG_CACHE_HOME"] == str(live_opencode.config.state_dir / "cache")
assert env["OPENCODE_DISABLE_PROJECT_CONFIG"] == "true"
assert env["GIT_CEILING_DIRECTORIES"] == f"{workdir}{os.pathsep}{live_opencode.config.data_dir.parent.resolve()}"
assert env["GIT_CEILING_DIRECTORIES"] == workdir
def test_opencode_state_dir_is_separate_from_workdir(tmp_path):

23
archivebox/tests/test_recursive_crawl.py
View File

@ -575,9 +575,6 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
.order_by("depth", "url")
.values_list("id", "url", "depth", "status", "parent_snapshot_id", "downloaded_at"),
)
snapshot_ids_by_output_dir = {
str(snapshot.output_dir): str(snapshot.id) for snapshot in Snapshot.objects.filter(crawl=crawl).order_by("depth", "url")
}
archive_results = list(
ArchiveResult.objects.filter(snapshot__crawl=crawl)
.select_related("snapshot")
@ -594,10 +591,17 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
"output_str",
),
)
process_snapshot_ids = {
process_id: str(snapshot_id)
for snapshot_id, process_id in ArchiveResult.objects.filter(
snapshot__crawl=crawl,
process_id__isnull=False,
).values_list("snapshot_id", "process_id")
}
processes = list(
Process.objects.filter(process_type=Process.TypeChoices.HOOK, pwd__contains=str(crawl.output_dir))
Process.objects.filter(process_type=Process.TypeChoices.HOOK, id__in=process_snapshot_ids)
.order_by("started_at")
.values_list("pwd", "cmd", "status", "exit_code", "started_at", "ended_at"),
.values_list("id", "pwd", "cmd", "status", "exit_code", "started_at", "ended_at"),
)
assert crawl.max_depth == 2
@ -681,16 +685,13 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia
if status == ArchiveResult.StatusChoices.FAILED and plugin != "archivedotorg"
]
assert not failed_hook_results
assert all(status == Process.StatusChoices.EXITED for _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
assert all(status == Process.StatusChoices.EXITED for _id, _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
intervals = []
for pwd, cmd, _status, _exit_code, started_at, ended_at in processes:
for process_id, pwd, cmd, _status, _exit_code, started_at, ended_at in processes:
if not started_at or not ended_at:
continue
process_snapshot_id = next(
(snapshot_id for output_dir, snapshot_id in snapshot_ids_by_output_dir.items() if str(pwd).startswith(output_dir)),
None,
)
process_snapshot_id = process_snapshot_ids.get(process_id)
if process_snapshot_id is None:
continue
intervals.append((process_snapshot_id, started_at, ended_at, pwd, cmd))

27
archivebox/tests/test_server_security_browser.py
View File

@ -14,7 +14,7 @@ from urllib.parse import urlencode
import pytest
from .conftest import _ensure_puppeteer, _find_cached_chrome, _find_system_browser, run_python_cwd
from .conftest import _find_cached_chrome, _find_system_browser, run_python_cwd
from .conftest import (
cli_env,
get_free_port,
@ -289,13 +289,28 @@ def _resolve_browser(shared_lib: Path) -> Path | None:
return None
@pytest.fixture(scope="session")
def browser_runtime(tmp_path_factory):
@pytest.fixture
def browser_runtime(initialized_archive: Path):
assert shutil.which("node") is not None, "Node.js is required for browser security tests"
assert shutil.which("pnpm") is not None, "pnpm is required for browser security tests"
shared_lib = tmp_path_factory.mktemp("archivebox_browser_lib")
_ensure_puppeteer(shared_lib)
shared_lib = initialized_archive / "lib"
env = cli_env(
ABXPKG_INSTALL_TIMEOUT="900",
ABXPKG_MIN_RELEASE_AGE="0",
LIB_DIR=str(shared_lib),
ABXPKG_LIB_DIR=str(shared_lib),
CHROME_HEADLESS="True",
CHROME_SANDBOX="False",
CHROME_ISOLATION="snapshot",
)
env.pop("CHROME_BINARY", None)
install_result = run_archivebox_cmd(
["install", "chrome"],
cwd=initialized_archive,
env=env,
timeout=900,
)
assert install_result.returncode == 0, install_result.stderr or install_result.stdout
browser = _resolve_browser(shared_lib)
assert browser, "No Chrome/Chromium binary available for browser security tests"

34
archivebox/tests/test_takeover_util.py
View File

@ -379,7 +379,7 @@ def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tm
)
assert "Stopping older ArchiveBox runner process" in update_stdout
deadline = time.time() + 90
deadline = time.time() + 180
runner_pid_after = runner_pid_before
while time.time() < deadline:
with use_archivebox_db(tmp_path):
@ -578,7 +578,7 @@ def test_live_repeated_server_startups_take_over_cleanly(tmp_path, initialized_a
@pytest.mark.timeout(420)
def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initialized_archive):
def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initialized_archive, recursive_test_site):
plugins_root = tmp_path / "runtime_plugins"
marker_dir = tmp_path / "slow-plugin-markers"
plugin_dir = plugins_root / "slow_exit"
@ -658,8 +658,8 @@ def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initi
"--max-urls=2",
"--crawl-max-size=50mb",
"--plugins=wget,parse_html_urls,slow_exit",
"https://example.com",
"https://blog.sweeting.me",
recursive_test_site["root_url"],
recursive_test_site["child_urls"][0],
],
cwd=tmp_path,
env=env,
@ -696,12 +696,15 @@ def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initi
assert add_proc.poll() is None, "foreground add should keep owning its crawl after the server exits"
assert "Got SIGTERM" in server_log.read_text(encoding="utf-8", errors="replace")
stop_archivebox_process(add_proc, signal.SIGKILL, timeout=30)
add_output = add_log.read_text(encoding="utf-8", errors="replace")
assert "Runner error" not in add_output
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
server2 = start_archivebox_server(tmp_path, port=port, log_name="server-add-owner-2.log", env=env)
_server2_log = server2.log_path
(marker_dir / "allow-finish").touch()
stop_archivebox_process(add_proc, signal.SIGTERM, timeout=30)
add_output = add_log.read_text(encoding="utf-8", errors="replace")
assert "Runner error" not in add_output
deadline = time.time() + 90
crawls = []
@ -751,17 +754,12 @@ def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initi
assert len(counter_runs) == len(set(counter_runs))
assert not (marker_dir / "counter-duplicates.txt").exists()
# TODO: improve abx-dl's ability to explicitly resume from a given plugin / hook and skip ones before that
# current behavior: on retry, earlier sealed results are left untouched; the interrupted result is marked skipped
# and may have partial output saved to fs
# assertions that enforce the current behavior (uncommented): previous results are not run twice + interrupted
# result is marked skipped
assert bad_results == [("slow_exit", ArchiveResult.StatusChoices.SKIPPED, "")]
# desired future behavior: earlier sealed results are left untouched, interrupted result is retried and cleanly
# overwrites on top of any previous partial output
# assert not bad_results
# assert (marker_dir / "hook-finished").exists()
# The interrupted hook should be retried directly without rerunning the
# previous hook in the same plugin. That keeps plugin-level shell hooks
# idempotent across runner takeover instead of depending on each hook to
# detect partial prior work itself.
assert not bad_results
assert (marker_dir / "hook-finished").exists()
finally:
for proc in (add_proc, add_proc2, server, server2, server3):
if proc is not None and proc.poll() is None:

16
archivebox/tests/test_urls.py
View File

@ -895,19 +895,19 @@ class TestUrlRouting:
assert f"http://{public_host}/static/archive.png" in live_html
assert "?preview=1" in live_html
assert "function createMainFrame(previousFrame)" in live_html
assert "function activateCardPreview(card, link)" in live_html
assert "ensureMainFrame(true)" in live_html
assert "function activateCardPreview(card, link, updateHash=true)" in live_html
assert "ensureMainFrame(currentSrc !== nextSrcAbs)" in live_html
assert "previousFrame.parentNode.replaceChild(frame, previousFrame)" in live_html
assert "previousFrame.src = 'about:blank'" in live_html
assert "event.stopImmediatePropagation()" in live_html
assert "const matchingLink = [...document.querySelectorAll('a[target=preview]')].find" in live_html
assert "const matchingLink = findPreviewLinkForHash(selectedPreviewHash)" in live_html
assert "jQuery(link).click()" not in live_html
assert "searchParams.delete('preview')" in live_html
assert "doc.body.style.flexDirection = 'column'" in live_html
assert "doc.body.style.alignItems = 'center'" in live_html
assert "img.style.margin = '0 auto'" in live_html
assert "window.location.hash = getPreviewHashValueFromHref(rawTarget)" in live_html
assert "const selectedPreviewHash = decodeURIComponent(window.location.hash.slice(1)).toLowerCase()" in live_html
assert "const selectedPreviewHash = window.location.hash ? decodeURIComponent(window.location.hash.slice(1)).toLowerCase() : ''" in live_html
assert "pointer-events: none;" in live_html
assert "pointer-events: auto;" in live_html
assert 'class="thumbnail-click-overlay"' in live_html
@ -921,19 +921,19 @@ class TestUrlRouting:
assert f"http://{public_host}/static/archive.png" in static_html
assert "?preview=1" in static_html
assert "function createMainFrame(previousFrame)" in static_html
assert "function activateCardPreview(card, link)" in static_html
assert "ensureMainFrame(true)" in static_html
assert "function activateCardPreview(card, link, updateHash=true)" in static_html
assert "ensureMainFrame(currentSrc !== nextSrcAbs)" in static_html
assert "previousFrame.parentNode.replaceChild(frame, previousFrame)" in static_html
assert "previousFrame.src = 'about:blank'" in static_html
assert "event.stopImmediatePropagation()" in static_html
assert "const matchingLink = [...document.querySelectorAll('a[target=preview]')].find" in static_html
assert "const matchingLink = findPreviewLinkForHash(selectedPreviewHash)" in static_html
assert "jQuery(link).click()" not in static_html
assert "searchParams.delete('preview')" in static_html
assert "doc.body.style.flexDirection = 'column'" in static_html
assert "doc.body.style.alignItems = 'center'" in static_html
assert "img.style.margin = '0 auto'" in static_html
assert "window.location.hash = getPreviewHashValueFromHref(rawTarget)" in static_html
assert "const selectedPreviewHash = decodeURIComponent(window.location.hash.slice(1)).toLowerCase()" in static_html
assert "const selectedPreviewHash = window.location.hash ? decodeURIComponent(window.location.hash.slice(1)).toLowerCase() : ''" in static_html
assert "pointer-events: none;" in static_html
assert "pointer-events: auto;" in static_html
assert 'class="thumbnail-click-overlay"' in static_html

90
archivebox/tests/test_version_metadata.py Normal file
View File

@ -0,0 +1,90 @@
from pathlib import Path
from archivebox.config import version
def _set_package_dir(monkeypatch, package_dir: Path) -> None:
monkeypatch.setattr(version, "PACKAGE_DIR", package_dir)
version.get_COMMIT_HASH.cache_clear()
def test_get_commit_hash_from_environment(monkeypatch) -> None:
commit_hash = "e" * 40
monkeypatch.setenv("ARCHIVEBOX_COMMIT_HASH", commit_hash)
version.get_COMMIT_HASH.cache_clear()
assert version.get_COMMIT_HASH() == commit_hash
def test_get_commit_hash_from_docker_version_file_ignores_short_hash(monkeypatch) -> None:
commit_hash = "f" * 40
class VersionPath:
def __init__(self, path: str):
self.path = path
def read_text(self) -> str:
assert self.path == "/VERSION.txt"
return f"COMMIT_HASH={commit_hash}\nArchiveBox COMMIT_HASH={commit_hash[:7]}\n"
monkeypatch.setattr(version, "IN_DOCKER", True)
monkeypatch.setattr(version, "Path", VersionPath)
version.get_COMMIT_HASH.cache_clear()
assert version.get_COMMIT_HASH() == commit_hash
def test_get_commit_hash_from_detached_head(monkeypatch, tmp_path) -> None:
commit_hash = "a" * 40
package_dir = tmp_path / "archivebox"
git_dir = tmp_path / ".git"
package_dir.mkdir()
git_dir.mkdir()
git_dir.joinpath("HEAD").write_text(commit_hash)
_set_package_dir(monkeypatch, package_dir)
assert version.get_COMMIT_HASH() == commit_hash
def test_get_commit_hash_from_branch_ref(monkeypatch, tmp_path) -> None:
commit_hash = "b" * 40
package_dir = tmp_path / "archivebox"
git_dir = tmp_path / ".git"
ref_path = git_dir / "refs" / "heads" / "dev"
package_dir.mkdir()
ref_path.parent.mkdir(parents=True)
git_dir.joinpath("HEAD").write_text("ref: refs/heads/dev")
ref_path.write_text(commit_hash)
_set_package_dir(monkeypatch, package_dir)
assert version.get_COMMIT_HASH() == commit_hash
def test_get_commit_hash_from_packed_ref(monkeypatch, tmp_path) -> None:
commit_hash = "c" * 40
package_dir = tmp_path / "archivebox"
git_dir = tmp_path / ".git"
package_dir.mkdir()
git_dir.mkdir()
git_dir.joinpath("HEAD").write_text("ref: refs/heads/dev")
git_dir.joinpath("packed-refs").write_text(f"{commit_hash} refs/heads/dev\n")
_set_package_dir(monkeypatch, package_dir)
assert version.get_COMMIT_HASH() == commit_hash
def test_get_commit_hash_from_worktree_gitdir(monkeypatch, tmp_path) -> None:
commit_hash = "d" * 40
package_dir = tmp_path / "worktree" / "archivebox"
real_git_dir = tmp_path / "repo" / ".git" / "worktrees" / "worktree"
package_dir.mkdir(parents=True)
real_git_dir.mkdir(parents=True)
package_dir.parent.joinpath(".git").write_text(f"gitdir: {real_git_dir}\n")
real_git_dir.joinpath("HEAD").write_text(commit_hash)
_set_package_dir(monkeypatch, package_dir)
assert version.get_COMMIT_HASH() == commit_hash

2
archivebox/workers/supervisord_util.py
View File

@ -170,7 +170,7 @@ RUNNER_WORKER = {
RUNNER_ONCE_WORKER = lambda args, name="worker_runner_once": {
**RUNNER_WORKER,
"name": name,
"command": _shell_join([sys.executable, "-m", "archivebox", "run", *args]),
"command": _shell_join([sys.executable, "-m", "archivebox", "run", "--no-stdin", *args]),
"environment": 'PYTHONUNBUFFERED="1",COLUMNS="200"',
"autorestart": "false",
"stopwaitsecs": "1",

8
bin/docker_entrypoint.sh
View File

@ -144,7 +144,7 @@ ensure_runtime_tree() {
run_as_archivebox() {
if [[ "$(id -u)" == "0" ]]; then
gosu "$ARCHIVEBOX_USER" "$@"
setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups "$@"
else
"$@"
fi
@ -283,12 +283,12 @@ if [[ "$1" == /* || "$1" == "bash" || "$1" == "sh" || "$1" == "echo" || "$1" ==
# "docker run archivebox /bin/bash -c '...'"
# "docker run archivebox cat /VERSION.txt"
if [[ "$(id -u)" == "0" ]]; then
exec gosu "$ARCHIVEBOX_USER" /bin/bash -c "exec $(printf ' %q' "$@")"
exec setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups /bin/bash -c "exec $(printf ' %q' "$@")"
else
exec /bin/bash -c "exec $(printf ' %q' "$@")"
fi
# printf requotes shell parameters properly https://stackoverflow.com/a/39463371/2156113
# gosu spawns an ephemeral bash process owned by archivebox user (bash wrapper is needed to load env vars, PATH, and setup terminal TTY)
# setpriv spawns an ephemeral bash process owned by archivebox user (bash wrapper is needed to load env vars, PATH, and setup terminal TTY)
# outermost exec hands over current process ID to inner bash process, inner exec hands over inner bash PID to user's command
else
# handle "docker run archivebox add some subcommand --with=args abc" by calling archivebox to run as args as CLI subcommand
@ -297,7 +297,7 @@ else
# "docker run archivebox manage createsupseruser"
# "docker run archivebox server 0.0.0.0:8000"
if [[ "$(id -u)" == "0" ]]; then
exec gosu "$ARCHIVEBOX_USER" "$ARCHIVEBOX_BIN_PATH" "$@"
exec setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups "$ARCHIVEBOX_BIN_PATH" "$@"
else
exec "$ARCHIVEBOX_BIN_PATH" "$@"
fi