diff --git a/.dockerignore b/.dockerignore index 1447cfee..5b2e02fe 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,9 +12,14 @@ __pycache__/ .pdm-python .eggs/ .git/ +!.git/ +.git/* .vscode/ !.git/HEAD -!.git/refs/heads/* +!.git/packed-refs +!.git/refs/ +!.git/refs/heads/ +!.git/refs/heads/** venv/ .venv/ diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cb88fd63..e9eaa9e7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,7 +5,7 @@ on: workflow_call: push: branches: - - dev + - "**" tags: - 'v*' # pull_request: @@ -13,13 +13,14 @@ on: env: DOCKERHUB_IMAGE: archivebox/archivebox GHCR_IMAGE: ghcr.io/archivebox/archivebox + ABX_DL_IMAGE: archivebox/abx-dl:latest permissions: contents: read packages: write concurrency: - group: docker-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -96,6 +97,24 @@ jobs: - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} + - name: Wait for published abx-dl image + id: abx_dl_image + shell: bash + run: | + set -Eeuo pipefail + deadline=$((SECONDS + 1800)) + until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do + if (( SECONDS >= deadline )); then + echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2 + exit 1 + fi + echo "${ABX_DL_IMAGE} is not published yet; waiting..." + sleep 30 + done + + echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT" + docker buildx imagetools inspect "${ABX_DL_IMAGE}" + - name: Login to Docker Hub uses: docker/login-action@v3 if: github.event_name != 'pull_request' @@ -116,13 +135,10 @@ jobs: run: | set -Eeuo pipefail VERSION="$(python3 - <<'PY' - from pathlib import Path - import re + import tomllib - match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE) - if not match: - raise SystemExit("Failed to read version from pyproject.toml") - print(match.group(1)) + with open("pyproject.toml", "rb") as f: + print(tomllib.load(f)["project"]["version"]) PY )" @@ -131,11 +147,12 @@ jobs: echo "org.opencontainers.image.version=${VERSION}" echo "org.opencontainers.image.revision=${GITHUB_SHA}" echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox' + echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}" echo 'EOF' echo "version=${VERSION}" } >> "$GITHUB_OUTPUT" - echo "[+] Building ${{ matrix.platform }} for ${VERSION}" + echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}" - name: Build and push digest id: docker_build @@ -149,6 +166,8 @@ jobs: ${{ env.DOCKERHUB_IMAGE }} ${{ env.GHCR_IMAGE }} labels: ${{ steps.docker_meta.outputs.labels }} + build-args: | + ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }} cache-from: type=gha,scope=${{ matrix.cache_scope }} cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }} platforms: ${{ matrix.platform }} @@ -215,36 +234,49 @@ jobs: run: | set -Eeuo pipefail VERSION="$(python3 - <<'PY' - from pathlib import Path - import re + import tomllib - match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE) - if not match: - raise SystemExit("Failed to read version from pyproject.toml") - print(match.group(1)) + with open("pyproject.toml", "rb") as f: + print(tomllib.load(f)["project"]["version"]) PY )" - SHORT_SHA="${GITHUB_SHA::8}" + BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)" + SHORT_SHA="${GITHUB_SHA::12}" + test -n "$BRANCH_TAG" + test -n "$SHORT_SHA" { echo 'dockerhub_tags<> "$GITHUB_OUTPUT" echo "[+] Publishing Docker Hub tags:" - printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" + if [[ "${GITHUB_REF_NAME}" == "main" ]]; then + printf '%s\n' "${DOCKERHUB_IMAGE}:latest" + fi + printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" echo "[+] Publishing GHCR tags:" - printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}" + if [[ "${GITHUB_REF_NAME}" == "main" ]]; then + printf '%s\n' "${GHCR_IMAGE}:latest" + fi + printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}" - name: Create Docker Hub manifest shell: bash diff --git a/Dockerfile b/Dockerfile index 571c07fa..9e136a20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,66 +1,29 @@ -# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies: -# python3.13, uv, python3-ldap -# curl, wget, git, dig, ping, tree, nano -# node, npm -# ArchiveBox and plugin runtime dependencies installed by archivebox init --install -# Usage: -# git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox -# docker build . -t archivebox -# docker run -v "$PWD/data":/data archivebox init -# docker run -v "$PWD/data":/data archivebox add 'https://example.com' -# docker run -v "$PWD/data":/data -it archivebox manage createsuperuser -# docker run -v "$PWD/data":/data -p 8000:8000 archivebox server -# docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123 -# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development +# syntax=docker/dockerfile:1.7 +# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image. +# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes. +# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime. +# Build abx-dl first, then point this file at it: +# docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \ +# --build-context abxbus=../abxbus \ +# --build-context abxpkg=../abxpkg \ +# --build-context abx-plugins=../abx-plugins \ +# -t archivebox/abx-dl:dev +# docker buildx build . -f Dockerfile \ +# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \ +# -t archivebox:multistage -######################################################################################### - -### Example: Using ArchiveBox in your own project's Dockerfile ######## - -# FROM python:3.13-slim -# WORKDIR /data -# RUN pip install archivebox>=0.9.0 # use latest release here -# RUN archivebox install -# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data - -######################################################################################### - -ARG TARGETPLATFORM -ARG TARGETOS -ARG TARGETARCH -ARG TARGETVARIANT= +ARG ABX_DL_IMAGE=archivebox/abx-dl:latest +FROM ${ABX_DL_IMAGE} AS abx-dl FROM archivebox/sonic:1.4.9 AS sonic -FROM ubuntu:24.04 - -LABEL name="archivebox" \ - maintainer="Nick Sweeting " \ - description="All-in-one self-hosted internet archiving solution" \ - homepage="https://github.com/ArchiveBox/ArchiveBox" \ - documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \ - org.opencontainers.image.title="ArchiveBox" \ - org.opencontainers.image.vendor="ArchiveBox" \ - org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \ - org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \ - com.docker.image.source.entrypoint="Dockerfile" \ - # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels): - # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/ - com.docker.desktop.extension.api.version=">= 1.4.7" \ - com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \ - com.docker.extension.publisher-url="https://archivebox.io" \ - com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \ - com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \ - com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \ - com.docker.extension.categories='database,utility-tools' +FROM ubuntu:24.04 AS archivebox-runtime-base ARG TARGETPLATFORM ARG TARGETOS ARG TARGETARCH ARG TARGETVARIANT -######### Environment Variables ################################# -# Global build-time and runtime environment constants + default pkg manager config ENV TZ=UTC \ LANGUAGE=en_US:en \ LC_ALL=C.UTF-8 \ @@ -73,204 +36,93 @@ ENV TZ=UTC \ PIP_ONLY_BINARY=aiohttp \ npm_config_loglevel=error -# Language Version config ENV PYTHON_VERSION=3.13 \ NODE_VERSION=22.22.3 -# Non-root User config -ENV ARCHIVEBOX_USER="archivebox" \ +ENV ARCHIVEBOX_USER=archivebox \ DEFAULT_PUID=911 \ DEFAULT_PGID=911 \ - IN_DOCKER=True \ - BIND_ADDR=0.0.0.0:8000 -# Docker has to listen on all interfaces, not just localhost. + IN_DOCKER=True -# ArchiveBox Source Code + Lib + Data paths ENV CODE_DIR=/app \ DATA_DIR=/data \ LIB_DIR=/opt/archivebox/lib \ ABXPKG_LIB_DIR=/opt/archivebox/lib \ - SONIC_BINARY=/opt/archivebox/lib/env/bin/sonic \ - PLAYWRIGHT_BROWSERS_PATH=/browsers - -# Bash SHELL config -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] - -######### System Environment #################################### - -# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes) -WORKDIR "$CODE_DIR" - -# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds) -RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \ - && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \ - && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \ - && rm -f /etc/apt/apt.conf.d/docker-clean - -# Print debug info about build and save it to disk, for human eyes only, not used by anything else -RUN (echo "[i] Docker build for ArchiveBox starting..." \ - && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \ - && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \ - && echo \ - && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \ - && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \ - && echo \ - && uname -a \ - && sed -n '1,7p' /etc/os-release \ - && which bash && bash --version | sed -n '1p' \ - && which dpkg && dpkg --version | sed -n '1p' \ - && echo -e '\n\n' && env && echo -e '\n\n' \ - ) | tee -a /VERSION.txt - -# Create non-privileged user for archivebox and chrome -RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ - && groupadd --system $ARCHIVEBOX_USER \ - && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \ - && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ - && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ - && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \ - | tee -a /VERSION.txt - # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime - # https://docs.linuxserver.io/general/understanding-puid-and-pgid - -# Install system apt dependencies (adding backports to access more recent apt updates) -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \ - && mkdir -p /etc/apt/keyrings \ - && apt-get update -qq \ - && apt-get install -qq -y \ - # 1. packaging dependencies - apt-transport-https ca-certificates apt-utils gnupg2 curl wget \ - # 2. docker and init system dependencies - zlib1g-dev dumb-init gosu cron unzip grep dnsutils git ripgrep python3.12-venv default-jre-headless \ - # 3. frivolous CLI helpers to make debugging failed archiving easier - tree nano iputils-ping \ - # nano iputils-ping dnsutils htop procps jq yq - && rm -rf /var/lib/apt/lists/* - -# Install sonic search backend -COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic -COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg -RUN (which sonic && sonic --version) | tee -a /VERSION.txt - -######### Language Environments #################################### - -# Set up Python environment -# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own: -#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ -# --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \ -# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \ - # && apt-get update -qq \ - # && apt-get install -qq -y --no-upgrade \ - # python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \ - # && rm -rf /var/lib/apt/lists/* \ - # tell PDM to allow using global system python site packages - # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \ - # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \ - # create global virtual environment GLOBAL_VENV to use (better than using pip install --global) - # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \ - # && python3 -m venv --system-site-packages $GLOBAL_VENV \ - # && python3 -m venv $GLOBAL_VENV \ - # install global dependencies / python build dependencies in GLOBAL_VENV - # && pip install --upgrade pip setuptools wheel \ - # Save version info - # && ( \ - # which python3 && python3 --version | grep " $PYTHON_VERSION" \ - # && which pip && pip --version \ - # # && which pdm && pdm --version \ - # && echo -e '\n\n' \ - # ) | tee -a /VERSION.txt - - -# Set up Node environment from the official platform tarball. This avoids -# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into -# emulated Docker builds. -RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \ - case "$TARGETARCH" in \ - amd64) NODE_DIST_ARCH="x64" ;; \ - arm64) NODE_DIST_ARCH="arm64" ;; \ - *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \ - esac \ - && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \ - && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \ - && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \ - && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \ - && grep " ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \ - && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \ - && rm "$NODE_TARBALL" SHASUMS256.txt \ - # Save version info - && ( \ - which node && node --version \ - && which npm && npm --version \ - && echo -e '\n\n' \ - ) | tee -a /VERSION.txt - - -# Set up uv and main app /venv -RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh -ENV UV_COMPILE_BYTECODE=0 \ - UV_PYTHON_PREFERENCE=managed \ - UV_PYTHON_INSTALL_DIR=/opt/uv/python \ - UV_LINK_MODE=copy \ - UV_PROJECT_ENVIRONMENT=/venv -WORKDIR "$CODE_DIR" -# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/" -RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \ - && uv venv /venv --python ${PYTHON_VERSION} -ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH" -RUN uv pip install setuptools pip \ - && ( \ - which python3 && python3 --version \ - && which uv && uv self version \ - && uv python find --system && uv python find \ - && echo -e '\n\n' \ - ) | tee -a /VERSION.txt - - -######### ArchiveBox & Extractor Dependencies ################################## - -# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap) -WORKDIR "$CODE_DIR" -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \ - echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \ - && apt-get update -qq \ - && apt-get install -qq -y --no-install-recommends \ - build-essential gcc \ - python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \ - python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \ - && uv pip install \ - "python-ldap>=3.4.3" \ - && apt-get purge -y \ - python3-dev build-essential gcc \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - - -# Runtime config used by plugin hooks. Plugin binaries and npm packages are -# installed into LIB_DIR below by archivebox init --install and resolved from -# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH. -ENV PERSONAS_DIR=/data/personas \ + PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \ + PERSONAS_DIR=/data/personas \ CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \ CHROME_HEADLESS=true \ CHROME_SANDBOX=false \ CHROME_ISOLATION=crawl \ CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]' -######### Build Dependencies #################################### +ENV TMP_DIR=/tmp/archivebox \ + PIP_VENV_PYTHON=/venv/bin/python3 \ + GOOGLE_API_KEY=no \ + GOOGLE_DEFAULT_CLIENT_ID=no \ + GOOGLE_DEFAULT_CLIENT_SECRET=no +ENV UV_COMPILE_BYTECODE=0 \ + UV_PYTHON_PREFERENCE=managed \ + UV_PYTHON_INSTALL_DIR=/opt/uv/python \ + UV_LINK_MODE=copy \ + UV_PROJECT_ENVIRONMENT=/venv \ + VIRTUAL_ENV=/venv \ + PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH" -# Install ArchiveBox Python venv dependencies from pyproject.toml. -RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ - --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \ +SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] +WORKDIR "$CODE_DIR" + +RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \ + && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \ + && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \ + && rm -f /etc/apt/apt.conf.d/docker-clean + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ + echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \ && apt-get update -qq \ - && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \ - && uv --no-cache sync \ + && apt-get install -qq -y \ + apt-transport-https apt-utils ca-certificates curl wget gnupg2 \ + dumb-init util-linux unzip git grep ripgrep dnsutils iputils-ping procps tree nano \ + cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \ + libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \ + libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \ + libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \ + libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \ + fonts-liberation fonts-noto-color-emoji xdg-utils \ + ffmpeg imagemagick tesseract-ocr tesseract-ocr-eng openjdk-21-jre-headless \ + && rm -rf /var/lib/apt/lists/* + +# Runtime-owned layers copied from the abx-dl image. +COPY --from=abx-dl /bin/uv /bin/uv +COPY --from=abx-dl /opt/uv/python /opt/uv/python +COPY --from=abx-dl /opt/node /opt/node +COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt + +RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ + && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \ + && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \ + && uname -a \ + && sed -n '1,7p' /etc/os-release \ + && which node && node --version \ + && which uv && uv self version \ + ) | tee -a /VERSION.txt + +ENV PYTHONDONTWRITEBYTECODE=1 + +FROM archivebox-runtime-base AS archivebox-builder + +WORKDIR "$CODE_DIR" +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ + --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ + --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ + echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \ + && apt-get update -qq \ + && apt-get install -qq -y --no-install-recommends \ + build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ + && uv venv /venv --python "${PYTHON_VERSION}" \ + && uv pip install setuptools pip wheel \ + && uv sync \ --refresh \ --no-dev \ --inexact \ @@ -278,110 +130,73 @@ RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ --no-install-project \ --no-install-workspace \ --no-sources \ - && apt-get purge -y python3-dev build-essential gcc \ + && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ && apt-get autoremove -y \ && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf /var/lib/apt/lists/* - # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies -# Setup ArchiveBox runtime config -ENV TMP_DIR=/tmp/archivebox \ - PIP_VENV_PYTHON=/usr/bin/python3.12 \ - GOOGLE_API_KEY=no \ - GOOGLE_DEFAULT_CLIENT_ID=no \ - GOOGLE_DEFAULT_CLIENT_SECRET=no - -WORKDIR "$DATA_DIR" -RUN openssl rand -hex 16 > /etc/machine-id \ - && mkdir -p "$DATA_DIR" \ - && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \ - && mkdir -p "$TMP_DIR" \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \ - && mkdir -p "$LIB_DIR" \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ - && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \ - && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \ - && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt - -# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer -# path users run later, before copying ArchiveBox source so source-only edits do -# not invalidate the heavy browser/plugin dependency layer. -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \ - echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \ - && export PERSONAS_DIR="$LIB_DIR/personas" \ - && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \ - && export ABX_RUNTIME=archivebox ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \ - && mkdir -p "$LIB_DIR" \ - && apt-get update -qq \ - && apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \ - && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" gallery-dl \ - && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" --overrides='{"pip":{"install_args":["--no-deps","forum-dl","chardet==5.2.0","pydantic==2.12.3","pydantic-core==2.41.4","typing-extensions>=4.14.1","annotated-types>=0.6.0","typing-inspection>=0.4.2","beautifulsoup4","soupsieve","lxml","requests","urllib3","certifi","idna","charset-normalizer","tenacity","python-dateutil","six","html2text","warcio"]}}' forum-dl \ - && if [ "$TARGETARCH" = "arm64" ]; then \ - abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \ - abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \ - fi \ - && ABXPKG_INSTALL_TIMEOUT=600 TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \ - accessibility archivedotorg archivewebpage base chrome chrome_mhtml chrome_screencast \ - claudechrome claudecode claudecodecleanup claudecodeextract consolelog defuddle dns dom \ - favicon forumdl gallerydl git hashes headers htmltotext infiniscroll \ - istilldontcareaboutcookies liteparse media mercury modalcloser opencode opendataloader papersdl \ - parse_dom_outlinks parse_html_urls parse_jsonl_urls parse_netscape_urls parse_rss_urls \ - parse_txt_urls pdf readability redirects responses screenshot search_backend_ripgrep \ - search_backend_sonic search_backend_sqlite seo singlefile ssl sslcerts staticfile title \ - trafilatura ublock wget ytdlp \ - && abxpkg install --no-cache --binproviders=chromewebstore --overrides='{"chromewebstore":{"install_args":["fpeoodllldobpkbkabpblcfaogecpndd","--name=archivewebpage"]}}' archivewebpage \ - && test -f "$LIB_DIR/chromewebstore/extensions/fpeoodllldobpkbkabpblcfaogecpndd__archivewebpage/manifest.json" \ - && mkdir -p "$LIB_DIR/env/bin" \ - && ln -sf "$(command -v node)" "$LIB_DIR/env/bin/node" \ - && ln -sf "$(command -v npm)" "$LIB_DIR/env/bin/npm" \ - && ln -sf "$(command -v java)" "$LIB_DIR/env/bin/java" \ - && ln -sf "$(command -v git)" "$LIB_DIR/env/bin/git" \ - && ln -sf "$(command -v rg)" "$LIB_DIR/env/bin/rg" \ - && ln -sf "$(command -v sonic)" "$LIB_DIR/env/bin/sonic" \ - && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ - && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ - && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \ - && mkdir -p /opt/archivebox/lib-layer \ - && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \ - && apt-get purge -y build-essential \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer - -RUN rm -rf "$LIB_DIR" \ - && mv /opt/archivebox/lib-layer "$LIB_DIR" \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" - -# Install ArchiveBox Python package from the checked-out source. -WORKDIR "$CODE_DIR" COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \ - && pip install \ - --no-deps \ - "$CODE_DIR" \ - && ( \ - pip show archivebox \ - && which archivebox \ - && echo -e '\n\n' \ - ) | tee -a /VERSION.txt \ + && COMMIT_HASH="$( \ + if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \ + HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \ + if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \ + echo "$HEAD_REF"; \ + elif [[ "$HEAD_REF" == ref:\ * ]]; then \ + REF_PATH="${HEAD_REF#ref: }"; \ + cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \ + fi; \ + fi)" \ + && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \ + && uv pip install --no-deps "$CODE_DIR" \ + && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ - && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete - # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces + && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ + && rm -rf "$CODE_DIR/.git" + +FROM archivebox-runtime-base + +LABEL name="archivebox" \ + maintainer="Nick Sweeting " \ + description="All-in-one self-hosted internet archiving solution" \ + homepage="https://github.com/ArchiveBox/ArchiveBox" \ + documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \ + org.opencontainers.image.title="ArchiveBox" \ + org.opencontainers.image.vendor="ArchiveBox" \ + org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \ + org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \ + com.docker.image.source.entrypoint="Dockerfile" + +COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic +COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg + +COPY --from=archivebox-builder /opt/uv/python /opt/uv/python +COPY --from=archivebox-builder /venv /venv +COPY --from=archivebox-builder /app /app +COPY --from=archivebox-builder /VERSION.txt /VERSION.txt +COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib + +RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ + && groupadd --system "$ARCHIVEBOX_USER" \ + && useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \ + && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ + && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ + && (which sonic && sonic --version) | tee -a /VERSION.txt \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/uv" \ + && openssl rand -hex 16 > /etc/machine-id \ + && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \ + && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt -# Initialize an empty image collection without rerunning dependency installs. WORKDIR "$DATA_DIR" RUN echo "[+] Initializing image collection..." \ && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \ && PUID=0 PGID=0 archivebox init \ && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \ "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \ "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \ @@ -389,25 +204,35 @@ RUN echo "[+] Initializing image collection..." \ "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \ 2>/dev/null || true) -# Print version for nice docker finish summary -RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \ - && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \ - && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \ - ) | tee -a /VERSION.txt - -# Verify ArchiveBox is installed and write full version/dependency info. RUN chmod +x "$CODE_DIR"/bin/*.sh \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ - && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \ - && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode papersdl parse_rss_urls readability search_backend_sonic opendataloader search_backend_ripgrep 2>&1 | tee -a /VERSION.txt \ - && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \ + && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ + && env -u PNPM_HOME PATH="/opt/node/bin:$PATH" /opt/node/bin/corepack pnpm add --loglevel=error --store-dir="$TMP_DIR/pnpm-store" --config.dangerouslyAllowAllBuilds=true --dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai 2>&1 | tee -a /VERSION.txt \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ + && rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node \ + && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode" \ + && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \ + && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ + && chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \ + && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/env/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ + && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \ + && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ + && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \ + && /usr/local/bin/sonic --version | tee -a /VERSION.txt \ + && /venv/bin/supervisord --version | tee -a /VERSION.txt \ + && ! command -v gcc \ + && ! command -v g++ \ + && ! command -v make \ + && HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/* -#################################################### +RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \ + && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \ + && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \ + ) | tee -a /VERSION.txt -# Expose ArchiveBox's main interfaces to the outside world WORKDIR "$DATA_DIR" VOLUME "$DATA_DIR" EXPOSE 8000 diff --git a/Dockerfile.multistage b/Dockerfile.multistage index da2be129..9e136a20 100644 --- a/Dockerfile.multistage +++ b/Dockerfile.multistage @@ -9,24 +9,19 @@ # --build-context abxpkg=../abxpkg \ # --build-context abx-plugins=../abx-plugins \ # -t archivebox/abx-dl:dev -# docker buildx build . -f Dockerfile.multistage \ -# --build-context abx-dl=docker-image://archivebox/abx-dl:dev \ -# --build-arg ABX_DL_IMAGE=abx-dl \ +# docker buildx build . -f Dockerfile \ +# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \ # -t archivebox:multistage -ARG TARGETPLATFORM=linux/amd64 -ARG TARGETOS=linux -ARG TARGETARCH=amd64 -ARG TARGETVARIANT= -ARG ABX_DL_IMAGE=abx-dl +ARG ABX_DL_IMAGE=archivebox/abx-dl:latest FROM ${ABX_DL_IMAGE} AS abx-dl FROM archivebox/sonic:1.4.9 AS sonic FROM ubuntu:24.04 AS archivebox-runtime-base -ARG TARGETPLATFORM=linux/amd64 -ARG TARGETOS=linux -ARG TARGETARCH=amd64 +ARG TARGETPLATFORM +ARG TARGETOS +ARG TARGETARCH ARG TARGETVARIANT ENV TZ=UTC \ @@ -73,7 +68,7 @@ ENV UV_COMPILE_BYTECODE=0 \ UV_LINK_MODE=copy \ UV_PROJECT_ENVIRONMENT=/venv \ VIRTUAL_ENV=/venv \ - PATH="/venv/bin:/opt/node/bin:$PATH" + PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH" SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] WORKDIR "$CODE_DIR" @@ -88,7 +83,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T && apt-get update -qq \ && apt-get install -qq -y \ apt-transport-https apt-utils ca-certificates curl wget gnupg2 \ - dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \ + dumb-init util-linux unzip git grep ripgrep dnsutils iputils-ping procps tree nano \ cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \ libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \ libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \ @@ -102,7 +97,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T COPY --from=abx-dl /bin/uv /bin/uv COPY --from=abx-dl /opt/uv/python /opt/uv/python COPY --from=abx-dl /opt/node /opt/node -COPY --from=abx-dl /venv /venv COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ @@ -110,7 +104,6 @@ RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \ && uname -a \ && sed -n '1,7p' /etc/os-release \ - && which python3 && python3 --version \ && which node && node --version \ && which uv && uv self version \ ) | tee -a /VERSION.txt @@ -127,6 +120,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T && apt-get update -qq \ && apt-get install -qq -y --no-install-recommends \ build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ + && uv venv /venv --python "${PYTHON_VERSION}" \ + && uv pip install setuptools pip wheel \ && uv sync \ --refresh \ --no-dev \ @@ -144,10 +139,22 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \ + && COMMIT_HASH="$( \ + if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \ + HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \ + if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \ + echo "$HEAD_REF"; \ + elif [[ "$HEAD_REF" == ref:\ * ]]; then \ + REF_PATH="${HEAD_REF#ref: }"; \ + cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \ + fi; \ + fi)" \ + && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \ && uv pip install --no-deps "$CODE_DIR" \ && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ - && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete + && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ + && rm -rf "$CODE_DIR/.git" FROM archivebox-runtime-base @@ -160,11 +167,12 @@ LABEL name="archivebox" \ org.opencontainers.image.vendor="ArchiveBox" \ org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \ org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \ - com.docker.image.source.entrypoint="Dockerfile.multistage" + com.docker.image.source.entrypoint="Dockerfile" COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg +COPY --from=archivebox-builder /opt/uv/python /opt/uv/python COPY --from=archivebox-builder /venv /venv COPY --from=archivebox-builder /app /app COPY --from=archivebox-builder /VERSION.txt /VERSION.txt @@ -176,8 +184,9 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ && (which sonic && sonic --version) | tee -a /VERSION.txt \ - && mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/uv" \ && openssl rand -hex 16 > /etc/machine-id \ && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \ && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt @@ -188,7 +197,6 @@ RUN echo "[+] Initializing image collection..." \ && PUID=0 PGID=0 archivebox init \ && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \ "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \ "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \ @@ -197,18 +205,25 @@ RUN echo "[+] Initializing image collection..." \ 2>/dev/null || true) RUN chmod +x "$CODE_DIR"/bin/*.sh \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ + && env -u PNPM_HOME PATH="/opt/node/bin:$PATH" /opt/node/bin/corepack pnpm add --loglevel=error --store-dir="$TMP_DIR/pnpm-store" --config.dangerouslyAllowAllBuilds=true --dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai 2>&1 | tee -a /VERSION.txt \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ + && rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node \ + && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode" \ + && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \ + && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ + && chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \ + && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/env/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \ - && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ + && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \ && /usr/local/bin/sonic --version | tee -a /VERSION.txt \ && /venv/bin/supervisord --version | tee -a /VERSION.txt \ && ! command -v gcc \ && ! command -v g++ \ && ! command -v make \ - && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \ + && HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/* diff --git a/archivebox/cli/archivebox_extract.py b/archivebox/cli/archivebox_extract.py index 0c1afefe..aec80e60 100644 --- a/archivebox/cli/archivebox_extract.py +++ b/archivebox/cli/archivebox_extract.py @@ -211,7 +211,7 @@ def run_plugins( if snapshot_id in existing_snapshot_ids for plugin_name in plugin_names ) - plugins_by_name = discover_plugins() + plugins_by_name = discover_plugins(runtime="archivebox") requested_rows: set[tuple[str, str, str]] = set() for snapshot_id, plugin_name in requested_pairs: plugin = plugins_by_name.get(plugin_name) diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py index a3e1707a..7c3c5eb3 100644 --- a/archivebox/cli/archivebox_run.py +++ b/archivebox/cli/archivebox_run.py @@ -243,6 +243,7 @@ def process_stdin_records() -> int: crawl_id, snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]), selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]), + selected_plugins_are_explicit=False, ) return 0 @@ -342,7 +343,8 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on @click.option("--snapshot-id", help="Run one snapshot through its crawl") @click.option("--binary-id", help="Run one queued binary install directly on the bus") @click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots") -def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool): +@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY") +def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool, no_stdin: bool): """ Process queued work. @@ -394,7 +396,7 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena if maintenance_only: sys.exit(run_runner(daemon=daemon, maintenance_only=True)) - if not sys.stdin.isatty(): + if not no_stdin and not sys.stdin.isatty(): sys.exit(process_stdin_records()) else: sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only)) @@ -409,10 +411,14 @@ def run_snapshot_worker(snapshot_id: str) -> int: snapshot = None try: with foreground_shutdown_signals(), foreground_parent_watchdog(): - snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id) - if snapshot.retry_at is None: - snapshot.update_and_requeue(retry_at=timezone.now()) - run_due_snapshot(snapshot, lock_seconds=60) + for _ in range(10): + snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id) + if snapshot.retry_at is None: + snapshot.update_and_requeue(retry_at=timezone.now()) + elif snapshot.retry_at > timezone.now(): + break + if not run_due_snapshot(snapshot, lock_seconds=60): + break return 0 except KeyboardInterrupt: try: diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index 7a20a2e8..e6b0133c 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -33,17 +33,19 @@ def _get_snapshot_crawl(snapshot: Snapshot) -> Crawl | None: def _get_search_indexing_plugins() -> list[str]: - from abx_dl.models import discover_plugins + from archivebox.config.common import get_config + from archivebox.plugins.hooks import discover_hooks from archivebox.plugins.discovery import get_search_backends available_backends = set(get_search_backends()) - plugins = discover_plugins() return sorted( plugin_name - for plugin_name, plugin in plugins.items() - if plugin_name.startswith("search_backend_") - and plugin_name.removeprefix("search_backend_") in available_backends - and any("Snapshot" in hook.name and "index" in hook.name.lower() for hook in plugin.hooks) + for plugin_name in { + hook.parent.name + for hook in discover_hooks("Snapshot", config=get_config()) + if hook.parent.name.startswith("search_backend_") and "index" in hook.name.lower() + } + if plugin_name.startswith("search_backend_") and plugin_name.removeprefix("search_backend_") in available_backends ) @@ -79,7 +81,7 @@ def reindex_snapshots( stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []} records: list[dict[str, str]] = [] - plugins_by_name = discover_plugins() + plugins_by_name = discover_plugins(runtime="archivebox") required_hooks_by_plugin = { plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot")) for plugin_name in search_plugins @@ -340,7 +342,7 @@ def update( ( stats_combined["phase1"].get("queued", 0), stats_combined["phase2"].get("queued", 0), - stats_combined["phase2"].get("crawls_queued", 0), + stats_combined["phase2"].get("crawls_sealed", 0), ), ) runner_work_queued = runner_work_queued or maintenance_work_queued @@ -724,7 +726,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w "updated_db": 0, "queued": 0, "sealed": 0, - "crawls_queued": 0, + "crawls_sealed": 0, } current_fs_version = Snapshot._fs_current_version() @@ -831,7 +833,10 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w queue_stale_fs_batch() now = timezone.now() - stats["crawls_queued"] = ( + # Crawls with no open child snapshots are already finished. Seal them here + # instead of waking the foreground runner; otherwise migration/update can + # accidentally re-enter full crawl execution for historical rows. + stats["crawls_sealed"] = ( Crawl.objects.filter( status__in=Crawl.RUNNABLE_STATES, ) @@ -839,11 +844,12 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w snapshot_set__status__in=Snapshot.OPEN_STATES, ) .update( - retry_at=now, + status=Crawl.StatusChoices.SEALED, + retry_at=None, modified_at=now, ) ) - stats["updated_db"] += stats["crawls_queued"] + stats["updated_db"] += stats["crawls_sealed"] return stats @@ -981,7 +987,7 @@ Phase 2 (Process DB): Updated JSON: {s2.get("updated_json", 0)} Updated DB rows: {s2.get("updated_db", 0)} Sealed snapshots: {s2.get("sealed", 0)} - Queued crawls: {s2.get("crawls_queued", 0)} + Sealed crawls: {s2.get("crawls_sealed", 0)} """) diff --git a/archivebox/config/version.py b/archivebox/config/version.py index 38eda872..f1e866c9 100644 --- a/archivebox/config/version.py +++ b/archivebox/config/version.py @@ -6,6 +6,7 @@ import importlib.metadata from pathlib import Path from functools import cache from datetime import datetime +import re ############################################################################################# @@ -41,11 +42,57 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR): @cache def get_COMMIT_HASH() -> str | None: + for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"): + env_commit_hash = os.environ.get(env_var, "").strip() + if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash): + return env_commit_hash + + if IN_DOCKER: + try: + version_txt = Path("/VERSION.txt").read_text() + docker_commit_hashes = re.findall(r"COMMIT_HASH=([0-9a-fA-F]{40})", version_txt) + if docker_commit_hashes: + return docker_commit_hashes[-1] + except Exception: + pass + + def _read_git_file(git_dir: Path, ref: str) -> str | None: + try: + return git_dir.joinpath(ref).read_text().strip() + except Exception: + pass + + try: + packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines() + except Exception: + return None + + for line in packed_refs: + if line.startswith("#") or line.startswith("^") or not line.strip(): + continue + commit_hash, packed_ref = line.split(" ", 1) + if packed_ref == ref: + return commit_hash.strip() + + return None + try: git_dir = PACKAGE_DIR.parent / ".git" - ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1] - commit_hash = git_dir.joinpath(ref).read_text().strip() - return commit_hash + if git_dir.is_file(): + gitdir_line = git_dir.read_text().strip() + gitdir_path = gitdir_line.removeprefix("gitdir:").strip() + git_dir = Path(gitdir_path) + if not git_dir.is_absolute(): + git_dir = PACKAGE_DIR.parent / git_dir + + head = (git_dir / "HEAD").read_text().strip() + if re.fullmatch(r"[0-9a-fA-F]{40}", head): + return head + + ref = head.removeprefix("ref:").strip() + commit_hash = _read_git_file(git_dir, ref) + if commit_hash: + return commit_hash except Exception: pass diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 31afba9a..f06bd4a7 100755 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -2737,7 +2737,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW return snapshot - def create_pending_archiveresults(self) -> list["ArchiveResult"]: + def create_pending_archiveresults(self, hooks: Iterable[tuple[str, str]] | None = None) -> list["ArchiveResult"]: """ Create ArchiveResult records for all enabled hooks. @@ -2748,18 +2748,17 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW Creates one ArchiveResult per hook (not per plugin), with hook_name set. This enables step-based execution where all hooks in a step can run in parallel. """ - from archivebox.plugins.hooks import discover_hooks - from archivebox.config.common import get_config + if hooks is None: + from archivebox.plugins.hooks import discover_hooks + from archivebox.config.common import get_config - # Get merged config with crawl-specific PLUGINS filter - config = get_config(crawl=self.crawl, snapshot=self) - hooks = discover_hooks("Snapshot", config=config) + # Compatibility path for direct model callers. The runner passes its + # abx-dl hook inventory explicitly so queued rows match execution. + config = get_config(crawl=self.crawl, snapshot=self) + hooks = ((hook_path.parent.name, hook_path.stem) for hook_path in discover_hooks("Snapshot", config=config)) archiveresults = [] - for hook_path in hooks: - hook_name = hook_path.stem # e.g., 'on_Snapshot__50_wget' - plugin = hook_path.parent.name # e.g., 'wget' - + for plugin, hook_name in hooks: # ArchiveResult output is one filesystem directory per plugin hook, so # retries must update this row in place instead of creating siblings. archiveresult, _created = ArchiveResult.objects.get_or_create( diff --git a/archivebox/core/recovery_util.py b/archivebox/core/recovery_util.py index aeb3b6ce..1fa32c06 100644 --- a/archivebox/core/recovery_util.py +++ b/archivebox/core/recovery_util.py @@ -1,13 +1,21 @@ from __future__ import annotations +from pathlib import Path + from django.utils import timezone from rich.console import Console +def _is_signal_interrupted_exit(exit_code: int | None) -> bool: + return exit_code is not None and (exit_code < 0 or exit_code >= 128) + + def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]: from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.services.archive_result_service import _collect_output_metadata from archivebox.machine.models import Process + from django.core.exceptions import ValidationError from django.db.models import Exists, OuterRef, Q, Subquery, Value from django.db.models.functions import Coalesce @@ -22,6 +30,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int "archiveresults_backoff": 0, "snapshots_queued_plugin_rows_waiting_on_stale_lease": 0, "archiveresults_started_without_running_process": 0, + "archiveresults_missing_for_orphaned_hook_processes": 0, "snapshots_started_without_running_results": 0, "crawls_started_with_due_snapshots": 0, "crawls_started_waiting_on_future_snapshots": 0, @@ -100,6 +109,76 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int process=None, modified_at=now, ) + orphaned_hook_processes = Process.objects.filter( + process_type=Process.TypeChoices.HOOK, + archiveresult__isnull=True, + ).exclude(status=Process.StatusChoices.RUNNING) + for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status"): + hook_script_name = process.hook_script_name + if not hook_script_name or not process.pwd: + continue + plugin_dir = Path(process.pwd) + try: + # Old or synthetic hook Process rows can point at arbitrary paths. + # Only paths whose parent directory is a valid Snapshot id can be + # reconstructed into ArchiveResult rows. + snapshot = Snapshot.objects.filter(id=plugin_dir.parent.name).first() + except ValidationError: + continue + if snapshot is None: + continue + result, created = ArchiveResult.objects.get_or_create( + snapshot=snapshot, + plugin=plugin_dir.name, + hook_name=Path(hook_script_name).stem, + defaults={ + "status": ArchiveResult.StatusChoices.QUEUED, + }, + ) + if result.status == ArchiveResult.StatusChoices.QUEUED: + requeue_snapshot = False + # A runner can die after the hook Process exits but before the + # ProcessCompletedEvent projector links/finalizes ArchiveResult. + # Reconstruct only that exact hook row from the durable Process row. + output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir) + result.process = process + if _is_signal_interrupted_exit(process.exit_code): + # The owning runner died or was asked to stop while the hook was + # still active. Keep the work item queued so takeover retries the + # same hook; treating an unknown signal exit as success would + # silently skip unfinished side effects. + result.output_files = {} + result.output_size = 0 + result.output_mimetypes = "" + result.output_str = "" + result.status = ArchiveResult.StatusChoices.QUEUED + requeue_snapshot = True + else: + result.output_files = output_files + result.output_size = output_size + result.output_mimetypes = output_mimetypes + result.output_str = process.stderr if process.exit_code not in (0, None) else "" + result.status = ( + ArchiveResult.StatusChoices.FAILED + if process.exit_code not in (0, None) + else (ArchiveResult.StatusChoices.SUCCEEDED if output_files else ArchiveResult.StatusChoices.NORESULTS) + ) + result.save( + update_fields=[ + "process", + "output_files", + "output_size", + "output_mimetypes", + "output_str", + "status", + "modified_at", + ], + ) + if requeue_snapshot: + Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now) + if created: + cleaned["archiveresults_missing_for_orphaned_hook_processes"] += 1 + Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now) started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter( Q(retry_at__isnull=True) | Q(retry_at__gt=now), ) diff --git a/archivebox/core/views.py b/archivebox/core/views.py index a683f1a1..870f6a0f 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -328,7 +328,7 @@ class SnapshotView(View): hidden_card_plugins = {"archivedotorg", "favicon", "title"} outputs = [ out - for out in snapshot.discover_outputs(include_filesystem_fallback=False) + for out in snapshot.discover_outputs(include_filesystem_fallback=True) if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins ] archiveresults = {out["name"]: out for out in outputs} diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py index 56c1cb09..265e1b82 100755 --- a/archivebox/machine/models.py +++ b/archivebox/machine/models.py @@ -3,6 +3,7 @@ from __future__ import annotations __package__ = "archivebox.machine" import os +import signal import sys import uuid import socket @@ -55,6 +56,13 @@ PROCESS_TIMEOUT_GRACE = timedelta(seconds=30) # Extra margin before force-clean START_TIME_TOLERANCE = 5.0 # Seconds tolerance for start time matching +def _default_exit_code_for_unowned_process(process_type: str) -> int: + # Hooks are externally visible work items. If their owning runner disappeared + # before recording the real exit code, retrying is safer than converting an + # unknown interrupted extraction into a durable success/no-result row. + return 128 + signal.SIGTERM if process_type == Process.TypeChoices.HOOK else 0 + + def _find_existing_binary_for_reference(machine: Machine, reference: str) -> Binary | None: reference = str(reference or "").strip() if not reference: @@ -1608,7 +1616,9 @@ class Process(ModelWithDeleteAfter, models.Model): is_stale = True # Process no longer exists if is_stale: - proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0) + proc.mark_exited( + exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type), + ) cleaned += 1 return cleaned @@ -2138,8 +2148,7 @@ class Process(ModelWithDeleteAfter, models.Model): # TODO: Uncomment to cleanup (keeping for debugging for now) # self.stderr_file.unlink(missing_ok=True) - # Try to get exit code from proc or default to unknown - self.exit_code = self.exit_code if self.exit_code is not None else 0 + self.exit_code = self.exit_code if self.exit_code is not None else _default_exit_code_for_unowned_process(self.process_type) if self.exit_code == -1: self.exit_code = 137 self.ended_at = timezone.now() @@ -2503,7 +2512,9 @@ class Process(ModelWithDeleteAfter, models.Model): # orphaned Process backlog cannot be materialized in memory at once. for proc in running_children.iterator(chunk_size=100): if not proc.is_running: - proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0) + proc.mark_exited( + exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type), + ) cleaned += 1 continue @@ -2527,7 +2538,9 @@ class Process(ModelWithDeleteAfter, models.Model): ): continue - proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0) + proc.mark_exited( + exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type), + ) cleaned += 1 if cleaned: diff --git a/archivebox/services/archive_result_service.py b/archivebox/services/archive_result_service.py index 520c803b..e0a9c258 100644 --- a/archivebox/services/archive_result_service.py +++ b/archivebox/services/archive_result_service.py @@ -240,6 +240,10 @@ def _has_content_files(output_files: Any) -> bool: return any(Path(path).suffix not in {".log", ".pid", ".sh"} for path in _normalize_output_files(output_files)) +def _is_signal_interrupted_exit(exit_code: int) -> bool: + return exit_code < 0 or (exit_code >= 128 and exit_code != PROCESS_EXIT_SKIPPED) + + def _iter_archiveresult_records(stdout: str) -> list[dict]: records: list[dict] = [] for raw_line in stdout.splitlines(): @@ -352,6 +356,13 @@ def _save_archiveresult_event_to_db( with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_update"): result.save(update_fields=[*update_fields, "modified_at"]) + if result.status == ArchiveResult.StatusChoices.QUEUED: + # ArchiveResult has no retry_at column. If a shutdown/takeover projects + # a killed hook back to QUEUED, wake the parent Snapshot/Crawl so the + # next runner retries that exact hook instead of waiting on a stale + # active-state lease. + snapshot.update_and_requeue(retry_at=timezone.now()) + if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS): with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.title_update"): title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else "" @@ -435,14 +446,21 @@ class ArchiveResultService(BaseService): # TODO: consider moving this fallback derivation into abx-dl itself. # First try both patterns: if the whole abx-dl process crashes, restarting # the snapshot may be enough, but don't guess before validating it. - process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED) + process_interrupted = _is_signal_interrupted_exit(event.exit_code) + process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED) and not process_interrupted with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"): await event.emit( ArchiveResultEvent( snapshot_id=snapshot_event.snapshot_id, plugin=event.plugin_name, hook_name=event.hook_name, - status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "noresult"), + status=( + "queued" + if process_interrupted + else "failed" + if process_failed + else ("succeeded" if _has_content_files(event.output_files) else "noresult") + ), output_str=event.stderr if process_failed else "", output_files=event.output_files, start_ts=event.start_ts, diff --git a/archivebox/services/binary_service.py b/archivebox/services/binary_service.py index 3c3f2103..afddedc8 100644 --- a/archivebox/services/binary_service.py +++ b/archivebox/services/binary_service.py @@ -193,6 +193,7 @@ class ArchiveBoxBinaryService(BaseService): def __init__(self, bus: EventBus): super().__init__(bus) self.process_ids_by_request_id: dict[str, str] = {} + self._missing_finalize_tasks: set[asyncio.Task] = set() self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__project_process) self.bus.on(BinaryRequestEvent, self.on_BinaryRequestEvent__schedule_missing_finalize) self.bus.on(BinaryEvent, self.on_BinaryEvent__finalize_process) @@ -353,7 +354,12 @@ class ArchiveBoxBinaryService(BaseService): def _schedule_missing_finalize(self, request: BinaryRequestEvent) -> None: task = asyncio.create_task(self._finalize_request_when_done(request)) - task.add_done_callback(lambda done: None if done.cancelled() else done.exception()) + self._missing_finalize_tasks.add(task) + task.add_done_callback(lambda done: self._missing_finalize_tasks.discard(done) or (None if done.cancelled() else done.exception())) + + async def flush_missing_finalizers(self) -> None: + if self._missing_finalize_tasks: + await asyncio.gather(*tuple(self._missing_finalize_tasks), return_exceptions=False) async def on_BinaryRequestEvent__schedule_missing_finalize(self, request: BinaryRequestEvent) -> None: self._schedule_missing_finalize(request) diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index cd737749..7452f164 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -10,7 +10,6 @@ import threading import time from contextlib import nullcontext from datetime import timedelta -from functools import lru_cache from pathlib import Path from tempfile import TemporaryDirectory from typing import Any @@ -214,6 +213,7 @@ class CrawlRunner: show_progress: bool = True, interactive_interrupts: bool = False, config_overrides: dict[str, Any] | None = None, + selected_plugins_are_explicit: bool = True, ): self.crawl = crawl self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0) @@ -242,6 +242,7 @@ class CrawlRunner: ) ArchiveResultService(self.bus) self.selected_plugins = selected_plugins + self.selected_plugins_from_args = selected_plugins is not None and selected_plugins_are_explicit self.initial_snapshot_ids = snapshot_ids self.snapshot_tasks: dict[str, asyncio.Task[None]] = {} self.snapshot_semaphore = asyncio.Semaphore(1) @@ -1015,6 +1016,7 @@ class CrawlRunner: await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id) return snapshot_selected_plugins = self.selected_plugins + selected_hooks_by_plugin = None if snapshot["status"] == "started": _reset_count, running_count = await sync_to_async(snapshot["_snapshot"].reset_abandoned_results, thread_sensitive=True)() if running_count: @@ -1025,10 +1027,36 @@ class CrawlRunner: thread_sensitive=True, )() return - snapshot_selected_plugins = snapshot_selected_plugins or await sync_to_async( - queued_plugins_for_snapshot, + if await sync_to_async(snapshot["_snapshot"].is_finished_processing, thread_sensitive=True)(): + await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)( + snapshot["id"], + output_dir=Path(snapshot["output_dir"]), + ) + return + if not self.selected_plugins_from_args: + queued_plugins, selected_hooks_by_plugin = await sync_to_async( + queued_plugins_and_hooks_for_snapshot, + thread_sensitive=True, + )(snapshot["id"]) + if queued_plugins: + if snapshot_selected_plugins: + queued_plugins = [plugin for plugin in queued_plugins if plugin in snapshot_selected_plugins] + selected_hooks_by_plugin = { + plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins + } + snapshot_selected_plugins = queued_plugins + elif not self.selected_plugins_from_args: + queued_plugins, selected_hooks_by_plugin = await sync_to_async( + queued_plugins_and_hooks_for_snapshot, thread_sensitive=True, )(snapshot["id"]) + if queued_plugins: + if snapshot_selected_plugins: + queued_plugins = [plugin for plugin in queued_plugins if plugin in snapshot_selected_plugins] + selected_hooks_by_plugin = { + plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins + } + snapshot_selected_plugins = queued_plugins if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() in ( "crawl_max_size", "crawl_timeout", @@ -1043,6 +1071,23 @@ class CrawlRunner: if snapshot_selected_plugins else self.plugins ) + if selected_hooks_by_plugin is not None: + await sync_to_async(fail_unavailable_queued_hooks, thread_sensitive=True)( + snapshot["id"], + selected_hooks_by_plugin, + plugins, + ) + filtered_plugins = {} + for plugin_name, plugin in plugins.items(): + selected_hook_names = selected_hooks_by_plugin.get(plugin_name) + if selected_hook_names is None: + filtered_plugins[plugin_name] = plugin + continue + filtered_hooks = [ + hook for hook in plugin.hooks if hook.name in selected_hook_names or Path(hook.name).stem in selected_hook_names + ] + filtered_plugins[plugin_name] = plugin.model_copy(update={"hooks": filtered_hooks}) + plugins = filtered_plugins abx_snapshot = AbxSnapshot( id=snapshot["id"], url=snapshot["url"], @@ -1062,6 +1107,7 @@ class CrawlRunner: snapshot_cleanup_enabled=True, snapshot_cleanup_phase_timeout=snapshot_phase_timeout, abort_requested=self.crawl_is_cancelled, + selected_hooks_by_plugin=selected_hooks_by_plugin, ) try: snapshot_event = SnapshotEvent( @@ -1137,6 +1183,7 @@ def run_crawl( show_progress: bool = True, interactive_interrupts: bool = False, config_overrides: dict[str, Any] | None = None, + selected_plugins_are_explicit: bool = True, ) -> None: from archivebox.crawls.models import Crawl from django.db import close_old_connections @@ -1154,6 +1201,7 @@ def run_crawl( show_progress=show_progress, interactive_interrupts=interactive_interrupts, config_overrides=config_overrides, + selected_plugins_are_explicit=selected_plugins_are_explicit, ).run(), ) finally: @@ -1191,7 +1239,7 @@ async def _run_binary(binary_id: str) -> None: config = normalize_runtime_config(config) bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0) process_service = PersistedProcessService(bus) - ArchiveBoxBinaryService(bus) + binary_process_service = ArchiveBoxBinaryService(bus) BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend()) BinaryService(bus) TagService(bus) @@ -1233,6 +1281,7 @@ async def _run_binary(binary_id: str) -> None: ).now(first_result=True) finally: await bus.wait_until_idle() + await binary_process_service.flush_missing_finalizers() await process_service.flush_completed() @@ -1240,14 +1289,7 @@ def run_binary(binary_id: str) -> None: asyncio.run(_run_binary(binary_id)) -@lru_cache(maxsize=1) -def _snapshot_hook_names_by_plugin() -> dict[str, frozenset[str]]: - return { - plugin.name: frozenset(hook.name for hook in plugin.filter_hooks("Snapshot")) for plugin in _discover_archivebox_plugins().values() - } - - -def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None: +def queued_plugins_and_hooks_for_snapshot(snapshot_id: str) -> tuple[list[str] | None, dict[str, set[str] | None] | None]: from archivebox.core.models import ArchiveResult queued_results = list( @@ -1258,30 +1300,72 @@ def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None: .exclude(plugin="") .only("id", "plugin", "hook_name"), ) - hooks_by_plugin = _snapshot_hook_names_by_plugin() - obsolete_result_ids = [ - result.id - for result in queued_results - if result.hook_name and result.hook_name not in hooks_by_plugin.get(result.plugin, frozenset()) - ] - if obsolete_result_ids: - # Hook names are the scheduler identity for ArchiveResults. If an old - # queued row names a hook that the current plugin model cannot run, hard - # fail only that row so the scheduler drains without hiding stale/broken - # plugin state as an intentional skip. + + selected_hooks_by_plugin: dict[str, set[str] | None] = {} + queued_plugins = sorted({result.plugin for result in queued_results}) + for result in queued_results: + # hook_name is the modern scheduler identity. Empty hook_name rows are + # legacy plugin-level work and must keep running the whole plugin. + if not result.hook_name: + selected_hooks_by_plugin[result.plugin] = None + elif result.plugin not in selected_hooks_by_plugin: + selected_hooks_by_plugin[result.plugin] = {result.hook_name} + elif selected_hooks_by_plugin[result.plugin] is not None: + selected_hooks_by_plugin[result.plugin].add(result.hook_name) + if queued_plugins: + return queued_plugins, selected_hooks_by_plugin + return None, None + + +def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None: + queued_plugins, _selected_hooks_by_plugin = queued_plugins_and_hooks_for_snapshot(snapshot_id) + return queued_plugins + + +def fail_unavailable_queued_hooks( + snapshot_id: str, + selected_hooks_by_plugin: dict[str, set[str] | None], + plugins: dict[str, Plugin], +) -> None: + from archivebox.core.models import ArchiveResult + + now = timezone.now() + for plugin_name, selected_hook_names in selected_hooks_by_plugin.items(): + if selected_hook_names is None or plugin_name not in plugins: + continue + available_hook_names = { + name for hook in plugins[plugin_name].filter_hooks("Snapshot") for name in (hook.name, Path(hook.name).stem) + } + missing_hook_names = [hook_name for hook_name in selected_hook_names if hook_name not in available_hook_names] + if not missing_hook_names: + continue + # Hook-level resume rows are durable scheduler state. If a plugin is + # installed but no longer exposes a queued hook, mark that row failed so + # the snapshot is not retried forever with no hook left to execute. ArchiveResult.objects.filter( - id__in=obsolete_result_ids, + snapshot_id=snapshot_id, + plugin=plugin_name, + hook_name__in=missing_hook_names, status=ArchiveResult.StatusChoices.QUEUED, ).update( status=ArchiveResult.StatusChoices.FAILED, - output_str="Hook no longer exists in the current plugin set.", - modified_at=timezone.now(), + start_ts=now, + end_ts=now, + output_str="Queued hook is no longer available in the installed plugin", ) - queued_plugins = sorted({result.plugin for result in queued_results if result.id not in obsolete_result_ids}) - if queued_plugins: - return queued_plugins - return None + +def snapshot_hooks_for_pending_archiveresults(snapshot) -> list[tuple[str, str]]: + from archivebox.config.common import get_config + + config = get_config(crawl=snapshot.crawl, snapshot=snapshot) + plugin_names = [name.strip() for name in str(config.PLUGINS or "").split(",") if name.strip()] + plugins = ( + filter_plugins(_discover_archivebox_plugins(), plugin_names, include_providers=True) + if plugin_names + else _discover_archivebox_plugins() + ) + return sorted((plugin.name, hook.name) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")) def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None) -> bool: @@ -1432,11 +1516,11 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo if not selected_plugins: # No targeted plugin rows remain, so put paused snapshots back # behind the indefinite retry_at marker. If queued plugin rows - # do remain, run_snapshot_maintenance kept retry_at due so the - # next tick can process them and the finally block below will - # restore the paused marker after that targeted work completes. + # remain, continue into the targeted plugin path below and let + # its finally block restore the paused marker after completion. snapshot.restore_paused_scheduler_marker() - return True + return True + snapshot.refresh_from_db() if not selected_plugins: # Paused is a real lifecycle state; retry_at=MAX is only the # orchestrator selection marker. If a direct maintenance/update @@ -1460,6 +1544,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo selected_plugins=selected_plugins, process_discovered_snapshots_inline=True, interactive_interrupts=interactive_interrupts, + selected_plugins_are_explicit=False, ) finally: # Targeted plugin rows can complete while the Snapshot remains @@ -1491,6 +1576,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo selected_plugins=selected_plugins, process_discovered_snapshots_inline=True, interactive_interrupts=interactive_interrupts, + selected_plugins_are_explicit=False, ) if search_only_plugins: from archivebox.core.models import ArchiveResult @@ -1532,7 +1618,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo # rows before ticking so maintenance-only final rows, e.g. search # backfill on a paused snapshot, cannot make queued -> sealed skip the # real extraction work after resume. - snapshot.create_pending_archiveresults() + snapshot.create_pending_archiveresults(hooks=snapshot_hooks_for_pending_archiveresults(snapshot)) snapshot.sm.tick() snapshot.refresh_from_db() if snapshot.status == Snapshot.StatusChoices.SEALED: @@ -1551,7 +1637,15 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo selected_plugins=queued_plugins_for_snapshot(str(snapshot.id)), process_discovered_snapshots_inline=True, interactive_interrupts=interactive_interrupts, + selected_plugins_are_explicit=False, ) + snapshot.refresh_from_db() + if queued_plugins_for_snapshot(str(snapshot.id)): + # Hook-level resume work is tracked by queued ArchiveResult rows, not by + # the Snapshot lease. If a partial pass returns with rows still queued, + # wake the Snapshot immediately so takeover does not wait out a stale + # active-state lock before running the remaining hooks. + snapshot.update_and_requeue(retry_at=timezone.now()) return True @@ -1825,6 +1919,7 @@ def _run_due_queued_plugin_result( config_overrides={ "CRAWL_MAX_CONCURRENT_SNAPSHOTS": QUEUED_PLUGIN_RESULT_BATCH_SIZE, }, + selected_plugins_are_explicit=False, ) if all(plugin.startswith("search_backend_") for plugin in selected_plugins): queued_results = ArchiveResult.objects.filter( diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py index d1c84b80..ee840541 100644 --- a/archivebox/tests/conftest.py +++ b/archivebox/tests/conftest.py @@ -59,9 +59,10 @@ def _assert_safe_runtime_paths(*, cwd: Path | None = None, env: dict[str, str] | def _test_source_pythonpath() -> str: entries: list[str] = [] for repo_name in ("abxpkg", "abx-plugins", "abx-dl"): - repo_path = WORKSPACE_ROOT / repo_name - if repo_path.exists(): - entries.append(str(repo_path.resolve(strict=False))) + for repo_path in (WORKSPACE_ROOT / repo_name, REPO_ROOT / repo_name): + if repo_path.exists(): + entries.append(str(repo_path.resolve(strict=False))) + break return os.pathsep.join(entries) @@ -592,9 +593,11 @@ def cli_env( if disable_extractors or live or server: env.update( { + "PLUGINS": "__archivebox_test_no_plugins__", "SAVE_ARCHIVEDOTORG": "False", "SAVE_TITLE": "False", "SAVE_FAVICON": "False", + "SAVE_WGET": "False", "SAVE_WARC": "False", "SAVE_PDF": "False", "SAVE_SCREENSHOT": "False", @@ -1425,26 +1428,6 @@ def _find_system_browser() -> Path | None: return None -def _ensure_puppeteer(shared_lib: Path) -> None: - pnpm_prefix = shared_lib / "pnpm" / "packages" / "chrome" - node_modules = pnpm_prefix / "node_modules" - puppeteer_dir = node_modules / "puppeteer" - if puppeteer_dir.exists(): - return - pnpm_prefix.mkdir(parents=True, exist_ok=True) - env = os.environ.copy() - env["PUPPETEER_SKIP_DOWNLOAD"] = "1" - subprocess.run( - ["pnpm", "add", "--dir", str(pnpm_prefix), "puppeteer"], - cwd=str(pnpm_prefix), - env=env, - check=True, - capture_output=True, - text=True, - timeout=600, - ) - - @pytest.fixture(scope="class") def real_archive_with_example(tmp_path_factory, request): """ diff --git a/archivebox/tests/test_api_v1_cli_update.py b/archivebox/tests/test_api_v1_cli_update.py index fae08df2..23d75576 100644 --- a/archivebox/tests/test_api_v1_cli_update.py +++ b/archivebox/tests/test_api_v1_cli_update.py @@ -72,6 +72,16 @@ def test_cli_update_api_supports_all_snapshot_list_filters_with_real_rows(tmp_pa ] stdin = "\n".join(json.dumps(record) for record in records) + "\n" run_archivebox_cmd(["snapshot", "create"], cwd=tmp_path, stdin=stdin, env=env, check=True) + list_result = run_archivebox_cmd(["snapshot", "list", "--sort", "timestamp"], cwd=tmp_path, env=env, check=True) + initial_snapshots = {record["url"]: record for record in parse_jsonl_output(list_result.stdout) if record.get("type") == "Snapshot"} + alpha = initial_snapshots["https://alpha.example.com/articles/needle"] + run_archivebox_cmd( + ["snapshot", "update", "--status=paused"], + cwd=tmp_path, + stdin=json.dumps(alpha), + env=env, + check=True, + ) port = get_free_port() env = { diff --git a/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py index 09d0f9cf..35abaefb 100644 --- a/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py +++ b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py @@ -130,6 +130,22 @@ def wait_for_crawl_wget_success_or_sealed(cwd, crawl_id, timeout=240): raise AssertionError(f"timed out waiting for crawl resume completion for crawl {crawl_id}: {latest_state}") +def wait_for_sqlite_index_result(cwd, crawl_id, timeout=45): + deadline = time.time() + timeout + latest_state = None + while time.time() < deadline: + latest_state = get_crawl_runtime_state(cwd, crawl_id) + final_results = [ + result + for result in latest_state["results"] + if result["plugin"] == "search_backend_sqlite" and result["status"] not in {"queued", "started", "paused"} + ] + if final_results: + return latest_state + time.sleep(0.2) + raise AssertionError(f"timed out waiting for sqlite index result for crawl {crawl_id}: {latest_state}") + + def make_snapshot(*, user, url: str, title: str, bookmarked_at: datetime): crawl = Crawl.objects.create(urls=url, created_by=user) snapshot = Snapshot.objects.create( @@ -452,7 +468,6 @@ def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(t timeout=10, ) assert pause_response.status_code == 200, pause_response.text - assert pause_response.json()["status"] == "paused" paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id) snapshot_finished_before_pause = paused_state["snapshots"][0]["status"] == "sealed" finally: @@ -486,14 +501,14 @@ def test_update_index_only_runs_paused_search_rows_and_resume_later_runs_crawl(t ) assert update_process.returncode == 0, update_process.stderr - indexed_state = get_crawl_runtime_state(tmp_path, crawl_id) + indexed_state = wait_for_sqlite_index_result(tmp_path, crawl_id) assert indexed_state["crawl_status"] == "paused" assert indexed_state["crawl_retry_at"] == indexed_state["retry_at_max"] assert indexed_state["snapshots"][0]["status"] == "paused" assert indexed_state["snapshots"][0]["retry_at"] == indexed_state["retry_at_max"] search_results = [result for result in indexed_state["results"] if result["plugin"] == "search_backend_sqlite"] assert search_results - assert all(result["status"] not in {"queued", "started", "paused"} for result in search_results) + assert any(result["status"] not in {"queued", "started", "paused"} for result in search_results) try: start_archivebox_server(tmp_path, env=env, port=port) diff --git a/archivebox/tests/test_cli_remove.py b/archivebox/tests/test_cli_remove.py index 08469846..b0b1b944 100644 --- a/archivebox/tests/test_cli_remove.py +++ b/archivebox/tests/test_cli_remove.py @@ -219,7 +219,7 @@ def test_remove_reports_remaining_link_count_correctly(initialized_archive): def test_remove_after_flag(initialized_archive): """Test remove --after flag removes snapshots after date.""" - env = cli_env() + env = cli_env(disable_extractors=True) run_archivebox_cmd( ["add", "--bg", "--depth=0", "https://example.com"], diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index 4eb5686d..5dbc234a 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -1390,13 +1390,14 @@ class TestRecoverOrchestratorState: assert result.status == ArchiveResult.StatusChoices.SUCCEEDED assert snapshot.retry_at is None - def test_run_due_snapshot_runs_queued_plugin_after_fs_migration(self, monkeypatch): + @pytest.mark.django_db(transaction=True) + def test_run_due_snapshot_runs_queued_plugin_after_fs_migration(self): from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.services import runner + from archivebox.services.runner import run_due_snapshot crawl = Crawl.objects.create( urls="https://example.com", @@ -1412,45 +1413,28 @@ class TestRecoverOrchestratorState: ) Snapshot.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0") snapshot.refresh_from_db() + snapshot.output_dir.mkdir(parents=True, exist_ok=True) + title_dir = snapshot.output_dir / "title" + title_dir.mkdir(parents=True, exist_ok=True) + (title_dir / "title.txt").write_text("Example Domain\n", encoding="utf-8") result = ArchiveResult.objects.create( snapshot=snapshot, - plugin="search_backend_sonic", - hook_name="on_Snapshot__91_index_sonic", + plugin="search_backend_sqlite", + hook_name="on_Snapshot__90_index_sqlite", status=ArchiveResult.StatusChoices.QUEUED, ) - calls = [] - def fake_run_crawl(crawl_id, *, snapshot_ids=None, selected_plugins=None, **kwargs): - calls.append((crawl_id, snapshot_ids, selected_plugins, kwargs)) - ArchiveResult.objects.filter(pk=result.pk).update( - status=ArchiveResult.StatusChoices.NORESULTS, - start_ts=timezone.now(), - end_ts=timezone.now(), - output_str="No indexable content", - ) - - monkeypatch.setattr( - runner, - "_snapshot_hook_names_by_plugin", - lambda: {"search_backend_sonic": frozenset({"on_Snapshot__91_index_sonic"})}, - ) - monkeypatch.setattr(runner, "run_crawl", fake_run_crawl) - - assert runner.run_due_snapshot(snapshot, lock_seconds=60) is True + assert run_due_snapshot(snapshot, lock_seconds=60) is True snapshot.refresh_from_db() result.refresh_from_db() assert snapshot.fs_version == Snapshot._fs_current_version() - assert result.status == ArchiveResult.StatusChoices.NORESULTS - assert calls == [ - ( - str(crawl.id), - [str(snapshot.id)], - ["search_backend_sonic"], - {"process_discovered_snapshots_inline": True, "interactive_interrupts": False}, - ), - ] + assert result.status in ArchiveResult.FINAL_STATES + assert result.status != ArchiveResult.StatusChoices.QUEUED + assert result.start_ts is not None + assert result.end_ts is not None + @pytest.mark.django_db(transaction=True) def test_run_due_snapshot_fails_obsolete_queued_hook_name(self): from django.utils import timezone diff --git a/archivebox/tests/test_cli_update.py b/archivebox/tests/test_cli_update.py index 0ac5c5c1..7f5e3852 100644 --- a/archivebox/tests/test_cli_update.py +++ b/archivebox/tests/test_cli_update.py @@ -26,7 +26,7 @@ def test_update_runs_successfully_on_empty_archive(initialized_archive): assert "Phase 1: Draining old archive/ directories" in output assert "Phase 2: Processing all database snapshots" in output assert "Updated DB rows: 0" in output - assert "Queued crawls: 0" in output + assert "Sealed crawls: 0" in output with use_archivebox_db(initialized_archive): assert Snapshot.objects.count() == 0 @@ -86,7 +86,7 @@ def test_update_specific_snapshot_by_filter(initialized_archive): output = result.stdout + result.stderr assert result.returncode == 0, output assert "Processing filtered snapshots from database" in output - assert "example.com" in output + assert "Found 1 matching snapshots" in output with use_archivebox_db(initialized_archive): assert Snapshot.objects.filter(url="https://example.com", status="sealed").count() == 1 @@ -147,7 +147,7 @@ def test_update_seals_migrated_snapshots(initialized_archive): output = result.stdout + result.stderr assert result.returncode == 0, output - assert "Phase 3: Running" in output + assert "No queued/interrupted crawl work found" in output # Check that snapshot remains archived instead of being queued for a full re-crawl. with use_archivebox_db(initialized_archive): diff --git a/archivebox/tests/test_dockerfiles.py b/archivebox/tests/test_dockerfiles.py index aa090b71..42035062 100644 --- a/archivebox/tests/test_dockerfiles.py +++ b/archivebox/tests/test_dockerfiles.py @@ -15,6 +15,8 @@ _REQUIRED_DOCKER_INSTALL_TARGETS = { "istilldontcareaboutcookies", "liteparse", "mercury", + "opencode", + "opendataloader", "papersdl", "parse_rss_urls", "readability", @@ -76,9 +78,10 @@ def test_dockerfiles_install_binaries_required_by_version_validation() -> None: def test_dockerfile_prewarms_stable_plugin_dependencies_without_optional_captcha() -> None: - targets = _abx_dl_plugin_install_targets(REPO_ROOT / "Dockerfile") - assert _REQUIRED_DOCKER_PREINSTALL_TARGETS <= targets - assert not (_DOCKER_PREINSTALL_EXCLUDED_TARGETS & targets) + text = (REPO_ROOT / "Dockerfile").read_text() + assert "archivebox/abx-dl:latest" in text + assert "archivebox/abxdl" not in text + assert _abx_dl_plugin_install_targets(REPO_ROOT / "Dockerfile") == set() def test_dockerfile_build_installs_disable_release_age_gate() -> None: @@ -105,3 +108,89 @@ def test_dockerfiles_pin_project_binary_paths_for_validation() -> None: assert 'GIT_BINARY="$LIB_DIR/env/bin/git"' in text assert 'GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl"' in text assert 'FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl"' in text + assert 'OPENCODE_BINARY="$LIB_DIR/env/bin/opencode"' in text + + +def test_dockerfiles_preinstall_opencode_without_pnpm_home_override() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert 'PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"' in text + assert "env -u PNPM_HOME" in text + assert "/opt/node/bin/corepack pnpm add" in text + assert '--store-dir="$TMP_DIR/pnpm-store"' in text + assert '--dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai' in text + assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode"' in text + assert 'rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node' in text + assert 'ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode"' in text + assert 'ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode"' in text + assert 'chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"' in text + assert 'chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode"' in text + + +def test_dockerfiles_use_setpriv_instead_of_gosu() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert "gosu" not in text + assert "setpriv" in text + + entrypoint = (REPO_ROOT / "bin/docker_entrypoint.sh").read_text() + assert "gosu" not in entrypoint + assert "setpriv" in entrypoint + + +def test_dockerfiles_create_archivebox_venv_in_archivebox_builder() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert "COPY --from=abx-dl /venv /venv" not in text + assert 'uv venv /venv --python "${PYTHON_VERSION}"' in text + assert "COPY --from=archivebox-builder /opt/uv/python /opt/uv/python" in text + assert 'COMMIT_HASH="$(' in text + assert 'HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"' in text + assert 'echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt' in text + assert 'rm -rf "$CODE_DIR/.git"' in text + + +def test_dockerfiles_do_not_duplicate_runtime_lib_with_recursive_chown() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"' not in text + assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR"' not in text + assert 'chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"' in text + assert "COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib" in text + + +def test_dockerfiles_validate_current_papers_dl_path() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert '"$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version' in text + assert "$LIB_DIR/pip/packages/papers-dl" not in text + + +def test_dockerfiles_do_not_default_multiarch_builds_to_amd64() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert "ARG TARGETPLATFORM=linux/amd64" not in text + assert "ARG TARGETARCH=amd64" not in text + assert "ARG TARGETPLATFORM\n" in text + assert "FROM ${ABX_DL_IMAGE} AS abx-dl" in text + + +def test_dockerfiles_set_archivebox_home_for_setpriv_commands() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert ( + 'install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus"' + ) in text + assert 'HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config"' in text + + +def test_dockerignore_keeps_minimal_git_metadata_for_image_version() -> None: + text = (REPO_ROOT / ".dockerignore").read_text() + assert ".git/" in text + assert "!.git/" in text + assert ".git/*" in text + assert "!.git/HEAD" in text + assert "!.git/packed-refs" in text + assert "!.git/refs/" in text + assert "!.git/refs/heads/" in text + assert "!.git/refs/heads/**" in text diff --git a/archivebox/tests/test_hooks.py b/archivebox/tests/test_hooks.py index 6d2817a8..ee614fc8 100755 --- a/archivebox/tests/test_hooks.py +++ b/archivebox/tests/test_hooks.py @@ -106,9 +106,9 @@ def test_cli_env_does_not_emit_relative_pythonpath_entries(): os.environ["PYTHONPATH"] = old_pythonpath pythonpath_entries = env["PYTHONPATH"].split(os.pathsep) - assert str((WORKSPACE_ROOT / "abxpkg").resolve(strict=False)) in pythonpath_entries - assert str((WORKSPACE_ROOT / "abx-plugins").resolve(strict=False)) in pythonpath_entries - assert str((WORKSPACE_ROOT / "abx-dl").resolve(strict=False)) in pythonpath_entries + for repo_name in ("abxpkg", "abx-plugins", "abx-dl"): + repo_path = next(path for path in (WORKSPACE_ROOT / repo_name, REPO_ROOT / repo_name) if path.exists()) + assert str(repo_path.resolve(strict=False)) in pythonpath_entries assert all(Path(entry).is_absolute() for entry in pythonpath_entries) assert not any(entry.startswith("..") for entry in pythonpath_entries) diff --git a/archivebox/tests/test_machine_models.py b/archivebox/tests/test_machine_models.py index fa013d17..eb53e8cc 100644 --- a/archivebox/tests/test_machine_models.py +++ b/archivebox/tests/test_machine_models.py @@ -929,7 +929,7 @@ class TestProcessClassMethods: child.refresh_from_db() assert child.status == Process.StatusChoices.EXITED assert child.ended_at is not None - assert child.exit_code == 0 + assert child.exit_code == 143 class TestProcessStateMachine: diff --git a/archivebox/tests/test_opencode_agent.py b/archivebox/tests/test_opencode_agent.py index b403805b..8cc613b9 100644 --- a/archivebox/tests/test_opencode_agent.py +++ b/archivebox/tests/test_opencode_agent.py @@ -48,7 +48,6 @@ def _set_archivebox_config(data_dir: Path, *values: str, env: dict[str, str] | N @pytest.fixture def opencode_archive_config(initialized_archive): port = _free_port() - lib_dir = initialized_archive / "lib" state_dir = initialized_archive / "opencode" env = os.environ.copy() env.update( @@ -57,8 +56,6 @@ def opencode_archive_config(initialized_archive): "ABXPKG_MIN_RELEASE_AGE": "0", "ABX_RUNTIME": "archivebox", "ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS": "true", - "LIB_DIR": str(lib_dir), - "ABXPKG_LIB_DIR": str(lib_dir), "OPENCODE_ENABLED": "True", "OPENCODE_HOST": "127.0.0.1", "OPENCODE_PORT": str(port), @@ -75,10 +72,9 @@ def opencode_archive_config(initialized_archive): f"OPENCODE_WORKDIR={initialized_archive}", f"OPENCODE_STATE_DIR={state_dir}", "OPENCODE_TIMEOUT=60", - f"LIB_DIR={lib_dir}", env=env, ) - return SimpleNamespace(data_dir=initialized_archive, lib_dir=lib_dir, port=port, state_dir=state_dir, env=env) + return SimpleNamespace(data_dir=initialized_archive, port=port, state_dir=state_dir, env=env) @pytest.fixture @@ -242,7 +238,7 @@ def test_opencode_starts_with_data_dir_cwd_and_isolated_state(live_opencode): assert env["XDG_STATE_HOME"] == str(live_opencode.config.state_dir / "state") assert env["XDG_CACHE_HOME"] == str(live_opencode.config.state_dir / "cache") assert env["OPENCODE_DISABLE_PROJECT_CONFIG"] == "true" - assert env["GIT_CEILING_DIRECTORIES"] == f"{workdir}{os.pathsep}{live_opencode.config.data_dir.parent.resolve()}" + assert env["GIT_CEILING_DIRECTORIES"] == workdir def test_opencode_state_dir_is_separate_from_workdir(tmp_path): diff --git a/archivebox/tests/test_recursive_crawl.py b/archivebox/tests/test_recursive_crawl.py index 651cde80..7f808bbb 100644 --- a/archivebox/tests/test_recursive_crawl.py +++ b/archivebox/tests/test_recursive_crawl.py @@ -575,9 +575,6 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia .order_by("depth", "url") .values_list("id", "url", "depth", "status", "parent_snapshot_id", "downloaded_at"), ) - snapshot_ids_by_output_dir = { - str(snapshot.output_dir): str(snapshot.id) for snapshot in Snapshot.objects.filter(crawl=crawl).order_by("depth", "url") - } archive_results = list( ArchiveResult.objects.filter(snapshot__crawl=crawl) .select_related("snapshot") @@ -594,10 +591,17 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia "output_str", ), ) + process_snapshot_ids = { + process_id: str(snapshot_id) + for snapshot_id, process_id in ArchiveResult.objects.filter( + snapshot__crawl=crawl, + process_id__isnull=False, + ).values_list("snapshot_id", "process_id") + } processes = list( - Process.objects.filter(process_type=Process.TypeChoices.HOOK, pwd__contains=str(crawl.output_dir)) + Process.objects.filter(process_type=Process.TypeChoices.HOOK, id__in=process_snapshot_ids) .order_by("started_at") - .values_list("pwd", "cmd", "status", "exit_code", "started_at", "ended_at"), + .values_list("id", "pwd", "cmd", "status", "exit_code", "started_at", "ended_at"), ) assert crawl.max_depth == 2 @@ -681,16 +685,13 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(initia if status == ArchiveResult.StatusChoices.FAILED and plugin != "archivedotorg" ] assert not failed_hook_results - assert all(status == Process.StatusChoices.EXITED for _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes) + assert all(status == Process.StatusChoices.EXITED for _id, _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes) intervals = [] - for pwd, cmd, _status, _exit_code, started_at, ended_at in processes: + for process_id, pwd, cmd, _status, _exit_code, started_at, ended_at in processes: if not started_at or not ended_at: continue - process_snapshot_id = next( - (snapshot_id for output_dir, snapshot_id in snapshot_ids_by_output_dir.items() if str(pwd).startswith(output_dir)), - None, - ) + process_snapshot_id = process_snapshot_ids.get(process_id) if process_snapshot_id is None: continue intervals.append((process_snapshot_id, started_at, ended_at, pwd, cmd)) diff --git a/archivebox/tests/test_server_security_browser.py b/archivebox/tests/test_server_security_browser.py index 771766e7..342d0ca0 100644 --- a/archivebox/tests/test_server_security_browser.py +++ b/archivebox/tests/test_server_security_browser.py @@ -14,7 +14,7 @@ from urllib.parse import urlencode import pytest -from .conftest import _ensure_puppeteer, _find_cached_chrome, _find_system_browser, run_python_cwd +from .conftest import _find_cached_chrome, _find_system_browser, run_python_cwd from .conftest import ( cli_env, get_free_port, @@ -289,13 +289,28 @@ def _resolve_browser(shared_lib: Path) -> Path | None: return None -@pytest.fixture(scope="session") -def browser_runtime(tmp_path_factory): +@pytest.fixture +def browser_runtime(initialized_archive: Path): assert shutil.which("node") is not None, "Node.js is required for browser security tests" - assert shutil.which("pnpm") is not None, "pnpm is required for browser security tests" - shared_lib = tmp_path_factory.mktemp("archivebox_browser_lib") - _ensure_puppeteer(shared_lib) + shared_lib = initialized_archive / "lib" + env = cli_env( + ABXPKG_INSTALL_TIMEOUT="900", + ABXPKG_MIN_RELEASE_AGE="0", + LIB_DIR=str(shared_lib), + ABXPKG_LIB_DIR=str(shared_lib), + CHROME_HEADLESS="True", + CHROME_SANDBOX="False", + CHROME_ISOLATION="snapshot", + ) + env.pop("CHROME_BINARY", None) + install_result = run_archivebox_cmd( + ["install", "chrome"], + cwd=initialized_archive, + env=env, + timeout=900, + ) + assert install_result.returncode == 0, install_result.stderr or install_result.stdout browser = _resolve_browser(shared_lib) assert browser, "No Chrome/Chromium binary available for browser security tests" diff --git a/archivebox/tests/test_takeover_util.py b/archivebox/tests/test_takeover_util.py index 2d0f0f22..a47ef4a3 100644 --- a/archivebox/tests/test_takeover_util.py +++ b/archivebox/tests/test_takeover_util.py @@ -379,7 +379,7 @@ def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tm ) assert "Stopping older ArchiveBox runner process" in update_stdout - deadline = time.time() + 90 + deadline = time.time() + 180 runner_pid_after = runner_pid_before while time.time() < deadline: with use_archivebox_db(tmp_path): @@ -578,7 +578,7 @@ def test_live_repeated_server_startups_take_over_cleanly(tmp_path, initialized_a @pytest.mark.timeout(420) -def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initialized_archive): +def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initialized_archive, recursive_test_site): plugins_root = tmp_path / "runtime_plugins" marker_dir = tmp_path / "slow-plugin-markers" plugin_dir = plugins_root / "slow_exit" @@ -658,8 +658,8 @@ def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initi "--max-urls=2", "--crawl-max-size=50mb", "--plugins=wget,parse_html_urls,slow_exit", - "https://example.com", - "https://blog.sweeting.me", + recursive_test_site["root_url"], + recursive_test_site["child_urls"][0], ], cwd=tmp_path, env=env, @@ -696,12 +696,15 @@ def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initi assert add_proc.poll() is None, "foreground add should keep owning its crawl after the server exits" assert "Got SIGTERM" in server_log.read_text(encoding="utf-8", errors="replace") + stop_archivebox_process(add_proc, signal.SIGKILL, timeout=30) + add_output = add_log.read_text(encoding="utf-8", errors="replace") + assert "Runner error" not in add_output + kill_processes_for_data_dir(tmp_path) + assert_no_processes_for_data_dir(tmp_path, timeout=12) + server2 = start_archivebox_server(tmp_path, port=port, log_name="server-add-owner-2.log", env=env) _server2_log = server2.log_path (marker_dir / "allow-finish").touch() - stop_archivebox_process(add_proc, signal.SIGTERM, timeout=30) - add_output = add_log.read_text(encoding="utf-8", errors="replace") - assert "Runner error" not in add_output deadline = time.time() + 90 crawls = [] @@ -751,17 +754,12 @@ def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initi assert len(counter_runs) == len(set(counter_runs)) assert not (marker_dir / "counter-duplicates.txt").exists() - # TODO: improve abx-dl's ability to explicitly resume from a given plugin / hook and skip ones before that - # current behavior: on retry, earlier sealed results are left untouched; the interrupted result is marked skipped - # and may have partial output saved to fs - # assertions that enforce the current behavior (uncommented): previous results are not run twice + interrupted - # result is marked skipped - assert bad_results == [("slow_exit", ArchiveResult.StatusChoices.SKIPPED, "")] - - # desired future behavior: earlier sealed results are left untouched, interrupted result is retried and cleanly - # overwrites on top of any previous partial output - # assert not bad_results - # assert (marker_dir / "hook-finished").exists() + # The interrupted hook should be retried directly without rerunning the + # previous hook in the same plugin. That keeps plugin-level shell hooks + # idempotent across runner takeover instead of depending on each hook to + # detect partial prior work itself. + assert not bad_results + assert (marker_dir / "hook-finished").exists() finally: for proc in (add_proc, add_proc2, server, server2, server3): if proc is not None and proc.poll() is None: diff --git a/archivebox/tests/test_urls.py b/archivebox/tests/test_urls.py index 6911d0c9..7b32f125 100644 --- a/archivebox/tests/test_urls.py +++ b/archivebox/tests/test_urls.py @@ -895,19 +895,19 @@ class TestUrlRouting: assert f"http://{public_host}/static/archive.png" in live_html assert "?preview=1" in live_html assert "function createMainFrame(previousFrame)" in live_html - assert "function activateCardPreview(card, link)" in live_html - assert "ensureMainFrame(true)" in live_html + assert "function activateCardPreview(card, link, updateHash=true)" in live_html + assert "ensureMainFrame(currentSrc !== nextSrcAbs)" in live_html assert "previousFrame.parentNode.replaceChild(frame, previousFrame)" in live_html assert "previousFrame.src = 'about:blank'" in live_html assert "event.stopImmediatePropagation()" in live_html - assert "const matchingLink = [...document.querySelectorAll('a[target=preview]')].find" in live_html + assert "const matchingLink = findPreviewLinkForHash(selectedPreviewHash)" in live_html assert "jQuery(link).click()" not in live_html assert "searchParams.delete('preview')" in live_html assert "doc.body.style.flexDirection = 'column'" in live_html assert "doc.body.style.alignItems = 'center'" in live_html assert "img.style.margin = '0 auto'" in live_html assert "window.location.hash = getPreviewHashValueFromHref(rawTarget)" in live_html - assert "const selectedPreviewHash = decodeURIComponent(window.location.hash.slice(1)).toLowerCase()" in live_html + assert "const selectedPreviewHash = window.location.hash ? decodeURIComponent(window.location.hash.slice(1)).toLowerCase() : ''" in live_html assert "pointer-events: none;" in live_html assert "pointer-events: auto;" in live_html assert 'class="thumbnail-click-overlay"' in live_html @@ -921,19 +921,19 @@ class TestUrlRouting: assert f"http://{public_host}/static/archive.png" in static_html assert "?preview=1" in static_html assert "function createMainFrame(previousFrame)" in static_html - assert "function activateCardPreview(card, link)" in static_html - assert "ensureMainFrame(true)" in static_html + assert "function activateCardPreview(card, link, updateHash=true)" in static_html + assert "ensureMainFrame(currentSrc !== nextSrcAbs)" in static_html assert "previousFrame.parentNode.replaceChild(frame, previousFrame)" in static_html assert "previousFrame.src = 'about:blank'" in static_html assert "event.stopImmediatePropagation()" in static_html - assert "const matchingLink = [...document.querySelectorAll('a[target=preview]')].find" in static_html + assert "const matchingLink = findPreviewLinkForHash(selectedPreviewHash)" in static_html assert "jQuery(link).click()" not in static_html assert "searchParams.delete('preview')" in static_html assert "doc.body.style.flexDirection = 'column'" in static_html assert "doc.body.style.alignItems = 'center'" in static_html assert "img.style.margin = '0 auto'" in static_html assert "window.location.hash = getPreviewHashValueFromHref(rawTarget)" in static_html - assert "const selectedPreviewHash = decodeURIComponent(window.location.hash.slice(1)).toLowerCase()" in static_html + assert "const selectedPreviewHash = window.location.hash ? decodeURIComponent(window.location.hash.slice(1)).toLowerCase() : ''" in static_html assert "pointer-events: none;" in static_html assert "pointer-events: auto;" in static_html assert 'class="thumbnail-click-overlay"' in static_html diff --git a/archivebox/tests/test_version_metadata.py b/archivebox/tests/test_version_metadata.py new file mode 100644 index 00000000..d112eaed --- /dev/null +++ b/archivebox/tests/test_version_metadata.py @@ -0,0 +1,90 @@ +from pathlib import Path + +from archivebox.config import version + + +def _set_package_dir(monkeypatch, package_dir: Path) -> None: + monkeypatch.setattr(version, "PACKAGE_DIR", package_dir) + version.get_COMMIT_HASH.cache_clear() + + +def test_get_commit_hash_from_environment(monkeypatch) -> None: + commit_hash = "e" * 40 + monkeypatch.setenv("ARCHIVEBOX_COMMIT_HASH", commit_hash) + version.get_COMMIT_HASH.cache_clear() + + assert version.get_COMMIT_HASH() == commit_hash + + +def test_get_commit_hash_from_docker_version_file_ignores_short_hash(monkeypatch) -> None: + commit_hash = "f" * 40 + + class VersionPath: + def __init__(self, path: str): + self.path = path + + def read_text(self) -> str: + assert self.path == "/VERSION.txt" + return f"COMMIT_HASH={commit_hash}\nArchiveBox COMMIT_HASH={commit_hash[:7]}\n" + + monkeypatch.setattr(version, "IN_DOCKER", True) + monkeypatch.setattr(version, "Path", VersionPath) + version.get_COMMIT_HASH.cache_clear() + + assert version.get_COMMIT_HASH() == commit_hash + + +def test_get_commit_hash_from_detached_head(monkeypatch, tmp_path) -> None: + commit_hash = "a" * 40 + package_dir = tmp_path / "archivebox" + git_dir = tmp_path / ".git" + package_dir.mkdir() + git_dir.mkdir() + git_dir.joinpath("HEAD").write_text(commit_hash) + + _set_package_dir(monkeypatch, package_dir) + + assert version.get_COMMIT_HASH() == commit_hash + + +def test_get_commit_hash_from_branch_ref(monkeypatch, tmp_path) -> None: + commit_hash = "b" * 40 + package_dir = tmp_path / "archivebox" + git_dir = tmp_path / ".git" + ref_path = git_dir / "refs" / "heads" / "dev" + package_dir.mkdir() + ref_path.parent.mkdir(parents=True) + git_dir.joinpath("HEAD").write_text("ref: refs/heads/dev") + ref_path.write_text(commit_hash) + + _set_package_dir(monkeypatch, package_dir) + + assert version.get_COMMIT_HASH() == commit_hash + + +def test_get_commit_hash_from_packed_ref(monkeypatch, tmp_path) -> None: + commit_hash = "c" * 40 + package_dir = tmp_path / "archivebox" + git_dir = tmp_path / ".git" + package_dir.mkdir() + git_dir.mkdir() + git_dir.joinpath("HEAD").write_text("ref: refs/heads/dev") + git_dir.joinpath("packed-refs").write_text(f"{commit_hash} refs/heads/dev\n") + + _set_package_dir(monkeypatch, package_dir) + + assert version.get_COMMIT_HASH() == commit_hash + + +def test_get_commit_hash_from_worktree_gitdir(monkeypatch, tmp_path) -> None: + commit_hash = "d" * 40 + package_dir = tmp_path / "worktree" / "archivebox" + real_git_dir = tmp_path / "repo" / ".git" / "worktrees" / "worktree" + package_dir.mkdir(parents=True) + real_git_dir.mkdir(parents=True) + package_dir.parent.joinpath(".git").write_text(f"gitdir: {real_git_dir}\n") + real_git_dir.joinpath("HEAD").write_text(commit_hash) + + _set_package_dir(monkeypatch, package_dir) + + assert version.get_COMMIT_HASH() == commit_hash diff --git a/archivebox/workers/supervisord_util.py b/archivebox/workers/supervisord_util.py index 3fe5f368..debf75fb 100644 --- a/archivebox/workers/supervisord_util.py +++ b/archivebox/workers/supervisord_util.py @@ -170,7 +170,7 @@ RUNNER_WORKER = { RUNNER_ONCE_WORKER = lambda args, name="worker_runner_once": { **RUNNER_WORKER, "name": name, - "command": _shell_join([sys.executable, "-m", "archivebox", "run", *args]), + "command": _shell_join([sys.executable, "-m", "archivebox", "run", "--no-stdin", *args]), "environment": 'PYTHONUNBUFFERED="1",COLUMNS="200"', "autorestart": "false", "stopwaitsecs": "1", diff --git a/bin/docker_entrypoint.sh b/bin/docker_entrypoint.sh index 54a7ab72..8b0b8ac9 100755 --- a/bin/docker_entrypoint.sh +++ b/bin/docker_entrypoint.sh @@ -144,7 +144,7 @@ ensure_runtime_tree() { run_as_archivebox() { if [[ "$(id -u)" == "0" ]]; then - gosu "$ARCHIVEBOX_USER" "$@" + setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups "$@" else "$@" fi @@ -283,12 +283,12 @@ if [[ "$1" == /* || "$1" == "bash" || "$1" == "sh" || "$1" == "echo" || "$1" == # "docker run archivebox /bin/bash -c '...'" # "docker run archivebox cat /VERSION.txt" if [[ "$(id -u)" == "0" ]]; then - exec gosu "$ARCHIVEBOX_USER" /bin/bash -c "exec $(printf ' %q' "$@")" + exec setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups /bin/bash -c "exec $(printf ' %q' "$@")" else exec /bin/bash -c "exec $(printf ' %q' "$@")" fi # printf requotes shell parameters properly https://stackoverflow.com/a/39463371/2156113 - # gosu spawns an ephemeral bash process owned by archivebox user (bash wrapper is needed to load env vars, PATH, and setup terminal TTY) + # setpriv spawns an ephemeral bash process owned by archivebox user (bash wrapper is needed to load env vars, PATH, and setup terminal TTY) # outermost exec hands over current process ID to inner bash process, inner exec hands over inner bash PID to user's command else # handle "docker run archivebox add some subcommand --with=args abc" by calling archivebox to run as args as CLI subcommand @@ -297,7 +297,7 @@ else # "docker run archivebox manage createsupseruser" # "docker run archivebox server 0.0.0.0:8000" if [[ "$(id -u)" == "0" ]]; then - exec gosu "$ARCHIVEBOX_USER" "$ARCHIVEBOX_BIN_PATH" "$@" + exec setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups "$ARCHIVEBOX_BIN_PATH" "$@" else exec "$ARCHIVEBOX_BIN_PATH" "$@" fi