From 232de59e9574ebb1e8a24ed189d782af1b4d8e9e Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Fri, 5 Jun 2026 00:53:33 -0700 Subject: [PATCH] Fix ArchiveBox Docker runtime layering --- .dockerignore | 7 +- Dockerfile | 52 ++++++++----- Dockerfile.multistage | 52 ++++++++----- archivebox/config/version.py | 53 ++++++++++++- archivebox/tests/test_dockerfiles.py | 75 +++++++++++++++++++ archivebox/tests/test_version_metadata.py | 90 +++++++++++++++++++++++ 6 files changed, 289 insertions(+), 40 deletions(-) create mode 100644 archivebox/tests/test_version_metadata.py diff --git a/.dockerignore b/.dockerignore index 1447cfee..5b2e02fe 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,9 +12,14 @@ __pycache__/ .pdm-python .eggs/ .git/ +!.git/ +.git/* .vscode/ !.git/HEAD -!.git/refs/heads/* +!.git/packed-refs +!.git/refs/ +!.git/refs/heads/ +!.git/refs/heads/** venv/ .venv/ diff --git a/Dockerfile b/Dockerfile index 0d9659bd..a1f8be54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,19 +13,15 @@ # --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \ # -t archivebox:multistage -ARG TARGETPLATFORM=linux/amd64 -ARG TARGETOS=linux -ARG TARGETARCH=amd64 -ARG TARGETVARIANT= ARG ABX_DL_IMAGE=archivebox/abx-dl:latest FROM ${ABX_DL_IMAGE} AS abx-dl FROM archivebox/sonic:1.4.9 AS sonic FROM ubuntu:24.04 AS archivebox-runtime-base -ARG TARGETPLATFORM=linux/amd64 -ARG TARGETOS=linux -ARG TARGETARCH=amd64 +ARG TARGETPLATFORM +ARG TARGETOS +ARG TARGETARCH ARG TARGETVARIANT ENV TZ=UTC \ @@ -72,7 +68,7 @@ ENV UV_COMPILE_BYTECODE=0 \ UV_LINK_MODE=copy \ UV_PROJECT_ENVIRONMENT=/venv \ VIRTUAL_ENV=/venv \ - PATH="/venv/bin:/opt/node/bin:$PATH" + PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH" SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] WORKDIR "$CODE_DIR" @@ -101,7 +97,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T COPY --from=abx-dl /bin/uv /bin/uv COPY --from=abx-dl /opt/uv/python /opt/uv/python COPY --from=abx-dl /opt/node /opt/node -COPY --from=abx-dl /venv /venv COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ @@ -109,7 +104,6 @@ RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \ && uname -a \ && sed -n '1,7p' /etc/os-release \ - && which python3 && python3 --version \ && which node && node --version \ && which uv && uv self version \ ) | tee -a /VERSION.txt @@ -126,6 +120,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T && apt-get update -qq \ && apt-get install -qq -y --no-install-recommends \ build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ + && uv venv /venv --python "${PYTHON_VERSION}" \ + && uv pip install setuptools pip wheel \ && uv sync \ --refresh \ --no-dev \ @@ -143,10 +139,22 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \ + && COMMIT_HASH="$( \ + if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \ + HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \ + if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \ + echo "$HEAD_REF"; \ + elif [[ "$HEAD_REF" == ref:\ * ]]; then \ + REF_PATH="${HEAD_REF#ref: }"; \ + cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \ + fi; \ + fi)" \ + && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \ && uv pip install --no-deps "$CODE_DIR" \ && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ - && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete + && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ + && rm -rf "$CODE_DIR/.git" FROM archivebox-runtime-base @@ -164,6 +172,7 @@ LABEL name="archivebox" \ COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg +COPY --from=archivebox-builder /opt/uv/python /opt/uv/python COPY --from=archivebox-builder /venv /venv COPY --from=archivebox-builder /app /app COPY --from=archivebox-builder /VERSION.txt /VERSION.txt @@ -175,8 +184,9 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ && (which sonic && sonic --version) | tee -a /VERSION.txt \ - && mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/uv" \ && openssl rand -hex 16 > /etc/machine-id \ && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \ && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt @@ -187,7 +197,6 @@ RUN echo "[+] Initializing image collection..." \ && PUID=0 PGID=0 archivebox init \ && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \ "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \ "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \ @@ -196,18 +205,25 @@ RUN echo "[+] Initializing image collection..." \ 2>/dev/null || true) RUN chmod +x "$CODE_DIR"/bin/*.sh \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ + && env -u PNPM_HOME PATH="/opt/node/bin:$PATH" /opt/node/bin/corepack pnpm add --loglevel=error --store-dir="$TMP_DIR/pnpm-store" --config.dangerouslyAllowAllBuilds=true --dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai 2>&1 | tee -a /VERSION.txt \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ + && rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node \ + && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode" \ + && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \ + && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ + && chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \ + && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \ - && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ + && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \ && /usr/local/bin/sonic --version | tee -a /VERSION.txt \ && /venv/bin/supervisord --version | tee -a /VERSION.txt \ && ! command -v gcc \ && ! command -v g++ \ && ! command -v make \ - && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ + && HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/* diff --git a/Dockerfile.multistage b/Dockerfile.multistage index 0d9659bd..a1f8be54 100644 --- a/Dockerfile.multistage +++ b/Dockerfile.multistage @@ -13,19 +13,15 @@ # --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \ # -t archivebox:multistage -ARG TARGETPLATFORM=linux/amd64 -ARG TARGETOS=linux -ARG TARGETARCH=amd64 -ARG TARGETVARIANT= ARG ABX_DL_IMAGE=archivebox/abx-dl:latest FROM ${ABX_DL_IMAGE} AS abx-dl FROM archivebox/sonic:1.4.9 AS sonic FROM ubuntu:24.04 AS archivebox-runtime-base -ARG TARGETPLATFORM=linux/amd64 -ARG TARGETOS=linux -ARG TARGETARCH=amd64 +ARG TARGETPLATFORM +ARG TARGETOS +ARG TARGETARCH ARG TARGETVARIANT ENV TZ=UTC \ @@ -72,7 +68,7 @@ ENV UV_COMPILE_BYTECODE=0 \ UV_LINK_MODE=copy \ UV_PROJECT_ENVIRONMENT=/venv \ VIRTUAL_ENV=/venv \ - PATH="/venv/bin:/opt/node/bin:$PATH" + PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH" SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] WORKDIR "$CODE_DIR" @@ -101,7 +97,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T COPY --from=abx-dl /bin/uv /bin/uv COPY --from=abx-dl /opt/uv/python /opt/uv/python COPY --from=abx-dl /opt/node /opt/node -COPY --from=abx-dl /venv /venv COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ @@ -109,7 +104,6 @@ RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \ && uname -a \ && sed -n '1,7p' /etc/os-release \ - && which python3 && python3 --version \ && which node && node --version \ && which uv && uv self version \ ) | tee -a /VERSION.txt @@ -126,6 +120,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T && apt-get update -qq \ && apt-get install -qq -y --no-install-recommends \ build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ + && uv venv /venv --python "${PYTHON_VERSION}" \ + && uv pip install setuptools pip wheel \ && uv sync \ --refresh \ --no-dev \ @@ -143,10 +139,22 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \ + && COMMIT_HASH="$( \ + if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \ + HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \ + if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \ + echo "$HEAD_REF"; \ + elif [[ "$HEAD_REF" == ref:\ * ]]; then \ + REF_PATH="${HEAD_REF#ref: }"; \ + cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \ + fi; \ + fi)" \ + && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \ && uv pip install --no-deps "$CODE_DIR" \ && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ - && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete + && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ + && rm -rf "$CODE_DIR/.git" FROM archivebox-runtime-base @@ -164,6 +172,7 @@ LABEL name="archivebox" \ COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg +COPY --from=archivebox-builder /opt/uv/python /opt/uv/python COPY --from=archivebox-builder /venv /venv COPY --from=archivebox-builder /app /app COPY --from=archivebox-builder /VERSION.txt /VERSION.txt @@ -175,8 +184,9 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ && (which sonic && sonic --version) | tee -a /VERSION.txt \ - && mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/uv" \ && openssl rand -hex 16 > /etc/machine-id \ && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \ && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt @@ -187,7 +197,6 @@ RUN echo "[+] Initializing image collection..." \ && PUID=0 PGID=0 archivebox init \ && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \ "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \ "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \ @@ -196,18 +205,25 @@ RUN echo "[+] Initializing image collection..." \ 2>/dev/null || true) RUN chmod +x "$CODE_DIR"/bin/*.sh \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ + && env -u PNPM_HOME PATH="/opt/node/bin:$PATH" /opt/node/bin/corepack pnpm add --loglevel=error --store-dir="$TMP_DIR/pnpm-store" --config.dangerouslyAllowAllBuilds=true --dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai 2>&1 | tee -a /VERSION.txt \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ + && rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node \ + && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode" \ + && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \ + && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ + && chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \ + && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \ - && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ + && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \ && /usr/local/bin/sonic --version | tee -a /VERSION.txt \ && /venv/bin/supervisord --version | tee -a /VERSION.txt \ && ! command -v gcc \ && ! command -v g++ \ && ! command -v make \ - && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ + && HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/* diff --git a/archivebox/config/version.py b/archivebox/config/version.py index 38eda872..f1e866c9 100644 --- a/archivebox/config/version.py +++ b/archivebox/config/version.py @@ -6,6 +6,7 @@ import importlib.metadata from pathlib import Path from functools import cache from datetime import datetime +import re ############################################################################################# @@ -41,11 +42,57 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR): @cache def get_COMMIT_HASH() -> str | None: + for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"): + env_commit_hash = os.environ.get(env_var, "").strip() + if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash): + return env_commit_hash + + if IN_DOCKER: + try: + version_txt = Path("/VERSION.txt").read_text() + docker_commit_hashes = re.findall(r"COMMIT_HASH=([0-9a-fA-F]{40})", version_txt) + if docker_commit_hashes: + return docker_commit_hashes[-1] + except Exception: + pass + + def _read_git_file(git_dir: Path, ref: str) -> str | None: + try: + return git_dir.joinpath(ref).read_text().strip() + except Exception: + pass + + try: + packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines() + except Exception: + return None + + for line in packed_refs: + if line.startswith("#") or line.startswith("^") or not line.strip(): + continue + commit_hash, packed_ref = line.split(" ", 1) + if packed_ref == ref: + return commit_hash.strip() + + return None + try: git_dir = PACKAGE_DIR.parent / ".git" - ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1] - commit_hash = git_dir.joinpath(ref).read_text().strip() - return commit_hash + if git_dir.is_file(): + gitdir_line = git_dir.read_text().strip() + gitdir_path = gitdir_line.removeprefix("gitdir:").strip() + git_dir = Path(gitdir_path) + if not git_dir.is_absolute(): + git_dir = PACKAGE_DIR.parent / git_dir + + head = (git_dir / "HEAD").read_text().strip() + if re.fullmatch(r"[0-9a-fA-F]{40}", head): + return head + + ref = head.removeprefix("ref:").strip() + commit_hash = _read_git_file(git_dir, ref) + if commit_hash: + return commit_hash except Exception: pass diff --git a/archivebox/tests/test_dockerfiles.py b/archivebox/tests/test_dockerfiles.py index 2696f7da..cd382427 100644 --- a/archivebox/tests/test_dockerfiles.py +++ b/archivebox/tests/test_dockerfiles.py @@ -108,6 +108,23 @@ def test_dockerfiles_pin_project_binary_paths_for_validation() -> None: assert 'GIT_BINARY="$LIB_DIR/env/bin/git"' in text assert 'GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl"' in text assert 'FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl"' in text + assert 'OPENCODE_BINARY="$LIB_DIR/bin/opencode"' in text + + +def test_dockerfiles_preinstall_opencode_without_pnpm_home_override() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert 'PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"' in text + assert "env -u PNPM_HOME" in text + assert "/opt/node/bin/corepack pnpm add" in text + assert '--store-dir="$TMP_DIR/pnpm-store"' in text + assert '--dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai' in text + assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode"' in text + assert 'rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node' in text + assert 'ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode"' in text + assert 'ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode"' in text + assert 'chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"' in text + assert 'chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode"' in text def test_dockerfiles_use_setpriv_instead_of_gosu() -> None: @@ -119,3 +136,61 @@ def test_dockerfiles_use_setpriv_instead_of_gosu() -> None: entrypoint = (REPO_ROOT / "bin/docker_entrypoint.sh").read_text() assert "gosu" not in entrypoint assert "setpriv" in entrypoint + + +def test_dockerfiles_create_archivebox_venv_in_archivebox_builder() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert "COPY --from=abx-dl /venv /venv" not in text + assert 'uv venv /venv --python "${PYTHON_VERSION}"' in text + assert "COPY --from=archivebox-builder /opt/uv/python /opt/uv/python" in text + assert 'COMMIT_HASH="$(' in text + assert 'HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"' in text + assert 'echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt' in text + assert 'rm -rf "$CODE_DIR/.git"' in text + + +def test_dockerfiles_do_not_duplicate_runtime_lib_with_recursive_chown() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"' not in text + assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR"' not in text + assert 'chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"' in text + assert "COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib" in text + + +def test_dockerfiles_validate_current_papers_dl_path() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert '"$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version' in text + assert "$LIB_DIR/pip/packages/papers-dl" not in text + + +def test_dockerfiles_do_not_default_multiarch_builds_to_amd64() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert "ARG TARGETPLATFORM=linux/amd64" not in text + assert "ARG TARGETARCH=amd64" not in text + assert "ARG TARGETPLATFORM\n" in text + assert "FROM ${ABX_DL_IMAGE} AS abx-dl" in text + + +def test_dockerfiles_set_archivebox_home_for_setpriv_commands() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert ( + 'install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus"' + ) in text + assert 'HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config"' in text + + +def test_dockerignore_keeps_minimal_git_metadata_for_image_version() -> None: + text = (REPO_ROOT / ".dockerignore").read_text() + assert ".git/" in text + assert "!.git/" in text + assert ".git/*" in text + assert "!.git/HEAD" in text + assert "!.git/packed-refs" in text + assert "!.git/refs/" in text + assert "!.git/refs/heads/" in text + assert "!.git/refs/heads/**" in text diff --git a/archivebox/tests/test_version_metadata.py b/archivebox/tests/test_version_metadata.py new file mode 100644 index 00000000..d112eaed --- /dev/null +++ b/archivebox/tests/test_version_metadata.py @@ -0,0 +1,90 @@ +from pathlib import Path + +from archivebox.config import version + + +def _set_package_dir(monkeypatch, package_dir: Path) -> None: + monkeypatch.setattr(version, "PACKAGE_DIR", package_dir) + version.get_COMMIT_HASH.cache_clear() + + +def test_get_commit_hash_from_environment(monkeypatch) -> None: + commit_hash = "e" * 40 + monkeypatch.setenv("ARCHIVEBOX_COMMIT_HASH", commit_hash) + version.get_COMMIT_HASH.cache_clear() + + assert version.get_COMMIT_HASH() == commit_hash + + +def test_get_commit_hash_from_docker_version_file_ignores_short_hash(monkeypatch) -> None: + commit_hash = "f" * 40 + + class VersionPath: + def __init__(self, path: str): + self.path = path + + def read_text(self) -> str: + assert self.path == "/VERSION.txt" + return f"COMMIT_HASH={commit_hash}\nArchiveBox COMMIT_HASH={commit_hash[:7]}\n" + + monkeypatch.setattr(version, "IN_DOCKER", True) + monkeypatch.setattr(version, "Path", VersionPath) + version.get_COMMIT_HASH.cache_clear() + + assert version.get_COMMIT_HASH() == commit_hash + + +def test_get_commit_hash_from_detached_head(monkeypatch, tmp_path) -> None: + commit_hash = "a" * 40 + package_dir = tmp_path / "archivebox" + git_dir = tmp_path / ".git" + package_dir.mkdir() + git_dir.mkdir() + git_dir.joinpath("HEAD").write_text(commit_hash) + + _set_package_dir(monkeypatch, package_dir) + + assert version.get_COMMIT_HASH() == commit_hash + + +def test_get_commit_hash_from_branch_ref(monkeypatch, tmp_path) -> None: + commit_hash = "b" * 40 + package_dir = tmp_path / "archivebox" + git_dir = tmp_path / ".git" + ref_path = git_dir / "refs" / "heads" / "dev" + package_dir.mkdir() + ref_path.parent.mkdir(parents=True) + git_dir.joinpath("HEAD").write_text("ref: refs/heads/dev") + ref_path.write_text(commit_hash) + + _set_package_dir(monkeypatch, package_dir) + + assert version.get_COMMIT_HASH() == commit_hash + + +def test_get_commit_hash_from_packed_ref(monkeypatch, tmp_path) -> None: + commit_hash = "c" * 40 + package_dir = tmp_path / "archivebox" + git_dir = tmp_path / ".git" + package_dir.mkdir() + git_dir.mkdir() + git_dir.joinpath("HEAD").write_text("ref: refs/heads/dev") + git_dir.joinpath("packed-refs").write_text(f"{commit_hash} refs/heads/dev\n") + + _set_package_dir(monkeypatch, package_dir) + + assert version.get_COMMIT_HASH() == commit_hash + + +def test_get_commit_hash_from_worktree_gitdir(monkeypatch, tmp_path) -> None: + commit_hash = "d" * 40 + package_dir = tmp_path / "worktree" / "archivebox" + real_git_dir = tmp_path / "repo" / ".git" / "worktrees" / "worktree" + package_dir.mkdir(parents=True) + real_git_dir.mkdir(parents=True) + package_dir.parent.joinpath(".git").write_text(f"gitdir: {real_git_dir}\n") + real_git_dir.joinpath("HEAD").write_text(commit_hash) + + _set_package_dir(monkeypatch, package_dir) + + assert version.get_COMMIT_HASH() == commit_hash