diff --git a/Dockerfile b/Dockerfile index 9e136a20..3c1d89b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,9 +15,8 @@ ARG ABX_DL_IMAGE=archivebox/abx-dl:latest -FROM ${ABX_DL_IMAGE} AS abx-dl FROM archivebox/sonic:1.4.9 AS sonic -FROM ubuntu:24.04 AS archivebox-runtime-base +FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base ARG TARGETPLATFORM ARG TARGETOS @@ -62,9 +61,18 @@ ENV TMP_DIR=/tmp/archivebox \ GOOGLE_DEFAULT_CLIENT_ID=no \ GOOGLE_DEFAULT_CLIENT_SECRET=no +ENV HOME=/home/archivebox \ + XDG_CONFIG_HOME=/home/archivebox/.config \ + XDG_CACHE_HOME=/home/archivebox/.cache \ + ABXPKG_INSTALL_TIMEOUT=600 \ + ABXPKG_POSTINSTALL_SCRIPTS=True \ + ABXPKG_MIN_RELEASE_AGE=0 \ + TIMEOUT=600 + ENV UV_COMPILE_BYTECODE=0 \ UV_PYTHON_PREFERENCE=managed \ UV_PYTHON_INSTALL_DIR=/opt/uv/python \ + ARCHIVEBOX_PYTHON_INSTALL_DIR=/opt/archivebox/python \ UV_LINK_MODE=copy \ UV_PROJECT_ENVIRONMENT=/venv \ VIRTUAL_ENV=/venv \ @@ -73,33 +81,8 @@ ENV UV_COMPILE_BYTECODE=0 \ SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] WORKDIR "$CODE_DIR" -RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \ - && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \ - && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \ - && rm -f /etc/apt/apt.conf.d/docker-clean - -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \ - && apt-get update -qq \ - && apt-get install -qq -y \ - apt-transport-https apt-utils ca-certificates curl wget gnupg2 \ - dumb-init util-linux unzip git grep ripgrep dnsutils iputils-ping procps tree nano \ - cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \ - libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \ - libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \ - libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \ - libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \ - fonts-liberation fonts-noto-color-emoji xdg-utils \ - ffmpeg imagemagick tesseract-ocr tesseract-ocr-eng openjdk-21-jre-headless \ - && rm -rf /var/lib/apt/lists/* - -# Runtime-owned layers copied from the abx-dl image. -COPY --from=abx-dl /bin/uv /bin/uv -COPY --from=abx-dl /opt/uv/python /opt/uv/python -COPY --from=abx-dl /opt/node /opt/node -COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt - -RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ +RUN cp /VERSION.txt /ABX-DL-VERSION.txt \ + && (echo "[i] Docker build for ArchiveBox multistage starting..." \ && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \ && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \ && uname -a \ @@ -110,6 +93,12 @@ RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ ENV PYTHONDONTWRITEBYTECODE=1 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ + echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \ + && apt-get update -qq \ + && apt-get install -qq -y --no-install-recommends ripgrep \ + && rm -rf /var/lib/apt/lists/* + FROM archivebox-runtime-base AS archivebox-builder WORKDIR "$CODE_DIR" @@ -117,12 +106,17 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \ + && export UV_PYTHON_INSTALL_DIR="$ARCHIVEBOX_PYTHON_INSTALL_DIR" \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \ + && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \ + && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \ + && rm -f /etc/apt/apt.conf.d/docker-clean \ && apt-get update -qq \ && apt-get install -qq -y --no-install-recommends \ build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ - && uv venv /venv --python "${PYTHON_VERSION}" \ - && uv pip install setuptools pip wheel \ - && uv sync \ + && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \ + && /usr/bin/uv pip install setuptools pip wheel \ + && /usr/bin/uv sync \ --refresh \ --no-dev \ --inexact \ @@ -150,8 +144,8 @@ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$T fi; \ fi)" \ && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \ - && uv pip install --no-deps "$CODE_DIR" \ - && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \ + && /usr/bin/uv pip install --no-deps "$CODE_DIR" \ + && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf "$CODE_DIR/.git" @@ -172,21 +166,22 @@ LABEL name="archivebox" \ COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg -COPY --from=archivebox-builder /opt/uv/python /opt/uv/python +COPY --from=archivebox-builder /opt/archivebox/python /opt/archivebox/python COPY --from=archivebox-builder /venv /venv COPY --from=archivebox-builder /app /app COPY --from=archivebox-builder /VERSION.txt /VERSION.txt -COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ - && groupadd --system "$ARCHIVEBOX_USER" \ - && useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \ - && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ - && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ + && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \ + && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \ + && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \ + && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_PUID" ]] || usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ + && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_PGID" ]] || groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ && (which sonic && sonic --version) | tee -a /VERSION.txt \ && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/uv" \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.config" "/home/$ARCHIVEBOX_USER/.cache" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \ && openssl rand -hex 16 > /etc/machine-id \ && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \ && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt @@ -204,18 +199,11 @@ RUN echo "[+] Initializing image collection..." \ "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \ 2>/dev/null || true) -RUN chmod +x "$CODE_DIR"/bin/*.sh \ - && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ - && env -u PNPM_HOME PATH="/opt/node/bin:$PATH" /opt/node/bin/corepack pnpm add --loglevel=error --store-dir="$TMP_DIR/pnpm-store" --config.dangerouslyAllowAllBuilds=true --dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai 2>&1 | tee -a /VERSION.txt \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ - && rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node \ - && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode" \ - && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \ - && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ - && chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \ - && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/env/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ - && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \ +RUN chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ + && chmod +x "$CODE_DIR"/bin/*.sh \ + && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" + +RUN "$LIB_DIR/bin/chromium" --version | tee -a /VERSION.txt \ && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \ && /usr/local/bin/sonic --version | tee -a /VERSION.txt \ @@ -223,7 +211,8 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \ && ! command -v gcc \ && ! command -v g++ \ && ! command -v make \ - && HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ + && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \ + && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/* diff --git a/Dockerfile.multistage b/Dockerfile.multistage index 9e136a20..3c1d89b2 100644 --- a/Dockerfile.multistage +++ b/Dockerfile.multistage @@ -15,9 +15,8 @@ ARG ABX_DL_IMAGE=archivebox/abx-dl:latest -FROM ${ABX_DL_IMAGE} AS abx-dl FROM archivebox/sonic:1.4.9 AS sonic -FROM ubuntu:24.04 AS archivebox-runtime-base +FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base ARG TARGETPLATFORM ARG TARGETOS @@ -62,9 +61,18 @@ ENV TMP_DIR=/tmp/archivebox \ GOOGLE_DEFAULT_CLIENT_ID=no \ GOOGLE_DEFAULT_CLIENT_SECRET=no +ENV HOME=/home/archivebox \ + XDG_CONFIG_HOME=/home/archivebox/.config \ + XDG_CACHE_HOME=/home/archivebox/.cache \ + ABXPKG_INSTALL_TIMEOUT=600 \ + ABXPKG_POSTINSTALL_SCRIPTS=True \ + ABXPKG_MIN_RELEASE_AGE=0 \ + TIMEOUT=600 + ENV UV_COMPILE_BYTECODE=0 \ UV_PYTHON_PREFERENCE=managed \ UV_PYTHON_INSTALL_DIR=/opt/uv/python \ + ARCHIVEBOX_PYTHON_INSTALL_DIR=/opt/archivebox/python \ UV_LINK_MODE=copy \ UV_PROJECT_ENVIRONMENT=/venv \ VIRTUAL_ENV=/venv \ @@ -73,33 +81,8 @@ ENV UV_COMPILE_BYTECODE=0 \ SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] WORKDIR "$CODE_DIR" -RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \ - && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \ - && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \ - && rm -f /etc/apt/apt.conf.d/docker-clean - -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \ - && apt-get update -qq \ - && apt-get install -qq -y \ - apt-transport-https apt-utils ca-certificates curl wget gnupg2 \ - dumb-init util-linux unzip git grep ripgrep dnsutils iputils-ping procps tree nano \ - cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \ - libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \ - libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \ - libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \ - libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \ - fonts-liberation fonts-noto-color-emoji xdg-utils \ - ffmpeg imagemagick tesseract-ocr tesseract-ocr-eng openjdk-21-jre-headless \ - && rm -rf /var/lib/apt/lists/* - -# Runtime-owned layers copied from the abx-dl image. -COPY --from=abx-dl /bin/uv /bin/uv -COPY --from=abx-dl /opt/uv/python /opt/uv/python -COPY --from=abx-dl /opt/node /opt/node -COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt - -RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ +RUN cp /VERSION.txt /ABX-DL-VERSION.txt \ + && (echo "[i] Docker build for ArchiveBox multistage starting..." \ && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \ && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \ && uname -a \ @@ -110,6 +93,12 @@ RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ ENV PYTHONDONTWRITEBYTECODE=1 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ + echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \ + && apt-get update -qq \ + && apt-get install -qq -y --no-install-recommends ripgrep \ + && rm -rf /var/lib/apt/lists/* + FROM archivebox-runtime-base AS archivebox-builder WORKDIR "$CODE_DIR" @@ -117,12 +106,17 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \ + && export UV_PYTHON_INSTALL_DIR="$ARCHIVEBOX_PYTHON_INSTALL_DIR" \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \ + && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \ + && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \ + && rm -f /etc/apt/apt.conf.d/docker-clean \ && apt-get update -qq \ && apt-get install -qq -y --no-install-recommends \ build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ - && uv venv /venv --python "${PYTHON_VERSION}" \ - && uv pip install setuptools pip wheel \ - && uv sync \ + && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \ + && /usr/bin/uv pip install setuptools pip wheel \ + && /usr/bin/uv sync \ --refresh \ --no-dev \ --inexact \ @@ -150,8 +144,8 @@ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$T fi; \ fi)" \ && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \ - && uv pip install --no-deps "$CODE_DIR" \ - && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \ + && /usr/bin/uv pip install --no-deps "$CODE_DIR" \ + && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf "$CODE_DIR/.git" @@ -172,21 +166,22 @@ LABEL name="archivebox" \ COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg -COPY --from=archivebox-builder /opt/uv/python /opt/uv/python +COPY --from=archivebox-builder /opt/archivebox/python /opt/archivebox/python COPY --from=archivebox-builder /venv /venv COPY --from=archivebox-builder /app /app COPY --from=archivebox-builder /VERSION.txt /VERSION.txt -COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ - && groupadd --system "$ARCHIVEBOX_USER" \ - && useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \ - && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ - && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ + && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \ + && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \ + && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \ + && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_PUID" ]] || usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ + && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_PGID" ]] || groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ && (which sonic && sonic --version) | tee -a /VERSION.txt \ && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/uv" \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.config" "/home/$ARCHIVEBOX_USER/.cache" \ + && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \ && openssl rand -hex 16 > /etc/machine-id \ && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \ && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt @@ -204,18 +199,11 @@ RUN echo "[+] Initializing image collection..." \ "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \ 2>/dev/null || true) -RUN chmod +x "$CODE_DIR"/bin/*.sh \ - && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ - && env -u PNPM_HOME PATH="/opt/node/bin:$PATH" /opt/node/bin/corepack pnpm add --loglevel=error --store-dir="$TMP_DIR/pnpm-store" --config.dangerouslyAllowAllBuilds=true --dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai 2>&1 | tee -a /VERSION.txt \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \ - && rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node \ - && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode" \ - && ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \ - && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ - && chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \ - && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/env/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ - && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \ +RUN chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ + && chmod +x "$CODE_DIR"/bin/*.sh \ + && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" + +RUN "$LIB_DIR/bin/chromium" --version | tee -a /VERSION.txt \ && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \ && /usr/local/bin/sonic --version | tee -a /VERSION.txt \ @@ -223,7 +211,8 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \ && ! command -v gcc \ && ! command -v g++ \ && ! command -v make \ - && HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ + && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \ + && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/* diff --git a/archivebox/tests/test_dockerfiles.py b/archivebox/tests/test_dockerfiles.py deleted file mode 100644 index 42035062..00000000 --- a/archivebox/tests/test_dockerfiles.py +++ /dev/null @@ -1,196 +0,0 @@ -from __future__ import annotations - -import re -from pathlib import Path - - -REPO_ROOT = Path(__file__).resolve().parents[2] - -_REQUIRED_DOCKER_INSTALL_TARGETS = { - "archivewebpage", - "defuddle", - "forumdl", - "gallerydl", - "git", - "istilldontcareaboutcookies", - "liteparse", - "mercury", - "opencode", - "opendataloader", - "papersdl", - "parse_rss_urls", - "readability", - "search_backend_ripgrep", - "search_backend_sonic", -} - -_REQUIRED_DOCKER_PREINSTALL_TARGETS = { - "archivewebpage", - "defuddle", - "forumdl", - "gallerydl", - "git", - "istilldontcareaboutcookies", - "liteparse", - "mercury", - "opendataloader", - "papersdl", - "parse_rss_urls", - "readability", - "search_backend_ripgrep", - "search_backend_sonic", - "ublock", - "wget", - "ytdlp", -} - -_DOCKER_PREINSTALL_EXCLUDED_TARGETS = { - "twocaptcha", -} - - -def _archivebox_install_commands(dockerfile: Path) -> list[str]: - text = dockerfile.read_text() - return re.findall(r"archivebox install ([^\\\n]+)", text) - - -def _bare_archivebox_install_commands(dockerfile: Path) -> list[str]: - text = dockerfile.read_text() - return re.findall(r"archivebox install(?:\s+2>&1|\s*\\)", text) - - -def _abx_dl_plugin_install_targets(dockerfile: Path) -> set[str]: - text = dockerfile.read_text() - commands = re.findall(r"abx-dl plugins --install((?: \\\n|[^\n])*)", text) - targets: set[str] = set() - for command in commands: - command_text = command.replace("\\", " ") - targets.update(re.findall(r"\b[a-z][a-z0-9_]*\b", command_text)) - return targets - - -def test_dockerfiles_install_binaries_required_by_version_validation() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - commands = _archivebox_install_commands(REPO_ROOT / dockerfile_name) - target_command = max(commands, key=lambda command: len(command.split())) - targets = set(target_command.split()) - assert _REQUIRED_DOCKER_INSTALL_TARGETS <= targets - - -def test_dockerfile_prewarms_stable_plugin_dependencies_without_optional_captcha() -> None: - text = (REPO_ROOT / "Dockerfile").read_text() - assert "archivebox/abx-dl:latest" in text - assert "archivebox/abxdl" not in text - assert _abx_dl_plugin_install_targets(REPO_ROOT / "Dockerfile") == set() - - -def test_dockerfile_build_installs_disable_release_age_gate() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - text = (REPO_ROOT / dockerfile_name).read_text() - assert "ABXPKG_POSTINSTALL_SCRIPTS=True" in text - assert "ABXPKG_MIN_RELEASE_AGE=0" in text - - -def test_dockerfiles_do_not_queue_optional_binaries_during_validation() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - assert _bare_archivebox_install_commands(REPO_ROOT / dockerfile_name) == [] - - -def test_dockerfiles_clean_build_data_dir_before_init() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - text = (REPO_ROOT / dockerfile_name).read_text() - assert 'find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +' in text - - -def test_dockerfiles_pin_project_binary_paths_for_validation() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - text = (REPO_ROOT / dockerfile_name).read_text() - assert 'GIT_BINARY="$LIB_DIR/env/bin/git"' in text - assert 'GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl"' in text - assert 'FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl"' in text - assert 'OPENCODE_BINARY="$LIB_DIR/env/bin/opencode"' in text - - -def test_dockerfiles_preinstall_opencode_without_pnpm_home_override() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - text = (REPO_ROOT / dockerfile_name).read_text() - assert 'PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"' in text - assert "env -u PNPM_HOME" in text - assert "/opt/node/bin/corepack pnpm add" in text - assert '--store-dir="$TMP_DIR/pnpm-store"' in text - assert '--dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai' in text - assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode"' in text - assert 'rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node' in text - assert 'ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode"' in text - assert 'ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode"' in text - assert 'chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"' in text - assert 'chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode"' in text - - -def test_dockerfiles_use_setpriv_instead_of_gosu() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - text = (REPO_ROOT / dockerfile_name).read_text() - assert "gosu" not in text - assert "setpriv" in text - - entrypoint = (REPO_ROOT / "bin/docker_entrypoint.sh").read_text() - assert "gosu" not in entrypoint - assert "setpriv" in entrypoint - - -def test_dockerfiles_create_archivebox_venv_in_archivebox_builder() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - text = (REPO_ROOT / dockerfile_name).read_text() - assert "COPY --from=abx-dl /venv /venv" not in text - assert 'uv venv /venv --python "${PYTHON_VERSION}"' in text - assert "COPY --from=archivebox-builder /opt/uv/python /opt/uv/python" in text - assert 'COMMIT_HASH="$(' in text - assert 'HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"' in text - assert 'echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt' in text - assert 'rm -rf "$CODE_DIR/.git"' in text - - -def test_dockerfiles_do_not_duplicate_runtime_lib_with_recursive_chown() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - text = (REPO_ROOT / dockerfile_name).read_text() - assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"' not in text - assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR"' not in text - assert 'chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"' in text - assert "COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib" in text - - -def test_dockerfiles_validate_current_papers_dl_path() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - text = (REPO_ROOT / dockerfile_name).read_text() - assert '"$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version' in text - assert "$LIB_DIR/pip/packages/papers-dl" not in text - - -def test_dockerfiles_do_not_default_multiarch_builds_to_amd64() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - text = (REPO_ROOT / dockerfile_name).read_text() - assert "ARG TARGETPLATFORM=linux/amd64" not in text - assert "ARG TARGETARCH=amd64" not in text - assert "ARG TARGETPLATFORM\n" in text - assert "FROM ${ABX_DL_IMAGE} AS abx-dl" in text - - -def test_dockerfiles_set_archivebox_home_for_setpriv_commands() -> None: - for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): - text = (REPO_ROOT / dockerfile_name).read_text() - assert ( - 'install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus"' - ) in text - assert 'HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config"' in text - - -def test_dockerignore_keeps_minimal_git_metadata_for_image_version() -> None: - text = (REPO_ROOT / ".dockerignore").read_text() - assert ".git/" in text - assert "!.git/" in text - assert ".git/*" in text - assert "!.git/HEAD" in text - assert "!.git/packed-refs" in text - assert "!.git/refs/" in text - assert "!.git/refs/heads/" in text - assert "!.git/refs/heads/**" in text