diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cb88fd63..e9eaa9e7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,7 +5,7 @@ on: workflow_call: push: branches: - - dev + - "**" tags: - 'v*' # pull_request: @@ -13,13 +13,14 @@ on: env: DOCKERHUB_IMAGE: archivebox/archivebox GHCR_IMAGE: ghcr.io/archivebox/archivebox + ABX_DL_IMAGE: archivebox/abx-dl:latest permissions: contents: read packages: write concurrency: - group: docker-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -96,6 +97,24 @@ jobs: - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} + - name: Wait for published abx-dl image + id: abx_dl_image + shell: bash + run: | + set -Eeuo pipefail + deadline=$((SECONDS + 1800)) + until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do + if (( SECONDS >= deadline )); then + echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2 + exit 1 + fi + echo "${ABX_DL_IMAGE} is not published yet; waiting..." + sleep 30 + done + + echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT" + docker buildx imagetools inspect "${ABX_DL_IMAGE}" + - name: Login to Docker Hub uses: docker/login-action@v3 if: github.event_name != 'pull_request' @@ -116,13 +135,10 @@ jobs: run: | set -Eeuo pipefail VERSION="$(python3 - <<'PY' - from pathlib import Path - import re + import tomllib - match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE) - if not match: - raise SystemExit("Failed to read version from pyproject.toml") - print(match.group(1)) + with open("pyproject.toml", "rb") as f: + print(tomllib.load(f)["project"]["version"]) PY )" @@ -131,11 +147,12 @@ jobs: echo "org.opencontainers.image.version=${VERSION}" echo "org.opencontainers.image.revision=${GITHUB_SHA}" echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox' + echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}" echo 'EOF' echo "version=${VERSION}" } >> "$GITHUB_OUTPUT" - echo "[+] Building ${{ matrix.platform }} for ${VERSION}" + echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}" - name: Build and push digest id: docker_build @@ -149,6 +166,8 @@ jobs: ${{ env.DOCKERHUB_IMAGE }} ${{ env.GHCR_IMAGE }} labels: ${{ steps.docker_meta.outputs.labels }} + build-args: | + ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }} cache-from: type=gha,scope=${{ matrix.cache_scope }} cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }} platforms: ${{ matrix.platform }} @@ -215,36 +234,49 @@ jobs: run: | set -Eeuo pipefail VERSION="$(python3 - <<'PY' - from pathlib import Path - import re + import tomllib - match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE) - if not match: - raise SystemExit("Failed to read version from pyproject.toml") - print(match.group(1)) + with open("pyproject.toml", "rb") as f: + print(tomllib.load(f)["project"]["version"]) PY )" - SHORT_SHA="${GITHUB_SHA::8}" + BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)" + SHORT_SHA="${GITHUB_SHA::12}" + test -n "$BRANCH_TAG" + test -n "$SHORT_SHA" { echo 'dockerhub_tags<> "$GITHUB_OUTPUT" echo "[+] Publishing Docker Hub tags:" - printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" + if [[ "${GITHUB_REF_NAME}" == "main" ]]; then + printf '%s\n' "${DOCKERHUB_IMAGE}:latest" + fi + printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" echo "[+] Publishing GHCR tags:" - printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}" + if [[ "${GITHUB_REF_NAME}" == "main" ]]; then + printf '%s\n' "${GHCR_IMAGE}:latest" + fi + printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}" - name: Create Docker Hub manifest shell: bash diff --git a/Dockerfile b/Dockerfile index 571c07fa..0d9659bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,66 +1,33 @@ -# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies: -# python3.13, uv, python3-ldap -# curl, wget, git, dig, ping, tree, nano -# node, npm -# ArchiveBox and plugin runtime dependencies installed by archivebox init --install -# Usage: -# git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox -# docker build . -t archivebox -# docker run -v "$PWD/data":/data archivebox init -# docker run -v "$PWD/data":/data archivebox add 'https://example.com' -# docker run -v "$PWD/data":/data -it archivebox manage createsuperuser -# docker run -v "$PWD/data":/data -p 8000:8000 archivebox server -# docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123 -# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development +# syntax=docker/dockerfile:1.7 +# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image. +# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes. +# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime. +# Build abx-dl first, then point this file at it: +# docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \ +# --build-context abxbus=../abxbus \ +# --build-context abxpkg=../abxpkg \ +# --build-context abx-plugins=../abx-plugins \ +# -t archivebox/abx-dl:dev +# docker buildx build . -f Dockerfile \ +# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \ +# -t archivebox:multistage -######################################################################################### - -### Example: Using ArchiveBox in your own project's Dockerfile ######## - -# FROM python:3.13-slim -# WORKDIR /data -# RUN pip install archivebox>=0.9.0 # use latest release here -# RUN archivebox install -# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data - -######################################################################################### - -ARG TARGETPLATFORM -ARG TARGETOS -ARG TARGETARCH +ARG TARGETPLATFORM=linux/amd64 +ARG TARGETOS=linux +ARG TARGETARCH=amd64 ARG TARGETVARIANT= +ARG ABX_DL_IMAGE=archivebox/abx-dl:latest +FROM ${ABX_DL_IMAGE} AS abx-dl FROM archivebox/sonic:1.4.9 AS sonic -FROM ubuntu:24.04 +FROM ubuntu:24.04 AS archivebox-runtime-base -LABEL name="archivebox" \ - maintainer="Nick Sweeting " \ - description="All-in-one self-hosted internet archiving solution" \ - homepage="https://github.com/ArchiveBox/ArchiveBox" \ - documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \ - org.opencontainers.image.title="ArchiveBox" \ - org.opencontainers.image.vendor="ArchiveBox" \ - org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \ - org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \ - com.docker.image.source.entrypoint="Dockerfile" \ - # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels): - # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/ - com.docker.desktop.extension.api.version=">= 1.4.7" \ - com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \ - com.docker.extension.publisher-url="https://archivebox.io" \ - com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \ - com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \ - com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \ - com.docker.extension.categories='database,utility-tools' - -ARG TARGETPLATFORM -ARG TARGETOS -ARG TARGETARCH +ARG TARGETPLATFORM=linux/amd64 +ARG TARGETOS=linux +ARG TARGETARCH=amd64 ARG TARGETVARIANT -######### Environment Variables ################################# -# Global build-time and runtime environment constants + default pkg manager config ENV TZ=UTC \ LANGUAGE=en_US:en \ LC_ALL=C.UTF-8 \ @@ -73,204 +40,93 @@ ENV TZ=UTC \ PIP_ONLY_BINARY=aiohttp \ npm_config_loglevel=error -# Language Version config ENV PYTHON_VERSION=3.13 \ NODE_VERSION=22.22.3 -# Non-root User config -ENV ARCHIVEBOX_USER="archivebox" \ +ENV ARCHIVEBOX_USER=archivebox \ DEFAULT_PUID=911 \ DEFAULT_PGID=911 \ - IN_DOCKER=True \ - BIND_ADDR=0.0.0.0:8000 -# Docker has to listen on all interfaces, not just localhost. + IN_DOCKER=True -# ArchiveBox Source Code + Lib + Data paths ENV CODE_DIR=/app \ DATA_DIR=/data \ LIB_DIR=/opt/archivebox/lib \ ABXPKG_LIB_DIR=/opt/archivebox/lib \ - SONIC_BINARY=/opt/archivebox/lib/env/bin/sonic \ - PLAYWRIGHT_BROWSERS_PATH=/browsers - -# Bash SHELL config -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] - -######### System Environment #################################### - -# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes) -WORKDIR "$CODE_DIR" - -# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds) -RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \ - && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \ - && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \ - && rm -f /etc/apt/apt.conf.d/docker-clean - -# Print debug info about build and save it to disk, for human eyes only, not used by anything else -RUN (echo "[i] Docker build for ArchiveBox starting..." \ - && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \ - && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \ - && echo \ - && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \ - && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \ - && echo \ - && uname -a \ - && sed -n '1,7p' /etc/os-release \ - && which bash && bash --version | sed -n '1p' \ - && which dpkg && dpkg --version | sed -n '1p' \ - && echo -e '\n\n' && env && echo -e '\n\n' \ - ) | tee -a /VERSION.txt - -# Create non-privileged user for archivebox and chrome -RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ - && groupadd --system $ARCHIVEBOX_USER \ - && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \ - && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ - && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ - && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \ - | tee -a /VERSION.txt - # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime - # https://docs.linuxserver.io/general/understanding-puid-and-pgid - -# Install system apt dependencies (adding backports to access more recent apt updates) -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \ - && mkdir -p /etc/apt/keyrings \ - && apt-get update -qq \ - && apt-get install -qq -y \ - # 1. packaging dependencies - apt-transport-https ca-certificates apt-utils gnupg2 curl wget \ - # 2. docker and init system dependencies - zlib1g-dev dumb-init gosu cron unzip grep dnsutils git ripgrep python3.12-venv default-jre-headless \ - # 3. frivolous CLI helpers to make debugging failed archiving easier - tree nano iputils-ping \ - # nano iputils-ping dnsutils htop procps jq yq - && rm -rf /var/lib/apt/lists/* - -# Install sonic search backend -COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic -COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg -RUN (which sonic && sonic --version) | tee -a /VERSION.txt - -######### Language Environments #################################### - -# Set up Python environment -# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own: -#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ -# --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \ -# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \ - # && apt-get update -qq \ - # && apt-get install -qq -y --no-upgrade \ - # python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \ - # && rm -rf /var/lib/apt/lists/* \ - # tell PDM to allow using global system python site packages - # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \ - # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \ - # create global virtual environment GLOBAL_VENV to use (better than using pip install --global) - # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \ - # && python3 -m venv --system-site-packages $GLOBAL_VENV \ - # && python3 -m venv $GLOBAL_VENV \ - # install global dependencies / python build dependencies in GLOBAL_VENV - # && pip install --upgrade pip setuptools wheel \ - # Save version info - # && ( \ - # which python3 && python3 --version | grep " $PYTHON_VERSION" \ - # && which pip && pip --version \ - # # && which pdm && pdm --version \ - # && echo -e '\n\n' \ - # ) | tee -a /VERSION.txt - - -# Set up Node environment from the official platform tarball. This avoids -# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into -# emulated Docker builds. -RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \ - case "$TARGETARCH" in \ - amd64) NODE_DIST_ARCH="x64" ;; \ - arm64) NODE_DIST_ARCH="arm64" ;; \ - *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \ - esac \ - && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \ - && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \ - && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \ - && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \ - && grep " ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \ - && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \ - && rm "$NODE_TARBALL" SHASUMS256.txt \ - # Save version info - && ( \ - which node && node --version \ - && which npm && npm --version \ - && echo -e '\n\n' \ - ) | tee -a /VERSION.txt - - -# Set up uv and main app /venv -RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh -ENV UV_COMPILE_BYTECODE=0 \ - UV_PYTHON_PREFERENCE=managed \ - UV_PYTHON_INSTALL_DIR=/opt/uv/python \ - UV_LINK_MODE=copy \ - UV_PROJECT_ENVIRONMENT=/venv -WORKDIR "$CODE_DIR" -# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/" -RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \ - && uv venv /venv --python ${PYTHON_VERSION} -ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH" -RUN uv pip install setuptools pip \ - && ( \ - which python3 && python3 --version \ - && which uv && uv self version \ - && uv python find --system && uv python find \ - && echo -e '\n\n' \ - ) | tee -a /VERSION.txt - - -######### ArchiveBox & Extractor Dependencies ################################## - -# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap) -WORKDIR "$CODE_DIR" -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \ - echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \ - && apt-get update -qq \ - && apt-get install -qq -y --no-install-recommends \ - build-essential gcc \ - python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \ - python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \ - && uv pip install \ - "python-ldap>=3.4.3" \ - && apt-get purge -y \ - python3-dev build-essential gcc \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - - -# Runtime config used by plugin hooks. Plugin binaries and npm packages are -# installed into LIB_DIR below by archivebox init --install and resolved from -# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH. -ENV PERSONAS_DIR=/data/personas \ + PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \ + PERSONAS_DIR=/data/personas \ CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \ CHROME_HEADLESS=true \ CHROME_SANDBOX=false \ CHROME_ISOLATION=crawl \ CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]' -######### Build Dependencies #################################### +ENV TMP_DIR=/tmp/archivebox \ + PIP_VENV_PYTHON=/venv/bin/python3 \ + GOOGLE_API_KEY=no \ + GOOGLE_DEFAULT_CLIENT_ID=no \ + GOOGLE_DEFAULT_CLIENT_SECRET=no +ENV UV_COMPILE_BYTECODE=0 \ + UV_PYTHON_PREFERENCE=managed \ + UV_PYTHON_INSTALL_DIR=/opt/uv/python \ + UV_LINK_MODE=copy \ + UV_PROJECT_ENVIRONMENT=/venv \ + VIRTUAL_ENV=/venv \ + PATH="/venv/bin:/opt/node/bin:$PATH" -# Install ArchiveBox Python venv dependencies from pyproject.toml. -RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ - --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \ +SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] +WORKDIR "$CODE_DIR" + +RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \ + && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \ + && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \ + && rm -f /etc/apt/apt.conf.d/docker-clean + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ + echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \ && apt-get update -qq \ - && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \ - && uv --no-cache sync \ + && apt-get install -qq -y \ + apt-transport-https apt-utils ca-certificates curl wget gnupg2 \ + dumb-init util-linux unzip git grep ripgrep dnsutils iputils-ping procps tree nano \ + cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \ + libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \ + libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \ + libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \ + libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \ + fonts-liberation fonts-noto-color-emoji xdg-utils \ + ffmpeg imagemagick tesseract-ocr tesseract-ocr-eng openjdk-21-jre-headless \ + && rm -rf /var/lib/apt/lists/* + +# Runtime-owned layers copied from the abx-dl image. +COPY --from=abx-dl /bin/uv /bin/uv +COPY --from=abx-dl /opt/uv/python /opt/uv/python +COPY --from=abx-dl /opt/node /opt/node +COPY --from=abx-dl /venv /venv +COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt + +RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \ + && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \ + && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \ + && uname -a \ + && sed -n '1,7p' /etc/os-release \ + && which python3 && python3 --version \ + && which node && node --version \ + && which uv && uv self version \ + ) | tee -a /VERSION.txt + +ENV PYTHONDONTWRITEBYTECODE=1 + +FROM archivebox-runtime-base AS archivebox-builder + +WORKDIR "$CODE_DIR" +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ + --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ + --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ + echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \ + && apt-get update -qq \ + && apt-get install -qq -y --no-install-recommends \ + build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ + && uv sync \ --refresh \ --no-dev \ --inexact \ @@ -278,103 +134,53 @@ RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ --no-install-project \ --no-install-workspace \ --no-sources \ - && apt-get purge -y python3-dev build-essential gcc \ + && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ && apt-get autoremove -y \ && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf /var/lib/apt/lists/* - # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies -# Setup ArchiveBox runtime config -ENV TMP_DIR=/tmp/archivebox \ - PIP_VENV_PYTHON=/usr/bin/python3.12 \ - GOOGLE_API_KEY=no \ - GOOGLE_DEFAULT_CLIENT_ID=no \ - GOOGLE_DEFAULT_CLIENT_SECRET=no - -WORKDIR "$DATA_DIR" -RUN openssl rand -hex 16 > /etc/machine-id \ - && mkdir -p "$DATA_DIR" \ - && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \ - && mkdir -p "$TMP_DIR" \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \ - && mkdir -p "$LIB_DIR" \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ - && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \ - && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \ - && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt - -# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer -# path users run later, before copying ArchiveBox source so source-only edits do -# not invalidate the heavy browser/plugin dependency layer. -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \ - echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \ - && export PERSONAS_DIR="$LIB_DIR/personas" \ - && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \ - && export ABX_RUNTIME=archivebox ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \ - && mkdir -p "$LIB_DIR" \ - && apt-get update -qq \ - && apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \ - && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" gallery-dl \ - && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" --overrides='{"pip":{"install_args":["--no-deps","forum-dl","chardet==5.2.0","pydantic==2.12.3","pydantic-core==2.41.4","typing-extensions>=4.14.1","annotated-types>=0.6.0","typing-inspection>=0.4.2","beautifulsoup4","soupsieve","lxml","requests","urllib3","certifi","idna","charset-normalizer","tenacity","python-dateutil","six","html2text","warcio"]}}' forum-dl \ - && if [ "$TARGETARCH" = "arm64" ]; then \ - abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \ - abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \ - fi \ - && ABXPKG_INSTALL_TIMEOUT=600 TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \ - accessibility archivedotorg archivewebpage base chrome chrome_mhtml chrome_screencast \ - claudechrome claudecode claudecodecleanup claudecodeextract consolelog defuddle dns dom \ - favicon forumdl gallerydl git hashes headers htmltotext infiniscroll \ - istilldontcareaboutcookies liteparse media mercury modalcloser opencode opendataloader papersdl \ - parse_dom_outlinks parse_html_urls parse_jsonl_urls parse_netscape_urls parse_rss_urls \ - parse_txt_urls pdf readability redirects responses screenshot search_backend_ripgrep \ - search_backend_sonic search_backend_sqlite seo singlefile ssl sslcerts staticfile title \ - trafilatura ublock wget ytdlp \ - && abxpkg install --no-cache --binproviders=chromewebstore --overrides='{"chromewebstore":{"install_args":["fpeoodllldobpkbkabpblcfaogecpndd","--name=archivewebpage"]}}' archivewebpage \ - && test -f "$LIB_DIR/chromewebstore/extensions/fpeoodllldobpkbkabpblcfaogecpndd__archivewebpage/manifest.json" \ - && mkdir -p "$LIB_DIR/env/bin" \ - && ln -sf "$(command -v node)" "$LIB_DIR/env/bin/node" \ - && ln -sf "$(command -v npm)" "$LIB_DIR/env/bin/npm" \ - && ln -sf "$(command -v java)" "$LIB_DIR/env/bin/java" \ - && ln -sf "$(command -v git)" "$LIB_DIR/env/bin/git" \ - && ln -sf "$(command -v rg)" "$LIB_DIR/env/bin/rg" \ - && ln -sf "$(command -v sonic)" "$LIB_DIR/env/bin/sonic" \ - && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ - && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ - && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \ - && mkdir -p /opt/archivebox/lib-layer \ - && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \ - && apt-get purge -y build-essential \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer - -RUN rm -rf "$LIB_DIR" \ - && mv /opt/archivebox/lib-layer "$LIB_DIR" \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" - -# Install ArchiveBox Python package from the checked-out source. -WORKDIR "$CODE_DIR" COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \ - && pip install \ - --no-deps \ - "$CODE_DIR" \ - && ( \ - pip show archivebox \ - && which archivebox \ - && echo -e '\n\n' \ - ) | tee -a /VERSION.txt \ + && uv pip install --no-deps "$CODE_DIR" \ + && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete - # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces -# Initialize an empty image collection without rerunning dependency installs. +FROM archivebox-runtime-base + +LABEL name="archivebox" \ + maintainer="Nick Sweeting " \ + description="All-in-one self-hosted internet archiving solution" \ + homepage="https://github.com/ArchiveBox/ArchiveBox" \ + documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \ + org.opencontainers.image.title="ArchiveBox" \ + org.opencontainers.image.vendor="ArchiveBox" \ + org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \ + org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \ + com.docker.image.source.entrypoint="Dockerfile" + +COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic +COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg + +COPY --from=archivebox-builder /venv /venv +COPY --from=archivebox-builder /app /app +COPY --from=archivebox-builder /VERSION.txt /VERSION.txt +COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib + +RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ + && groupadd --system "$ARCHIVEBOX_USER" \ + && useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \ + && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ + && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ + && (which sonic && sonic --version) | tee -a /VERSION.txt \ + && mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && openssl rand -hex 16 > /etc/machine-id \ + && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \ + && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt + WORKDIR "$DATA_DIR" RUN echo "[+] Initializing image collection..." \ && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \ @@ -389,25 +195,28 @@ RUN echo "[+] Initializing image collection..." \ "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \ 2>/dev/null || true) -# Print version for nice docker finish summary -RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \ - && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \ - && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \ - ) | tee -a /VERSION.txt - -# Verify ArchiveBox is installed and write full version/dependency info. RUN chmod +x "$CODE_DIR"/bin/*.sh \ && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ - && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \ - && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode papersdl parse_rss_urls readability search_backend_sonic opendataloader search_backend_ripgrep 2>&1 | tee -a /VERSION.txt \ - && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \ + && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ + && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ + && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \ + && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ + && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \ + && /usr/local/bin/sonic --version | tee -a /VERSION.txt \ + && /venv/bin/supervisord --version | tee -a /VERSION.txt \ + && ! command -v gcc \ + && ! command -v g++ \ + && ! command -v make \ + && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/* -#################################################### +RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \ + && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \ + && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \ + ) | tee -a /VERSION.txt -# Expose ArchiveBox's main interfaces to the outside world WORKDIR "$DATA_DIR" VOLUME "$DATA_DIR" EXPOSE 8000 diff --git a/Dockerfile.multistage b/Dockerfile.multistage index da2be129..0d9659bd 100644 --- a/Dockerfile.multistage +++ b/Dockerfile.multistage @@ -9,16 +9,15 @@ # --build-context abxpkg=../abxpkg \ # --build-context abx-plugins=../abx-plugins \ # -t archivebox/abx-dl:dev -# docker buildx build . -f Dockerfile.multistage \ -# --build-context abx-dl=docker-image://archivebox/abx-dl:dev \ -# --build-arg ABX_DL_IMAGE=abx-dl \ +# docker buildx build . -f Dockerfile \ +# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \ # -t archivebox:multistage ARG TARGETPLATFORM=linux/amd64 ARG TARGETOS=linux ARG TARGETARCH=amd64 ARG TARGETVARIANT= -ARG ABX_DL_IMAGE=abx-dl +ARG ABX_DL_IMAGE=archivebox/abx-dl:latest FROM ${ABX_DL_IMAGE} AS abx-dl FROM archivebox/sonic:1.4.9 AS sonic @@ -88,7 +87,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T && apt-get update -qq \ && apt-get install -qq -y \ apt-transport-https apt-utils ca-certificates curl wget gnupg2 \ - dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \ + dumb-init util-linux unzip git grep ripgrep dnsutils iputils-ping procps tree nano \ cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \ libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \ libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \ @@ -160,7 +159,7 @@ LABEL name="archivebox" \ org.opencontainers.image.vendor="ArchiveBox" \ org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \ org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \ - com.docker.image.source.entrypoint="Dockerfile.multistage" + com.docker.image.source.entrypoint="Dockerfile" COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg @@ -199,7 +198,7 @@ RUN echo "[+] Initializing image collection..." \ RUN chmod +x "$CODE_DIR"/bin/*.sh \ && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \ && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ + && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \ && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \ && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \ @@ -208,7 +207,7 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \ && ! command -v gcc \ && ! command -v g++ \ && ! command -v make \ - && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \ + && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/* diff --git a/archivebox/tests/test_dockerfiles.py b/archivebox/tests/test_dockerfiles.py index aa090b71..2696f7da 100644 --- a/archivebox/tests/test_dockerfiles.py +++ b/archivebox/tests/test_dockerfiles.py @@ -15,6 +15,8 @@ _REQUIRED_DOCKER_INSTALL_TARGETS = { "istilldontcareaboutcookies", "liteparse", "mercury", + "opencode", + "opendataloader", "papersdl", "parse_rss_urls", "readability", @@ -76,9 +78,10 @@ def test_dockerfiles_install_binaries_required_by_version_validation() -> None: def test_dockerfile_prewarms_stable_plugin_dependencies_without_optional_captcha() -> None: - targets = _abx_dl_plugin_install_targets(REPO_ROOT / "Dockerfile") - assert _REQUIRED_DOCKER_PREINSTALL_TARGETS <= targets - assert not (_DOCKER_PREINSTALL_EXCLUDED_TARGETS & targets) + text = (REPO_ROOT / "Dockerfile").read_text() + assert "archivebox/abx-dl:latest" in text + assert "archivebox/abxdl" not in text + assert _abx_dl_plugin_install_targets(REPO_ROOT / "Dockerfile") == set() def test_dockerfile_build_installs_disable_release_age_gate() -> None: @@ -105,3 +108,14 @@ def test_dockerfiles_pin_project_binary_paths_for_validation() -> None: assert 'GIT_BINARY="$LIB_DIR/env/bin/git"' in text assert 'GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl"' in text assert 'FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl"' in text + + +def test_dockerfiles_use_setpriv_instead_of_gosu() -> None: + for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"): + text = (REPO_ROOT / dockerfile_name).read_text() + assert "gosu" not in text + assert "setpriv" in text + + entrypoint = (REPO_ROOT / "bin/docker_entrypoint.sh").read_text() + assert "gosu" not in entrypoint + assert "setpriv" in entrypoint diff --git a/bin/docker_entrypoint.sh b/bin/docker_entrypoint.sh index 54a7ab72..8b0b8ac9 100755 --- a/bin/docker_entrypoint.sh +++ b/bin/docker_entrypoint.sh @@ -144,7 +144,7 @@ ensure_runtime_tree() { run_as_archivebox() { if [[ "$(id -u)" == "0" ]]; then - gosu "$ARCHIVEBOX_USER" "$@" + setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups "$@" else "$@" fi @@ -283,12 +283,12 @@ if [[ "$1" == /* || "$1" == "bash" || "$1" == "sh" || "$1" == "echo" || "$1" == # "docker run archivebox /bin/bash -c '...'" # "docker run archivebox cat /VERSION.txt" if [[ "$(id -u)" == "0" ]]; then - exec gosu "$ARCHIVEBOX_USER" /bin/bash -c "exec $(printf ' %q' "$@")" + exec setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups /bin/bash -c "exec $(printf ' %q' "$@")" else exec /bin/bash -c "exec $(printf ' %q' "$@")" fi # printf requotes shell parameters properly https://stackoverflow.com/a/39463371/2156113 - # gosu spawns an ephemeral bash process owned by archivebox user (bash wrapper is needed to load env vars, PATH, and setup terminal TTY) + # setpriv spawns an ephemeral bash process owned by archivebox user (bash wrapper is needed to load env vars, PATH, and setup terminal TTY) # outermost exec hands over current process ID to inner bash process, inner exec hands over inner bash PID to user's command else # handle "docker run archivebox add some subcommand --with=args abc" by calling archivebox to run as args as CLI subcommand @@ -297,7 +297,7 @@ else # "docker run archivebox manage createsupseruser" # "docker run archivebox server 0.0.0.0:8000" if [[ "$(id -u)" == "0" ]]; then - exec gosu "$ARCHIVEBOX_USER" "$ARCHIVEBOX_BIN_PATH" "$@" + exec setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups "$ARCHIVEBOX_BIN_PATH" "$@" else exec "$ARCHIVEBOX_BIN_PATH" "$@" fi