diff --git a/.dockerignore b/.dockerignore index 4138f523..ba8c4d5c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -75,6 +75,15 @@ docker/ website/ typings/ +# The runtime COPY needs ArchiveBox's package and bin/ entrypoint, not the +# development UI screenshots or test trees. PNGs are already compressed, so +# including them creates a large image layer that later cleanup cannot remove. +publicsite/ +old/ +test/ +tests/ +archivebox/tests/ + tmp/ .tmp/ data/ diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a9dbe0f1..76ec0d30 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -210,9 +210,11 @@ jobs: platforms: ${{ matrix.platform }} cache-from: type=gha,scope=${{ matrix.cache_scope }} cache-to: ${{ inputs.full_tests && format('type=gha,mode=max,scope={0}', matrix.cache_scope) || '' }} + # Keep the abx-dl base layers in zstd form: it is both smaller and + # faster to export than gzip, and ArchiveBox can reuse them by digest. outputs: | - ${{ inputs.full_tests && format('type=oci,dest={0}/archivebox-image.tar', runner.temp) || '' }} - ${{ inputs.push_digests && format('type=image,"name={0},{1}",push-by-digest=true,name-canonical=true,push=true', env.DOCKERHUB_IMAGE, env.GHCR_IMAGE) || '' }} + ${{ inputs.full_tests && format('type=oci,dest={0}/archivebox-image.tar,compression=zstd,compression-level=3,force-compression=true', runner.temp) || '' }} + ${{ inputs.push_digests && format('type=image,"name={0},{1}",push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3,force-compression=true', env.DOCKERHUB_IMAGE, env.GHCR_IMAGE) || '' }} - name: Build pull request image id: docker_build_pr diff --git a/Dockerfile b/Dockerfile index 575f4b84..81795c61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,7 +103,9 @@ rm -f /etc/apt/apt.conf.d/docker-clean apt-get update -qq apt-get install -qq -y --no-install-recommends \ build-essential gcc libldap2-dev libsasl2-dev libssl-dev -/usr/bin/uv venv --no-cache --clear /venv --python "${PYTHON_VERSION}" +# Extend abx-dl's existing venv instead of clearing it. Clearing and later +# copying a complete replacement over the base duplicates every venv byte in +# the final overlay history even when most packages are unchanged. /usr/bin/uv pip install --no-cache setuptools pip wheel mkdir -p /tmp/archivebox-uv-project @@ -167,10 +169,16 @@ rm -f /venv/bin/uv /venv/bin/uvx abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get autoremove -y /usr/bin/find "$ABXPKG_LIB_DIR/env/bin" -maxdepth 1 -type l -name strip -delete -rm -rf /var/lib/apt/lists/* +rm -rf /venv/lib/python3.*/site-packages/pip* \ + /venv/lib/python3.*/site-packages/wheel* \ + /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel +echo 'Binary::apt::APT::Keep-Downloaded-Packages "0";' > /etc/apt/apt.conf.d/99keep-cache +rm -rf /var/lib/apt/lists/* /tmp/archivebox-uv-project EOF COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" +# Compile only ArchiveBox's installed package. Touching all of /venv here would +# copy every inherited abx-dl file into a new layer merely to change its mtime. RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \ && COMMIT_HASH="$( \ if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \ @@ -190,11 +198,14 @@ RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \ && cd / \ && ARCHIVEBOX_PY_DIR="$(/venv/bin/python -c 'import pathlib, archivebox; print(pathlib.Path(archivebox.__file__).parent)')" \ && /venv/bin/python -m compileall --invalidation-mode checked-hash -q "$ARCHIVEBOX_PY_DIR" \ - && find /venv -exec touch -h -d "@$(date +%s)" {} + \ && test -f "$(/venv/bin/python -c 'import archivebox; print(archivebox.__cached__)')" \ && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt -FROM archivebox-runtime-base +# The builder installs and purges compilers in one layer, so its final +# filesystem is already runtime-clean. Preserve that ancestry: starting again +# from archivebox-runtime-base and COPYing /venv would bake a second full venv +# over the inherited one instead of recording only ArchiveBox's package delta. +FROM archivebox-builder LABEL name="archivebox" \ maintainer="Nick Sweeting " \ @@ -210,10 +221,9 @@ LABEL name="archivebox" \ COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg -COPY --from=archivebox-builder /venv /venv -COPY --from=archivebox-builder /app /app -COPY --from=archivebox-builder /VERSION.txt /VERSION.txt - +# The builder invokes abxpkg as root for temporary ELF stripping, which can +# rewrite derived state ownership. Restore UID 911 before resolving binaries +# as the runtime user; the old copy-based final stage hid this dependency. RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \ && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \ && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \ @@ -225,11 +235,11 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \ && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \ && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \ - && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \ && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \ && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \ && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \ && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \ + && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \ && openssl rand -hex 16 > /etc/machine-id \ && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \ && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt