mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-14 11:06:13 +05:00
Fix ArchiveBox Docker runtime layering
This commit is contained in:
Nick Sweeting
parent
466238d7dd
commit
232de59e95
@ -12,9 +12,14 @@ __pycache__/
|
||||
.pdm-python
|
||||
.eggs/
|
||||
.git/
|
||||
!.git/
|
||||
.git/*
|
||||
.vscode/
|
||||
!.git/HEAD
|
||||
!.git/refs/heads/*
|
||||
!.git/packed-refs
|
||||
!.git/refs/
|
||||
!.git/refs/heads/
|
||||
!.git/refs/heads/**
|
||||
|
||||
venv/
|
||||
.venv/
|
||||
|
||||
52
Dockerfile
52
Dockerfile
@ -13,19 +13,15 @@
|
||||
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
|
||||
# -t archivebox:multistage
|
||||
|
||||
ARG TARGETPLATFORM=linux/amd64
|
||||
ARG TARGETOS=linux
|
||||
ARG TARGETARCH=amd64
|
||||
ARG TARGETVARIANT=
|
||||
ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
|
||||
|
||||
FROM ${ABX_DL_IMAGE} AS abx-dl
|
||||
FROM archivebox/sonic:1.4.9 AS sonic
|
||||
FROM ubuntu:24.04 AS archivebox-runtime-base
|
||||
|
||||
ARG TARGETPLATFORM=linux/amd64
|
||||
ARG TARGETOS=linux
|
||||
ARG TARGETARCH=amd64
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
|
||||
ENV TZ=UTC \
|
||||
@ -72,7 +68,7 @@ ENV UV_COMPILE_BYTECODE=0 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PROJECT_ENVIRONMENT=/venv \
|
||||
VIRTUAL_ENV=/venv \
|
||||
PATH="/venv/bin:/opt/node/bin:$PATH"
|
||||
PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
|
||||
WORKDIR "$CODE_DIR"
|
||||
@ -101,7 +97,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
|
||||
COPY --from=abx-dl /bin/uv /bin/uv
|
||||
COPY --from=abx-dl /opt/uv/python /opt/uv/python
|
||||
COPY --from=abx-dl /opt/node /opt/node
|
||||
COPY --from=abx-dl /venv /venv
|
||||
COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
|
||||
|
||||
RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
|
||||
@ -109,7 +104,6 @@ RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
|
||||
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
|
||||
&& uname -a \
|
||||
&& sed -n '1,7p' /etc/os-release \
|
||||
&& which python3 && python3 --version \
|
||||
&& which node && node --version \
|
||||
&& which uv && uv self version \
|
||||
) | tee -a /VERSION.txt
|
||||
@ -126,6 +120,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y --no-install-recommends \
|
||||
build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
|
||||
&& uv venv /venv --python "${PYTHON_VERSION}" \
|
||||
&& uv pip install setuptools pip wheel \
|
||||
&& uv sync \
|
||||
--refresh \
|
||||
--no-dev \
|
||||
@ -143,10 +139,22 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
|
||||
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
|
||||
RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
|
||||
&& COMMIT_HASH="$( \
|
||||
if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
|
||||
HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
|
||||
if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
|
||||
echo "$HEAD_REF"; \
|
||||
elif [[ "$HEAD_REF" == ref:\ * ]]; then \
|
||||
REF_PATH="${HEAD_REF#ref: }"; \
|
||||
cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
|
||||
fi; \
|
||||
fi)" \
|
||||
&& if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
|
||||
&& uv pip install --no-deps "$CODE_DIR" \
|
||||
&& (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
|
||||
&& find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
|
||||
&& find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& rm -rf "$CODE_DIR/.git"
|
||||
|
||||
FROM archivebox-runtime-base
|
||||
|
||||
@ -164,6 +172,7 @@ LABEL name="archivebox" \
|
||||
COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
|
||||
COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
|
||||
|
||||
COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
|
||||
COPY --from=archivebox-builder /venv /venv
|
||||
COPY --from=archivebox-builder /app /app
|
||||
COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
|
||||
@ -175,8 +184,9 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
|
||||
&& usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
|
||||
&& groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
|
||||
&& (which sonic && sonic --version) | tee -a /VERSION.txt \
|
||||
&& mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/uv" \
|
||||
&& openssl rand -hex 16 > /etc/machine-id \
|
||||
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
|
||||
&& echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
|
||||
@ -187,7 +197,6 @@ RUN echo "[+] Initializing image collection..." \
|
||||
&& PUID=0 PGID=0 archivebox init \
|
||||
&& find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
|
||||
&& (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
|
||||
"$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
|
||||
"$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
|
||||
@ -196,18 +205,25 @@ RUN echo "[+] Initializing image collection..." \
|
||||
2>/dev/null || true)
|
||||
|
||||
RUN chmod +x "$CODE_DIR"/bin/*.sh \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
|
||||
&& chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
|
||||
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \
|
||||
&& env -u PNPM_HOME PATH="/opt/node/bin:$PATH" /opt/node/bin/corepack pnpm add --loglevel=error --store-dir="$TMP_DIR/pnpm-store" --config.dangerouslyAllowAllBuilds=true --dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai 2>&1 | tee -a /VERSION.txt \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \
|
||||
&& rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node \
|
||||
&& ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode" \
|
||||
&& ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \
|
||||
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
|
||||
&& chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \
|
||||
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
|
||||
&& "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
|
||||
&& "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
|
||||
&& "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
|
||||
&& /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
|
||||
&& /usr/local/bin/sonic --version | tee -a /VERSION.txt \
|
||||
&& /venv/bin/supervisord --version | tee -a /VERSION.txt \
|
||||
&& ! command -v gcc \
|
||||
&& ! command -v g++ \
|
||||
&& ! command -v make \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
|
||||
&& HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
|
||||
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
|
||||
|
||||
@ -13,19 +13,15 @@
|
||||
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
|
||||
# -t archivebox:multistage
|
||||
|
||||
ARG TARGETPLATFORM=linux/amd64
|
||||
ARG TARGETOS=linux
|
||||
ARG TARGETARCH=amd64
|
||||
ARG TARGETVARIANT=
|
||||
ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
|
||||
|
||||
FROM ${ABX_DL_IMAGE} AS abx-dl
|
||||
FROM archivebox/sonic:1.4.9 AS sonic
|
||||
FROM ubuntu:24.04 AS archivebox-runtime-base
|
||||
|
||||
ARG TARGETPLATFORM=linux/amd64
|
||||
ARG TARGETOS=linux
|
||||
ARG TARGETARCH=amd64
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
|
||||
ENV TZ=UTC \
|
||||
@ -72,7 +68,7 @@ ENV UV_COMPILE_BYTECODE=0 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PROJECT_ENVIRONMENT=/venv \
|
||||
VIRTUAL_ENV=/venv \
|
||||
PATH="/venv/bin:/opt/node/bin:$PATH"
|
||||
PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
|
||||
WORKDIR "$CODE_DIR"
|
||||
@ -101,7 +97,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
|
||||
COPY --from=abx-dl /bin/uv /bin/uv
|
||||
COPY --from=abx-dl /opt/uv/python /opt/uv/python
|
||||
COPY --from=abx-dl /opt/node /opt/node
|
||||
COPY --from=abx-dl /venv /venv
|
||||
COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
|
||||
|
||||
RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
|
||||
@ -109,7 +104,6 @@ RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
|
||||
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
|
||||
&& uname -a \
|
||||
&& sed -n '1,7p' /etc/os-release \
|
||||
&& which python3 && python3 --version \
|
||||
&& which node && node --version \
|
||||
&& which uv && uv self version \
|
||||
) | tee -a /VERSION.txt
|
||||
@ -126,6 +120,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y --no-install-recommends \
|
||||
build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
|
||||
&& uv venv /venv --python "${PYTHON_VERSION}" \
|
||||
&& uv pip install setuptools pip wheel \
|
||||
&& uv sync \
|
||||
--refresh \
|
||||
--no-dev \
|
||||
@ -143,10 +139,22 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
|
||||
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
|
||||
RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
|
||||
&& COMMIT_HASH="$( \
|
||||
if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
|
||||
HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
|
||||
if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
|
||||
echo "$HEAD_REF"; \
|
||||
elif [[ "$HEAD_REF" == ref:\ * ]]; then \
|
||||
REF_PATH="${HEAD_REF#ref: }"; \
|
||||
cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
|
||||
fi; \
|
||||
fi)" \
|
||||
&& if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
|
||||
&& uv pip install --no-deps "$CODE_DIR" \
|
||||
&& (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
|
||||
&& find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
|
||||
&& find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& rm -rf "$CODE_DIR/.git"
|
||||
|
||||
FROM archivebox-runtime-base
|
||||
|
||||
@ -164,6 +172,7 @@ LABEL name="archivebox" \
|
||||
COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
|
||||
COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
|
||||
|
||||
COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
|
||||
COPY --from=archivebox-builder /venv /venv
|
||||
COPY --from=archivebox-builder /app /app
|
||||
COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
|
||||
@ -175,8 +184,9 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
|
||||
&& usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
|
||||
&& groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
|
||||
&& (which sonic && sonic --version) | tee -a /VERSION.txt \
|
||||
&& mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/uv" \
|
||||
&& openssl rand -hex 16 > /etc/machine-id \
|
||||
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
|
||||
&& echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
|
||||
@ -187,7 +197,6 @@ RUN echo "[+] Initializing image collection..." \
|
||||
&& PUID=0 PGID=0 archivebox init \
|
||||
&& find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
|
||||
&& (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
|
||||
"$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
|
||||
"$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
|
||||
@ -196,18 +205,25 @@ RUN echo "[+] Initializing image collection..." \
|
||||
2>/dev/null || true)
|
||||
|
||||
RUN chmod +x "$CODE_DIR"/bin/*.sh \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
|
||||
&& chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
|
||||
&& install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \
|
||||
&& env -u PNPM_HOME PATH="/opt/node/bin:$PATH" /opt/node/bin/corepack pnpm add --loglevel=error --store-dir="$TMP_DIR/pnpm-store" --config.dangerouslyAllowAllBuilds=true --dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai 2>&1 | tee -a /VERSION.txt \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode" \
|
||||
&& rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node \
|
||||
&& ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode" \
|
||||
&& ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode" \
|
||||
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
|
||||
&& chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode" \
|
||||
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" OPENCODE_BINARY="$LIB_DIR/bin/opencode" HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
|
||||
&& "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
|
||||
&& "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
|
||||
&& "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
|
||||
&& /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
|
||||
&& /usr/local/bin/sonic --version | tee -a /VERSION.txt \
|
||||
&& /venv/bin/supervisord --version | tee -a /VERSION.txt \
|
||||
&& ! command -v gcc \
|
||||
&& ! command -v g++ \
|
||||
&& ! command -v make \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
|
||||
&& HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config" XDG_CACHE_HOME="/home/$ARCHIVEBOX_USER/.cache" setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
|
||||
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
|
||||
|
||||
@ -6,6 +6,7 @@ import importlib.metadata
|
||||
from pathlib import Path
|
||||
from functools import cache
|
||||
from datetime import datetime
|
||||
import re
|
||||
|
||||
#############################################################################################
|
||||
|
||||
@ -41,11 +42,57 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):
|
||||
|
||||
@cache
|
||||
def get_COMMIT_HASH() -> str | None:
|
||||
for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"):
|
||||
env_commit_hash = os.environ.get(env_var, "").strip()
|
||||
if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
|
||||
return env_commit_hash
|
||||
|
||||
if IN_DOCKER:
|
||||
try:
|
||||
version_txt = Path("/VERSION.txt").read_text()
|
||||
docker_commit_hashes = re.findall(r"COMMIT_HASH=([0-9a-fA-F]{40})", version_txt)
|
||||
if docker_commit_hashes:
|
||||
return docker_commit_hashes[-1]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _read_git_file(git_dir: Path, ref: str) -> str | None:
|
||||
try:
|
||||
return git_dir.joinpath(ref).read_text().strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
for line in packed_refs:
|
||||
if line.startswith("#") or line.startswith("^") or not line.strip():
|
||||
continue
|
||||
commit_hash, packed_ref = line.split(" ", 1)
|
||||
if packed_ref == ref:
|
||||
return commit_hash.strip()
|
||||
|
||||
return None
|
||||
|
||||
try:
|
||||
git_dir = PACKAGE_DIR.parent / ".git"
|
||||
ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1]
|
||||
commit_hash = git_dir.joinpath(ref).read_text().strip()
|
||||
return commit_hash
|
||||
if git_dir.is_file():
|
||||
gitdir_line = git_dir.read_text().strip()
|
||||
gitdir_path = gitdir_line.removeprefix("gitdir:").strip()
|
||||
git_dir = Path(gitdir_path)
|
||||
if not git_dir.is_absolute():
|
||||
git_dir = PACKAGE_DIR.parent / git_dir
|
||||
|
||||
head = (git_dir / "HEAD").read_text().strip()
|
||||
if re.fullmatch(r"[0-9a-fA-F]{40}", head):
|
||||
return head
|
||||
|
||||
ref = head.removeprefix("ref:").strip()
|
||||
commit_hash = _read_git_file(git_dir, ref)
|
||||
if commit_hash:
|
||||
return commit_hash
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@ -108,6 +108,23 @@ def test_dockerfiles_pin_project_binary_paths_for_validation() -> None:
|
||||
assert 'GIT_BINARY="$LIB_DIR/env/bin/git"' in text
|
||||
assert 'GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl"' in text
|
||||
assert 'FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl"' in text
|
||||
assert 'OPENCODE_BINARY="$LIB_DIR/bin/opencode"' in text
|
||||
|
||||
|
||||
def test_dockerfiles_preinstall_opencode_without_pnpm_home_override() -> None:
|
||||
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
|
||||
text = (REPO_ROOT / dockerfile_name).read_text()
|
||||
assert 'PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"' in text
|
||||
assert "env -u PNPM_HOME" in text
|
||||
assert "/opt/node/bin/corepack pnpm add" in text
|
||||
assert '--store-dir="$TMP_DIR/pnpm-store"' in text
|
||||
assert '--dir="$LIB_DIR/pnpm/packages/opencode" opencode-ai' in text
|
||||
assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/pnpm/packages/opencode"' in text
|
||||
assert 'rm -rf "$TMP_DIR/pnpm-store" /root/.cache/node' in text
|
||||
assert 'ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/bin/opencode"' in text
|
||||
assert 'ln -sf "$LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" "$LIB_DIR/env/bin/opencode"' in text
|
||||
assert 'chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"' in text
|
||||
assert 'chown -h "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR/bin/opencode" "$LIB_DIR/env/bin/opencode"' in text
|
||||
|
||||
|
||||
def test_dockerfiles_use_setpriv_instead_of_gosu() -> None:
|
||||
@ -119,3 +136,61 @@ def test_dockerfiles_use_setpriv_instead_of_gosu() -> None:
|
||||
entrypoint = (REPO_ROOT / "bin/docker_entrypoint.sh").read_text()
|
||||
assert "gosu" not in entrypoint
|
||||
assert "setpriv" in entrypoint
|
||||
|
||||
|
||||
def test_dockerfiles_create_archivebox_venv_in_archivebox_builder() -> None:
|
||||
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
|
||||
text = (REPO_ROOT / dockerfile_name).read_text()
|
||||
assert "COPY --from=abx-dl /venv /venv" not in text
|
||||
assert 'uv venv /venv --python "${PYTHON_VERSION}"' in text
|
||||
assert "COPY --from=archivebox-builder /opt/uv/python /opt/uv/python" in text
|
||||
assert 'COMMIT_HASH="$(' in text
|
||||
assert 'HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"' in text
|
||||
assert 'echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt' in text
|
||||
assert 'rm -rf "$CODE_DIR/.git"' in text
|
||||
|
||||
|
||||
def test_dockerfiles_do_not_duplicate_runtime_lib_with_recursive_chown() -> None:
|
||||
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
|
||||
text = (REPO_ROOT / dockerfile_name).read_text()
|
||||
assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"' not in text
|
||||
assert 'chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR"' not in text
|
||||
assert 'chown "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"' in text
|
||||
assert "COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib" in text
|
||||
|
||||
|
||||
def test_dockerfiles_validate_current_papers_dl_path() -> None:
|
||||
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
|
||||
text = (REPO_ROOT / dockerfile_name).read_text()
|
||||
assert '"$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version' in text
|
||||
assert "$LIB_DIR/pip/packages/papers-dl" not in text
|
||||
|
||||
|
||||
def test_dockerfiles_do_not_default_multiarch_builds_to_amd64() -> None:
|
||||
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
|
||||
text = (REPO_ROOT / dockerfile_name).read_text()
|
||||
assert "ARG TARGETPLATFORM=linux/amd64" not in text
|
||||
assert "ARG TARGETARCH=amd64" not in text
|
||||
assert "ARG TARGETPLATFORM\n" in text
|
||||
assert "FROM ${ABX_DL_IMAGE} AS abx-dl" in text
|
||||
|
||||
|
||||
def test_dockerfiles_set_archivebox_home_for_setpriv_commands() -> None:
|
||||
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
|
||||
text = (REPO_ROOT / dockerfile_name).read_text()
|
||||
assert (
|
||||
'install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus"'
|
||||
) in text
|
||||
assert 'HOME="/home/$ARCHIVEBOX_USER" XDG_CONFIG_HOME="/home/$ARCHIVEBOX_USER/.config"' in text
|
||||
|
||||
|
||||
def test_dockerignore_keeps_minimal_git_metadata_for_image_version() -> None:
|
||||
text = (REPO_ROOT / ".dockerignore").read_text()
|
||||
assert ".git/" in text
|
||||
assert "!.git/" in text
|
||||
assert ".git/*" in text
|
||||
assert "!.git/HEAD" in text
|
||||
assert "!.git/packed-refs" in text
|
||||
assert "!.git/refs/" in text
|
||||
assert "!.git/refs/heads/" in text
|
||||
assert "!.git/refs/heads/**" in text
|
||||
|
||||
90
archivebox/tests/test_version_metadata.py
Normal file
90
archivebox/tests/test_version_metadata.py
Normal file
@ -0,0 +1,90 @@
|
||||
from pathlib import Path
|
||||
|
||||
from archivebox.config import version
|
||||
|
||||
|
||||
def _set_package_dir(monkeypatch, package_dir: Path) -> None:
|
||||
monkeypatch.setattr(version, "PACKAGE_DIR", package_dir)
|
||||
version.get_COMMIT_HASH.cache_clear()
|
||||
|
||||
|
||||
def test_get_commit_hash_from_environment(monkeypatch) -> None:
|
||||
commit_hash = "e" * 40
|
||||
monkeypatch.setenv("ARCHIVEBOX_COMMIT_HASH", commit_hash)
|
||||
version.get_COMMIT_HASH.cache_clear()
|
||||
|
||||
assert version.get_COMMIT_HASH() == commit_hash
|
||||
|
||||
|
||||
def test_get_commit_hash_from_docker_version_file_ignores_short_hash(monkeypatch) -> None:
|
||||
commit_hash = "f" * 40
|
||||
|
||||
class VersionPath:
|
||||
def __init__(self, path: str):
|
||||
self.path = path
|
||||
|
||||
def read_text(self) -> str:
|
||||
assert self.path == "/VERSION.txt"
|
||||
return f"COMMIT_HASH={commit_hash}\nArchiveBox COMMIT_HASH={commit_hash[:7]}\n"
|
||||
|
||||
monkeypatch.setattr(version, "IN_DOCKER", True)
|
||||
monkeypatch.setattr(version, "Path", VersionPath)
|
||||
version.get_COMMIT_HASH.cache_clear()
|
||||
|
||||
assert version.get_COMMIT_HASH() == commit_hash
|
||||
|
||||
|
||||
def test_get_commit_hash_from_detached_head(monkeypatch, tmp_path) -> None:
|
||||
commit_hash = "a" * 40
|
||||
package_dir = tmp_path / "archivebox"
|
||||
git_dir = tmp_path / ".git"
|
||||
package_dir.mkdir()
|
||||
git_dir.mkdir()
|
||||
git_dir.joinpath("HEAD").write_text(commit_hash)
|
||||
|
||||
_set_package_dir(monkeypatch, package_dir)
|
||||
|
||||
assert version.get_COMMIT_HASH() == commit_hash
|
||||
|
||||
|
||||
def test_get_commit_hash_from_branch_ref(monkeypatch, tmp_path) -> None:
|
||||
commit_hash = "b" * 40
|
||||
package_dir = tmp_path / "archivebox"
|
||||
git_dir = tmp_path / ".git"
|
||||
ref_path = git_dir / "refs" / "heads" / "dev"
|
||||
package_dir.mkdir()
|
||||
ref_path.parent.mkdir(parents=True)
|
||||
git_dir.joinpath("HEAD").write_text("ref: refs/heads/dev")
|
||||
ref_path.write_text(commit_hash)
|
||||
|
||||
_set_package_dir(monkeypatch, package_dir)
|
||||
|
||||
assert version.get_COMMIT_HASH() == commit_hash
|
||||
|
||||
|
||||
def test_get_commit_hash_from_packed_ref(monkeypatch, tmp_path) -> None:
|
||||
commit_hash = "c" * 40
|
||||
package_dir = tmp_path / "archivebox"
|
||||
git_dir = tmp_path / ".git"
|
||||
package_dir.mkdir()
|
||||
git_dir.mkdir()
|
||||
git_dir.joinpath("HEAD").write_text("ref: refs/heads/dev")
|
||||
git_dir.joinpath("packed-refs").write_text(f"{commit_hash} refs/heads/dev\n")
|
||||
|
||||
_set_package_dir(monkeypatch, package_dir)
|
||||
|
||||
assert version.get_COMMIT_HASH() == commit_hash
|
||||
|
||||
|
||||
def test_get_commit_hash_from_worktree_gitdir(monkeypatch, tmp_path) -> None:
|
||||
commit_hash = "d" * 40
|
||||
package_dir = tmp_path / "worktree" / "archivebox"
|
||||
real_git_dir = tmp_path / "repo" / ".git" / "worktrees" / "worktree"
|
||||
package_dir.mkdir(parents=True)
|
||||
real_git_dir.mkdir(parents=True)
|
||||
package_dir.parent.joinpath(".git").write_text(f"gitdir: {real_git_dir}\n")
|
||||
real_git_dir.joinpath("HEAD").write_text(commit_hash)
|
||||
|
||||
_set_package_dir(monkeypatch, package_dir)
|
||||
|
||||
assert version.get_COMMIT_HASH() == commit_hash
|
||||
Loading…
Reference in New Issue
Block a user