mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-13 18:46:17 +05:00
Use published abx-dl image for Docker builds
This commit is contained in:
Nick Sweeting
parent
16cd3837e4
commit
466238d7dd
72
.github/workflows/docker.yml
vendored
72
.github/workflows/docker.yml
vendored
@ -5,7 +5,7 @@ on:
|
||||
workflow_call:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
- "**"
|
||||
tags:
|
||||
- 'v*'
|
||||
# pull_request:
|
||||
@ -13,13 +13,14 @@ on:
|
||||
env:
|
||||
DOCKERHUB_IMAGE: archivebox/archivebox
|
||||
GHCR_IMAGE: ghcr.io/archivebox/archivebox
|
||||
ABX_DL_IMAGE: archivebox/abx-dl:latest
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
concurrency:
|
||||
group: docker-${{ github.ref }}
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
@ -96,6 +97,24 @@ jobs:
|
||||
- name: Available platforms
|
||||
run: echo ${{ steps.buildx.outputs.platforms }}
|
||||
|
||||
- name: Wait for published abx-dl image
|
||||
id: abx_dl_image
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
deadline=$((SECONDS + 1800))
|
||||
until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
|
||||
if (( SECONDS >= deadline )); then
|
||||
echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "${ABX_DL_IMAGE} is not published yet; waiting..."
|
||||
sleep 30
|
||||
done
|
||||
|
||||
echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
|
||||
docker buildx imagetools inspect "${ABX_DL_IMAGE}"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
if: github.event_name != 'pull_request'
|
||||
@ -116,13 +135,10 @@ jobs:
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION="$(python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tomllib
|
||||
|
||||
match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
|
||||
if not match:
|
||||
raise SystemExit("Failed to read version from pyproject.toml")
|
||||
print(match.group(1))
|
||||
with open("pyproject.toml", "rb") as f:
|
||||
print(tomllib.load(f)["project"]["version"])
|
||||
PY
|
||||
)"
|
||||
|
||||
@ -131,11 +147,12 @@ jobs:
|
||||
echo "org.opencontainers.image.version=${VERSION}"
|
||||
echo "org.opencontainers.image.revision=${GITHUB_SHA}"
|
||||
echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
|
||||
echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
|
||||
echo 'EOF'
|
||||
echo "version=${VERSION}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "[+] Building ${{ matrix.platform }} for ${VERSION}"
|
||||
echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
|
||||
|
||||
- name: Build and push digest
|
||||
id: docker_build
|
||||
@ -149,6 +166,8 @@ jobs:
|
||||
${{ env.DOCKERHUB_IMAGE }}
|
||||
${{ env.GHCR_IMAGE }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
build-args: |
|
||||
ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
|
||||
cache-from: type=gha,scope=${{ matrix.cache_scope }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
@ -215,36 +234,49 @@ jobs:
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION="$(python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tomllib
|
||||
|
||||
match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
|
||||
if not match:
|
||||
raise SystemExit("Failed to read version from pyproject.toml")
|
||||
print(match.group(1))
|
||||
with open("pyproject.toml", "rb") as f:
|
||||
print(tomllib.load(f)["project"]["version"])
|
||||
PY
|
||||
)"
|
||||
SHORT_SHA="${GITHUB_SHA::8}"
|
||||
BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
|
||||
SHORT_SHA="${GITHUB_SHA::12}"
|
||||
test -n "$BRANCH_TAG"
|
||||
test -n "$SHORT_SHA"
|
||||
|
||||
{
|
||||
echo 'dockerhub_tags<<EOF'
|
||||
echo "${DOCKERHUB_IMAGE}:dev"
|
||||
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
|
||||
echo "${DOCKERHUB_IMAGE}:latest"
|
||||
fi
|
||||
echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
|
||||
echo "${DOCKERHUB_IMAGE}:${VERSION}"
|
||||
echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
|
||||
echo 'EOF'
|
||||
echo 'ghcr_tags<<EOF'
|
||||
echo "${GHCR_IMAGE}:dev"
|
||||
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
|
||||
echo "${GHCR_IMAGE}:latest"
|
||||
fi
|
||||
echo "${GHCR_IMAGE}:${BRANCH_TAG}"
|
||||
echo "${GHCR_IMAGE}:${VERSION}"
|
||||
echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
|
||||
echo 'EOF'
|
||||
echo "version=${VERSION}"
|
||||
echo "branch_tag=${BRANCH_TAG}"
|
||||
echo "short_sha=${SHORT_SHA}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "[+] Publishing Docker Hub tags:"
|
||||
printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
|
||||
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
|
||||
printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
|
||||
fi
|
||||
printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
|
||||
echo "[+] Publishing GHCR tags:"
|
||||
printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
|
||||
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
|
||||
printf '%s\n' "${GHCR_IMAGE}:latest"
|
||||
fi
|
||||
printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
|
||||
|
||||
- name: Create Docker Hub manifest
|
||||
shell: bash
|
||||
|
||||
473
Dockerfile
473
Dockerfile
@ -1,66 +1,33 @@
|
||||
# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
|
||||
# python3.13, uv, python3-ldap
|
||||
# curl, wget, git, dig, ping, tree, nano
|
||||
# node, npm
|
||||
# ArchiveBox and plugin runtime dependencies installed by archivebox init --install
|
||||
# Usage:
|
||||
# git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
|
||||
# docker build . -t archivebox
|
||||
# docker run -v "$PWD/data":/data archivebox init
|
||||
# docker run -v "$PWD/data":/data archivebox add 'https://example.com'
|
||||
# docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
|
||||
# docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
|
||||
# docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
|
||||
# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
|
||||
# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
|
||||
# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
|
||||
# Build abx-dl first, then point this file at it:
|
||||
# docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
|
||||
# --build-context abxbus=../abxbus \
|
||||
# --build-context abxpkg=../abxpkg \
|
||||
# --build-context abx-plugins=../abx-plugins \
|
||||
# -t archivebox/abx-dl:dev
|
||||
# docker buildx build . -f Dockerfile \
|
||||
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
|
||||
# -t archivebox:multistage
|
||||
|
||||
#########################################################################################
|
||||
|
||||
### Example: Using ArchiveBox in your own project's Dockerfile ########
|
||||
|
||||
# FROM python:3.13-slim
|
||||
# WORKDIR /data
|
||||
# RUN pip install archivebox>=0.9.0 # use latest release here
|
||||
# RUN archivebox install
|
||||
# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
|
||||
|
||||
#########################################################################################
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
ARG TARGETPLATFORM=linux/amd64
|
||||
ARG TARGETOS=linux
|
||||
ARG TARGETARCH=amd64
|
||||
ARG TARGETVARIANT=
|
||||
ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
|
||||
|
||||
FROM ${ABX_DL_IMAGE} AS abx-dl
|
||||
FROM archivebox/sonic:1.4.9 AS sonic
|
||||
FROM ubuntu:24.04
|
||||
FROM ubuntu:24.04 AS archivebox-runtime-base
|
||||
|
||||
LABEL name="archivebox" \
|
||||
maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
|
||||
description="All-in-one self-hosted internet archiving solution" \
|
||||
homepage="https://github.com/ArchiveBox/ArchiveBox" \
|
||||
documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
|
||||
org.opencontainers.image.title="ArchiveBox" \
|
||||
org.opencontainers.image.vendor="ArchiveBox" \
|
||||
org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
|
||||
org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
|
||||
com.docker.image.source.entrypoint="Dockerfile" \
|
||||
# TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
|
||||
# https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
|
||||
com.docker.desktop.extension.api.version=">= 1.4.7" \
|
||||
com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
|
||||
com.docker.extension.publisher-url="https://archivebox.io" \
|
||||
com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
|
||||
com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
|
||||
com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
|
||||
com.docker.extension.categories='database,utility-tools'
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
ARG TARGETPLATFORM=linux/amd64
|
||||
ARG TARGETOS=linux
|
||||
ARG TARGETARCH=amd64
|
||||
ARG TARGETVARIANT
|
||||
######### Environment Variables #################################
|
||||
|
||||
# Global build-time and runtime environment constants + default pkg manager config
|
||||
ENV TZ=UTC \
|
||||
LANGUAGE=en_US:en \
|
||||
LC_ALL=C.UTF-8 \
|
||||
@ -73,204 +40,93 @@ ENV TZ=UTC \
|
||||
PIP_ONLY_BINARY=aiohttp \
|
||||
npm_config_loglevel=error
|
||||
|
||||
# Language Version config
|
||||
ENV PYTHON_VERSION=3.13 \
|
||||
NODE_VERSION=22.22.3
|
||||
|
||||
# Non-root User config
|
||||
ENV ARCHIVEBOX_USER="archivebox" \
|
||||
ENV ARCHIVEBOX_USER=archivebox \
|
||||
DEFAULT_PUID=911 \
|
||||
DEFAULT_PGID=911 \
|
||||
IN_DOCKER=True \
|
||||
BIND_ADDR=0.0.0.0:8000
|
||||
# Docker has to listen on all interfaces, not just localhost.
|
||||
IN_DOCKER=True
|
||||
|
||||
# ArchiveBox Source Code + Lib + Data paths
|
||||
ENV CODE_DIR=/app \
|
||||
DATA_DIR=/data \
|
||||
LIB_DIR=/opt/archivebox/lib \
|
||||
ABXPKG_LIB_DIR=/opt/archivebox/lib \
|
||||
SONIC_BINARY=/opt/archivebox/lib/env/bin/sonic \
|
||||
PLAYWRIGHT_BROWSERS_PATH=/browsers
|
||||
|
||||
# Bash SHELL config
|
||||
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
|
||||
|
||||
######### System Environment ####################################
|
||||
|
||||
# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
|
||||
WORKDIR "$CODE_DIR"
|
||||
|
||||
# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
|
||||
RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
|
||||
&& echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
|
||||
&& echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
|
||||
&& rm -f /etc/apt/apt.conf.d/docker-clean
|
||||
|
||||
# Print debug info about build and save it to disk, for human eyes only, not used by anything else
|
||||
RUN (echo "[i] Docker build for ArchiveBox starting..." \
|
||||
&& echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
|
||||
&& echo \
|
||||
&& echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
|
||||
&& echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
|
||||
&& echo \
|
||||
&& uname -a \
|
||||
&& sed -n '1,7p' /etc/os-release \
|
||||
&& which bash && bash --version | sed -n '1p' \
|
||||
&& which dpkg && dpkg --version | sed -n '1p' \
|
||||
&& echo -e '\n\n' && env && echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Create non-privileged user for archivebox and chrome
|
||||
RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
|
||||
&& groupadd --system $ARCHIVEBOX_USER \
|
||||
&& useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
|
||||
&& usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
|
||||
&& groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
|
||||
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
|
||||
| tee -a /VERSION.txt
|
||||
# DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
|
||||
# https://docs.linuxserver.io/general/understanding-puid-and-pgid
|
||||
|
||||
# Install system apt dependencies (adding backports to access more recent apt updates)
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
|
||||
&& mkdir -p /etc/apt/keyrings \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y \
|
||||
# 1. packaging dependencies
|
||||
apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
|
||||
# 2. docker and init system dependencies
|
||||
zlib1g-dev dumb-init gosu cron unzip grep dnsutils git ripgrep python3.12-venv default-jre-headless \
|
||||
# 3. frivolous CLI helpers to make debugging failed archiving easier
|
||||
tree nano iputils-ping \
|
||||
# nano iputils-ping dnsutils htop procps jq yq
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install sonic search backend
|
||||
COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
|
||||
COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
|
||||
RUN (which sonic && sonic --version) | tee -a /VERSION.txt
|
||||
|
||||
######### Language Environments ####################################
|
||||
|
||||
# Set up Python environment
|
||||
# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
|
||||
#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
# --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
|
||||
# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
|
||||
# && apt-get update -qq \
|
||||
# && apt-get install -qq -y --no-upgrade \
|
||||
# python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
|
||||
# && rm -rf /var/lib/apt/lists/* \
|
||||
# tell PDM to allow using global system python site packages
|
||||
# && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
|
||||
# && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
|
||||
# create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
|
||||
# && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
|
||||
# && python3 -m venv --system-site-packages $GLOBAL_VENV \
|
||||
# && python3 -m venv $GLOBAL_VENV \
|
||||
# install global dependencies / python build dependencies in GLOBAL_VENV
|
||||
# && pip install --upgrade pip setuptools wheel \
|
||||
# Save version info
|
||||
# && ( \
|
||||
# which python3 && python3 --version | grep " $PYTHON_VERSION" \
|
||||
# && which pip && pip --version \
|
||||
# # && which pdm && pdm --version \
|
||||
# && echo -e '\n\n' \
|
||||
# ) | tee -a /VERSION.txt
|
||||
|
||||
|
||||
# Set up Node environment from the official platform tarball. This avoids
|
||||
# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
|
||||
# emulated Docker builds.
|
||||
RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
|
||||
case "$TARGETARCH" in \
|
||||
amd64) NODE_DIST_ARCH="x64" ;; \
|
||||
arm64) NODE_DIST_ARCH="arm64" ;; \
|
||||
*) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
|
||||
esac \
|
||||
&& NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
|
||||
&& echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
|
||||
&& curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
|
||||
&& curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
|
||||
&& grep " ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
|
||||
&& tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
|
||||
&& rm "$NODE_TARBALL" SHASUMS256.txt \
|
||||
# Save version info
|
||||
&& ( \
|
||||
which node && node --version \
|
||||
&& which npm && npm --version \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
|
||||
# Set up uv and main app /venv
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
|
||||
ENV UV_COMPILE_BYTECODE=0 \
|
||||
UV_PYTHON_PREFERENCE=managed \
|
||||
UV_PYTHON_INSTALL_DIR=/opt/uv/python \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PROJECT_ENVIRONMENT=/venv
|
||||
WORKDIR "$CODE_DIR"
|
||||
# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
|
||||
RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
|
||||
&& uv venv /venv --python ${PYTHON_VERSION}
|
||||
ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
|
||||
RUN uv pip install setuptools pip \
|
||||
&& ( \
|
||||
which python3 && python3 --version \
|
||||
&& which uv && uv self version \
|
||||
&& uv python find --system && uv python find \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
|
||||
######### ArchiveBox & Extractor Dependencies ##################################
|
||||
|
||||
# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
|
||||
WORKDIR "$CODE_DIR"
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
|
||||
#--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y --no-install-recommends \
|
||||
build-essential gcc \
|
||||
python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
|
||||
python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
|
||||
&& uv pip install \
|
||||
"python-ldap>=3.4.3" \
|
||||
&& apt-get purge -y \
|
||||
python3-dev build-essential gcc \
|
||||
&& apt-get autoremove -y \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
# Runtime config used by plugin hooks. Plugin binaries and npm packages are
|
||||
# installed into LIB_DIR below by archivebox init --install and resolved from
|
||||
# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
|
||||
ENV PERSONAS_DIR=/data/personas \
|
||||
PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
|
||||
PERSONAS_DIR=/data/personas \
|
||||
CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
|
||||
CHROME_HEADLESS=true \
|
||||
CHROME_SANDBOX=false \
|
||||
CHROME_ISOLATION=crawl \
|
||||
CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
|
||||
|
||||
######### Build Dependencies ####################################
|
||||
ENV TMP_DIR=/tmp/archivebox \
|
||||
PIP_VENV_PYTHON=/venv/bin/python3 \
|
||||
GOOGLE_API_KEY=no \
|
||||
GOOGLE_DEFAULT_CLIENT_ID=no \
|
||||
GOOGLE_DEFAULT_CLIENT_SECRET=no
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=0 \
|
||||
UV_PYTHON_PREFERENCE=managed \
|
||||
UV_PYTHON_INSTALL_DIR=/opt/uv/python \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PROJECT_ENVIRONMENT=/venv \
|
||||
VIRTUAL_ENV=/venv \
|
||||
PATH="/venv/bin:/opt/node/bin:$PATH"
|
||||
|
||||
# Install ArchiveBox Python venv dependencies from pyproject.toml.
|
||||
RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
|
||||
--mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
|
||||
WORKDIR "$CODE_DIR"
|
||||
|
||||
RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
|
||||
&& echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
|
||||
&& echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
|
||||
&& rm -f /etc/apt/apt.conf.d/docker-clean
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
|
||||
&& uv --no-cache sync \
|
||||
&& apt-get install -qq -y \
|
||||
apt-transport-https apt-utils ca-certificates curl wget gnupg2 \
|
||||
dumb-init util-linux unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
|
||||
cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \
|
||||
libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \
|
||||
libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
|
||||
libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \
|
||||
libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \
|
||||
fonts-liberation fonts-noto-color-emoji xdg-utils \
|
||||
ffmpeg imagemagick tesseract-ocr tesseract-ocr-eng openjdk-21-jre-headless \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Runtime-owned layers copied from the abx-dl image.
|
||||
COPY --from=abx-dl /bin/uv /bin/uv
|
||||
COPY --from=abx-dl /opt/uv/python /opt/uv/python
|
||||
COPY --from=abx-dl /opt/node /opt/node
|
||||
COPY --from=abx-dl /venv /venv
|
||||
COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
|
||||
|
||||
RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
|
||||
&& echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
|
||||
&& uname -a \
|
||||
&& sed -n '1,7p' /etc/os-release \
|
||||
&& which python3 && python3 --version \
|
||||
&& which node && node --version \
|
||||
&& which uv && uv self version \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
FROM archivebox-runtime-base AS archivebox-builder
|
||||
|
||||
WORKDIR "$CODE_DIR"
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
|
||||
echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y --no-install-recommends \
|
||||
build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
|
||||
&& uv sync \
|
||||
--refresh \
|
||||
--no-dev \
|
||||
--inexact \
|
||||
@ -278,103 +134,53 @@ RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
|
||||
--no-install-project \
|
||||
--no-install-workspace \
|
||||
--no-sources \
|
||||
&& apt-get purge -y python3-dev build-essential gcc \
|
||||
&& apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
|
||||
&& apt-get autoremove -y \
|
||||
&& find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
|
||||
|
||||
# Setup ArchiveBox runtime config
|
||||
ENV TMP_DIR=/tmp/archivebox \
|
||||
PIP_VENV_PYTHON=/usr/bin/python3.12 \
|
||||
GOOGLE_API_KEY=no \
|
||||
GOOGLE_DEFAULT_CLIENT_ID=no \
|
||||
GOOGLE_DEFAULT_CLIENT_SECRET=no
|
||||
|
||||
WORKDIR "$DATA_DIR"
|
||||
RUN openssl rand -hex 16 > /etc/machine-id \
|
||||
&& mkdir -p "$DATA_DIR" \
|
||||
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
|
||||
&& mkdir -p "$TMP_DIR" \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
|
||||
&& mkdir -p "$LIB_DIR" \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
|
||||
&& mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
|
||||
|
||||
# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
|
||||
# path users run later, before copying ArchiveBox source so source-only edits do
|
||||
# not invalidate the heavy browser/plugin dependency layer.
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
|
||||
--mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
|
||||
&& export PERSONAS_DIR="$LIB_DIR/personas" \
|
||||
&& export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
|
||||
&& export ABX_RUNTIME=archivebox ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 \
|
||||
&& mkdir -p "$LIB_DIR" \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \
|
||||
&& abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" gallery-dl \
|
||||
&& abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" --overrides='{"pip":{"install_args":["--no-deps","forum-dl","chardet==5.2.0","pydantic==2.12.3","pydantic-core==2.41.4","typing-extensions>=4.14.1","annotated-types>=0.6.0","typing-inspection>=0.4.2","beautifulsoup4","soupsieve","lxml","requests","urllib3","certifi","idna","charset-normalizer","tenacity","python-dateutil","six","html2text","warcio"]}}' forum-dl \
|
||||
&& if [ "$TARGETARCH" = "arm64" ]; then \
|
||||
abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
|
||||
abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
|
||||
fi \
|
||||
&& ABXPKG_INSTALL_TIMEOUT=600 TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
|
||||
accessibility archivedotorg archivewebpage base chrome chrome_mhtml chrome_screencast \
|
||||
claudechrome claudecode claudecodecleanup claudecodeextract consolelog defuddle dns dom \
|
||||
favicon forumdl gallerydl git hashes headers htmltotext infiniscroll \
|
||||
istilldontcareaboutcookies liteparse media mercury modalcloser opencode opendataloader papersdl \
|
||||
parse_dom_outlinks parse_html_urls parse_jsonl_urls parse_netscape_urls parse_rss_urls \
|
||||
parse_txt_urls pdf readability redirects responses screenshot search_backend_ripgrep \
|
||||
search_backend_sonic search_backend_sqlite seo singlefile ssl sslcerts staticfile title \
|
||||
trafilatura ublock wget ytdlp \
|
||||
&& abxpkg install --no-cache --binproviders=chromewebstore --overrides='{"chromewebstore":{"install_args":["fpeoodllldobpkbkabpblcfaogecpndd","--name=archivewebpage"]}}' archivewebpage \
|
||||
&& test -f "$LIB_DIR/chromewebstore/extensions/fpeoodllldobpkbkabpblcfaogecpndd__archivewebpage/manifest.json" \
|
||||
&& mkdir -p "$LIB_DIR/env/bin" \
|
||||
&& ln -sf "$(command -v node)" "$LIB_DIR/env/bin/node" \
|
||||
&& ln -sf "$(command -v npm)" "$LIB_DIR/env/bin/npm" \
|
||||
&& ln -sf "$(command -v java)" "$LIB_DIR/env/bin/java" \
|
||||
&& ln -sf "$(command -v git)" "$LIB_DIR/env/bin/git" \
|
||||
&& ln -sf "$(command -v rg)" "$LIB_DIR/env/bin/rg" \
|
||||
&& ln -sf "$(command -v sonic)" "$LIB_DIR/env/bin/sonic" \
|
||||
&& find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
|
||||
&& mkdir -p /opt/archivebox/lib-layer \
|
||||
&& cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
|
||||
&& apt-get purge -y build-essential \
|
||||
&& apt-get autoremove -y \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
|
||||
|
||||
RUN rm -rf "$LIB_DIR" \
|
||||
&& mv /opt/archivebox/lib-layer "$LIB_DIR" \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
|
||||
|
||||
# Install ArchiveBox Python package from the checked-out source.
|
||||
WORKDIR "$CODE_DIR"
|
||||
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
|
||||
RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
|
||||
&& pip install \
|
||||
--no-deps \
|
||||
"$CODE_DIR" \
|
||||
&& ( \
|
||||
pip show archivebox \
|
||||
&& which archivebox \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt \
|
||||
&& uv pip install --no-deps "$CODE_DIR" \
|
||||
&& (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
|
||||
&& find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
|
||||
# installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
|
||||
|
||||
# Initialize an empty image collection without rerunning dependency installs.
|
||||
FROM archivebox-runtime-base
|
||||
|
||||
LABEL name="archivebox" \
|
||||
maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
|
||||
description="All-in-one self-hosted internet archiving solution" \
|
||||
homepage="https://github.com/ArchiveBox/ArchiveBox" \
|
||||
documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
|
||||
org.opencontainers.image.title="ArchiveBox" \
|
||||
org.opencontainers.image.vendor="ArchiveBox" \
|
||||
org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
|
||||
org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
|
||||
com.docker.image.source.entrypoint="Dockerfile"
|
||||
|
||||
COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
|
||||
COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
|
||||
|
||||
COPY --from=archivebox-builder /venv /venv
|
||||
COPY --from=archivebox-builder /app /app
|
||||
COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
|
||||
COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib
|
||||
|
||||
RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
|
||||
&& groupadd --system "$ARCHIVEBOX_USER" \
|
||||
&& useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
|
||||
&& usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
|
||||
&& groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
|
||||
&& (which sonic && sonic --version) | tee -a /VERSION.txt \
|
||||
&& mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& openssl rand -hex 16 > /etc/machine-id \
|
||||
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
|
||||
&& echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
|
||||
|
||||
WORKDIR "$DATA_DIR"
|
||||
RUN echo "[+] Initializing image collection..." \
|
||||
&& find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
|
||||
@ -389,25 +195,28 @@ RUN echo "[+] Initializing image collection..." \
|
||||
"$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
|
||||
2>/dev/null || true)
|
||||
|
||||
# Print version for nice docker finish summary
|
||||
RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
|
||||
&& echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
|
||||
&& echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Verify ArchiveBox is installed and write full version/dependency info.
|
||||
RUN chmod +x "$CODE_DIR"/bin/*.sh \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
|
||||
&& chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode papersdl parse_rss_urls readability search_backend_sonic opendataloader search_backend_ripgrep 2>&1 | tee -a /VERSION.txt \
|
||||
&& gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
|
||||
&& chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
|
||||
&& "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
|
||||
&& "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
|
||||
&& /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
|
||||
&& /usr/local/bin/sonic --version | tee -a /VERSION.txt \
|
||||
&& /venv/bin/supervisord --version | tee -a /VERSION.txt \
|
||||
&& ! command -v gcc \
|
||||
&& ! command -v g++ \
|
||||
&& ! command -v make \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
|
||||
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
|
||||
|
||||
####################################################
|
||||
RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
|
||||
&& echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Expose ArchiveBox's main interfaces to the outside world
|
||||
WORKDIR "$DATA_DIR"
|
||||
VOLUME "$DATA_DIR"
|
||||
EXPOSE 8000
|
||||
|
||||
@ -9,16 +9,15 @@
|
||||
# --build-context abxpkg=../abxpkg \
|
||||
# --build-context abx-plugins=../abx-plugins \
|
||||
# -t archivebox/abx-dl:dev
|
||||
# docker buildx build . -f Dockerfile.multistage \
|
||||
# --build-context abx-dl=docker-image://archivebox/abx-dl:dev \
|
||||
# --build-arg ABX_DL_IMAGE=abx-dl \
|
||||
# docker buildx build . -f Dockerfile \
|
||||
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
|
||||
# -t archivebox:multistage
|
||||
|
||||
ARG TARGETPLATFORM=linux/amd64
|
||||
ARG TARGETOS=linux
|
||||
ARG TARGETARCH=amd64
|
||||
ARG TARGETVARIANT=
|
||||
ARG ABX_DL_IMAGE=abx-dl
|
||||
ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
|
||||
|
||||
FROM ${ABX_DL_IMAGE} AS abx-dl
|
||||
FROM archivebox/sonic:1.4.9 AS sonic
|
||||
@ -88,7 +87,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y \
|
||||
apt-transport-https apt-utils ca-certificates curl wget gnupg2 \
|
||||
dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
|
||||
dumb-init util-linux unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
|
||||
cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \
|
||||
libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \
|
||||
libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
|
||||
@ -160,7 +159,7 @@ LABEL name="archivebox" \
|
||||
org.opencontainers.image.vendor="ArchiveBox" \
|
||||
org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
|
||||
org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
|
||||
com.docker.image.source.entrypoint="Dockerfile.multistage"
|
||||
com.docker.image.source.entrypoint="Dockerfile"
|
||||
|
||||
COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
|
||||
COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
|
||||
@ -199,7 +198,7 @@ RUN echo "[+] Initializing image collection..." \
|
||||
RUN chmod +x "$CODE_DIR"/bin/*.sh \
|
||||
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
|
||||
&& chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
|
||||
&& GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opencode opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
|
||||
&& "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
|
||||
&& "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
|
||||
&& /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
|
||||
@ -208,7 +207,7 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \
|
||||
&& ! command -v gcc \
|
||||
&& ! command -v g++ \
|
||||
&& ! command -v make \
|
||||
&& gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
|
||||
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
|
||||
&& find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
|
||||
&& rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
|
||||
|
||||
@ -15,6 +15,8 @@ _REQUIRED_DOCKER_INSTALL_TARGETS = {
|
||||
"istilldontcareaboutcookies",
|
||||
"liteparse",
|
||||
"mercury",
|
||||
"opencode",
|
||||
"opendataloader",
|
||||
"papersdl",
|
||||
"parse_rss_urls",
|
||||
"readability",
|
||||
@ -76,9 +78,10 @@ def test_dockerfiles_install_binaries_required_by_version_validation() -> None:
|
||||
|
||||
|
||||
def test_dockerfile_prewarms_stable_plugin_dependencies_without_optional_captcha() -> None:
|
||||
targets = _abx_dl_plugin_install_targets(REPO_ROOT / "Dockerfile")
|
||||
assert _REQUIRED_DOCKER_PREINSTALL_TARGETS <= targets
|
||||
assert not (_DOCKER_PREINSTALL_EXCLUDED_TARGETS & targets)
|
||||
text = (REPO_ROOT / "Dockerfile").read_text()
|
||||
assert "archivebox/abx-dl:latest" in text
|
||||
assert "archivebox/abxdl" not in text
|
||||
assert _abx_dl_plugin_install_targets(REPO_ROOT / "Dockerfile") == set()
|
||||
|
||||
|
||||
def test_dockerfile_build_installs_disable_release_age_gate() -> None:
|
||||
@ -105,3 +108,14 @@ def test_dockerfiles_pin_project_binary_paths_for_validation() -> None:
|
||||
assert 'GIT_BINARY="$LIB_DIR/env/bin/git"' in text
|
||||
assert 'GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl"' in text
|
||||
assert 'FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl"' in text
|
||||
|
||||
|
||||
def test_dockerfiles_use_setpriv_instead_of_gosu() -> None:
|
||||
for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
|
||||
text = (REPO_ROOT / dockerfile_name).read_text()
|
||||
assert "gosu" not in text
|
||||
assert "setpriv" in text
|
||||
|
||||
entrypoint = (REPO_ROOT / "bin/docker_entrypoint.sh").read_text()
|
||||
assert "gosu" not in entrypoint
|
||||
assert "setpriv" in entrypoint
|
||||
|
||||
@ -144,7 +144,7 @@ ensure_runtime_tree() {
|
||||
|
||||
run_as_archivebox() {
|
||||
if [[ "$(id -u)" == "0" ]]; then
|
||||
gosu "$ARCHIVEBOX_USER" "$@"
|
||||
setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups "$@"
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
@ -283,12 +283,12 @@ if [[ "$1" == /* || "$1" == "bash" || "$1" == "sh" || "$1" == "echo" || "$1" ==
|
||||
# "docker run archivebox /bin/bash -c '...'"
|
||||
# "docker run archivebox cat /VERSION.txt"
|
||||
if [[ "$(id -u)" == "0" ]]; then
|
||||
exec gosu "$ARCHIVEBOX_USER" /bin/bash -c "exec $(printf ' %q' "$@")"
|
||||
exec setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups /bin/bash -c "exec $(printf ' %q' "$@")"
|
||||
else
|
||||
exec /bin/bash -c "exec $(printf ' %q' "$@")"
|
||||
fi
|
||||
# printf requotes shell parameters properly https://stackoverflow.com/a/39463371/2156113
|
||||
# gosu spawns an ephemeral bash process owned by archivebox user (bash wrapper is needed to load env vars, PATH, and setup terminal TTY)
|
||||
# setpriv spawns an ephemeral bash process owned by archivebox user (bash wrapper is needed to load env vars, PATH, and setup terminal TTY)
|
||||
# outermost exec hands over current process ID to inner bash process, inner exec hands over inner bash PID to user's command
|
||||
else
|
||||
# handle "docker run archivebox add some subcommand --with=args abc" by calling archivebox to run as args as CLI subcommand
|
||||
@ -297,7 +297,7 @@ else
|
||||
# "docker run archivebox manage createsupseruser"
|
||||
# "docker run archivebox server 0.0.0.0:8000"
|
||||
if [[ "$(id -u)" == "0" ]]; then
|
||||
exec gosu "$ARCHIVEBOX_USER" "$ARCHIVEBOX_BIN_PATH" "$@"
|
||||
exec setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups "$ARCHIVEBOX_BIN_PATH" "$@"
|
||||
else
|
||||
exec "$ARCHIVEBOX_BIN_PATH" "$@"
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user