From 943d4ee64b43af7125ef01a77ebc2ccb38b13cb6 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 23 May 2026 15:51:53 -0700 Subject: [PATCH] Simplify Docker runtime dependency install --- Dockerfile | 136 ++++++++++----------------------------- bin/docker_entrypoint.sh | 3 +- bin/setup.sh | 35 +++++----- docker-compose.yml | 4 +- etc/package.json | 2 +- pyproject.toml | 2 +- 6 files changed, 56 insertions(+), 126 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1d6bd228..b6d11a27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # This is the Dockerfile for ArchiveBox, it bundles the following main dependencies: # python3.13, uv, python3-ldap # curl, wget, git, dig, ping, tree, nano -# node, npm, single-file, readability-extractor, postlight-parser -# ArchiveBox, yt-dlp, Google Chrome, ffmpeg +# node, npm +# ArchiveBox and plugin runtime dependencies installed by archivebox init --install # Usage: # git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox # docker build . -t archivebox @@ -10,8 +10,7 @@ # docker run -v "$PWD/data":/data archivebox add 'https://example.com' # docker run -v "$PWD/data":/data -it archivebox manage createsuperuser # docker run -v "$PWD/data":/data -p 8000:8000 archivebox server -# Chrome for Linux is only distributed for amd64, so the Docker image targets linux/amd64. -# docker buildx build . --platform=linux/amd64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123 +# docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123 # Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development @@ -27,7 +26,13 @@ ######################################################################################### -FROM --platform=linux/amd64 ubuntu:24.04 +ARG TARGETPLATFORM +ARG TARGETOS +ARG TARGETARCH +ARG TARGETVARIANT= + +FROM archivebox/sonic:1.4.9 AS sonic +FROM ubuntu:24.04 LABEL name="archivebox" \ maintainer="Nick Sweeting " \ @@ -81,6 +86,7 @@ ENV ARCHIVEBOX_USER="archivebox" \ ENV CODE_DIR=/app \ DATA_DIR=/data \ LIB_DIR=/opt/archivebox/lib \ + ABXPKG_LIB_DIR=/opt/archivebox/lib \ PLAYWRIGHT_BROWSERS_PATH=/browsers # Bash SHELL config @@ -133,35 +139,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T # 1. packaging dependencies apt-transport-https ca-certificates apt-utils gnupg2 curl wget \ # 2. docker and init system dependencies - zlib1g-dev dumb-init gosu cron unzip grep dnsutils python3.12-venv \ + zlib1g-dev dumb-init gosu cron unzip grep dnsutils git python3.12-venv \ # 3. frivolous CLI helpers to make debugging failed archiving easier tree nano iputils-ping \ # nano iputils-ping dnsutils htop procps jq yq && rm -rf /var/lib/apt/lists/* -# Install apt binary dependencies for extractors -# COPY --from=selenium/ffmpeg:latest /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - echo "[+] APT Installing extractor dependencies for $TARGETPLATFORM..." \ - && apt-get update -qq \ - && apt-get install -qq -y --no-install-recommends \ - git ripgrep default-jre \ - # Packages we have also needed in the past: - # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \ - # curl wget (already installed above) - && rm -rf /var/lib/apt/lists/* \ - # Save version info - && ( \ - which curl && curl --version | sed -n '1p' \ - && which wget && wget --version 2>&1 | sed -n '1p' \ - && which git && git --version 2>&1 | sed -n '1p' \ - # && which ffmpeg && (ffmpeg --version 2>&1 | sed -n '1p') || true \ - && which rg && rg --version 2>&1 | sed -n '1p' \ - && echo -e '\n\n' \ - ) | tee -a /VERSION.txt - # Install sonic search backend -COPY --from=archivebox/sonic:1.4.9 /usr/local/bin/sonic /usr/local/bin/sonic +COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg RUN (which sonic && sonic --version) | tee -a /VERSION.txt @@ -194,15 +179,22 @@ RUN (which sonic && sonic --version) | tee -a /VERSION.txt # ) | tee -a /VERSION.txt -# Set up Node environment from the official linux-x64 tarball. This avoids +# Set up Node environment from the official platform tarball. This avoids # NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into -# emulated linux/amd64 Docker builds. +# emulated Docker builds. RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \ - echo "[+] Installing NODE $NODE_VERSION for linux/amd64..." \ - && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz" \ - && echo "c7a10d6816da8eaaa7534dd73c71c6e2b2c391dbbf845e364902d156615dd1b8 node-v${NODE_VERSION}-linux-x64.tar.gz" | sha256sum -c - \ - && tar -xzf "node-v${NODE_VERSION}-linux-x64.tar.gz" -C /usr/local --strip-components=1 --no-same-owner \ - && rm "node-v${NODE_VERSION}-linux-x64.tar.gz" \ + case "$TARGETARCH" in \ + amd64) NODE_DIST_ARCH="x64" ;; \ + arm64) NODE_DIST_ARCH="arm64" ;; \ + *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \ + esac \ + && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \ + && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \ + && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \ + && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \ + && grep " ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \ + && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "$NODE_TARBALL" SHASUMS256.txt \ # Save version info && ( \ which node && node --version \ @@ -254,79 +246,16 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T && rm -rf /var/lib/apt/lists/* -# Install apt font & rendering dependencies for Google Chrome -# RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - -# Install Google Chrome inside the container. Docker must use the package pulled -# during the image build, never a browser from the host checkout. -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - echo "[+] APT Installing Google Chrome, fonts, and display libraries for $TARGETPLATFORM..." \ - && curl -fsSL "https://dl.google.com/linux/linux_signing_key.pub" | gpg --dearmor -o /etc/apt/keyrings/google-linux.gpg \ - && echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/google-linux.gpg] https://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \ - && apt-get update -qq \ - && apt-get install -qq -y \ - #fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \ - #at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \ - #libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \ - #libxaw7 libxcomposite1 libxdamage1 libxfont2 \ - google-chrome-stable ffmpeg \ - libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \ - # xfonts-scalable xfonts-utils xserver-common xvfb \ - # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway - # libxss1 dbus dbus-x11 upower \ - # && service dbus start \ - && ln -sf /usr/bin/google-chrome-stable /usr/bin/chrome \ - && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/google-chrome/Crash Reports/pending/" \ - && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \ - && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \ - && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \ - # Save version info - && rm -rf /var/lib/apt/lists/* \ - && ( \ - which google-chrome-stable && google-chrome-stable --version \ - && which chrome && chrome --version \ - && which ffmpeg && ffmpeg -version | sed -n '1p' \ - && echo -e '\n\n' \ - ) | tee -a /VERSION.txt - -# Install Node extractor dependencies -ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH" \ +# Runtime config used by plugin hooks. Plugin binaries and npm packages are +# installed into LIB_DIR below by archivebox init --install. +ENV PATH="$LIB_DIR/npm/node_modules/.bin:$PATH" \ PERSONAS_DIR=/data/personas \ - NODE_PATH="/home/$ARCHIVEBOX_USER/.npm/lib/node_modules:/usr/lib/node_modules:$LIB_DIR/npm/node_modules:/data/personas/Default/node_modules" \ - PUPPETEER_EXECUTABLE_PATH=/usr/bin/google-chrome-stable \ - CHROME_BIN=/usr/bin/google-chrome-stable \ - CHROME_BINARY=/usr/bin/google-chrome-stable \ + NODE_PATH="$LIB_DIR/npm/node_modules:/data/personas/Default/node_modules" \ CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \ CHROME_HEADLESS=true \ CHROME_SANDBOX=false \ CHROME_ISOLATION=crawl \ CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]' -USER $ARCHIVEBOX_USER -WORKDIR "/home/$ARCHIVEBOX_USER/.npm" -RUN --mount=type=cache,target=/home/archivebox/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \ - echo "[+] NPM Installing node extractor dependencies into /home/$ARCHIVEBOX_USER/.npm..." \ - && npm config set prefix "/home/$ARCHIVEBOX_USER/.npm" \ - && npm install --global --prefer-offline --no-fund --no-audit --cache "/home/$ARCHIVEBOX_USER/.npm_cache" \ - "@postlight/parser@^2.2.3" \ - "readability-extractor@github:ArchiveBox/readability-extractor" \ - "single-file-cli@^1.1.54" \ - "abxbus@^2.5.4" \ - "puppeteer-core@^23.5.0" \ - "puppeteer@^23.5.0" \ - "@puppeteer/browsers@^2.4.0" \ - && rm -Rf "/home/$ARCHIVEBOX_USER/.cache/puppeteer" -USER root -WORKDIR "$CODE_DIR" -RUN ( \ - which node && node --version \ - && which npm && npm version \ - && which postlight-parser \ - && which readability-extractor && readability-extractor --version \ - && which single-file && single-file --version \ - && which puppeteer && puppeteer --version \ - && echo -e '\n\n' \ - ) | tee -a /VERSION.txt ######### Build Dependencies #################################### @@ -386,7 +315,8 @@ RUN openssl rand -hex 16 > /etc/machine-id \ # Pre-bake plugin-managed runtime dependencies using the same installer path # users run later via archivebox init --install / archivebox install. RUN echo "[+] Initializing image collection and installing plugin runtime dependencies into $LIB_DIR..." \ - && gosu "$DEFAULT_PUID" archivebox init --install + && gosu "$DEFAULT_PUID" archivebox init --install \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$LIB_DIR" # Print version for nice docker finish summary RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \ diff --git a/bin/docker_entrypoint.sh b/bin/docker_entrypoint.sh index 349d4151..55b3da7e 100755 --- a/bin/docker_entrypoint.sh +++ b/bin/docker_entrypoint.sh @@ -31,6 +31,7 @@ ulimit -c 0 >/dev/null 2>&1 || true export DATA_DIR="${DATA_DIR:-/data}" export TMP_DIR="${TMP_DIR:-/tmp/archivebox}" export LIB_DIR="${LIB_DIR:-/opt/archivebox/lib}" +export ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-$LIB_DIR}" export ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}" # Global default PUID and PGID if data dir is empty and no intended PUID+PGID is set manually by user @@ -134,7 +135,7 @@ find "$DATA_DIR/personas" -type f \( \ find /tmp "$TMP_DIR" -maxdepth 1 -type d -name "archivebox-chrome-profile.*" -mmin +30 -exec rm -rf {} + >/dev/null 2>&1 || true -# also chown BROWSERS_DIR because otherwise 'archivebox setup' wont be able to 'playwright install chromium' at runtime +# also chown BROWSERS_DIR because browser providers may manage files there at runtime export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-/browsers}" mkdir -p "$PLAYWRIGHT_BROWSERS_PATH/permissions_test_safe_to_delete" rm -Rf "$PLAYWRIGHT_BROWSERS_PATH/permissions_test_safe_to_delete" diff --git a/bin/setup.sh b/bin/setup.sh index 46964651..f473ca91 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -22,8 +22,12 @@ clear ARCHIVEBOX_BRANCH="${ARCHIVEBOX_BRANCH:-dev}" ARCHIVEBOX_IMAGE="${ARCHIVEBOX_IMAGE:-archivebox/archivebox:dev}" -ARCHIVEBOX_PLATFORM="${ARCHIVEBOX_PLATFORM:-linux/amd64}" +ARCHIVEBOX_PLATFORM="${ARCHIVEBOX_PLATFORM:-}" ARCHIVEBOX_COMPOSE_URL="${ARCHIVEBOX_COMPOSE_URL:-https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/${ARCHIVEBOX_BRANCH}/docker-compose.yml}" +DOCKER_PLATFORM_ARGS=() +if [ -n "$ARCHIVEBOX_PLATFORM" ]; then + DOCKER_PLATFORM_ARGS=(--platform "$ARCHIVEBOX_PLATFORM") +fi wait_for_archivebox() { local url="http://127.0.0.1:8000/health/" @@ -61,7 +65,7 @@ if [ "$(id -u)" -eq 0 ]; then exit 2 fi -if (command -v docker > /dev/null && docker compose version > /dev/null && docker pull --platform "$ARCHIVEBOX_PLATFORM" "$ARCHIVEBOX_IMAGE"); then +if (command -v docker > /dev/null && docker compose version > /dev/null && docker pull "${DOCKER_PLATFORM_ARGS[@]}" "$ARCHIVEBOX_IMAGE"); then echo "[+] Initializing an ArchiveBox data folder at ~/archivebox/data using Docker Compose..." mkdir -p ~/archivebox/data || exit 1 cd ~/archivebox @@ -70,7 +74,7 @@ if (command -v docker > /dev/null && docker compose version > /dev/null && docke fi curl -fsSL "$ARCHIVEBOX_COMPOSE_URL" > docker-compose.yml export ARCHIVEBOX_IMAGE ARCHIVEBOX_PLATFORM - docker compose run --rm archivebox init + docker compose run --rm archivebox init --install echo echo "[+] Starting ArchiveBox server using: docker compose up -d..." docker compose up -d @@ -88,7 +92,7 @@ if (command -v docker > /dev/null && docker compose version > /dev/null && docke echo " docker compose run archivebox list" echo " docker compose run archivebox help" exit 0 -elif (command -v docker > /dev/null && docker pull --platform "$ARCHIVEBOX_PLATFORM" "$ARCHIVEBOX_IMAGE"); then +elif (command -v docker > /dev/null && docker pull "${DOCKER_PLATFORM_ARGS[@]}" "$ARCHIVEBOX_IMAGE"); then echo "[+] Initializing an ArchiveBox data folder at ~/archivebox/data using Docker..." mkdir -p ~/archivebox/data || exit 1 cd ~/archivebox @@ -96,10 +100,10 @@ elif (command -v docker > /dev/null && docker pull --platform "$ARCHIVEBOX_PLATF mv -i ~/archivebox/* ~/archivebox/data/ fi cd ./data - docker run --platform "$ARCHIVEBOX_PLATFORM" -v "$PWD":/data -it --rm "$ARCHIVEBOX_IMAGE" init + docker run "${DOCKER_PLATFORM_ARGS[@]}" -v "$PWD":/data -it --rm "$ARCHIVEBOX_IMAGE" init --install echo echo "[+] Starting ArchiveBox server using: docker run -d archivebox/archivebox..." - docker run --platform "$ARCHIVEBOX_PLATFORM" -v "$PWD":/data -it -d -p 8000:8000 --name=archivebox "$ARCHIVEBOX_IMAGE" + docker run "${DOCKER_PLATFORM_ARGS[@]}" -v "$PWD":/data -it -d -p 8000:8000 --name=archivebox "$ARCHIVEBOX_IMAGE" wait_for_archivebox open_archivebox echo @@ -108,11 +112,11 @@ elif (command -v docker > /dev/null && docker pull --platform "$ARCHIVEBOX_PLATF echo " docker ps --filter name=archivebox" echo " docker kill archivebox" echo " docker pull $ARCHIVEBOX_IMAGE" - echo " docker run --platform $ARCHIVEBOX_PLATFORM -v $PWD:/data -d -p 8000:8000 --name=archivebox $ARCHIVEBOX_IMAGE" - echo " docker run --platform $ARCHIVEBOX_PLATFORM -v $PWD:/data -it $ARCHIVEBOX_IMAGE manage createsuperuser" - echo " docker run --platform $ARCHIVEBOX_PLATFORM -v $PWD:/data -it $ARCHIVEBOX_IMAGE add 'https://example.com'" - echo " docker run --platform $ARCHIVEBOX_PLATFORM -v $PWD:/data -it $ARCHIVEBOX_IMAGE list" - echo " docker run --platform $ARCHIVEBOX_PLATFORM -v $PWD:/data -it $ARCHIVEBOX_IMAGE help" + echo " docker run ${DOCKER_PLATFORM_ARGS[*]} -v $PWD:/data -d -p 8000:8000 --name=archivebox $ARCHIVEBOX_IMAGE" + echo " docker run ${DOCKER_PLATFORM_ARGS[*]} -v $PWD:/data -it $ARCHIVEBOX_IMAGE manage createsuperuser" + echo " docker run ${DOCKER_PLATFORM_ARGS[*]} -v $PWD:/data -it $ARCHIVEBOX_IMAGE add 'https://example.com'" + echo " docker run ${DOCKER_PLATFORM_ARGS[*]} -v $PWD:/data -it $ARCHIVEBOX_IMAGE list" + echo " docker run ${DOCKER_PLATFORM_ARGS[*]} -v $PWD:/data -it $ARCHIVEBOX_IMAGE help" exit 0 fi @@ -137,7 +141,7 @@ echo echo " - archivebox" echo " - python3, pip, nodejs, npm (languages used by ArchiveBox, and its extractor modules)" echo " - curl, wget, git, youtube-dl, yt-dlp (used for extracting title, favicon, git, media, and more)" -echo " - chromium (skips this if any Chrome/Chromium version is already installed)" +echo " - chromium (installed/discovered by archivebox init --install)" echo echo " If you'd rather install these manually as-needed, you can find detailed documentation here:" echo " https://github.com/ArchiveBox/ArchiveBox/wiki/Install" @@ -153,7 +157,7 @@ if which apt-get > /dev/null; then echo "[+] Installing ArchiveBox system dependencies using apt..." sudo apt-get update -qq sudo apt-get install -y git python3 python3-pip python3-venv wget curl yt-dlp ffmpeg git nodejs npm ripgrep - sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libnss3 libxss1 libasound2 libxtst6 xauth xvfb libgbm-dev || sudo apt-get install -y chromium || sudo apt-get install -y chromium-browser || true + sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libnss3 libxss1 libasound2 libxtst6 xauth xvfb libgbm-dev || true echo echo "[+] Installing ArchiveBox python dependencies using pip3..." sudo python3 -m pip install --upgrade --ignore-installed archivebox yt-dlp @@ -213,11 +217,6 @@ fi # sudo npm i -g npm || true # sudo python3 -m pip install --upgrade pip setuptools || true -echo -echo "[+] Installing Chromium binary using playwright..." -python3 -m playwright install --with-deps chromium || true -echo - echo echo "[+] Initializing ArchiveBox data folder at ~/archivebox/data..." mkdir -p ~/archivebox/data || exit 1 diff --git a/docker-compose.yml b/docker-compose.yml index 0234e3b2..5fcdc688 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ # Usage: # mkdir -p ~/archivebox/data && cd ~/archivebox # curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml -# docker compose run archivebox init +# docker compose run archivebox init --install # docker compose run archivebox add --depth=1 'https://news.ycombinator.com' # docker compose run -T archivebox add < bookmarks.txt # docker compose up -d && open 'http://web.archivebox.localhost:8000' @@ -12,7 +12,7 @@ services: archivebox: image: ${ARCHIVEBOX_IMAGE:-archivebox/archivebox:dev} - # platform: linux/amd64 + # platform: ${ARCHIVEBOX_PLATFORM:-linux/amd64} # optional override; omit for native arch ports: - 8000:8000 volumes: diff --git a/etc/package.json b/etc/package.json index 8db01db3..c1911045 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.31rc8", + "version": "0.9.31rc9", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index fdc5fbfe..856f135b 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.31rc8" +version = "0.9.31rc9" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]