From f3fcc1584c3c4c82b2e861663da239b2a45b2cd4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 00:19:50 +0000 Subject: [PATCH 01/19] Restore Homebrew and Debian package manager support - Add Homebrew formula (brew_dist/archivebox.rb) using virtualenv pattern with auto-generation via homebrew-pypi-poet in bin/build_brew.sh - Add Debian packaging via nFPM (pkg/debian/) with thin .deb that pip-installs archivebox into /opt/archivebox/venv on postinstall - Add build/release scripts: bin/{build,release}_{brew,deb}.sh - Update CI workflows to build packages on release and test them - Update README apt/brew install instructions with working commands - Update bin/setup.sh to use .deb download instead of old Launchpad PPA https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 102 ++++++++++++++++-------------- .github/workflows/homebrew.yml | 88 ++++++++++++++++---------- README.md | 23 ++----- bin/build.sh | 8 ++- bin/build_brew.sh | 101 +++++++++++++++++++++++++++++ bin/build_deb.sh | 43 +++++++++++++ bin/release.sh | 2 + bin/release_brew.sh | 48 ++++++++++++++ bin/release_deb.sh | 33 ++++++++++ bin/setup.sh | 32 ++++------ brew_dist/archivebox.rb | 51 +++++++++++++++ pkg/debian/archivebox | 13 ++++ pkg/debian/archivebox.service | 16 +++++ pkg/debian/install.sh | 33 ++++++++++ pkg/debian/nfpm.yaml | 67 ++++++++++++++++++++ pkg/debian/scripts/postinstall.sh | 5 ++ pkg/debian/scripts/preremove.sh | 9 +++ 17 files changed, 552 insertions(+), 122 deletions(-) create mode 100755 bin/build_brew.sh create mode 100755 bin/build_deb.sh create mode 100755 bin/release_brew.sh create mode 100755 bin/release_deb.sh create mode 100644 brew_dist/archivebox.rb create mode 100755 pkg/debian/archivebox create mode 100644 pkg/debian/archivebox.service create mode 100755 pkg/debian/install.sh create mode 100644 pkg/debian/nfpm.yaml create mode 100755 pkg/debian/scripts/postinstall.sh create mode 100755 pkg/debian/scripts/preremove.sh diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 9b95071e..709817a5 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -2,63 +2,67 @@ name: Build Debian package on: workflow_dispatch: - push: - -env: - DEB_BUILD_OPTIONS: nocheck + release: + types: [published] jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 + strategy: + matrix: + arch: [amd64, arm64] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: - submodules: true fetch-depth: 1 - - name: Install packaging dependencies + - name: Get version + id: version + run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Install nfpm + run: | + curl -sfL https://install.goreleaser.com/github.com/goreleaser/nfpm.sh | sh -s -- -b /usr/local/bin + + - name: Build .deb package + run: | + export VERSION="${{ steps.version.outputs.version }}" + export ARCH="${{ matrix.arch }}" + ./bin/build_deb.sh + + - name: Upload .deb artifact + uses: actions/upload-artifact@v4 + with: + name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb + path: dist/*.deb + + - name: Upload .deb to GitHub Release + if: github.event_name == 'release' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload "${{ github.event.release.tag_name }}" dist/*.deb --clobber + + test: + needs: build + runs-on: ubuntu-24.04 + + steps: + - name: Download .deb artifact + uses: actions/download-artifact@v4 + with: + pattern: archivebox-*-amd64.deb + merge-multiple: true + + - name: Install .deb package run: | sudo apt-get update -qq - sudo apt-get install -y \ - python3 python3-dev python3-pip python3-venv python3-all \ - dh-python debhelper devscripts dput software-properties-common \ - python3-distutils python3-setuptools python3-wheel python3-stdeb + sudo apt-get install -y python3 python3-pip python3-venv nodejs npm git wget curl ripgrep + sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y - # - name: Build Debian/Apt sdist_dsc - # run: | - # ./bin/build_pip.sh - - # - name: Check ArchiveBox version - # run: | - # # must create dir needed for snaps to run as non-root on github actions - # sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001 - # mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data" - # archivebox --version - # archivebox init --setup - - # - name: Add some links to test - # run: | - # cd "${{ github.workspace }}/data" - # archivebox add 'https://example.com' - # archivebox status - - # - name: Commit built package - # run: | - # cd deb_dist/ - # git config --local user.email "action@github.com" - # git config --local user.name "GitHub Action" - # git commit -m "Debian package autobuild" -a - - # - name: Push build to Github - # uses: ad-m/github-push-action@master - # with: - # github_token: ${{ secrets.GITHUB_TOKEN }} - # repository: ArchiveBox/debian-archivebox - # branch: ${{ github.ref }} - # directory: deb_dist - - # - name: Push build to Launchpad PPA - # run: | - # debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes" - # dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes" + - name: Test archivebox CLI + run: | + archivebox version + mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test + archivebox init --setup diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index af7a0795..6c5924ea 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -1,51 +1,71 @@ -name: Build Homebrew package +name: Build Homebrew formula on: workflow_dispatch: - push: - + release: + types: [published] jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: - submodules: true fetch-depth: 1 - # TODO: modify archivebox.rb to update src url, hashes, and dependencies + - name: Get version + id: version + run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT" - - name: Build Homebrew Bottle + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Generate Homebrew formula run: | - pip3 install --upgrade pip setuptools wheel - cd brew_dist/ - brew install --build-bottle ./archivebox.rb - # brew bottle archivebox - archivebox version + python3 -m venv /tmp/poet-venv + source /tmp/poet-venv/bin/activate + pip install --quiet "archivebox==${{ steps.version.outputs.version }}" homebrew-pypi-poet + poet -f archivebox > /tmp/archivebox-generated.rb + deactivate - - name: Add some links to test + - name: Upload formula artifact + uses: actions/upload-artifact@v4 + with: + name: archivebox.rb + path: /tmp/archivebox-generated.rb + + - name: Push to homebrew-archivebox tap + if: github.event_name == 'release' + env: + GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} run: | - mkdir data && cd data - archivebox init --setup - archivebox add 'https://example.com' + # Clone the tap repo and update the formula + git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap + cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb 2>/dev/null || cp /tmp/archivebox-generated.rb /tmp/tap/archivebox.rb + cd /tmp/tap + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add archivebox.rb + git diff --cached --quiet || git commit -m "Update archivebox to v${{ steps.version.outputs.version }}" + git push origin HEAD + + test: + needs: build + runs-on: macos-latest + + steps: + - uses: actions/checkout@v4 + + - name: Download formula artifact + uses: actions/download-artifact@v4 + with: + name: archivebox.rb + path: /tmp/ + + - name: Test Homebrew formula + run: | + brew install --build-from-source /tmp/archivebox-generated.rb || true archivebox version - archivebox status - - # - name: Commit built package - # run: | - # cd brew_dist/ - # git config --local user.email "action@github.com" - # git config --local user.name "GitHub Action" - # git commit -m "Homebrew package autobuild" -a - - # - name: Push build to Github - # uses: ad-m/github-push-action@master - # with: - # github_token: ${{ secrets.GITHUB_TOKEN }} - # repository: ArchiveBox/homebrew-archivebox - # branch: ${{ github.ref }} - # directory: brew_dist - - # TODO: push bottle homebrew core PR with latest changes diff --git a/README.md b/README.md index 1804ecf8..66b9126f 100644 --- a/README.md +++ b/README.md @@ -294,19 +294,11 @@ See below for more usage examples using the C
aptitude apt (Ubuntu/Debian/etc.)
-See the Install: Bare Metal Wiki for instructions. ➡️ - +See the debian-archivebox repo for more details about this distribution.

@@ -337,9 +329,6 @@ See below for more usage examples using the C
  • Install the ArchiveBox package using brew. 
    brew tap archivebox/archivebox
 brew install archivebox
-# update to newest version with pip (sometimes brew package is outdated)
-pip install --upgrade --ignore-installed archivebox yt-dlp playwright
-playwright install --with-deps chromium    # install chromium and its system dependencies
 archivebox version                         # make sure all dependencies are installed
    See the Install: Bare Metal Wiki for more granular instructions for macOS... ➡️ diff --git a/bin/build.sh b/bin/build.sh index b3271873..1ebf2d3d 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -19,9 +19,13 @@ cd "$REPO_DIR" # the order matters ./bin/build_docs.sh ./bin/build_pip.sh +./bin/build_deb.sh +./bin/build_brew.sh ./bin/build_docker.sh echo "[√] Done. Install the built package by running:" -echo " python3 setup.py install" +echo " pip install archivebox" echo " # or" -echo " pip3 install ." +echo " sudo apt install ./dist/archivebox*.deb" +echo " # or" +echo " brew tap archivebox/archivebox && brew install archivebox" diff --git a/bin/build_brew.sh b/bin/build_brew.sh new file mode 100755 index 00000000..e56c2a2b --- /dev/null +++ b/bin/build_brew.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +cd "$REPO_DIR" + +VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" +FORMULA_FILE="$REPO_DIR/brew_dist/archivebox.rb" + +echo "[+] Building Homebrew formula for archivebox==${VERSION}..." + +# Create a temporary virtualenv for generating the formula +TMPDIR="$(mktemp -d)" +trap "rm -rf $TMPDIR" EXIT + +python3 -m venv "$TMPDIR/venv" +source "$TMPDIR/venv/bin/activate" + +pip install --quiet "archivebox==${VERSION}" homebrew-pypi-poet 2>/dev/null + +echo "[+] Generating resource stanzas with homebrew-pypi-poet..." +RESOURCES="$(poet archivebox)" + +# Get the sdist URL and SHA256 from PyPI +SDIST_URL="$(pip download --no-binary :all: --no-deps -d "$TMPDIR/sdist" "archivebox==${VERSION}" 2>&1 | grep -oP 'https://\S+\.tar\.gz' | head -1 || true)" +if [ -z "$SDIST_URL" ]; then + SDIST_URL="https://files.pythonhosted.org/packages/source/a/archivebox/archivebox-${VERSION}.tar.gz" +fi +SDIST_SHA256="$(pip hash "$TMPDIR/sdist/"*.tar.gz 2>/dev/null | grep 'sha256:' | cut -d: -f2 || echo '')" + +deactivate + +echo "[+] Updating formula file: $FORMULA_FILE" + +# Build the formula from the template +cat > "$FORMULA_FILE" << RUBY +# This formula is auto-generated by bin/build_brew.sh using homebrew-pypi-poet. +# To update: run bin/build_brew.sh, or trigger the GitHub Actions homebrew workflow. +# +# Users install with: +# brew tap archivebox/archivebox +# brew install archivebox + +class Archivebox < Formula + include Language::Python::Virtualenv + + desc "Self-hosted internet archiving solution" + homepage "https://github.com/ArchiveBox/ArchiveBox" + url "${SDIST_URL}" + sha256 "${SDIST_SHA256}" + license "MIT" + head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" + + depends_on "python@3.13" + depends_on "node" + depends_on "git" + depends_on "wget" + depends_on "curl" + depends_on "ripgrep" + depends_on "yt-dlp" + + # Python dependency resource blocks auto-generated by homebrew-pypi-poet + # AUTOGENERATED_RESOURCES_START +${RESOURCES} + # AUTOGENERATED_RESOURCES_END + + def install + virtualenv_install_with_resources + end + + def post_install + # Install runtime dependencies (plugins, JS extractors, etc.) + system bin/"archivebox", "install", "--binproviders", "pip,npm" + end + + service do + run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] + keep_alive crashed: true + working_dir var/"archivebox" + log_path var/"log/archivebox.log" + error_log_path var/"log/archivebox.log" + end + + test do + assert_match version.to_s, shell_output("#{bin}/archivebox version") + end +end +RUBY + +echo "[√] Formula updated: $FORMULA_FILE" +echo " Version: ${VERSION}" +echo " URL: ${SDIST_URL}" diff --git a/bin/build_deb.sh b/bin/build_deb.sh new file mode 100755 index 00000000..08c0950d --- /dev/null +++ b/bin/build_deb.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +cd "$REPO_DIR" + +VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" +export VERSION + +# Default to amd64, can be overridden with ARCH=arm64 +export ARCH="${ARCH:-amd64}" + +echo "[+] Building .deb package for archivebox_${VERSION}_${ARCH}..." + +# Check for nfpm +if ! command -v nfpm &>/dev/null; then + echo "[!] nfpm not found. Install it with one of:" + echo " go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest" + echo " uv tool install nfpm" + echo " brew install goreleaser/tap/nfpm" + echo " curl -sfL https://install.goreleaser.com/github.com/goreleaser/nfpm.sh | sh" + exit 1 +fi + +mkdir -p "$REPO_DIR/dist" + +nfpm package \ + --config "$REPO_DIR/pkg/debian/nfpm.yaml" \ + --packager deb \ + --target "$REPO_DIR/dist/" + +echo +echo "[√] Built .deb package:" +ls -la "$REPO_DIR/dist/"archivebox*.deb diff --git a/bin/release.sh b/bin/release.sh index 4170b0d2..a2fd719a 100755 --- a/bin/release.sh +++ b/bin/release.sh @@ -28,6 +28,8 @@ cd "$REPO_DIR" # ./bin/release_docs.sh ./bin/release_git.sh "$@" ./bin/release_pip.sh "$@" +./bin/release_deb.sh "$@" +./bin/release_brew.sh "$@" ./bin/release_docker.sh "$@" VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" diff --git a/bin/release_brew.sh b/bin/release_brew.sh new file mode 100755 index 00000000..19fbe90a --- /dev/null +++ b/bin/release_brew.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +cd "$REPO_DIR" + +VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" +FORMULA_FILE="$REPO_DIR/brew_dist/archivebox.rb" +TAP_REPO="ArchiveBox/homebrew-archivebox" + +if [ ! -f "$FORMULA_FILE" ]; then + echo "[!] Formula not found at $FORMULA_FILE" + echo " Run ./bin/build_brew.sh first to generate it." + exit 1 +fi + +echo "[+] Releasing Homebrew formula for archivebox==${VERSION} to ${TAP_REPO}..." + +# Clone the tap repo, update formula, commit, and push +TMPDIR="$(mktemp -d)" +trap "rm -rf $TMPDIR" EXIT + +git clone "https://github.com/${TAP_REPO}.git" "$TMPDIR/tap" +cp "$FORMULA_FILE" "$TMPDIR/tap/archivebox.rb" + +cd "$TMPDIR/tap" +git add archivebox.rb +if git diff --cached --quiet; then + echo "[i] No changes to formula, skipping release." + exit 0 +fi + +git commit -m "Update archivebox to v${VERSION}" +git push origin HEAD + +echo "[√] Homebrew formula pushed to ${TAP_REPO}" +echo " Users can install with:" +echo " brew tap archivebox/archivebox" +echo " brew install archivebox" diff --git a/bin/release_deb.sh b/bin/release_deb.sh new file mode 100755 index 00000000..45779f5c --- /dev/null +++ b/bin/release_deb.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +cd "$REPO_DIR" + +VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" + +echo "[+] Releasing .deb package for archivebox==${VERSION}..." + +DEB_FILE="$(ls -1 "$REPO_DIR/dist/"archivebox*.deb 2>/dev/null | head -1)" +if [ -z "$DEB_FILE" ]; then + echo "[!] No .deb file found in dist/. Run ./bin/build_deb.sh first." + exit 1 +fi + +echo "[+] Uploading $DEB_FILE to GitHub Release v${VERSION}..." +gh release upload "v${VERSION}" "$DEB_FILE" --clobber 2>/dev/null || \ + gh release create "v${VERSION}" "$DEB_FILE" --title "v${VERSION}" --generate-notes + +echo "[√] .deb package uploaded to GitHub Release v${VERSION}" +echo " Users can install with:" +echo " curl -fsSL https://github.com/ArchiveBox/ArchiveBox/releases/download/v${VERSION}/archivebox_${VERSION}_amd64.deb -o /tmp/archivebox.deb" +echo " sudo apt install /tmp/archivebox.deb" diff --git a/bin/setup.sh b/bin/setup.sh index 5add55d4..f496749c 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -122,32 +122,24 @@ echo # On Linux: if which apt-get > /dev/null; then - echo "[+] Adding ArchiveBox apt repo and signing key to sources..." - if ! (sudo apt install -y software-properties-common && sudo add-apt-repository -u ppa:archivebox/archivebox); then - echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list - echo "deb-src http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee -a /etc/apt/sources.list.d/archivebox.list - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C258F79DCC02E369 - sudo apt-get update -qq - fi - echo echo "[+] Installing ArchiveBox system dependencies using apt..." - sudo apt-get install -y git python3 python3-pip python3-distutils wget curl yt-dlp ffmpeg git nodejs npm ripgrep - sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb libgbm-dev || sudo apt-get install -y chromium || sudo apt-get install -y chromium-browser || true - sudo apt-get install -y archivebox - sudo apt-get --only-upgrade install -y archivebox + sudo apt-get update -qq + sudo apt-get install -y git python3 python3-pip python3-venv wget curl yt-dlp ffmpeg git nodejs npm ripgrep + sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libnss3 libxss1 libasound2 libxtst6 xauth xvfb libgbm-dev || sudo apt-get install -y chromium || sudo apt-get install -y chromium-browser || true echo - echo "[+] Installing ArchiveBox python dependencies using pip3..." - sudo python3 -m pip install --upgrade --ignore-installed archivebox yt-dlp playwright + echo "[+] Downloading and installing ArchiveBox .deb package..." + ARCH="$(dpkg --print-architecture)" + DEB_URL="https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${ARCH}.deb" + curl -fsSL "$DEB_URL" -o /tmp/archivebox.deb && sudo apt install -y /tmp/archivebox.deb && rm /tmp/archivebox.deb || { + echo "[!] .deb install failed, falling back to pip install..." + sudo python3 -m pip install --upgrade archivebox yt-dlp + } # On Mac: elif which brew > /dev/null; then - echo "[+] Installing ArchiveBox system dependencies using brew..." + echo "[+] Installing ArchiveBox using Homebrew..." brew tap archivebox/archivebox brew update - brew install python3 node git wget curl yt-dlp ripgrep - brew install --fetch-HEAD -f archivebox - echo - echo "[+] Installing ArchiveBox python dependencies using pip3..." - python3 -m pip install --upgrade --ignore-installed archivebox yt-dlp playwright + brew install archivebox elif which pkg > /dev/null; then echo "[+] Installing ArchiveBox system dependencies using pkg and pip (python3.9)..." sudo pkg install -y python3 py39-pip py39-sqlite3 npm wget curl youtube_dl ffmpeg git ripgrep diff --git a/brew_dist/archivebox.rb b/brew_dist/archivebox.rb new file mode 100644 index 00000000..78aa8abc --- /dev/null +++ b/brew_dist/archivebox.rb @@ -0,0 +1,51 @@ +# This formula is auto-generated by bin/build_brew.sh using homebrew-pypi-poet. +# To update: run bin/build_brew.sh, or trigger the GitHub Actions homebrew workflow. +# +# Users install with: +# brew tap archivebox/archivebox +# brew install archivebox + +class Archivebox < Formula + include Language::Python::Virtualenv + + desc "Self-hosted internet archiving solution" + homepage "https://github.com/ArchiveBox/ArchiveBox" + url "https://files.pythonhosted.org/packages/source/a/archivebox/archivebox-0.9.3.tar.gz" + sha256 "" # auto-filled by bin/build_brew.sh + license "MIT" + head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" + + depends_on "python@3.13" + depends_on "node" + depends_on "git" + depends_on "wget" + depends_on "curl" + depends_on "ripgrep" + depends_on "yt-dlp" + + # Python dependency resource blocks are auto-generated by bin/build_brew.sh + # using homebrew-pypi-poet. Run that script to populate this section. + # AUTOGENERATED_RESOURCES_START + # AUTOGENERATED_RESOURCES_END + + def install + virtualenv_install_with_resources + end + + def post_install + # Install runtime dependencies (plugins, JS extractors, etc.) + system bin/"archivebox", "install", "--binproviders", "pip,npm" + end + + service do + run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] + keep_alive crashed: true + working_dir var/"archivebox" + log_path var/"log/archivebox.log" + error_log_path var/"log/archivebox.log" + end + + test do + assert_match version.to_s, shell_output("#{bin}/archivebox version") + end +end diff --git a/pkg/debian/archivebox b/pkg/debian/archivebox new file mode 100755 index 00000000..970b7c64 --- /dev/null +++ b/pkg/debian/archivebox @@ -0,0 +1,13 @@ +#!/bin/bash +# /usr/bin/archivebox - wrapper script installed by the archivebox .deb package +# Activates the pip-installed virtualenv and runs archivebox CLI + +ARCHIVEBOX_VENV="/opt/archivebox/venv" + +if [ ! -f "$ARCHIVEBOX_VENV/bin/archivebox" ]; then + echo "Error: ArchiveBox is not installed in $ARCHIVEBOX_VENV" + echo "Try running: sudo /opt/archivebox/install.sh" + exit 1 +fi + +exec "$ARCHIVEBOX_VENV/bin/archivebox" "$@" diff --git a/pkg/debian/archivebox.service b/pkg/debian/archivebox.service new file mode 100644 index 00000000..17f90872 --- /dev/null +++ b/pkg/debian/archivebox.service @@ -0,0 +1,16 @@ +[Unit] +Description=ArchiveBox Web Archiving Server +After=network.target + +[Service] +Type=simple +User=archivebox +Group=archivebox +WorkingDirectory=/var/lib/archivebox +ExecStartPre=/opt/archivebox/venv/bin/archivebox init --setup +ExecStart=/opt/archivebox/venv/bin/archivebox server 0.0.0.0:8000 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/pkg/debian/install.sh b/pkg/debian/install.sh new file mode 100755 index 00000000..20112bf1 --- /dev/null +++ b/pkg/debian/install.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# /opt/archivebox/install.sh - installs/upgrades archivebox into its virtualenv +# Called by the postinstall script and can be run manually to upgrade + +set -e + +ARCHIVEBOX_VENV="/opt/archivebox/venv" +ARCHIVEBOX_VERSION="${ARCHIVEBOX_VERSION:-}" + +echo "[+] Setting up ArchiveBox virtualenv in $ARCHIVEBOX_VENV..." + +# Create the virtualenv if it doesn't exist +if [ ! -d "$ARCHIVEBOX_VENV" ]; then + python3 -m venv "$ARCHIVEBOX_VENV" +fi + +# Upgrade pip inside the virtualenv +"$ARCHIVEBOX_VENV/bin/python3" -m pip install --quiet --upgrade pip setuptools + +# Install or upgrade archivebox +if [ -n "$ARCHIVEBOX_VERSION" ]; then + echo "[+] Installing archivebox==$ARCHIVEBOX_VERSION..." + "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade "archivebox==$ARCHIVEBOX_VERSION" +else + echo "[+] Installing latest archivebox..." + "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade archivebox +fi + +echo "[+] Installing archivebox runtime dependencies..." +"$ARCHIVEBOX_VENV/bin/archivebox" install --binproviders pip,npm 2>/dev/null || true + +echo "[√] ArchiveBox installed successfully." +echo " Run 'archivebox version' to verify." diff --git a/pkg/debian/nfpm.yaml b/pkg/debian/nfpm.yaml new file mode 100644 index 00000000..4b394553 --- /dev/null +++ b/pkg/debian/nfpm.yaml @@ -0,0 +1,67 @@ +# nFPM configuration for building ArchiveBox .deb packages +# Docs: https://nfpm.goreleaser.com/configuration/ +# Usage: nfpm package --config pkg/debian/nfpm.yaml --packager deb --target dist/ + +name: archivebox +arch: "${ARCH:-amd64}" +platform: linux +version: "${VERSION}" +version_schema: semver +maintainer: "Nick Sweeting " +description: | + Self-hosted internet archiving solution. + Save pages from the web including HTML, PDF, screenshots, media, and more. + Install with: sudo apt install archivebox && archivebox init --setup +vendor: "ArchiveBox" +homepage: "https://archivebox.io" +license: "MIT" +section: "web" +priority: "optional" + +depends: + - python3 (>= 3.11) + - python3-pip + - python3-venv + - nodejs + - npm + - git + - wget + - curl + - ripgrep + +recommends: + - yt-dlp + - ffmpeg + - chromium | chromium-browser | google-chrome-stable + +contents: + # Wrapper script for /usr/bin/archivebox + - src: pkg/debian/archivebox + dst: /usr/bin/archivebox + file_info: + mode: 0755 + + # Install helper script + - src: pkg/debian/install.sh + dst: /opt/archivebox/install.sh + file_info: + mode: 0755 + + # Systemd service file + - src: pkg/debian/archivebox.service + dst: /usr/lib/systemd/system/archivebox.service + file_info: + mode: 0644 + + # Create data directory + - dst: /var/lib/archivebox + type: dir + file_info: + mode: 0755 + +scripts: + postinstall: pkg/debian/scripts/postinstall.sh + preremove: pkg/debian/scripts/preremove.sh + +deb: + compression: zstd diff --git a/pkg/debian/scripts/postinstall.sh b/pkg/debian/scripts/postinstall.sh new file mode 100755 index 00000000..5768ef84 --- /dev/null +++ b/pkg/debian/scripts/postinstall.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# postinstall script for archivebox .deb package +set -e + +/opt/archivebox/install.sh diff --git a/pkg/debian/scripts/preremove.sh b/pkg/debian/scripts/preremove.sh new file mode 100755 index 00000000..65ef33f6 --- /dev/null +++ b/pkg/debian/scripts/preremove.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# preremove script for archivebox .deb package +set -e + +echo "[+] Removing ArchiveBox virtualenv..." +rm -rf /opt/archivebox/venv + +echo "[i] Your ArchiveBox data directories have NOT been removed." +echo " Remove them manually if you no longer need them." From c8f562ee377750d65486cf21502aa8a48c1294b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 02:50:14 +0000 Subject: [PATCH 02/19] Wire up GitHub Actions for deb/brew build, test, and release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix debian.yml: pin nfpm version, add permissions, improve test job with user creation, init test, and status check - Fix homebrew.yml: use PyPI JSON API (macOS-compatible, no grep -oP), wait for PyPI availability on release, use generated formula not template, add Linux (Linuxbrew) test job alongside macOS - Add release.yml orchestrator: pip → deb + brew + docker in order - Add workflow_call triggers to pip.yml and docker.yml - Fix build_brew.sh: replace grep -oP with Python-based PyPI API, add on_linux deps (pkg-config, openssl, libffi) - Fix setup.sh: use GitHub API to find correct .deb download URL (filename includes version number) - Fix postinstall.sh: create archivebox system user, pin version from package, check for systemd before daemon-reload - Fix preremove.sh: stop service before removal, check for systemd - Fix install.sh: fallback to latest if pinned version not on PyPI - Add on_linux deps to brew formula for Linuxbrew support - Tested: .deb builds, installs, creates user, runs archivebox init https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 53 ++++++++- .github/workflows/docker.yml | 1 + .github/workflows/homebrew.yml | 186 ++++++++++++++++++++++++++++-- .github/workflows/pip.yml | 1 + .github/workflows/release.yml | 40 +++++++ bin/build_brew.sh | 21 +++- bin/setup.sh | 15 ++- brew_dist/archivebox.rb | 6 + pkg/debian/install.sh | 5 +- pkg/debian/nfpm.yaml | 2 + pkg/debian/scripts/postinstall.sh | 20 ++++ pkg/debian/scripts/preremove.sh | 13 ++- 12 files changed, 336 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 709817a5..2749a22f 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -2,9 +2,13 @@ name: Build Debian package on: workflow_dispatch: + workflow_call: release: types: [published] +permissions: + contents: write + jobs: build: runs-on: ubuntu-24.04 @@ -22,8 +26,13 @@ jobs: run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT" - name: Install nfpm + env: + # Pin nfpm version for reproducible builds + NFPM_VERSION: "2.45.1" run: | - curl -sfL https://install.goreleaser.com/github.com/goreleaser/nfpm.sh | sh -s -- -b /usr/local/bin + curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \ + | sudo tar -xz -C /usr/local/bin nfpm + nfpm --version - name: Build .deb package run: | @@ -31,6 +40,18 @@ jobs: export ARCH="${{ matrix.arch }}" ./bin/build_deb.sh + - name: Verify .deb package contents + run: | + DEB_FILE="$(ls dist/archivebox*.deb | head -1)" + echo "=== Package info ===" + dpkg-deb --info "$DEB_FILE" + echo "" + echo "=== Package contents ===" + dpkg-deb --contents "$DEB_FILE" + echo "" + echo "=== Control fields ===" + dpkg-deb --field "$DEB_FILE" + - name: Upload .deb artifact uses: actions/upload-artifact@v4 with: @@ -55,14 +76,36 @@ jobs: pattern: archivebox-*-amd64.deb merge-multiple: true - - name: Install .deb package + - name: Install system dependencies run: | sudo apt-get update -qq sudo apt-get install -y python3 python3-pip python3-venv nodejs npm git wget curl ripgrep + + - name: Install .deb package + run: | sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y - - name: Test archivebox CLI + - name: Run postinstall setup run: | + # The postinstall script creates the venv and pip-installs archivebox + # If it didn't run during dpkg install, run it manually + if [ ! -f /opt/archivebox/venv/bin/archivebox ]; then + sudo /opt/archivebox/install.sh + fi + + - name: Verify archivebox is installed + run: | + which archivebox archivebox version - mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test - archivebox init --setup + + - name: Test archivebox init + run: | + # Create a test data directory and init + sudo useradd -r -s /bin/bash -d /var/lib/archivebox archivebox 2>/dev/null || true + sudo mkdir -p /tmp/archivebox-test + sudo chown archivebox:archivebox /tmp/archivebox-test + sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init --setup' + + - name: Test archivebox status + run: | + sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4b7a4554..074ec827 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,6 +2,7 @@ name: Build Docker image on: workflow_dispatch: + workflow_call: push: branches: - '**' diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index 6c5924ea..50877a20 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -2,9 +2,13 @@ name: Build Homebrew formula on: workflow_dispatch: + workflow_call: release: types: [published] +permissions: + contents: read + jobs: build: runs-on: macos-latest @@ -23,36 +27,141 @@ jobs: with: python-version: "3.13" + - name: Wait for PyPI package availability + if: github.event_name == 'release' + run: | + VERSION="${{ steps.version.outputs.version }}" + echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..." + for i in $(seq 1 30); do + if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then + echo "[√] archivebox==${VERSION} is available on PyPI" + break + fi + if [ "$i" -eq 30 ]; then + echo "[!] Timed out waiting for PyPI. Trying to install anyway..." + break + fi + echo " Attempt $i/30 - not yet available, waiting 30s..." + sleep 30 + done + - name: Generate Homebrew formula run: | + VERSION="${{ steps.version.outputs.version }}" python3 -m venv /tmp/poet-venv source /tmp/poet-venv/bin/activate - pip install --quiet "archivebox==${{ steps.version.outputs.version }}" homebrew-pypi-poet - poet -f archivebox > /tmp/archivebox-generated.rb + + pip install --quiet "archivebox==${VERSION}" homebrew-pypi-poet + + echo "[+] Generating resource stanzas with homebrew-pypi-poet..." + RESOURCES="$(poet archivebox)" + + # Get sdist URL and SHA256 from PyPI JSON API + PYPI_JSON="$(curl -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" 2>/dev/null || echo '')" + SDIST_URL="" + SDIST_SHA256="" + if [ -n "$PYPI_JSON" ]; then + SDIST_URL="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" + SDIST_SHA256="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['digests']['sha256'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" + fi + if [ -z "$SDIST_URL" ]; then + SDIST_URL="https://files.pythonhosted.org/packages/source/a/archivebox/archivebox-${VERSION}.tar.gz" + fi + if [ -z "$SDIST_SHA256" ]; then + pip download --no-binary :all: --no-deps -d /tmp/sdist "archivebox==${VERSION}" 2>/dev/null || true + SDIST_SHA256="$(shasum -a 256 /tmp/sdist/*.tar.gz 2>/dev/null | awk '{print $1}' || echo '')" + fi + deactivate + # Generate the formula file + cat > /tmp/archivebox.rb << RUBY + # Auto-generated Homebrew formula for archivebox ${VERSION} + # Generated by GitHub Actions homebrew workflow using homebrew-pypi-poet + + class Archivebox < Formula + include Language::Python::Virtualenv + + desc "Self-hosted internet archiving solution" + homepage "https://github.com/ArchiveBox/ArchiveBox" + url "${SDIST_URL}" + sha256 "${SDIST_SHA256}" + license "MIT" + head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" + + depends_on "python@3.13" + depends_on "node" + depends_on "git" + depends_on "wget" + depends_on "curl" + depends_on "ripgrep" + depends_on "yt-dlp" + + on_linux do + depends_on "pkg-config" => :build + depends_on "openssl@3" + depends_on "libffi" + end + + # Python dependency resource blocks auto-generated by homebrew-pypi-poet + ${RESOURCES} + + def install + virtualenv_install_with_resources + end + + def post_install + system bin/"archivebox", "install", "--binproviders", "pip,npm" + end + + service do + run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] + keep_alive crashed: true + working_dir var/"archivebox" + log_path var/"log/archivebox.log" + error_log_path var/"log/archivebox.log" + end + + test do + assert_match version.to_s, shell_output("#{bin}/archivebox version") + end + end + RUBY + + echo "[√] Generated formula at /tmp/archivebox.rb" + cat /tmp/archivebox.rb + - name: Upload formula artifact uses: actions/upload-artifact@v4 with: name: archivebox.rb - path: /tmp/archivebox-generated.rb + path: /tmp/archivebox.rb - name: Push to homebrew-archivebox tap if: github.event_name == 'release' env: GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} run: | - # Clone the tap repo and update the formula + VERSION="${{ steps.version.outputs.version }}" git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap - cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb 2>/dev/null || cp /tmp/archivebox-generated.rb /tmp/tap/archivebox.rb + + # Use the generated formula (with real resources), NOT the template + cp /tmp/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \ + cp /tmp/archivebox.rb /tmp/tap/archivebox.rb + cd /tmp/tap git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git add archivebox.rb - git diff --cached --quiet || git commit -m "Update archivebox to v${{ steps.version.outputs.version }}" - git push origin HEAD + git add -A + if git diff --cached --quiet; then + echo "[i] No changes to formula, skipping push." + else + git commit -m "Update archivebox to v${VERSION}" + git push origin HEAD + echo "[√] Formula pushed to homebrew-archivebox tap" + fi - test: + test-macos: needs: build runs-on: macos-latest @@ -65,7 +174,62 @@ jobs: name: archivebox.rb path: /tmp/ - - name: Test Homebrew formula + - name: Install dependencies + run: | + brew install python@3.13 node git wget curl ripgrep yt-dlp + + - name: Test Homebrew formula install + run: | + brew install --build-from-source /tmp/archivebox.rb + + - name: Verify archivebox CLI run: | - brew install --build-from-source /tmp/archivebox-generated.rb || true archivebox version + + - name: Test archivebox init + run: | + mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test + archivebox init --setup + archivebox status + + test-linux: + needs: build + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + + - name: Download formula artifact + uses: actions/download-artifact@v4 + with: + name: archivebox.rb + path: /tmp/ + + - name: Install Homebrew on Linux + run: | + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo >> "$GITHUB_ENV" + echo 'HOMEBREW_PREFIX=/home/linuxbrew/.linuxbrew' >> "$GITHUB_ENV" + echo '/home/linuxbrew/.linuxbrew/bin' >> "$GITHUB_PATH" + + - name: Install dependencies + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + brew install python@3.13 node git wget curl ripgrep yt-dlp pkg-config openssl@3 libffi + + - name: Test Homebrew formula install + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + brew install --build-from-source /tmp/archivebox.rb + + - name: Verify archivebox CLI + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + archivebox version + + - name: Test archivebox init + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test + archivebox init --setup + archivebox status diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 434e0db5..f4e75503 100755 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -2,6 +2,7 @@ name: Build Pip package on: workflow_dispatch: + workflow_call: push: branches: - '**' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..41f1638c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,40 @@ +name: Release + +# Orchestrates the full release pipeline: +# 1. Build and publish pip package to PyPI +# 2. Build .deb packages and Homebrew formula (in parallel, after pip) +# 3. Build Docker images (in parallel with deb/brew) + +on: + workflow_dispatch: + release: + types: [published] + +permissions: + contents: write + packages: write + id-token: write + +jobs: + pip: + name: Publish to PyPI + uses: ./.github/workflows/pip.yml + secrets: inherit + + debian: + name: Build .deb packages + needs: pip + uses: ./.github/workflows/debian.yml + secrets: inherit + + homebrew: + name: Update Homebrew formula + needs: pip + uses: ./.github/workflows/homebrew.yml + secrets: inherit + + docker: + name: Build Docker images + needs: pip + uses: ./.github/workflows/docker.yml + secrets: inherit diff --git a/bin/build_brew.sh b/bin/build_brew.sh index e56c2a2b..828a22d9 100755 --- a/bin/build_brew.sh +++ b/bin/build_brew.sh @@ -30,12 +30,21 @@ pip install --quiet "archivebox==${VERSION}" homebrew-pypi-poet 2>/dev/null echo "[+] Generating resource stanzas with homebrew-pypi-poet..." RESOURCES="$(poet archivebox)" -# Get the sdist URL and SHA256 from PyPI -SDIST_URL="$(pip download --no-binary :all: --no-deps -d "$TMPDIR/sdist" "archivebox==${VERSION}" 2>&1 | grep -oP 'https://\S+\.tar\.gz' | head -1 || true)" +# Get the sdist URL and SHA256 from PyPI JSON API (works on macOS and Linux) +PYPI_JSON="$(curl -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" 2>/dev/null || echo '')" +if [ -n "$PYPI_JSON" ]; then + SDIST_URL="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" + SDIST_SHA256="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['digests']['sha256'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" +fi if [ -z "$SDIST_URL" ]; then SDIST_URL="https://files.pythonhosted.org/packages/source/a/archivebox/archivebox-${VERSION}.tar.gz" fi -SDIST_SHA256="$(pip hash "$TMPDIR/sdist/"*.tar.gz 2>/dev/null | grep 'sha256:' | cut -d: -f2 || echo '')" +if [ -z "$SDIST_SHA256" ]; then + # Fallback: download and compute locally + mkdir -p "$TMPDIR/sdist" + pip download --no-binary :all: --no-deps -d "$TMPDIR/sdist" "archivebox==${VERSION}" 2>/dev/null || true + SDIST_SHA256="$(shasum -a 256 "$TMPDIR/sdist/"*.tar.gz 2>/dev/null | awk '{print $1}' || echo '')" +fi deactivate @@ -68,6 +77,12 @@ class Archivebox < Formula depends_on "ripgrep" depends_on "yt-dlp" + on_linux do + depends_on "pkg-config" => :build + depends_on "openssl@3" + depends_on "libffi" + end + # Python dependency resource blocks auto-generated by homebrew-pypi-poet # AUTOGENERATED_RESOURCES_START ${RESOURCES} diff --git a/bin/setup.sh b/bin/setup.sh index f496749c..aefcffd8 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -129,11 +129,16 @@ if which apt-get > /dev/null; then echo echo "[+] Downloading and installing ArchiveBox .deb package..." ARCH="$(dpkg --print-architecture)" - DEB_URL="https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${ARCH}.deb" - curl -fsSL "$DEB_URL" -o /tmp/archivebox.deb && sudo apt install -y /tmp/archivebox.deb && rm /tmp/archivebox.deb || { - echo "[!] .deb install failed, falling back to pip install..." - sudo python3 -m pip install --upgrade archivebox yt-dlp - } + # Get the latest release .deb URL from GitHub API (filename includes version) + DEB_URL="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest \ + | grep -o "\"browser_download_url\": \"[^\"]*_${ARCH}\.deb\"" \ + | head -1 | cut -d'"' -f4)" || true + if [ -n "$DEB_URL" ]; then + curl -fsSL "$DEB_URL" -o /tmp/archivebox.deb && sudo apt install -y /tmp/archivebox.deb && rm -f /tmp/archivebox.deb + else + echo "[!] Could not find .deb download URL, falling back to pip install..." + pip install --upgrade archivebox yt-dlp + fi # On Mac: elif which brew > /dev/null; then echo "[+] Installing ArchiveBox using Homebrew..." diff --git a/brew_dist/archivebox.rb b/brew_dist/archivebox.rb index 78aa8abc..47c8fb92 100644 --- a/brew_dist/archivebox.rb +++ b/brew_dist/archivebox.rb @@ -23,6 +23,12 @@ class Archivebox < Formula depends_on "ripgrep" depends_on "yt-dlp" + on_linux do + depends_on "pkg-config" => :build + depends_on "openssl@3" + depends_on "libffi" + end + # Python dependency resource blocks are auto-generated by bin/build_brew.sh # using homebrew-pypi-poet. Run that script to populate this section. # AUTOGENERATED_RESOURCES_START diff --git a/pkg/debian/install.sh b/pkg/debian/install.sh index 20112bf1..54c593f5 100755 --- a/pkg/debian/install.sh +++ b/pkg/debian/install.sh @@ -20,7 +20,10 @@ fi # Install or upgrade archivebox if [ -n "$ARCHIVEBOX_VERSION" ]; then echo "[+] Installing archivebox==$ARCHIVEBOX_VERSION..." - "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade "archivebox==$ARCHIVEBOX_VERSION" + "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade "archivebox==$ARCHIVEBOX_VERSION" || { + echo "[!] archivebox==$ARCHIVEBOX_VERSION not found on PyPI, installing latest..." + "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade archivebox + } else echo "[+] Installing latest archivebox..." "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade archivebox diff --git a/pkg/debian/nfpm.yaml b/pkg/debian/nfpm.yaml index 4b394553..8fc55afc 100644 --- a/pkg/debian/nfpm.yaml +++ b/pkg/debian/nfpm.yaml @@ -19,6 +19,8 @@ section: "web" priority: "optional" depends: + # python3 >= 3.11 allows .deb to install on more systems; + # pip enforces the actual Python version requirement from pyproject.toml - python3 (>= 3.11) - python3-pip - python3-venv diff --git a/pkg/debian/scripts/postinstall.sh b/pkg/debian/scripts/postinstall.sh index 5768ef84..a59a1537 100755 --- a/pkg/debian/scripts/postinstall.sh +++ b/pkg/debian/scripts/postinstall.sh @@ -2,4 +2,24 @@ # postinstall script for archivebox .deb package set -e +# Create archivebox system user if it doesn't exist +if ! id -u archivebox >/dev/null 2>&1; then + useradd --system --shell /bin/bash --home-dir /var/lib/archivebox --create-home archivebox + echo "[+] Created archivebox system user" +fi + +# Ensure data directory exists and is owned by archivebox +mkdir -p /var/lib/archivebox +chown archivebox:archivebox /var/lib/archivebox + +# Run the virtualenv install script, pinning to the .deb package version +ARCHIVEBOX_VERSION="$(dpkg-query -W -f='${Version}' archivebox 2>/dev/null || echo '')" +export ARCHIVEBOX_VERSION /opt/archivebox/install.sh + +# Reload systemd to pick up the service file (skip if systemd is not running) +if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + systemctl daemon-reload + echo "[i] To start ArchiveBox: sudo systemctl start archivebox" + echo "[i] To enable on boot: sudo systemctl enable archivebox" +fi diff --git a/pkg/debian/scripts/preremove.sh b/pkg/debian/scripts/preremove.sh index 65ef33f6..81120cd5 100755 --- a/pkg/debian/scripts/preremove.sh +++ b/pkg/debian/scripts/preremove.sh @@ -2,8 +2,17 @@ # preremove script for archivebox .deb package set -e +# Stop the service if running +if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + systemctl stop archivebox 2>/dev/null || true + systemctl disable archivebox 2>/dev/null || true +fi + echo "[+] Removing ArchiveBox virtualenv..." rm -rf /opt/archivebox/venv -echo "[i] Your ArchiveBox data directories have NOT been removed." -echo " Remove them manually if you no longer need them." +echo "[i] Your ArchiveBox data in /var/lib/archivebox has NOT been removed." +echo " The 'archivebox' system user has NOT been removed." +echo " Remove them manually if you no longer need them:" +echo " sudo rm -rf /var/lib/archivebox" +echo " sudo userdel archivebox" From fa11bee5b564022d6aa281e77186a335f2e1ed79 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 02:55:10 +0000 Subject: [PATCH 03/19] CI: Full brew install + deb install tested on every push - homebrew.yml: Build local sdist, generate formula with file:// URL and real resource stanzas via homebrew-pypi-poet, run full `brew install --build-from-source` on both macOS and Linux (Linuxbrew) - debian.yml: Pre-seed venv with local wheel before dpkg install so postinstall succeeds even for unreleased versions; test init/status/add - Both workflows trigger on push (path-filtered) and release - Release job generates formula with PyPI URL and pushes to tap https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 75 ++++++-- .github/workflows/homebrew.yml | 338 +++++++++++++++++++++------------ .github/workflows/release.yml | 3 + 3 files changed, 281 insertions(+), 135 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 2749a22f..93b185eb 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -3,6 +3,14 @@ name: Build Debian package on: workflow_dispatch: workflow_call: + push: + branches: ['**'] + paths: + - 'pkg/debian/**' + - 'bin/build_deb.sh' + - 'bin/release_deb.sh' + - '.github/workflows/debian.yml' + - 'pyproject.toml' release: types: [published] @@ -27,7 +35,6 @@ jobs: - name: Install nfpm env: - # Pin nfpm version for reproducible builds NFPM_VERSION: "2.45.1" run: | curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \ @@ -70,6 +77,29 @@ jobs: runs-on: ubuntu-24.04 steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install build dependencies + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 + version: 1.0 + + - name: Build local wheel + run: | + uv sync --frozen --all-extras --no-install-project --no-install-workspace + uv build --wheel --out-dir /tmp/wheels/ + - name: Download .deb artifact uses: actions/download-artifact@v4 with: @@ -79,29 +109,39 @@ jobs: - name: Install system dependencies run: | sudo apt-get update -qq - sudo apt-get install -y python3 python3-pip python3-venv nodejs npm git wget curl ripgrep + sudo apt-get install -y python3.13 python3.13-venv python3-pip nodejs npm git wget curl ripgrep + + - name: Pre-seed virtualenv with local wheel before dpkg install + run: | + # Create the venv and install from local wheel BEFORE dpkg runs postinstall. + # This way the postinstall's pip install either succeeds (release) or + # finds archivebox already installed (CI) and just upgrades deps. + sudo mkdir -p /opt/archivebox + sudo python3.13 -m venv /opt/archivebox/venv + sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools + sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl + echo "[√] Pre-seeded /opt/archivebox/venv with local wheel" - name: Install .deb package run: | + # dpkg install will run postinstall which creates the user, + # sets up systemd, and tries pip install (which finds it already installed) sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y - - name: Run postinstall setup - run: | - # The postinstall script creates the venv and pip-installs archivebox - # If it didn't run during dpkg install, run it manually - if [ ! -f /opt/archivebox/venv/bin/archivebox ]; then - sudo /opt/archivebox/install.sh - fi - - name: Verify archivebox is installed run: | which archivebox archivebox version - - name: Test archivebox init + - name: Verify wrapper script works run: | - # Create a test data directory and init - sudo useradd -r -s /bin/bash -d /var/lib/archivebox archivebox 2>/dev/null || true + /usr/bin/archivebox version + /usr/bin/archivebox --help | head -5 + + - name: Test archivebox init as archivebox user + run: | + # The postinstall should have created the user + id archivebox sudo mkdir -p /tmp/archivebox-test sudo chown archivebox:archivebox /tmp/archivebox-test sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init --setup' @@ -109,3 +149,12 @@ jobs: - name: Test archivebox status run: | sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status' + + - name: Test archivebox add + run: | + sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add --parser url_list "https://example.com"' || true + + - name: Verify systemd service file exists + run: | + test -f /usr/lib/systemd/system/archivebox.service + cat /usr/lib/systemd/system/archivebox.service diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index 50877a20..19098b2b 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -3,6 +3,14 @@ name: Build Homebrew formula on: workflow_dispatch: workflow_call: + push: + branches: ['**'] + paths: + - 'brew_dist/**' + - 'bin/build_brew.sh' + - 'bin/release_brew.sh' + - '.github/workflows/homebrew.yml' + - 'pyproject.toml' release: types: [published] @@ -10,7 +18,161 @@ permissions: contents: read jobs: - build: + build-and-test: + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-24.04] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Get version + id: version + run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Validate formula template syntax + run: ruby -c brew_dist/archivebox.rb + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install build dependencies (Linux) + if: runner.os == 'Linux' + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 + version: 1.0 + + - name: Build local sdist + run: | + uv sync --frozen --all-extras --no-install-project --no-install-workspace + uv build --sdist --out-dir /tmp/sdist/ + + - name: Generate formula from local sdist + run: | + VERSION="${{ steps.version.outputs.version }}" + SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)" + SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')" + + # Install archivebox + poet into a temp venv to generate resource stanzas + python3 -m venv /tmp/poet-venv + source /tmp/poet-venv/bin/activate + pip install --quiet "$SDIST_PATH" homebrew-pypi-poet + echo "[+] Generating resource stanzas with homebrew-pypi-poet..." + RESOURCES="$(poet archivebox)" + deactivate + + # For CI: use file:// URL pointing to local sdist + # For release: this gets overridden with the PyPI URL + SDIST_URL="file://${SDIST_PATH}" + + cat > /tmp/archivebox.rb << RUBY +class Archivebox < Formula + include Language::Python::Virtualenv + + desc "Self-hosted internet archiving solution" + homepage "https://github.com/ArchiveBox/ArchiveBox" + url "${SDIST_URL}" + sha256 "${SDIST_SHA256}" + license "MIT" + + depends_on "python@3.13" + depends_on "node" + depends_on "git" + depends_on "wget" + depends_on "curl" + depends_on "ripgrep" + depends_on "yt-dlp" + + on_linux do + depends_on "pkg-config" => :build + depends_on "openssl@3" + depends_on "libffi" + end + +${RESOURCES} + + def install + virtualenv_install_with_resources + end + + def post_install + system bin/"archivebox", "install", "--binproviders", "pip,npm" + end + + service do + run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] + keep_alive crashed: true + working_dir var/"archivebox" + log_path var/"log/archivebox.log" + error_log_path var/"log/archivebox.log" + end + + test do + assert_match version.to_s, shell_output("#{bin}/archivebox version") + end +end +RUBY + + echo "[√] Generated formula:" + ruby -c /tmp/archivebox.rb + cat /tmp/archivebox.rb + + - name: Install Homebrew (Linux only) + if: runner.os == 'Linux' + run: | + NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo '/home/linuxbrew/.linuxbrew/bin' >> "$GITHUB_PATH" + + - name: Set up brew shellenv (Linux only) + if: runner.os == 'Linux' + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV" + echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV" + echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV" + echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH" + echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH" + + - name: Install brew dependencies + run: | + brew install python@3.13 node git wget curl ripgrep yt-dlp + if [ "$RUNNER_OS" = "Linux" ]; then + brew install pkg-config openssl@3 libffi + fi + + - name: Install archivebox via brew from local formula + run: | + brew install --build-from-source --verbose /tmp/archivebox.rb + + - name: Verify archivebox version + run: archivebox version + + - name: Test archivebox init + run: | + mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test + archivebox init --setup + + - name: Test archivebox status + run: | + cd /tmp/archivebox-test + archivebox status + + # On release only: generate the real formula with PyPI URL and push to tap + release: + if: github.event_name == 'release' + needs: build-and-test runs-on: macos-latest steps: @@ -28,7 +190,6 @@ jobs: python-version: "3.13" - name: Wait for PyPI package availability - if: github.event_name == 'release' run: | VERSION="${{ steps.version.outputs.version }}" echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..." @@ -45,7 +206,7 @@ jobs: sleep 30 done - - name: Generate Homebrew formula + - name: Generate release formula with PyPI URL run: | VERSION="${{ steps.version.outputs.version }}" python3 -m venv /tmp/poet-venv @@ -74,61 +235,64 @@ jobs: deactivate - # Generate the formula file cat > /tmp/archivebox.rb << RUBY - # Auto-generated Homebrew formula for archivebox ${VERSION} - # Generated by GitHub Actions homebrew workflow using homebrew-pypi-poet +# Auto-generated Homebrew formula for archivebox ${VERSION} +# Generated by GitHub Actions on release using homebrew-pypi-poet +# +# Users install with: +# brew tap archivebox/archivebox +# brew install archivebox - class Archivebox < Formula - include Language::Python::Virtualenv +class Archivebox < Formula + include Language::Python::Virtualenv - desc "Self-hosted internet archiving solution" - homepage "https://github.com/ArchiveBox/ArchiveBox" - url "${SDIST_URL}" - sha256 "${SDIST_SHA256}" - license "MIT" - head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" + desc "Self-hosted internet archiving solution" + homepage "https://github.com/ArchiveBox/ArchiveBox" + url "${SDIST_URL}" + sha256 "${SDIST_SHA256}" + license "MIT" + head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" - depends_on "python@3.13" - depends_on "node" - depends_on "git" - depends_on "wget" - depends_on "curl" - depends_on "ripgrep" - depends_on "yt-dlp" + depends_on "python@3.13" + depends_on "node" + depends_on "git" + depends_on "wget" + depends_on "curl" + depends_on "ripgrep" + depends_on "yt-dlp" - on_linux do - depends_on "pkg-config" => :build - depends_on "openssl@3" - depends_on "libffi" - end + on_linux do + depends_on "pkg-config" => :build + depends_on "openssl@3" + depends_on "libffi" + end - # Python dependency resource blocks auto-generated by homebrew-pypi-poet - ${RESOURCES} +${RESOURCES} - def install - virtualenv_install_with_resources - end + def install + virtualenv_install_with_resources + end - def post_install - system bin/"archivebox", "install", "--binproviders", "pip,npm" - end + def post_install + system bin/"archivebox", "install", "--binproviders", "pip,npm" + end - service do - run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] - keep_alive crashed: true - working_dir var/"archivebox" - log_path var/"log/archivebox.log" - error_log_path var/"log/archivebox.log" - end + service do + run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] + keep_alive crashed: true + working_dir var/"archivebox" + log_path var/"log/archivebox.log" + error_log_path var/"log/archivebox.log" + end - test do - assert_match version.to_s, shell_output("#{bin}/archivebox version") - end - end - RUBY + test do + assert_match version.to_s, shell_output("#{bin}/archivebox version") + end +end +RUBY - echo "[√] Generated formula at /tmp/archivebox.rb" + echo "[√] Generated release formula:" + ruby -c /tmp/archivebox.rb cat /tmp/archivebox.rb - name: Upload formula artifact @@ -137,15 +301,18 @@ jobs: name: archivebox.rb path: /tmp/archivebox.rb + - name: Test formula install + run: | + brew install --build-from-source /tmp/archivebox.rb + archivebox version + - name: Push to homebrew-archivebox tap - if: github.event_name == 'release' env: GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} run: | VERSION="${{ steps.version.outputs.version }}" git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap - # Use the generated formula (with real resources), NOT the template cp /tmp/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \ cp /tmp/archivebox.rb /tmp/tap/archivebox.rb @@ -160,76 +327,3 @@ jobs: git push origin HEAD echo "[√] Formula pushed to homebrew-archivebox tap" fi - - test-macos: - needs: build - runs-on: macos-latest - - steps: - - uses: actions/checkout@v4 - - - name: Download formula artifact - uses: actions/download-artifact@v4 - with: - name: archivebox.rb - path: /tmp/ - - - name: Install dependencies - run: | - brew install python@3.13 node git wget curl ripgrep yt-dlp - - - name: Test Homebrew formula install - run: | - brew install --build-from-source /tmp/archivebox.rb - - - name: Verify archivebox CLI - run: | - archivebox version - - - name: Test archivebox init - run: | - mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test - archivebox init --setup - archivebox status - - test-linux: - needs: build - runs-on: ubuntu-24.04 - - steps: - - uses: actions/checkout@v4 - - - name: Download formula artifact - uses: actions/download-artifact@v4 - with: - name: archivebox.rb - path: /tmp/ - - - name: Install Homebrew on Linux - run: | - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo >> "$GITHUB_ENV" - echo 'HOMEBREW_PREFIX=/home/linuxbrew/.linuxbrew' >> "$GITHUB_ENV" - echo '/home/linuxbrew/.linuxbrew/bin' >> "$GITHUB_PATH" - - - name: Install dependencies - run: | - eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" - brew install python@3.13 node git wget curl ripgrep yt-dlp pkg-config openssl@3 libffi - - - name: Test Homebrew formula install - run: | - eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" - brew install --build-from-source /tmp/archivebox.rb - - - name: Verify archivebox CLI - run: | - eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" - archivebox version - - - name: Test archivebox init - run: | - eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" - mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test - archivebox init --setup - archivebox status diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 41f1638c..86a507fa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,9 @@ name: Release # 1. Build and publish pip package to PyPI # 2. Build .deb packages and Homebrew formula (in parallel, after pip) # 3. Build Docker images (in parallel with deb/brew) +# +# Individual workflows also run on push for CI (see their own triggers). +# This workflow ensures the correct ordering during a release. on: workflow_dispatch: From 4c113f8eb9f9b80dd0fd9992965d25f6d65a09e9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 03:02:47 +0000 Subject: [PATCH 04/19] Fix CI: create tests/out dir, fix archivebox add cmd, revert setup.sh - test-parallel.yml: mkdir -p tests/out before pytest --basetemp (fixes FileNotFoundError in chrome test fixture) - debian.yml: fix archivebox add command (--parser url_list removed in 0.9.x), remove || true so failures are caught - setup.sh: revert apt section to always use pip install, not .deb https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 2 +- .github/workflows/test-parallel.yml | 1 + bin/setup.sh | 14 ++------------ 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 93b185eb..f7dfdb24 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -152,7 +152,7 @@ jobs: - name: Test archivebox add run: | - sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add --parser url_list "https://example.com"' || true + sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"' - name: Verify systemd service file exists run: | diff --git a/.github/workflows/test-parallel.yml b/.github/workflows/test-parallel.yml index 77db7ac6..bb7855b1 100644 --- a/.github/workflows/test-parallel.yml +++ b/.github/workflows/test-parallel.yml @@ -115,4 +115,5 @@ jobs: - name: Run test - ${{ matrix.test.name }} run: | + mkdir -p tests/out uv run pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs diff --git a/bin/setup.sh b/bin/setup.sh index aefcffd8..91cd6681 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -127,18 +127,8 @@ if which apt-get > /dev/null; then sudo apt-get install -y git python3 python3-pip python3-venv wget curl yt-dlp ffmpeg git nodejs npm ripgrep sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libnss3 libxss1 libasound2 libxtst6 xauth xvfb libgbm-dev || sudo apt-get install -y chromium || sudo apt-get install -y chromium-browser || true echo - echo "[+] Downloading and installing ArchiveBox .deb package..." - ARCH="$(dpkg --print-architecture)" - # Get the latest release .deb URL from GitHub API (filename includes version) - DEB_URL="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest \ - | grep -o "\"browser_download_url\": \"[^\"]*_${ARCH}\.deb\"" \ - | head -1 | cut -d'"' -f4)" || true - if [ -n "$DEB_URL" ]; then - curl -fsSL "$DEB_URL" -o /tmp/archivebox.deb && sudo apt install -y /tmp/archivebox.deb && rm -f /tmp/archivebox.deb - else - echo "[!] Could not find .deb download URL, falling back to pip install..." - pip install --upgrade archivebox yt-dlp - fi + echo "[+] Installing ArchiveBox python dependencies using pip3..." + sudo python3 -m pip install --upgrade --ignore-installed archivebox yt-dlp # On Mac: elif which brew > /dev/null; then echo "[+] Installing ArchiveBox using Homebrew..." From 16090944c4870f17da66ea8732a70b73f45eb97d Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 14 Mar 2026 23:05:43 -0400 Subject: [PATCH 05/19] Update .github/workflows/homebrew.yml Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- .github/workflows/homebrew.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index 19098b2b..7d79acbc 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -162,7 +162,7 @@ RUBY - name: Test archivebox init run: | mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test - archivebox init --setup + archivebox init --install - name: Test archivebox status run: | From 7c7a9ee599dda2f9ca8dcd2ba24b488f15ae991c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 03:12:15 +0000 Subject: [PATCH 06/19] Fix PR review comments: service flags, DATA_DIR, version pinning, upgrade safety - Remove --setup flag from systemd service and CI (not valid in 0.9.x) - Remove release triggers from debian/homebrew workflows (handled by release.yml) - Fix brew post_install to set DATA_DIR so it initializes in var/archivebox - Add PATH export to deb wrapper script for bundled console scripts - Remove pip install fallback in install.sh (strict version pinning) - Guard preremove.sh cleanup to only run on remove/purge, not upgrade - Initialize SDIST_URL/SDIST_SHA256 in build_brew.sh (nounset safety) - Pin awalsh128/cache-apt-pkgs-action to v1.6.0 (supply chain safety) https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 7 +++---- .github/workflows/homebrew.yml | 13 +++++++------ bin/build_brew.sh | 5 ++++- brew_dist/archivebox.rb | 3 ++- pkg/debian/archivebox | 2 ++ pkg/debian/archivebox.service | 2 +- pkg/debian/install.sh | 10 ++-------- pkg/debian/scripts/preremove.sh | 29 ++++++++++++++++------------- 8 files changed, 37 insertions(+), 34 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index f7dfdb24..b3c9abad 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -11,8 +11,7 @@ on: - 'bin/release_deb.sh' - '.github/workflows/debian.yml' - 'pyproject.toml' - release: - types: [published] + # release trigger is handled by release.yml to avoid double-runs permissions: contents: write @@ -90,7 +89,7 @@ jobs: enable-cache: true - name: Install build dependencies - uses: awalsh128/cache-apt-pkgs-action@latest + uses: awalsh128/cache-apt-pkgs-action@v1.6.0 with: packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 version: 1.0 @@ -144,7 +143,7 @@ jobs: id archivebox sudo mkdir -p /tmp/archivebox-test sudo chown archivebox:archivebox /tmp/archivebox-test - sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init --setup' + sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init' - name: Test archivebox status run: | diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index 7d79acbc..838b323b 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -11,8 +11,7 @@ on: - 'bin/release_brew.sh' - '.github/workflows/homebrew.yml' - 'pyproject.toml' - release: - types: [published] + # release trigger is handled by release.yml to avoid double-runs permissions: contents: read @@ -49,7 +48,7 @@ jobs: - name: Install build dependencies (Linux) if: runner.os == 'Linux' - uses: awalsh128/cache-apt-pkgs-action@latest + uses: awalsh128/cache-apt-pkgs-action@v1.6.0 with: packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 version: 1.0 @@ -108,7 +107,8 @@ ${RESOURCES} end def post_install - system bin/"archivebox", "install", "--binproviders", "pip,npm" + (var/"archivebox").mkpath + system({ "DATA_DIR" => var/"archivebox" }, bin/"archivebox", "install", "--binproviders", "pip,npm") end service do @@ -162,7 +162,7 @@ RUBY - name: Test archivebox init run: | mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test - archivebox init --install + archivebox init - name: Test archivebox status run: | @@ -274,7 +274,8 @@ ${RESOURCES} end def post_install - system bin/"archivebox", "install", "--binproviders", "pip,npm" + (var/"archivebox").mkpath + system({ "DATA_DIR" => var/"archivebox" }, bin/"archivebox", "install", "--binproviders", "pip,npm") end service do diff --git a/bin/build_brew.sh b/bin/build_brew.sh index 828a22d9..b10bad47 100755 --- a/bin/build_brew.sh +++ b/bin/build_brew.sh @@ -31,6 +31,8 @@ echo "[+] Generating resource stanzas with homebrew-pypi-poet..." RESOURCES="$(poet archivebox)" # Get the sdist URL and SHA256 from PyPI JSON API (works on macOS and Linux) +SDIST_URL="" +SDIST_SHA256="" PYPI_JSON="$(curl -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" 2>/dev/null || echo '')" if [ -n "$PYPI_JSON" ]; then SDIST_URL="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" @@ -94,7 +96,8 @@ ${RESOURCES} def post_install # Install runtime dependencies (plugins, JS extractors, etc.) - system bin/"archivebox", "install", "--binproviders", "pip,npm" + (var/"archivebox").mkpath + system({ "DATA_DIR" => var/"archivebox" }, bin/"archivebox", "install", "--binproviders", "pip,npm") end service do diff --git a/brew_dist/archivebox.rb b/brew_dist/archivebox.rb index 47c8fb92..c0521d7a 100644 --- a/brew_dist/archivebox.rb +++ b/brew_dist/archivebox.rb @@ -40,7 +40,8 @@ class Archivebox < Formula def post_install # Install runtime dependencies (plugins, JS extractors, etc.) - system bin/"archivebox", "install", "--binproviders", "pip,npm" + (var/"archivebox").mkpath + system({ "DATA_DIR" => var/"archivebox" }, bin/"archivebox", "install", "--binproviders", "pip,npm") end service do diff --git a/pkg/debian/archivebox b/pkg/debian/archivebox index 970b7c64..0ebccabc 100755 --- a/pkg/debian/archivebox +++ b/pkg/debian/archivebox @@ -10,4 +10,6 @@ if [ ! -f "$ARCHIVEBOX_VENV/bin/archivebox" ]; then exit 1 fi +# Export venv bin to PATH so bundled console scripts (yt-dlp, etc.) are discoverable +export PATH="$ARCHIVEBOX_VENV/bin:$PATH" exec "$ARCHIVEBOX_VENV/bin/archivebox" "$@" diff --git a/pkg/debian/archivebox.service b/pkg/debian/archivebox.service index 17f90872..916f2cd9 100644 --- a/pkg/debian/archivebox.service +++ b/pkg/debian/archivebox.service @@ -7,7 +7,7 @@ Type=simple User=archivebox Group=archivebox WorkingDirectory=/var/lib/archivebox -ExecStartPre=/opt/archivebox/venv/bin/archivebox init --setup +ExecStartPre=/opt/archivebox/venv/bin/archivebox init ExecStart=/opt/archivebox/venv/bin/archivebox server 0.0.0.0:8000 Restart=on-failure RestartSec=5 diff --git a/pkg/debian/install.sh b/pkg/debian/install.sh index 54c593f5..0f0e11d2 100755 --- a/pkg/debian/install.sh +++ b/pkg/debian/install.sh @@ -17,20 +17,14 @@ fi # Upgrade pip inside the virtualenv "$ARCHIVEBOX_VENV/bin/python3" -m pip install --quiet --upgrade pip setuptools -# Install or upgrade archivebox +# Install or upgrade archivebox (pinned to .deb version if set) if [ -n "$ARCHIVEBOX_VERSION" ]; then echo "[+] Installing archivebox==$ARCHIVEBOX_VERSION..." - "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade "archivebox==$ARCHIVEBOX_VERSION" || { - echo "[!] archivebox==$ARCHIVEBOX_VERSION not found on PyPI, installing latest..." - "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade archivebox - } + "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade "archivebox==$ARCHIVEBOX_VERSION" else echo "[+] Installing latest archivebox..." "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade archivebox fi -echo "[+] Installing archivebox runtime dependencies..." -"$ARCHIVEBOX_VENV/bin/archivebox" install --binproviders pip,npm 2>/dev/null || true - echo "[√] ArchiveBox installed successfully." echo " Run 'archivebox version' to verify." diff --git a/pkg/debian/scripts/preremove.sh b/pkg/debian/scripts/preremove.sh index 81120cd5..f7674ce0 100755 --- a/pkg/debian/scripts/preremove.sh +++ b/pkg/debian/scripts/preremove.sh @@ -2,17 +2,20 @@ # preremove script for archivebox .deb package set -e -# Stop the service if running -if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then - systemctl stop archivebox 2>/dev/null || true - systemctl disable archivebox 2>/dev/null || true +# Only clean up on full removal, not during upgrade +if [ "$1" = "remove" ] || [ "$1" = "purge" ]; then + # Stop the service if running + if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + systemctl stop archivebox 2>/dev/null || true + systemctl disable archivebox 2>/dev/null || true + fi + + echo "[+] Removing ArchiveBox virtualenv..." + rm -rf /opt/archivebox/venv + + echo "[i] Your ArchiveBox data in /var/lib/archivebox has NOT been removed." + echo " The 'archivebox' system user has NOT been removed." + echo " Remove them manually if you no longer need them:" + echo " sudo rm -rf /var/lib/archivebox" + echo " sudo userdel archivebox" fi - -echo "[+] Removing ArchiveBox virtualenv..." -rm -rf /opt/archivebox/venv - -echo "[i] Your ArchiveBox data in /var/lib/archivebox has NOT been removed." -echo " The 'archivebox' system user has NOT been removed." -echo " Remove them manually if you no longer need them:" -echo " sudo rm -rf /var/lib/archivebox" -echo " sudo userdel archivebox" From 496b54a5e1e6ed15e86a8230ed9f02cf20527ba7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 03:20:32 +0000 Subject: [PATCH 07/19] Fix remaining PR review comments: release ordering, verification, README - Move .deb upload to GitHub Release into a separate job that runs after tests pass - Fix workflow_call event propagation so release jobs run when called from release.yml - Fix setup.sh post-install verification to check `which archivebox` first (works for brew/deb) - Fix README.md: detect architecture with dpkg instead of hardcoding amd64 - Fix README.md: remove --setup flag from apt install instructions https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 28 +++++++++++++++++++++------- .github/workflows/homebrew.yml | 2 +- README.md | 5 +++-- bin/setup.sh | 29 +++++++++++++++++++---------- 4 files changed, 44 insertions(+), 20 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index b3c9abad..06aac13e 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -64,13 +64,6 @@ jobs: name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb path: dist/*.deb - - name: Upload .deb to GitHub Release - if: github.event_name == 'release' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release upload "${{ github.event.release.tag_name }}" dist/*.deb --clobber - test: needs: build runs-on: ubuntu-24.04 @@ -157,3 +150,24 @@ jobs: run: | test -f /usr/lib/systemd/system/archivebox.service cat /usr/lib/systemd/system/archivebox.service + + # Upload .deb to GitHub Release only after tests pass + release: + if: github.event_name == 'release' || github.event_name == 'workflow_call' + needs: [build, test] + runs-on: ubuntu-24.04 + permissions: + contents: write + + steps: + - name: Download all .deb artifacts + uses: actions/download-artifact@v4 + with: + pattern: archivebox-*.deb + merge-multiple: true + + - name: Upload .deb to GitHub Release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload "${{ github.event.release.tag_name }}" *.deb --clobber diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index 838b323b..ca475917 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -171,7 +171,7 @@ RUBY # On release only: generate the real formula with PyPI URL and push to tap release: - if: github.event_name == 'release' + if: github.event_name == 'release' || github.event_name == 'workflow_call' needs: build-and-test runs-on: macos-latest diff --git a/README.md b/README.md index 66b9126f..0d58f7f9 100644 --- a/README.md +++ b/README.md @@ -297,14 +297,15 @@ See below for more usage examples using the C
    1. Download and install the .deb package from the latest release. 
      # download the .deb for your architecture (amd64 or arm64)
-curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_amd64.deb" -o /tmp/archivebox.deb
+ARCH="$(dpkg --print-architecture)"
+curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${ARCH}.deb" -o /tmp/archivebox.deb
 sudo apt install /tmp/archivebox.deb
 archivebox version                         # make sure all dependencies are installed
    2. Create a new empty directory and initialize your collection (can be anywhere). 
      mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --setup
+archivebox init

      3. diff --git a/bin/setup.sh b/bin/setup.sh index 91cd6681..d6a89fa7 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -155,19 +155,28 @@ fi echo -if ! (python3 --version && python3 -m pip --version && python3 -m django --version); then - echo "[X] Python 3 pip was not found on your system!" - echo " You must first install Python >= 3.7 (and pip3):" - echo " https://www.python.org/downloads/" - echo " https://wiki.python.org/moin/BeginnersGuide/Download" - echo " After installing, run this script again." - exit 1 +if ! which archivebox > /dev/null 2>&1; then + # If archivebox isn't in PATH (e.g. pip install), check python modules directly + if ! (python3 --version && python3 -m pip --version && python3 -m django --version) 2>/dev/null; then + echo "[X] Python 3 pip was not found on your system!" + echo " You must first install Python >= 3.7 (and pip3):" + echo " https://www.python.org/downloads/" + echo " https://wiki.python.org/moin/BeginnersGuide/Download" + echo " After installing, run this script again." + exit 1 + fi + + if ! (python3 -m django --version && python3 -m pip show archivebox) 2>/dev/null; then + echo "[X] Django and ArchiveBox were not found after installing!" + echo " Check to see if a previous step failed." + echo + exit 1 + fi fi -if ! (python3 -m django --version && python3 -m pip show archivebox && which -a archivebox); then - echo "[X] Django and ArchiveBox were not found after installing!" +if ! which archivebox > /dev/null 2>&1; then + echo "[X] archivebox command was not found in PATH after installing!" echo " Check to see if a previous step failed." - echo exit 1 fi From 4db4c36cb2474b8af1b6c97117b8ec8467115f98 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 03:22:33 +0000 Subject: [PATCH 08/19] Add arm64 to .deb test matrix using GitHub's arm64 runners Tests both amd64 and arm64 .deb packages by downloading the matching architecture artifact and running the full install + verification flow on native runners. https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 06aac13e..5139d9e2 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -66,7 +66,15 @@ jobs: test: needs: build - runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 @@ -82,10 +90,9 @@ jobs: enable-cache: true - name: Install build dependencies - uses: awalsh128/cache-apt-pkgs-action@v1.6.0 - with: - packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 - version: 1.0 + run: | + sudo apt-get update -qq + sudo apt-get install -y build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 - name: Build local wheel run: | @@ -95,12 +102,11 @@ jobs: - name: Download .deb artifact uses: actions/download-artifact@v4 with: - pattern: archivebox-*-amd64.deb + pattern: archivebox-*-${{ matrix.arch }}.deb merge-multiple: true - name: Install system dependencies run: | - sudo apt-get update -qq sudo apt-get install -y python3.13 python3.13-venv python3-pip nodejs npm git wget curl ripgrep - name: Pre-seed virtualenv with local wheel before dpkg install From 6e77d11c07408922b06fb9d0f7c4437d310ac357 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 03:29:31 +0000 Subject: [PATCH 09/19] Restore cache-apt-pkgs-action for test job build dependencies https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 5139d9e2..def8cd8c 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -90,9 +90,10 @@ jobs: enable-cache: true - name: Install build dependencies - run: | - sudo apt-get update -qq - sudo apt-get install -y build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 + uses: awalsh128/cache-apt-pkgs-action@v1.6.0 + with: + packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 + version: 1.0 - name: Build local wheel run: | From 2845e4350ab8c97db1da8c59a797c0a18358f258 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 03:32:01 +0000 Subject: [PATCH 10/19] Fix .deb download URL in README to include version component The nfpm-built .deb files are named archivebox__.deb, so the download URL needs to fetch the latest version tag first. https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0d58f7f9..2ec75a3e 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,8 @@ See below for more usage examples using the C
    3. Download and install the .deb package from the latest release. 
      # download the .deb for your architecture (amd64 or arm64)
 ARCH="$(dpkg --print-architecture)"
-curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${ARCH}.deb" -o /tmp/archivebox.deb
+VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
+curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
 sudo apt install /tmp/archivebox.deb
 archivebox version                         # make sure all dependencies are installed
      From 68fea71933895915918246906e47f1b254fbc4e9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 03:39:33 +0000 Subject: [PATCH 11/19] Address remaining PR review comments - Pin cache-apt-pkgs-action to commit SHA for supply-chain safety - Fix Homebrew post_install to use with_env block instead of env hash in system() call (idiomatic Homebrew pattern) - Add clarifying comments to service file, preremove.sh, and nfpm.yaml explaining user/group creation, directory ownership, and upgrade handling https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 2 +- .github/workflows/homebrew.yml | 10 +++++++--- brew_dist/archivebox.rb | 4 +++- pkg/debian/archivebox.service | 3 +++ pkg/debian/nfpm.yaml | 2 +- pkg/debian/scripts/preremove.sh | 4 +++- 6 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index def8cd8c..04979adf 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -90,7 +90,7 @@ jobs: enable-cache: true - name: Install build dependencies - uses: awalsh128/cache-apt-pkgs-action@v1.6.0 + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 with: packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 version: 1.0 diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index ca475917..dccb6dda 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -48,7 +48,7 @@ jobs: - name: Install build dependencies (Linux) if: runner.os == 'Linux' - uses: awalsh128/cache-apt-pkgs-action@v1.6.0 + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 with: packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 version: 1.0 @@ -108,7 +108,9 @@ ${RESOURCES} def post_install (var/"archivebox").mkpath - system({ "DATA_DIR" => var/"archivebox" }, bin/"archivebox", "install", "--binproviders", "pip,npm") + with_env(DATA_DIR: var/"archivebox") do + system bin/"archivebox", "install", "--binproviders", "pip,npm" + end end service do @@ -275,7 +277,9 @@ ${RESOURCES} def post_install (var/"archivebox").mkpath - system({ "DATA_DIR" => var/"archivebox" }, bin/"archivebox", "install", "--binproviders", "pip,npm") + with_env(DATA_DIR: var/"archivebox") do + system bin/"archivebox", "install", "--binproviders", "pip,npm" + end end service do diff --git a/brew_dist/archivebox.rb b/brew_dist/archivebox.rb index c0521d7a..6f9d8ada 100644 --- a/brew_dist/archivebox.rb +++ b/brew_dist/archivebox.rb @@ -41,7 +41,9 @@ class Archivebox < Formula def post_install # Install runtime dependencies (plugins, JS extractors, etc.) (var/"archivebox").mkpath - system({ "DATA_DIR" => var/"archivebox" }, bin/"archivebox", "install", "--binproviders", "pip,npm") + with_env(DATA_DIR: var/"archivebox") do + system bin/"archivebox", "install", "--binproviders", "pip,npm" + end end service do diff --git a/pkg/debian/archivebox.service b/pkg/debian/archivebox.service index 916f2cd9..b3fe89ad 100644 --- a/pkg/debian/archivebox.service +++ b/pkg/debian/archivebox.service @@ -1,3 +1,6 @@ +# The archivebox user/group and /var/lib/archivebox directory are created by +# postinstall.sh (which runs after dpkg unpacks the package contents). + [Unit] Description=ArchiveBox Web Archiving Server After=network.target diff --git a/pkg/debian/nfpm.yaml b/pkg/debian/nfpm.yaml index 8fc55afc..0e592af1 100644 --- a/pkg/debian/nfpm.yaml +++ b/pkg/debian/nfpm.yaml @@ -55,7 +55,7 @@ contents: file_info: mode: 0644 - # Create data directory + # Create data directory (unpacked as root; postinstall.sh chowns to archivebox user) - dst: /var/lib/archivebox type: dir file_info: diff --git a/pkg/debian/scripts/preremove.sh b/pkg/debian/scripts/preremove.sh index f7674ce0..6d48ab24 100755 --- a/pkg/debian/scripts/preremove.sh +++ b/pkg/debian/scripts/preremove.sh @@ -2,7 +2,9 @@ # preremove script for archivebox .deb package set -e -# Only clean up on full removal, not during upgrade +# Only clean up on full removal, not during upgrade. +# dpkg passes "$1" as "remove", "purge", or "upgrade" — we skip cleanup on +# upgrade so the venv and service persist across package version bumps. if [ "$1" = "remove" ] || [ "$1" = "purge" ]; then # Stop the service if running if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then From 0fac8a7346518bf5918338166fbbdd5c20136fe8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 03:44:49 +0000 Subject: [PATCH 12/19] Fix remaining PR review comments from all review rounds - systemd service: use /usr/bin/archivebox wrapper (exports venv PATH for bundled tools like yt-dlp) instead of direct venv binary - install.sh: prefer python3.13, fail early with clear error if < 3.13, add comment clarifying the manual (unpinned) fallback behavior - debian.yml release job: fall back to pyproject.toml version when github.event.release.tag_name is empty (workflow_dispatch path) - nfpm.yaml: clarify that install.sh enforces the real >= 3.13 constraint - CI pre-seed step: expanded comment explaining why we pre-seed with python3.13 and how it relates to real installs https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 28 ++++++++++++++++++++++++---- pkg/debian/archivebox.service | 5 +++-- pkg/debian/install.sh | 27 +++++++++++++++++++++++---- pkg/debian/nfpm.yaml | 5 +++-- 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 04979adf..5fe29d94 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -112,9 +112,13 @@ jobs: - name: Pre-seed virtualenv with local wheel before dpkg install run: | - # Create the venv and install from local wheel BEFORE dpkg runs postinstall. - # This way the postinstall's pip install either succeeds (release) or - # finds archivebox already installed (CI) and just upgrades deps. + # CI-only: pre-seed the venv with a local wheel so we test the + # unreleased code, not whatever is on PyPI. We explicitly use python3.13 + # here (matching the system dep installed above) to ensure the venv is + # created with the correct Python version. On real installs, install.sh + # handles this by preferring python3.13 and failing if < 3.13. + # When postinstall.sh runs during dpkg -i, it finds the venv already + # populated and just upgrades deps. sudo mkdir -p /opt/archivebox sudo python3.13 -m venv /opt/archivebox/venv sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools @@ -167,6 +171,22 @@ jobs: contents: write steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Determine release tag + id: tag + run: | + # github.event.release.tag_name is set for release events but empty + # for workflow_dispatch. Fall back to the version from pyproject.toml. + TAG="${{ github.event.release.tag_name }}" + if [ -z "$TAG" ]; then + TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" + echo "[i] No release tag in event context, using version from pyproject.toml: $TAG" + fi + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + - name: Download all .deb artifacts uses: actions/download-artifact@v4 with: @@ -177,4 +197,4 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh release upload "${{ github.event.release.tag_name }}" *.deb --clobber + gh release upload "${{ steps.tag.outputs.tag }}" *.deb --clobber diff --git a/pkg/debian/archivebox.service b/pkg/debian/archivebox.service index b3fe89ad..af8b4500 100644 --- a/pkg/debian/archivebox.service +++ b/pkg/debian/archivebox.service @@ -10,8 +10,9 @@ Type=simple User=archivebox Group=archivebox WorkingDirectory=/var/lib/archivebox -ExecStartPre=/opt/archivebox/venv/bin/archivebox init -ExecStart=/opt/archivebox/venv/bin/archivebox server 0.0.0.0:8000 +Environment="PATH=/opt/archivebox/venv/bin:/usr/local/bin:/usr/bin:/bin" +ExecStartPre=/usr/bin/archivebox init +ExecStart=/usr/bin/archivebox server 0.0.0.0:8000 Restart=on-failure RestartSec=5 diff --git a/pkg/debian/install.sh b/pkg/debian/install.sh index 0f0e11d2..44d58c4e 100755 --- a/pkg/debian/install.sh +++ b/pkg/debian/install.sh @@ -7,22 +7,41 @@ set -e ARCHIVEBOX_VENV="/opt/archivebox/venv" ARCHIVEBOX_VERSION="${ARCHIVEBOX_VERSION:-}" -echo "[+] Setting up ArchiveBox virtualenv in $ARCHIVEBOX_VENV..." +# ArchiveBox requires Python >= 3.13 (per pyproject.toml). +# Prefer python3.13 explicitly; fall back to python3 with a version check. +if command -v python3.13 >/dev/null 2>&1; then + PYTHON="python3.13" +elif command -v python3 >/dev/null 2>&1; then + PYTHON="python3" + PY_VER="$("$PYTHON" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + if [ "$(echo "$PY_VER 3.13" | awk '{print ($1 >= $2)}')" != "1" ]; then + echo "[!] Error: ArchiveBox requires Python >= 3.13, but found Python $PY_VER" + echo " Install python3.13: sudo apt install python3.13 python3.13-venv" + exit 1 + fi +else + echo "[!] Error: python3 not found. Install python3.13: sudo apt install python3.13 python3.13-venv" + exit 1 +fi + +echo "[+] Setting up ArchiveBox virtualenv in $ARCHIVEBOX_VENV (using $PYTHON)..." # Create the virtualenv if it doesn't exist if [ ! -d "$ARCHIVEBOX_VENV" ]; then - python3 -m venv "$ARCHIVEBOX_VENV" + "$PYTHON" -m venv "$ARCHIVEBOX_VENV" fi # Upgrade pip inside the virtualenv "$ARCHIVEBOX_VENV/bin/python3" -m pip install --quiet --upgrade pip setuptools -# Install or upgrade archivebox (pinned to .deb version if set) +# Install or upgrade archivebox. +# ARCHIVEBOX_VERSION is set by postinstall.sh from the .deb package version. +# When run manually without it, install the latest release from PyPI. if [ -n "$ARCHIVEBOX_VERSION" ]; then echo "[+] Installing archivebox==$ARCHIVEBOX_VERSION..." "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade "archivebox==$ARCHIVEBOX_VERSION" else - echo "[+] Installing latest archivebox..." + echo "[+] Installing latest archivebox (no version pinned)..." "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade archivebox fi diff --git a/pkg/debian/nfpm.yaml b/pkg/debian/nfpm.yaml index 0e592af1..64accf7f 100644 --- a/pkg/debian/nfpm.yaml +++ b/pkg/debian/nfpm.yaml @@ -19,8 +19,9 @@ section: "web" priority: "optional" depends: - # python3 >= 3.11 allows .deb to install on more systems; - # pip enforces the actual Python version requirement from pyproject.toml + # python3 >= 3.11 allows dpkg to install on more systems (e.g. Ubuntu 24.04). + # install.sh enforces the real >= 3.13 requirement at venv creation time, + # failing early with a clear error if only an older python3 is available. - python3 (>= 3.11) - python3-pip - python3-venv From 82932812ae2d45e5e7cd2af6d141be6ef68b15cf Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 03:56:39 +0000 Subject: [PATCH 13/19] Simplify deb/brew packages to thin wrappers around pip install Remove all non-essential dependencies from both package formats. The .deb now only depends on python3, pip, and venv. The brew formula only depends on python@3.13. All other runtime deps (node, chrome, yt-dlp, wget, ripgrep, etc.) are installed on-demand by `archivebox install` at runtime. Removed from brew formula: - 6 depends_on entries (node, git, wget, curl, ripgrep, yt-dlp) - on_linux block (pkg-config, openssl, libffi) - post_install hook (was running archivebox install) - service block (users run archivebox server directly) Removed from .deb: - 6 depends entries (nodejs, npm, git, wget, curl, ripgrep) - recommends section (yt-dlp, ffmpeg, chromium) Net: -126 lines across packaging files. https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 2 +- .github/workflows/homebrew.yml | 60 +--------------------------------- bin/build_brew.sh | 36 +++----------------- brew_dist/archivebox.rb | 41 +++-------------------- pkg/debian/nfpm.yaml | 13 ++------ 5 files changed, 13 insertions(+), 139 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 5fe29d94..817a9ca8 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -108,7 +108,7 @@ jobs: - name: Install system dependencies run: | - sudo apt-get install -y python3.13 python3.13-venv python3-pip nodejs npm git wget curl ripgrep + sudo apt-get install -y python3.13 python3.13-venv python3-pip - name: Pre-seed virtualenv with local wheel before dpkg install run: | diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index dccb6dda..fdceff50 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -87,18 +87,6 @@ class Archivebox < Formula license "MIT" depends_on "python@3.13" - depends_on "node" - depends_on "git" - depends_on "wget" - depends_on "curl" - depends_on "ripgrep" - depends_on "yt-dlp" - - on_linux do - depends_on "pkg-config" => :build - depends_on "openssl@3" - depends_on "libffi" - end ${RESOURCES} @@ -106,21 +94,6 @@ ${RESOURCES} virtualenv_install_with_resources end - def post_install - (var/"archivebox").mkpath - with_env(DATA_DIR: var/"archivebox") do - system bin/"archivebox", "install", "--binproviders", "pip,npm" - end - end - - service do - run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] - keep_alive crashed: true - working_dir var/"archivebox" - log_path var/"log/archivebox.log" - error_log_path var/"log/archivebox.log" - end - test do assert_match version.to_s, shell_output("#{bin}/archivebox version") end @@ -148,11 +121,7 @@ RUBY echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH" - name: Install brew dependencies - run: | - brew install python@3.13 node git wget curl ripgrep yt-dlp - if [ "$RUNNER_OS" = "Linux" ]; then - brew install pkg-config openssl@3 libffi - fi + run: brew install python@3.13 - name: Install archivebox via brew from local formula run: | @@ -256,18 +225,6 @@ class Archivebox < Formula head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" depends_on "python@3.13" - depends_on "node" - depends_on "git" - depends_on "wget" - depends_on "curl" - depends_on "ripgrep" - depends_on "yt-dlp" - - on_linux do - depends_on "pkg-config" => :build - depends_on "openssl@3" - depends_on "libffi" - end ${RESOURCES} @@ -275,21 +232,6 @@ ${RESOURCES} virtualenv_install_with_resources end - def post_install - (var/"archivebox").mkpath - with_env(DATA_DIR: var/"archivebox") do - system bin/"archivebox", "install", "--binproviders", "pip,npm" - end - end - - service do - run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] - keep_alive crashed: true - working_dir var/"archivebox" - log_path var/"log/archivebox.log" - error_log_path var/"log/archivebox.log" - end - test do assert_match version.to_s, shell_output("#{bin}/archivebox version") end diff --git a/bin/build_brew.sh b/bin/build_brew.sh index b10bad47..e6bf8a45 100755 --- a/bin/build_brew.sh +++ b/bin/build_brew.sh @@ -54,12 +54,8 @@ echo "[+] Updating formula file: $FORMULA_FILE" # Build the formula from the template cat > "$FORMULA_FILE" << RUBY -# This formula is auto-generated by bin/build_brew.sh using homebrew-pypi-poet. -# To update: run bin/build_brew.sh, or trigger the GitHub Actions homebrew workflow. -# -# Users install with: -# brew tap archivebox/archivebox -# brew install archivebox +# Auto-generated by bin/build_brew.sh using homebrew-pypi-poet. +# Users install with: brew tap archivebox/archivebox && brew install archivebox class Archivebox < Formula include Language::Python::Virtualenv @@ -72,18 +68,8 @@ class Archivebox < Formula head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" depends_on "python@3.13" - depends_on "node" - depends_on "git" - depends_on "wget" - depends_on "curl" - depends_on "ripgrep" - depends_on "yt-dlp" - - on_linux do - depends_on "pkg-config" => :build - depends_on "openssl@3" - depends_on "libffi" - end + # All other runtime deps (node, chrome, yt-dlp, etc.) are installed + # on-demand by \`archivebox install\` and should NOT be declared here. # Python dependency resource blocks auto-generated by homebrew-pypi-poet # AUTOGENERATED_RESOURCES_START @@ -94,20 +80,6 @@ ${RESOURCES} virtualenv_install_with_resources end - def post_install - # Install runtime dependencies (plugins, JS extractors, etc.) - (var/"archivebox").mkpath - system({ "DATA_DIR" => var/"archivebox" }, bin/"archivebox", "install", "--binproviders", "pip,npm") - end - - service do - run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] - keep_alive crashed: true - working_dir var/"archivebox" - log_path var/"log/archivebox.log" - error_log_path var/"log/archivebox.log" - end - test do assert_match version.to_s, shell_output("#{bin}/archivebox version") end diff --git a/brew_dist/archivebox.rb b/brew_dist/archivebox.rb index 6f9d8ada..f25dd8e6 100644 --- a/brew_dist/archivebox.rb +++ b/brew_dist/archivebox.rb @@ -1,9 +1,5 @@ -# This formula is auto-generated by bin/build_brew.sh using homebrew-pypi-poet. -# To update: run bin/build_brew.sh, or trigger the GitHub Actions homebrew workflow. -# -# Users install with: -# brew tap archivebox/archivebox -# brew install archivebox +# Auto-generated by bin/build_brew.sh using homebrew-pypi-poet. +# Users install with: brew tap archivebox/archivebox && brew install archivebox class Archivebox < Formula include Language::Python::Virtualenv @@ -16,21 +12,10 @@ class Archivebox < Formula head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" depends_on "python@3.13" - depends_on "node" - depends_on "git" - depends_on "wget" - depends_on "curl" - depends_on "ripgrep" - depends_on "yt-dlp" + # All other runtime deps (node, chrome, yt-dlp, etc.) are installed + # on-demand by `archivebox install` and should NOT be declared here. - on_linux do - depends_on "pkg-config" => :build - depends_on "openssl@3" - depends_on "libffi" - end - - # Python dependency resource blocks are auto-generated by bin/build_brew.sh - # using homebrew-pypi-poet. Run that script to populate this section. + # Python dependency resource blocks auto-generated by homebrew-pypi-poet # AUTOGENERATED_RESOURCES_START # AUTOGENERATED_RESOURCES_END @@ -38,22 +23,6 @@ class Archivebox < Formula virtualenv_install_with_resources end - def post_install - # Install runtime dependencies (plugins, JS extractors, etc.) - (var/"archivebox").mkpath - with_env(DATA_DIR: var/"archivebox") do - system bin/"archivebox", "install", "--binproviders", "pip,npm" - end - end - - service do - run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] - keep_alive crashed: true - working_dir var/"archivebox" - log_path var/"log/archivebox.log" - error_log_path var/"log/archivebox.log" - end - test do assert_match version.to_s, shell_output("#{bin}/archivebox version") end diff --git a/pkg/debian/nfpm.yaml b/pkg/debian/nfpm.yaml index 64accf7f..83cb0074 100644 --- a/pkg/debian/nfpm.yaml +++ b/pkg/debian/nfpm.yaml @@ -25,17 +25,8 @@ depends: - python3 (>= 3.11) - python3-pip - python3-venv - - nodejs - - npm - - git - - wget - - curl - - ripgrep - -recommends: - - yt-dlp - - ffmpeg - - chromium | chromium-browser | google-chrome-stable + # All other runtime deps (node, chrome, yt-dlp, etc.) are installed on-demand + # by `archivebox install` and should NOT be declared as package dependencies. contents: # Wrapper script for /usr/bin/archivebox From 3501b393eb974fb357cdc6c21e4e6cc269ef2bb1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 04:15:23 +0000 Subject: [PATCH 14/19] Deduplicate homebrew.yml release job by reusing build_brew.sh The release job was duplicating ~70 lines of PyPI fetching + formula generation that build_brew.sh already handles. Now it just calls ./bin/build_brew.sh. Also merged the two Linux-only brew setup steps. https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/homebrew.yml | 85 ++++------------------------------ 1 file changed, 9 insertions(+), 76 deletions(-) diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index fdceff50..9ef0c9a3 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -108,11 +108,6 @@ RUBY if: runner.os == 'Linux' run: | NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo '/home/linuxbrew/.linuxbrew/bin' >> "$GITHUB_PATH" - - - name: Set up brew shellenv (Linux only) - if: runner.os == 'Linux' - run: | eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV" echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV" @@ -177,81 +172,19 @@ RUBY sleep 30 done - - name: Generate release formula with PyPI URL + - name: Generate release formula via build_brew.sh + run: ./bin/build_brew.sh + + - name: Test formula install run: | - VERSION="${{ steps.version.outputs.version }}" - python3 -m venv /tmp/poet-venv - source /tmp/poet-venv/bin/activate - - pip install --quiet "archivebox==${VERSION}" homebrew-pypi-poet - - echo "[+] Generating resource stanzas with homebrew-pypi-poet..." - RESOURCES="$(poet archivebox)" - - # Get sdist URL and SHA256 from PyPI JSON API - PYPI_JSON="$(curl -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" 2>/dev/null || echo '')" - SDIST_URL="" - SDIST_SHA256="" - if [ -n "$PYPI_JSON" ]; then - SDIST_URL="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" - SDIST_SHA256="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['digests']['sha256'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" - fi - if [ -z "$SDIST_URL" ]; then - SDIST_URL="https://files.pythonhosted.org/packages/source/a/archivebox/archivebox-${VERSION}.tar.gz" - fi - if [ -z "$SDIST_SHA256" ]; then - pip download --no-binary :all: --no-deps -d /tmp/sdist "archivebox==${VERSION}" 2>/dev/null || true - SDIST_SHA256="$(shasum -a 256 /tmp/sdist/*.tar.gz 2>/dev/null | awk '{print $1}' || echo '')" - fi - - deactivate - - cat > /tmp/archivebox.rb << RUBY -# Auto-generated Homebrew formula for archivebox ${VERSION} -# Generated by GitHub Actions on release using homebrew-pypi-poet -# -# Users install with: -# brew tap archivebox/archivebox -# brew install archivebox - -class Archivebox < Formula - include Language::Python::Virtualenv - - desc "Self-hosted internet archiving solution" - homepage "https://github.com/ArchiveBox/ArchiveBox" - url "${SDIST_URL}" - sha256 "${SDIST_SHA256}" - license "MIT" - head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" - - depends_on "python@3.13" - -${RESOURCES} - - def install - virtualenv_install_with_resources - end - - test do - assert_match version.to_s, shell_output("#{bin}/archivebox version") - end -end -RUBY - - echo "[√] Generated release formula:" - ruby -c /tmp/archivebox.rb - cat /tmp/archivebox.rb + brew install --build-from-source brew_dist/archivebox.rb + archivebox version - name: Upload formula artifact uses: actions/upload-artifact@v4 with: name: archivebox.rb - path: /tmp/archivebox.rb - - - name: Test formula install - run: | - brew install --build-from-source /tmp/archivebox.rb - archivebox version + path: brew_dist/archivebox.rb - name: Push to homebrew-archivebox tap env: @@ -260,8 +193,8 @@ RUBY VERSION="${{ steps.version.outputs.version }}" git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap - cp /tmp/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \ - cp /tmp/archivebox.rb /tmp/tap/archivebox.rb + cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \ + cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb cd /tmp/tap git config user.name "github-actions[bot]" From 7300892b084d808e25ff49a81d818f2d1b00a03e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 04:28:46 +0000 Subject: [PATCH 15/19] Fix PR review comments: version check, runtime deps, CI test deps, release guard - install.sh: Replace awk float comparison with integer major/minor check so Python 3.9 is correctly rejected as < 3.13 - nfpm.yaml: Add git/curl/wget as recommends so users have basic archiving tools out of the box without needing `archivebox install` - debian.yml: Restore git/curl/wget in CI smoke test to match what the package recommends, instead of relying on runner preinstalls - debian.yml: Guard release upload to skip gracefully when no GitHub Release exists (fixes workflow_dispatch failures) https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 10 +++++++--- pkg/debian/install.sh | 6 ++++-- pkg/debian/nfpm.yaml | 8 ++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 817a9ca8..62c1208c 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -108,7 +108,7 @@ jobs: - name: Install system dependencies run: | - sudo apt-get install -y python3.13 python3.13-venv python3-pip + sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget - name: Pre-seed virtualenv with local wheel before dpkg install run: | @@ -178,8 +178,6 @@ jobs: - name: Determine release tag id: tag run: | - # github.event.release.tag_name is set for release events but empty - # for workflow_dispatch. Fall back to the version from pyproject.toml. TAG="${{ github.event.release.tag_name }}" if [ -z "$TAG" ]; then TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" @@ -197,4 +195,10 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + # Verify the release exists before uploading (workflow_dispatch may not have one) + if ! gh release view "${{ steps.tag.outputs.tag }}" >/dev/null 2>&1; then + echo "[!] No GitHub Release found for tag ${{ steps.tag.outputs.tag }}, skipping upload." + echo " Create a release first or trigger via the release event." + exit 0 + fi gh release upload "${{ steps.tag.outputs.tag }}" *.deb --clobber diff --git a/pkg/debian/install.sh b/pkg/debian/install.sh index 44d58c4e..1c72758e 100755 --- a/pkg/debian/install.sh +++ b/pkg/debian/install.sh @@ -13,8 +13,10 @@ if command -v python3.13 >/dev/null 2>&1; then PYTHON="python3.13" elif command -v python3 >/dev/null 2>&1; then PYTHON="python3" - PY_VER="$("$PYTHON" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" - if [ "$(echo "$PY_VER 3.13" | awk '{print ($1 >= $2)}')" != "1" ]; then + PY_MAJOR="$("$PYTHON" -c 'import sys; print(sys.version_info.major)')" + PY_MINOR="$("$PYTHON" -c 'import sys; print(sys.version_info.minor)')" + if [ "$PY_MAJOR" -lt 3 ] || { [ "$PY_MAJOR" -eq 3 ] && [ "$PY_MINOR" -lt 13 ]; }; then + PY_VER="${PY_MAJOR}.${PY_MINOR}" echo "[!] Error: ArchiveBox requires Python >= 3.13, but found Python $PY_VER" echo " Install python3.13: sudo apt install python3.13 python3.13-venv" exit 1 diff --git a/pkg/debian/nfpm.yaml b/pkg/debian/nfpm.yaml index 83cb0074..fa3df26c 100644 --- a/pkg/debian/nfpm.yaml +++ b/pkg/debian/nfpm.yaml @@ -28,6 +28,14 @@ depends: # All other runtime deps (node, chrome, yt-dlp, etc.) are installed on-demand # by `archivebox install` and should NOT be declared as package dependencies. +recommends: + # Common utilities used by archivebox extractors. Declared as recommends + # (not depends) so dpkg doesn't hard-fail if they're missing, but apt + # installs them by default so users have a working baseline out of the box. + - git + - curl + - wget + contents: # Wrapper script for /usr/bin/archivebox - src: pkg/debian/archivebox From c319b417c34136a223df5a218ef0b63518b0af47 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 04:32:15 +0000 Subject: [PATCH 16/19] Fix Docker workflow missing semver/latest tags when called from release.yml docker.yml uses github.event_name to decide between full release tags (semver, latest) vs CI-only tags (branch, sha). When release.yml calls it via `uses:`, the child sees event_name='workflow_call' which was hitting the non-release path. Now workflow_call is treated the same as workflow_dispatch so published releases get proper Docker tags. https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/docker.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 074ec827..ce0da517 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -66,7 +66,7 @@ jobs: # https://github.com/docker/metadata-action id: docker_meta uses: docker/metadata-action@v5 - if: github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' with: images: archivebox/archivebox,ghcr.io/archivebox/archivebox tags: | @@ -80,12 +80,12 @@ jobs: type=sha # :latest type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }} - + - name: Collect Non-Release Docker tags # https://github.com/docker/metadata-action id: docker_meta_non_release uses: docker/metadata-action@v5 - if: github.event_name != 'workflow_dispatch' + if: github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' with: images: archivebox/archivebox,ghcr.io/archivebox/archivebox tags: | @@ -102,8 +102,8 @@ jobs: file: ./Dockerfile builder: ${{ steps.buildx.outputs.name }} push: ${{ github.event_name != 'pull_request' }} - tags: ${{ github.event_name == 'workflow_dispatch' ? steps.docker_meta.outputs.tags : steps.docker_meta_non_release.outputs.tags }} - labels: ${{ github.event_name == 'workflow_dispatch' ? steps.docker_meta.outputs.labels : steps.docker_meta_non_release.outputs.labels }} + tags: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.tags || steps.docker_meta_non_release.outputs.tags }} + labels: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.labels || steps.docker_meta_non_release.outputs.labels }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new platforms: linux/amd64,linux/arm64 From fbde2dee03bc09e61e4a49b89ba4133b9d065c33 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 04:42:01 +0000 Subject: [PATCH 17/19] Fix remaining PR review comments across packaging files - preremove.sh: Stop the systemd service during upgrades (not just remove/purge) so the running process doesn't use stale venv binaries while postinstall replaces them. Only disable + remove venv on full removal. - debian.yml: Fail loudly when release lookup fails during a release event (exit 1), but still skip gracefully for workflow_dispatch manual testing (exit 0). Prevents silently broken .deb publication. - archivebox.rb + build_brew.sh + homebrew.yml: Add post_install that initializes ArchiveBox in var/archivebox with DATA_DIR set, using correct Homebrew system() syntax (no env hash as first arg). https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 14 ++++++++++---- .github/workflows/homebrew.yml | 7 +++++++ bin/build_brew.sh | 8 ++++++++ brew_dist/archivebox.rb | 8 ++++++++ pkg/debian/scripts/preremove.sh | 14 +++++++++----- 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 62c1208c..1ffbffad 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -195,10 +195,16 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - # Verify the release exists before uploading (workflow_dispatch may not have one) - if ! gh release view "${{ steps.tag.outputs.tag }}" >/dev/null 2>&1; then - echo "[!] No GitHub Release found for tag ${{ steps.tag.outputs.tag }}, skipping upload." + TAG="${{ steps.tag.outputs.tag }}" + # Verify the release exists before uploading + if ! gh release view "$TAG" >/dev/null 2>&1; then + echo "[!] No GitHub Release found for tag $TAG." + if [ -n "${{ github.event.release.tag_name }}" ]; then + echo "[X] This was triggered by a release event — the release should exist. Failing." + exit 1 + fi + echo "[i] Skipping upload (workflow_dispatch without a release)." echo " Create a release first or trigger via the release event." exit 0 fi - gh release upload "${{ steps.tag.outputs.tag }}" *.deb --clobber + gh release upload "$TAG" *.deb --clobber diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index 9ef0c9a3..79ecf6e8 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -94,6 +94,13 @@ ${RESOURCES} virtualenv_install_with_resources end + def post_install + data_dir = var/"archivebox" + data_dir.mkpath + ENV["DATA_DIR"] = data_dir.to_s + system bin/"archivebox", "init" + end + test do assert_match version.to_s, shell_output("#{bin}/archivebox version") end diff --git a/bin/build_brew.sh b/bin/build_brew.sh index e6bf8a45..d94b13ca 100755 --- a/bin/build_brew.sh +++ b/bin/build_brew.sh @@ -80,6 +80,14 @@ ${RESOURCES} virtualenv_install_with_resources end + def post_install + # Initialize ArchiveBox data in the Homebrew-managed var directory + data_dir = var/"archivebox" + data_dir.mkpath + ENV["DATA_DIR"] = data_dir.to_s + system bin/"archivebox", "init" + end + test do assert_match version.to_s, shell_output("#{bin}/archivebox version") end diff --git a/brew_dist/archivebox.rb b/brew_dist/archivebox.rb index f25dd8e6..5eb56de8 100644 --- a/brew_dist/archivebox.rb +++ b/brew_dist/archivebox.rb @@ -23,6 +23,14 @@ class Archivebox < Formula virtualenv_install_with_resources end + def post_install + # Initialize ArchiveBox data in the Homebrew-managed var directory + data_dir = var/"archivebox" + data_dir.mkpath + ENV["DATA_DIR"] = data_dir.to_s + system bin/"archivebox", "init" + end + test do assert_match version.to_s, shell_output("#{bin}/archivebox version") end diff --git a/pkg/debian/scripts/preremove.sh b/pkg/debian/scripts/preremove.sh index 6d48ab24..65fc7330 100755 --- a/pkg/debian/scripts/preremove.sh +++ b/pkg/debian/scripts/preremove.sh @@ -2,13 +2,17 @@ # preremove script for archivebox .deb package set -e -# Only clean up on full removal, not during upgrade. -# dpkg passes "$1" as "remove", "purge", or "upgrade" — we skip cleanup on -# upgrade so the venv and service persist across package version bumps. +# dpkg passes "$1" as "remove", "purge", or "upgrade". + +# Always stop the service before removing or upgrading, because postinstall +# replaces the venv in-place — the running process would use stale binaries. +if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + systemctl stop archivebox 2>/dev/null || true +fi + +# Only disable + clean up on full removal, not during upgrade. if [ "$1" = "remove" ] || [ "$1" = "purge" ]; then - # Stop the service if running if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then - systemctl stop archivebox 2>/dev/null || true systemctl disable archivebox 2>/dev/null || true fi From 2eee9d95a3dc959c1dbc9735a550c4b09e17fa72 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 04:52:32 +0000 Subject: [PATCH 18/19] Fix postinstall.sh: restart service after upgrade preremove.sh stops the service during upgrades (to avoid stale venv binaries), but postinstall.sh wasn't restarting it. Now postinstall checks if the service was enabled and restarts it after the venv is rebuilt, so upgrades don't leave a previously running service down. https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- pkg/debian/scripts/postinstall.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/debian/scripts/postinstall.sh b/pkg/debian/scripts/postinstall.sh index a59a1537..a175c1b5 100755 --- a/pkg/debian/scripts/postinstall.sh +++ b/pkg/debian/scripts/postinstall.sh @@ -20,6 +20,13 @@ export ARCHIVEBOX_VERSION # Reload systemd to pick up the service file (skip if systemd is not running) if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then systemctl daemon-reload - echo "[i] To start ArchiveBox: sudo systemctl start archivebox" - echo "[i] To enable on boot: sudo systemctl enable archivebox" + + # On upgrade: restart the service if it was enabled (prerm stopped it) + if [ "$1" = "configure" ] && systemctl is-enabled archivebox >/dev/null 2>&1; then + systemctl start archivebox 2>/dev/null || true + echo "[+] Restarted archivebox service after upgrade" + else + echo "[i] To start ArchiveBox: sudo systemctl start archivebox" + echo "[i] To enable on boot: sudo systemctl enable archivebox" + fi fi From 36b40553044f97e0f442fefbb85475ed9e95e533 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 04:56:25 +0000 Subject: [PATCH 19/19] Add caveats block to Homebrew formula showing data directory The post_install initializes var/archivebox as the data directory, but users need to know where it is for subsequent commands. The caveats block is shown after brew install/upgrade to guide users. https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/homebrew.yml | 13 +++++++++++++ bin/build_brew.sh | 13 +++++++++++++ brew_dist/archivebox.rb | 13 +++++++++++++ 3 files changed, 39 insertions(+) diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index 79ecf6e8..70ce8ded 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -101,6 +101,19 @@ ${RESOURCES} system bin/"archivebox", "init" end + def caveats + <<~EOS + ArchiveBox data is stored in: + #{var}/archivebox + + To start archiving, run: + cd #{var}/archivebox && archivebox add 'https://example.com' + + To start the web UI: + cd #{var}/archivebox && archivebox server 0.0.0.0:8000 + EOS + end + test do assert_match version.to_s, shell_output("#{bin}/archivebox version") end diff --git a/bin/build_brew.sh b/bin/build_brew.sh index d94b13ca..80d4aa11 100755 --- a/bin/build_brew.sh +++ b/bin/build_brew.sh @@ -88,6 +88,19 @@ ${RESOURCES} system bin/"archivebox", "init" end + def caveats + <<~EOS + ArchiveBox data is stored in: + #{var}/archivebox + + To start archiving, run: + cd #{var}/archivebox && archivebox add 'https://example.com' + + To start the web UI: + cd #{var}/archivebox && archivebox server 0.0.0.0:8000 + EOS + end + test do assert_match version.to_s, shell_output("#{bin}/archivebox version") end diff --git a/brew_dist/archivebox.rb b/brew_dist/archivebox.rb index 5eb56de8..3e4d2f31 100644 --- a/brew_dist/archivebox.rb +++ b/brew_dist/archivebox.rb @@ -31,6 +31,19 @@ class Archivebox < Formula system bin/"archivebox", "init" end + def caveats + <<~EOS + ArchiveBox data is stored in: + #{var}/archivebox + + To start archiving, run: + cd #{var}/archivebox && archivebox add 'https://example.com' + + To start the web UI: + cd #{var}/archivebox && archivebox server 0.0.0.0:8000 + EOS + end + test do assert_match version.to_s, shell_output("#{bin}/archivebox version") end