From c8f562ee377750d65486cf21502aa8a48c1294b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Mar 2026 02:50:14 +0000 Subject: [PATCH] Wire up GitHub Actions for deb/brew build, test, and release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix debian.yml: pin nfpm version, add permissions, improve test job with user creation, init test, and status check - Fix homebrew.yml: use PyPI JSON API (macOS-compatible, no grep -oP), wait for PyPI availability on release, use generated formula not template, add Linux (Linuxbrew) test job alongside macOS - Add release.yml orchestrator: pip → deb + brew + docker in order - Add workflow_call triggers to pip.yml and docker.yml - Fix build_brew.sh: replace grep -oP with Python-based PyPI API, add on_linux deps (pkg-config, openssl, libffi) - Fix setup.sh: use GitHub API to find correct .deb download URL (filename includes version number) - Fix postinstall.sh: create archivebox system user, pin version from package, check for systemd before daemon-reload - Fix preremove.sh: stop service before removal, check for systemd - Fix install.sh: fallback to latest if pinned version not on PyPI - Add on_linux deps to brew formula for Linuxbrew support - Tested: .deb builds, installs, creates user, runs archivebox init https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn --- .github/workflows/debian.yml | 53 ++++++++- .github/workflows/docker.yml | 1 + .github/workflows/homebrew.yml | 186 ++++++++++++++++++++++++++++-- .github/workflows/pip.yml | 1 + .github/workflows/release.yml | 40 +++++++ bin/build_brew.sh | 21 +++- bin/setup.sh | 15 ++- brew_dist/archivebox.rb | 6 + pkg/debian/install.sh | 5 +- pkg/debian/nfpm.yaml | 2 + pkg/debian/scripts/postinstall.sh | 20 ++++ pkg/debian/scripts/preremove.sh | 13 ++- 12 files changed, 336 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 709817a5..2749a22f 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -2,9 +2,13 @@ name: Build Debian package on: workflow_dispatch: + workflow_call: release: types: [published] +permissions: + contents: write + jobs: build: runs-on: ubuntu-24.04 @@ -22,8 +26,13 @@ jobs: run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT" - name: Install nfpm + env: + # Pin nfpm version for reproducible builds + NFPM_VERSION: "2.45.1" run: | - curl -sfL https://install.goreleaser.com/github.com/goreleaser/nfpm.sh | sh -s -- -b /usr/local/bin + curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \ + | sudo tar -xz -C /usr/local/bin nfpm + nfpm --version - name: Build .deb package run: | @@ -31,6 +40,18 @@ jobs: export ARCH="${{ matrix.arch }}" ./bin/build_deb.sh + - name: Verify .deb package contents + run: | + DEB_FILE="$(ls dist/archivebox*.deb | head -1)" + echo "=== Package info ===" + dpkg-deb --info "$DEB_FILE" + echo "" + echo "=== Package contents ===" + dpkg-deb --contents "$DEB_FILE" + echo "" + echo "=== Control fields ===" + dpkg-deb --field "$DEB_FILE" + - name: Upload .deb artifact uses: actions/upload-artifact@v4 with: @@ -55,14 +76,36 @@ jobs: pattern: archivebox-*-amd64.deb merge-multiple: true - - name: Install .deb package + - name: Install system dependencies run: | sudo apt-get update -qq sudo apt-get install -y python3 python3-pip python3-venv nodejs npm git wget curl ripgrep + + - name: Install .deb package + run: | sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y - - name: Test archivebox CLI + - name: Run postinstall setup run: | + # The postinstall script creates the venv and pip-installs archivebox + # If it didn't run during dpkg install, run it manually + if [ ! -f /opt/archivebox/venv/bin/archivebox ]; then + sudo /opt/archivebox/install.sh + fi + + - name: Verify archivebox is installed + run: | + which archivebox archivebox version - mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test - archivebox init --setup + + - name: Test archivebox init + run: | + # Create a test data directory and init + sudo useradd -r -s /bin/bash -d /var/lib/archivebox archivebox 2>/dev/null || true + sudo mkdir -p /tmp/archivebox-test + sudo chown archivebox:archivebox /tmp/archivebox-test + sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init --setup' + + - name: Test archivebox status + run: | + sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4b7a4554..074ec827 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,6 +2,7 @@ name: Build Docker image on: workflow_dispatch: + workflow_call: push: branches: - '**' diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index 6c5924ea..50877a20 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -2,9 +2,13 @@ name: Build Homebrew formula on: workflow_dispatch: + workflow_call: release: types: [published] +permissions: + contents: read + jobs: build: runs-on: macos-latest @@ -23,36 +27,141 @@ jobs: with: python-version: "3.13" + - name: Wait for PyPI package availability + if: github.event_name == 'release' + run: | + VERSION="${{ steps.version.outputs.version }}" + echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..." + for i in $(seq 1 30); do + if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then + echo "[√] archivebox==${VERSION} is available on PyPI" + break + fi + if [ "$i" -eq 30 ]; then + echo "[!] Timed out waiting for PyPI. Trying to install anyway..." + break + fi + echo " Attempt $i/30 - not yet available, waiting 30s..." + sleep 30 + done + - name: Generate Homebrew formula run: | + VERSION="${{ steps.version.outputs.version }}" python3 -m venv /tmp/poet-venv source /tmp/poet-venv/bin/activate - pip install --quiet "archivebox==${{ steps.version.outputs.version }}" homebrew-pypi-poet - poet -f archivebox > /tmp/archivebox-generated.rb + + pip install --quiet "archivebox==${VERSION}" homebrew-pypi-poet + + echo "[+] Generating resource stanzas with homebrew-pypi-poet..." + RESOURCES="$(poet archivebox)" + + # Get sdist URL and SHA256 from PyPI JSON API + PYPI_JSON="$(curl -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" 2>/dev/null || echo '')" + SDIST_URL="" + SDIST_SHA256="" + if [ -n "$PYPI_JSON" ]; then + SDIST_URL="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" + SDIST_SHA256="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['digests']['sha256'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" + fi + if [ -z "$SDIST_URL" ]; then + SDIST_URL="https://files.pythonhosted.org/packages/source/a/archivebox/archivebox-${VERSION}.tar.gz" + fi + if [ -z "$SDIST_SHA256" ]; then + pip download --no-binary :all: --no-deps -d /tmp/sdist "archivebox==${VERSION}" 2>/dev/null || true + SDIST_SHA256="$(shasum -a 256 /tmp/sdist/*.tar.gz 2>/dev/null | awk '{print $1}' || echo '')" + fi + deactivate + # Generate the formula file + cat > /tmp/archivebox.rb << RUBY + # Auto-generated Homebrew formula for archivebox ${VERSION} + # Generated by GitHub Actions homebrew workflow using homebrew-pypi-poet + + class Archivebox < Formula + include Language::Python::Virtualenv + + desc "Self-hosted internet archiving solution" + homepage "https://github.com/ArchiveBox/ArchiveBox" + url "${SDIST_URL}" + sha256 "${SDIST_SHA256}" + license "MIT" + head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" + + depends_on "python@3.13" + depends_on "node" + depends_on "git" + depends_on "wget" + depends_on "curl" + depends_on "ripgrep" + depends_on "yt-dlp" + + on_linux do + depends_on "pkg-config" => :build + depends_on "openssl@3" + depends_on "libffi" + end + + # Python dependency resource blocks auto-generated by homebrew-pypi-poet + ${RESOURCES} + + def install + virtualenv_install_with_resources + end + + def post_install + system bin/"archivebox", "install", "--binproviders", "pip,npm" + end + + service do + run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] + keep_alive crashed: true + working_dir var/"archivebox" + log_path var/"log/archivebox.log" + error_log_path var/"log/archivebox.log" + end + + test do + assert_match version.to_s, shell_output("#{bin}/archivebox version") + end + end + RUBY + + echo "[√] Generated formula at /tmp/archivebox.rb" + cat /tmp/archivebox.rb + - name: Upload formula artifact uses: actions/upload-artifact@v4 with: name: archivebox.rb - path: /tmp/archivebox-generated.rb + path: /tmp/archivebox.rb - name: Push to homebrew-archivebox tap if: github.event_name == 'release' env: GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} run: | - # Clone the tap repo and update the formula + VERSION="${{ steps.version.outputs.version }}" git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap - cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb 2>/dev/null || cp /tmp/archivebox-generated.rb /tmp/tap/archivebox.rb + + # Use the generated formula (with real resources), NOT the template + cp /tmp/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \ + cp /tmp/archivebox.rb /tmp/tap/archivebox.rb + cd /tmp/tap git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git add archivebox.rb - git diff --cached --quiet || git commit -m "Update archivebox to v${{ steps.version.outputs.version }}" - git push origin HEAD + git add -A + if git diff --cached --quiet; then + echo "[i] No changes to formula, skipping push." + else + git commit -m "Update archivebox to v${VERSION}" + git push origin HEAD + echo "[√] Formula pushed to homebrew-archivebox tap" + fi - test: + test-macos: needs: build runs-on: macos-latest @@ -65,7 +174,62 @@ jobs: name: archivebox.rb path: /tmp/ - - name: Test Homebrew formula + - name: Install dependencies + run: | + brew install python@3.13 node git wget curl ripgrep yt-dlp + + - name: Test Homebrew formula install + run: | + brew install --build-from-source /tmp/archivebox.rb + + - name: Verify archivebox CLI run: | - brew install --build-from-source /tmp/archivebox-generated.rb || true archivebox version + + - name: Test archivebox init + run: | + mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test + archivebox init --setup + archivebox status + + test-linux: + needs: build + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + + - name: Download formula artifact + uses: actions/download-artifact@v4 + with: + name: archivebox.rb + path: /tmp/ + + - name: Install Homebrew on Linux + run: | + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo >> "$GITHUB_ENV" + echo 'HOMEBREW_PREFIX=/home/linuxbrew/.linuxbrew' >> "$GITHUB_ENV" + echo '/home/linuxbrew/.linuxbrew/bin' >> "$GITHUB_PATH" + + - name: Install dependencies + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + brew install python@3.13 node git wget curl ripgrep yt-dlp pkg-config openssl@3 libffi + + - name: Test Homebrew formula install + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + brew install --build-from-source /tmp/archivebox.rb + + - name: Verify archivebox CLI + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + archivebox version + + - name: Test archivebox init + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test + archivebox init --setup + archivebox status diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 434e0db5..f4e75503 100755 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -2,6 +2,7 @@ name: Build Pip package on: workflow_dispatch: + workflow_call: push: branches: - '**' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..41f1638c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,40 @@ +name: Release + +# Orchestrates the full release pipeline: +# 1. Build and publish pip package to PyPI +# 2. Build .deb packages and Homebrew formula (in parallel, after pip) +# 3. Build Docker images (in parallel with deb/brew) + +on: + workflow_dispatch: + release: + types: [published] + +permissions: + contents: write + packages: write + id-token: write + +jobs: + pip: + name: Publish to PyPI + uses: ./.github/workflows/pip.yml + secrets: inherit + + debian: + name: Build .deb packages + needs: pip + uses: ./.github/workflows/debian.yml + secrets: inherit + + homebrew: + name: Update Homebrew formula + needs: pip + uses: ./.github/workflows/homebrew.yml + secrets: inherit + + docker: + name: Build Docker images + needs: pip + uses: ./.github/workflows/docker.yml + secrets: inherit diff --git a/bin/build_brew.sh b/bin/build_brew.sh index e56c2a2b..828a22d9 100755 --- a/bin/build_brew.sh +++ b/bin/build_brew.sh @@ -30,12 +30,21 @@ pip install --quiet "archivebox==${VERSION}" homebrew-pypi-poet 2>/dev/null echo "[+] Generating resource stanzas with homebrew-pypi-poet..." RESOURCES="$(poet archivebox)" -# Get the sdist URL and SHA256 from PyPI -SDIST_URL="$(pip download --no-binary :all: --no-deps -d "$TMPDIR/sdist" "archivebox==${VERSION}" 2>&1 | grep -oP 'https://\S+\.tar\.gz' | head -1 || true)" +# Get the sdist URL and SHA256 from PyPI JSON API (works on macOS and Linux) +PYPI_JSON="$(curl -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" 2>/dev/null || echo '')" +if [ -n "$PYPI_JSON" ]; then + SDIST_URL="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" + SDIST_SHA256="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['digests']['sha256'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" +fi if [ -z "$SDIST_URL" ]; then SDIST_URL="https://files.pythonhosted.org/packages/source/a/archivebox/archivebox-${VERSION}.tar.gz" fi -SDIST_SHA256="$(pip hash "$TMPDIR/sdist/"*.tar.gz 2>/dev/null | grep 'sha256:' | cut -d: -f2 || echo '')" +if [ -z "$SDIST_SHA256" ]; then + # Fallback: download and compute locally + mkdir -p "$TMPDIR/sdist" + pip download --no-binary :all: --no-deps -d "$TMPDIR/sdist" "archivebox==${VERSION}" 2>/dev/null || true + SDIST_SHA256="$(shasum -a 256 "$TMPDIR/sdist/"*.tar.gz 2>/dev/null | awk '{print $1}' || echo '')" +fi deactivate @@ -68,6 +77,12 @@ class Archivebox < Formula depends_on "ripgrep" depends_on "yt-dlp" + on_linux do + depends_on "pkg-config" => :build + depends_on "openssl@3" + depends_on "libffi" + end + # Python dependency resource blocks auto-generated by homebrew-pypi-poet # AUTOGENERATED_RESOURCES_START ${RESOURCES} diff --git a/bin/setup.sh b/bin/setup.sh index f496749c..aefcffd8 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -129,11 +129,16 @@ if which apt-get > /dev/null; then echo echo "[+] Downloading and installing ArchiveBox .deb package..." ARCH="$(dpkg --print-architecture)" - DEB_URL="https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${ARCH}.deb" - curl -fsSL "$DEB_URL" -o /tmp/archivebox.deb && sudo apt install -y /tmp/archivebox.deb && rm /tmp/archivebox.deb || { - echo "[!] .deb install failed, falling back to pip install..." - sudo python3 -m pip install --upgrade archivebox yt-dlp - } + # Get the latest release .deb URL from GitHub API (filename includes version) + DEB_URL="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest \ + | grep -o "\"browser_download_url\": \"[^\"]*_${ARCH}\.deb\"" \ + | head -1 | cut -d'"' -f4)" || true + if [ -n "$DEB_URL" ]; then + curl -fsSL "$DEB_URL" -o /tmp/archivebox.deb && sudo apt install -y /tmp/archivebox.deb && rm -f /tmp/archivebox.deb + else + echo "[!] Could not find .deb download URL, falling back to pip install..." + pip install --upgrade archivebox yt-dlp + fi # On Mac: elif which brew > /dev/null; then echo "[+] Installing ArchiveBox using Homebrew..." diff --git a/brew_dist/archivebox.rb b/brew_dist/archivebox.rb index 78aa8abc..47c8fb92 100644 --- a/brew_dist/archivebox.rb +++ b/brew_dist/archivebox.rb @@ -23,6 +23,12 @@ class Archivebox < Formula depends_on "ripgrep" depends_on "yt-dlp" + on_linux do + depends_on "pkg-config" => :build + depends_on "openssl@3" + depends_on "libffi" + end + # Python dependency resource blocks are auto-generated by bin/build_brew.sh # using homebrew-pypi-poet. Run that script to populate this section. # AUTOGENERATED_RESOURCES_START diff --git a/pkg/debian/install.sh b/pkg/debian/install.sh index 20112bf1..54c593f5 100755 --- a/pkg/debian/install.sh +++ b/pkg/debian/install.sh @@ -20,7 +20,10 @@ fi # Install or upgrade archivebox if [ -n "$ARCHIVEBOX_VERSION" ]; then echo "[+] Installing archivebox==$ARCHIVEBOX_VERSION..." - "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade "archivebox==$ARCHIVEBOX_VERSION" + "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade "archivebox==$ARCHIVEBOX_VERSION" || { + echo "[!] archivebox==$ARCHIVEBOX_VERSION not found on PyPI, installing latest..." + "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade archivebox + } else echo "[+] Installing latest archivebox..." "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade archivebox diff --git a/pkg/debian/nfpm.yaml b/pkg/debian/nfpm.yaml index 4b394553..8fc55afc 100644 --- a/pkg/debian/nfpm.yaml +++ b/pkg/debian/nfpm.yaml @@ -19,6 +19,8 @@ section: "web" priority: "optional" depends: + # python3 >= 3.11 allows .deb to install on more systems; + # pip enforces the actual Python version requirement from pyproject.toml - python3 (>= 3.11) - python3-pip - python3-venv diff --git a/pkg/debian/scripts/postinstall.sh b/pkg/debian/scripts/postinstall.sh index 5768ef84..a59a1537 100755 --- a/pkg/debian/scripts/postinstall.sh +++ b/pkg/debian/scripts/postinstall.sh @@ -2,4 +2,24 @@ # postinstall script for archivebox .deb package set -e +# Create archivebox system user if it doesn't exist +if ! id -u archivebox >/dev/null 2>&1; then + useradd --system --shell /bin/bash --home-dir /var/lib/archivebox --create-home archivebox + echo "[+] Created archivebox system user" +fi + +# Ensure data directory exists and is owned by archivebox +mkdir -p /var/lib/archivebox +chown archivebox:archivebox /var/lib/archivebox + +# Run the virtualenv install script, pinning to the .deb package version +ARCHIVEBOX_VERSION="$(dpkg-query -W -f='${Version}' archivebox 2>/dev/null || echo '')" +export ARCHIVEBOX_VERSION /opt/archivebox/install.sh + +# Reload systemd to pick up the service file (skip if systemd is not running) +if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + systemctl daemon-reload + echo "[i] To start ArchiveBox: sudo systemctl start archivebox" + echo "[i] To enable on boot: sudo systemctl enable archivebox" +fi diff --git a/pkg/debian/scripts/preremove.sh b/pkg/debian/scripts/preremove.sh index 65ef33f6..81120cd5 100755 --- a/pkg/debian/scripts/preremove.sh +++ b/pkg/debian/scripts/preremove.sh @@ -2,8 +2,17 @@ # preremove script for archivebox .deb package set -e +# Stop the service if running +if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + systemctl stop archivebox 2>/dev/null || true + systemctl disable archivebox 2>/dev/null || true +fi + echo "[+] Removing ArchiveBox virtualenv..." rm -rf /opt/archivebox/venv -echo "[i] Your ArchiveBox data directories have NOT been removed." -echo " Remove them manually if you no longer need them." +echo "[i] Your ArchiveBox data in /var/lib/archivebox has NOT been removed." +echo " The 'archivebox' system user has NOT been removed." +echo " Remove them manually if you no longer need them:" +echo " sudo rm -rf /var/lib/archivebox" +echo " sudo userdel archivebox"