diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 9b95071e..1ffbffad 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -2,63 +2,209 @@ name: Build Debian package on: workflow_dispatch: + workflow_call: push: + branches: ['**'] + paths: + - 'pkg/debian/**' + - 'bin/build_deb.sh' + - 'bin/release_deb.sh' + - '.github/workflows/debian.yml' + - 'pyproject.toml' + # release trigger is handled by release.yml to avoid double-runs -env: - DEB_BUILD_OPTIONS: nocheck +permissions: + contents: write jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 + strategy: + matrix: + arch: [amd64, arm64] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: - submodules: true fetch-depth: 1 - - name: Install packaging dependencies + - name: Get version + id: version + run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Install nfpm + env: + NFPM_VERSION: "2.45.1" run: | - sudo apt-get update -qq - sudo apt-get install -y \ - python3 python3-dev python3-pip python3-venv python3-all \ - dh-python debhelper devscripts dput software-properties-common \ - python3-distutils python3-setuptools python3-wheel python3-stdeb + curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \ + | sudo tar -xz -C /usr/local/bin nfpm + nfpm --version - # - name: Build Debian/Apt sdist_dsc - # run: | - # ./bin/build_pip.sh + - name: Build .deb package + run: | + export VERSION="${{ steps.version.outputs.version }}" + export ARCH="${{ matrix.arch }}" + ./bin/build_deb.sh - # - name: Check ArchiveBox version - # run: | - # # must create dir needed for snaps to run as non-root on github actions - # sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001 - # mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data" - # archivebox --version - # archivebox init --setup + - name: Verify .deb package contents + run: | + DEB_FILE="$(ls dist/archivebox*.deb | head -1)" + echo "=== Package info ===" + dpkg-deb --info "$DEB_FILE" + echo "" + echo "=== Package contents ===" + dpkg-deb --contents "$DEB_FILE" + echo "" + echo "=== Control fields ===" + dpkg-deb --field "$DEB_FILE" - # - name: Add some links to test - # run: | - # cd "${{ github.workspace }}/data" - # archivebox add 'https://example.com' - # archivebox status + - name: Upload .deb artifact + uses: actions/upload-artifact@v4 + with: + name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb + path: dist/*.deb - # - name: Commit built package - # run: | - # cd deb_dist/ - # git config --local user.email "action@github.com" - # git config --local user.name "GitHub Action" - # git commit -m "Debian package autobuild" -a - - # - name: Push build to Github - # uses: ad-m/github-push-action@master - # with: - # github_token: ${{ secrets.GITHUB_TOKEN }} - # repository: ArchiveBox/debian-archivebox - # branch: ${{ github.ref }} - # directory: deb_dist + test: + needs: build + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} - # - name: Push build to Launchpad PPA - # run: | - # debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes" - # dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes" + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install build dependencies + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 + with: + packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 + version: 1.0 + + - name: Build local wheel + run: | + uv sync --frozen --all-extras --no-install-project --no-install-workspace + uv build --wheel --out-dir /tmp/wheels/ + + - name: Download .deb artifact + uses: actions/download-artifact@v4 + with: + pattern: archivebox-*-${{ matrix.arch }}.deb + merge-multiple: true + + - name: Install system dependencies + run: | + sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget + + - name: Pre-seed virtualenv with local wheel before dpkg install + run: | + # CI-only: pre-seed the venv with a local wheel so we test the + # unreleased code, not whatever is on PyPI. We explicitly use python3.13 + # here (matching the system dep installed above) to ensure the venv is + # created with the correct Python version. On real installs, install.sh + # handles this by preferring python3.13 and failing if < 3.13. + # When postinstall.sh runs during dpkg -i, it finds the venv already + # populated and just upgrades deps. + sudo mkdir -p /opt/archivebox + sudo python3.13 -m venv /opt/archivebox/venv + sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools + sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl + echo "[√] Pre-seeded /opt/archivebox/venv with local wheel" + + - name: Install .deb package + run: | + # dpkg install will run postinstall which creates the user, + # sets up systemd, and tries pip install (which finds it already installed) + sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y + + - name: Verify archivebox is installed + run: | + which archivebox + archivebox version + + - name: Verify wrapper script works + run: | + /usr/bin/archivebox version + /usr/bin/archivebox --help | head -5 + + - name: Test archivebox init as archivebox user + run: | + # The postinstall should have created the user + id archivebox + sudo mkdir -p /tmp/archivebox-test + sudo chown archivebox:archivebox /tmp/archivebox-test + sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init' + + - name: Test archivebox status + run: | + sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status' + + - name: Test archivebox add + run: | + sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"' + + - name: Verify systemd service file exists + run: | + test -f /usr/lib/systemd/system/archivebox.service + cat /usr/lib/systemd/system/archivebox.service + + # Upload .deb to GitHub Release only after tests pass + release: + if: github.event_name == 'release' || github.event_name == 'workflow_call' + needs: [build, test] + runs-on: ubuntu-24.04 + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Determine release tag + id: tag + run: | + TAG="${{ github.event.release.tag_name }}" + if [ -z "$TAG" ]; then + TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" + echo "[i] No release tag in event context, using version from pyproject.toml: $TAG" + fi + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + + - name: Download all .deb artifacts + uses: actions/download-artifact@v4 + with: + pattern: archivebox-*.deb + merge-multiple: true + + - name: Upload .deb to GitHub Release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + TAG="${{ steps.tag.outputs.tag }}" + # Verify the release exists before uploading + if ! gh release view "$TAG" >/dev/null 2>&1; then + echo "[!] No GitHub Release found for tag $TAG." + if [ -n "${{ github.event.release.tag_name }}" ]; then + echo "[X] This was triggered by a release event — the release should exist. Failing." + exit 1 + fi + echo "[i] Skipping upload (workflow_dispatch without a release)." + echo " Create a release first or trigger via the release event." + exit 0 + fi + gh release upload "$TAG" *.deb --clobber diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4b7a4554..ce0da517 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,6 +2,7 @@ name: Build Docker image on: workflow_dispatch: + workflow_call: push: branches: - '**' @@ -65,7 +66,7 @@ jobs: # https://github.com/docker/metadata-action id: docker_meta uses: docker/metadata-action@v5 - if: github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' with: images: archivebox/archivebox,ghcr.io/archivebox/archivebox tags: | @@ -79,12 +80,12 @@ jobs: type=sha # :latest type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }} - + - name: Collect Non-Release Docker tags # https://github.com/docker/metadata-action id: docker_meta_non_release uses: docker/metadata-action@v5 - if: github.event_name != 'workflow_dispatch' + if: github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call' with: images: archivebox/archivebox,ghcr.io/archivebox/archivebox tags: | @@ -101,8 +102,8 @@ jobs: file: ./Dockerfile builder: ${{ steps.buildx.outputs.name }} push: ${{ github.event_name != 'pull_request' }} - tags: ${{ github.event_name == 'workflow_dispatch' ? steps.docker_meta.outputs.tags : steps.docker_meta_non_release.outputs.tags }} - labels: ${{ github.event_name == 'workflow_dispatch' ? steps.docker_meta.outputs.labels : steps.docker_meta_non_release.outputs.labels }} + tags: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.tags || steps.docker_meta_non_release.outputs.tags }} + labels: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.labels || steps.docker_meta_non_release.outputs.labels }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index af7a0795..70ce8ded 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -1,51 +1,229 @@ -name: Build Homebrew package +name: Build Homebrew formula on: workflow_dispatch: + workflow_call: push: + branches: ['**'] + paths: + - 'brew_dist/**' + - 'bin/build_brew.sh' + - 'bin/release_brew.sh' + - '.github/workflows/homebrew.yml' + - 'pyproject.toml' + # release trigger is handled by release.yml to avoid double-runs +permissions: + contents: read jobs: - build: + build-and-test: + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-24.04] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Get version + id: version + run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Validate formula template syntax + run: ruby -c brew_dist/archivebox.rb + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install build dependencies (Linux) + if: runner.os == 'Linux' + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 + with: + packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 + version: 1.0 + + - name: Build local sdist + run: | + uv sync --frozen --all-extras --no-install-project --no-install-workspace + uv build --sdist --out-dir /tmp/sdist/ + + - name: Generate formula from local sdist + run: | + VERSION="${{ steps.version.outputs.version }}" + SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)" + SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')" + + # Install archivebox + poet into a temp venv to generate resource stanzas + python3 -m venv /tmp/poet-venv + source /tmp/poet-venv/bin/activate + pip install --quiet "$SDIST_PATH" homebrew-pypi-poet + echo "[+] Generating resource stanzas with homebrew-pypi-poet..." + RESOURCES="$(poet archivebox)" + deactivate + + # For CI: use file:// URL pointing to local sdist + # For release: this gets overridden with the PyPI URL + SDIST_URL="file://${SDIST_PATH}" + + cat > /tmp/archivebox.rb << RUBY +class Archivebox < Formula + include Language::Python::Virtualenv + + desc "Self-hosted internet archiving solution" + homepage "https://github.com/ArchiveBox/ArchiveBox" + url "${SDIST_URL}" + sha256 "${SDIST_SHA256}" + license "MIT" + + depends_on "python@3.13" + +${RESOURCES} + + def install + virtualenv_install_with_resources + end + + def post_install + data_dir = var/"archivebox" + data_dir.mkpath + ENV["DATA_DIR"] = data_dir.to_s + system bin/"archivebox", "init" + end + + def caveats + <<~EOS + ArchiveBox data is stored in: + #{var}/archivebox + + To start archiving, run: + cd #{var}/archivebox && archivebox add 'https://example.com' + + To start the web UI: + cd #{var}/archivebox && archivebox server 0.0.0.0:8000 + EOS + end + + test do + assert_match version.to_s, shell_output("#{bin}/archivebox version") + end +end +RUBY + + echo "[√] Generated formula:" + ruby -c /tmp/archivebox.rb + cat /tmp/archivebox.rb + + - name: Install Homebrew (Linux only) + if: runner.os == 'Linux' + run: | + NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV" + echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV" + echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV" + echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH" + echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH" + + - name: Install brew dependencies + run: brew install python@3.13 + + - name: Install archivebox via brew from local formula + run: | + brew install --build-from-source --verbose /tmp/archivebox.rb + + - name: Verify archivebox version + run: archivebox version + + - name: Test archivebox init + run: | + mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test + archivebox init + + - name: Test archivebox status + run: | + cd /tmp/archivebox-test + archivebox status + + # On release only: generate the real formula with PyPI URL and push to tap + release: + if: github.event_name == 'release' || github.event_name == 'workflow_call' + needs: build-and-test runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: - submodules: true fetch-depth: 1 - # TODO: modify archivebox.rb to update src url, hashes, and dependencies + - name: Get version + id: version + run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT" - - name: Build Homebrew Bottle + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Wait for PyPI package availability run: | - pip3 install --upgrade pip setuptools wheel - cd brew_dist/ - brew install --build-bottle ./archivebox.rb - # brew bottle archivebox + VERSION="${{ steps.version.outputs.version }}" + echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..." + for i in $(seq 1 30); do + if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then + echo "[√] archivebox==${VERSION} is available on PyPI" + break + fi + if [ "$i" -eq 30 ]; then + echo "[!] Timed out waiting for PyPI. Trying to install anyway..." + break + fi + echo " Attempt $i/30 - not yet available, waiting 30s..." + sleep 30 + done + + - name: Generate release formula via build_brew.sh + run: ./bin/build_brew.sh + + - name: Test formula install + run: | + brew install --build-from-source brew_dist/archivebox.rb archivebox version - - name: Add some links to test + - name: Upload formula artifact + uses: actions/upload-artifact@v4 + with: + name: archivebox.rb + path: brew_dist/archivebox.rb + + - name: Push to homebrew-archivebox tap + env: + GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} run: | - mkdir data && cd data - archivebox init --setup - archivebox add 'https://example.com' - archivebox version - archivebox status + VERSION="${{ steps.version.outputs.version }}" + git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap - # - name: Commit built package - # run: | - # cd brew_dist/ - # git config --local user.email "action@github.com" - # git config --local user.name "GitHub Action" - # git commit -m "Homebrew package autobuild" -a - - # - name: Push build to Github - # uses: ad-m/github-push-action@master - # with: - # github_token: ${{ secrets.GITHUB_TOKEN }} - # repository: ArchiveBox/homebrew-archivebox - # branch: ${{ github.ref }} - # directory: brew_dist + cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \ + cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb - # TODO: push bottle homebrew core PR with latest changes + cd /tmp/tap + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add -A + if git diff --cached --quiet; then + echo "[i] No changes to formula, skipping push." + else + git commit -m "Update archivebox to v${VERSION}" + git push origin HEAD + echo "[√] Formula pushed to homebrew-archivebox tap" + fi diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 434e0db5..f4e75503 100755 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -2,6 +2,7 @@ name: Build Pip package on: workflow_dispatch: + workflow_call: push: branches: - '**' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..86a507fa --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,43 @@ +name: Release + +# Orchestrates the full release pipeline: +# 1. Build and publish pip package to PyPI +# 2. Build .deb packages and Homebrew formula (in parallel, after pip) +# 3. Build Docker images (in parallel with deb/brew) +# +# Individual workflows also run on push for CI (see their own triggers). +# This workflow ensures the correct ordering during a release. + +on: + workflow_dispatch: + release: + types: [published] + +permissions: + contents: write + packages: write + id-token: write + +jobs: + pip: + name: Publish to PyPI + uses: ./.github/workflows/pip.yml + secrets: inherit + + debian: + name: Build .deb packages + needs: pip + uses: ./.github/workflows/debian.yml + secrets: inherit + + homebrew: + name: Update Homebrew formula + needs: pip + uses: ./.github/workflows/homebrew.yml + secrets: inherit + + docker: + name: Build Docker images + needs: pip + uses: ./.github/workflows/docker.yml + secrets: inherit diff --git a/.github/workflows/test-parallel.yml b/.github/workflows/test-parallel.yml index 77db7ac6..bb7855b1 100644 --- a/.github/workflows/test-parallel.yml +++ b/.github/workflows/test-parallel.yml @@ -115,4 +115,5 @@ jobs: - name: Run test - ${{ matrix.test.name }} run: | + mkdir -p tests/out uv run pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs diff --git a/README.md b/README.md index 051ac2ff..9a74338e 100644 --- a/README.md +++ b/README.md @@ -293,19 +293,13 @@ See below for more usage examples using the C
aptitude apt (Ubuntu/Debian/etc.)
-See the Install: Bare Metal Wiki for instructions. ➡️ - +See the debian-archivebox repo for more details about this distribution.

@@ -336,9 +330,6 @@ See below for more usage examples using the C
  • Install the ArchiveBox package using brew. 
    brew tap archivebox/archivebox
 brew install archivebox
-# update to newest version with pip (sometimes brew package is outdated)
-pip install --upgrade --ignore-installed archivebox yt-dlp playwright
-playwright install --with-deps chromium    # install chromium and its system dependencies
 archivebox version                         # make sure all dependencies are installed
    See the Install: Bare Metal Wiki for more granular instructions for macOS... ➡️ diff --git a/bin/build.sh b/bin/build.sh index b3271873..1ebf2d3d 100755 --- a/bin/build.sh +++ b/bin/build.sh @@ -19,9 +19,13 @@ cd "$REPO_DIR" # the order matters ./bin/build_docs.sh ./bin/build_pip.sh +./bin/build_deb.sh +./bin/build_brew.sh ./bin/build_docker.sh echo "[√] Done. Install the built package by running:" -echo " python3 setup.py install" +echo " pip install archivebox" echo " # or" -echo " pip3 install ." +echo " sudo apt install ./dist/archivebox*.deb" +echo " # or" +echo " brew tap archivebox/archivebox && brew install archivebox" diff --git a/bin/build_brew.sh b/bin/build_brew.sh new file mode 100755 index 00000000..80d4aa11 --- /dev/null +++ b/bin/build_brew.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +cd "$REPO_DIR" + +VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" +FORMULA_FILE="$REPO_DIR/brew_dist/archivebox.rb" + +echo "[+] Building Homebrew formula for archivebox==${VERSION}..." + +# Create a temporary virtualenv for generating the formula +TMPDIR="$(mktemp -d)" +trap "rm -rf $TMPDIR" EXIT + +python3 -m venv "$TMPDIR/venv" +source "$TMPDIR/venv/bin/activate" + +pip install --quiet "archivebox==${VERSION}" homebrew-pypi-poet 2>/dev/null + +echo "[+] Generating resource stanzas with homebrew-pypi-poet..." +RESOURCES="$(poet archivebox)" + +# Get the sdist URL and SHA256 from PyPI JSON API (works on macOS and Linux) +SDIST_URL="" +SDIST_SHA256="" +PYPI_JSON="$(curl -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" 2>/dev/null || echo '')" +if [ -n "$PYPI_JSON" ]; then + SDIST_URL="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" + SDIST_SHA256="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['digests']['sha256'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')" +fi +if [ -z "$SDIST_URL" ]; then + SDIST_URL="https://files.pythonhosted.org/packages/source/a/archivebox/archivebox-${VERSION}.tar.gz" +fi +if [ -z "$SDIST_SHA256" ]; then + # Fallback: download and compute locally + mkdir -p "$TMPDIR/sdist" + pip download --no-binary :all: --no-deps -d "$TMPDIR/sdist" "archivebox==${VERSION}" 2>/dev/null || true + SDIST_SHA256="$(shasum -a 256 "$TMPDIR/sdist/"*.tar.gz 2>/dev/null | awk '{print $1}' || echo '')" +fi + +deactivate + +echo "[+] Updating formula file: $FORMULA_FILE" + +# Build the formula from the template +cat > "$FORMULA_FILE" << RUBY +# Auto-generated by bin/build_brew.sh using homebrew-pypi-poet. +# Users install with: brew tap archivebox/archivebox && brew install archivebox + +class Archivebox < Formula + include Language::Python::Virtualenv + + desc "Self-hosted internet archiving solution" + homepage "https://github.com/ArchiveBox/ArchiveBox" + url "${SDIST_URL}" + sha256 "${SDIST_SHA256}" + license "MIT" + head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" + + depends_on "python@3.13" + # All other runtime deps (node, chrome, yt-dlp, etc.) are installed + # on-demand by \`archivebox install\` and should NOT be declared here. + + # Python dependency resource blocks auto-generated by homebrew-pypi-poet + # AUTOGENERATED_RESOURCES_START +${RESOURCES} + # AUTOGENERATED_RESOURCES_END + + def install + virtualenv_install_with_resources + end + + def post_install + # Initialize ArchiveBox data in the Homebrew-managed var directory + data_dir = var/"archivebox" + data_dir.mkpath + ENV["DATA_DIR"] = data_dir.to_s + system bin/"archivebox", "init" + end + + def caveats + <<~EOS + ArchiveBox data is stored in: + #{var}/archivebox + + To start archiving, run: + cd #{var}/archivebox && archivebox add 'https://example.com' + + To start the web UI: + cd #{var}/archivebox && archivebox server 0.0.0.0:8000 + EOS + end + + test do + assert_match version.to_s, shell_output("#{bin}/archivebox version") + end +end +RUBY + +echo "[√] Formula updated: $FORMULA_FILE" +echo " Version: ${VERSION}" +echo " URL: ${SDIST_URL}" diff --git a/bin/build_deb.sh b/bin/build_deb.sh new file mode 100755 index 00000000..08c0950d --- /dev/null +++ b/bin/build_deb.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +cd "$REPO_DIR" + +VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" +export VERSION + +# Default to amd64, can be overridden with ARCH=arm64 +export ARCH="${ARCH:-amd64}" + +echo "[+] Building .deb package for archivebox_${VERSION}_${ARCH}..." + +# Check for nfpm +if ! command -v nfpm &>/dev/null; then + echo "[!] nfpm not found. Install it with one of:" + echo " go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest" + echo " uv tool install nfpm" + echo " brew install goreleaser/tap/nfpm" + echo " curl -sfL https://install.goreleaser.com/github.com/goreleaser/nfpm.sh | sh" + exit 1 +fi + +mkdir -p "$REPO_DIR/dist" + +nfpm package \ + --config "$REPO_DIR/pkg/debian/nfpm.yaml" \ + --packager deb \ + --target "$REPO_DIR/dist/" + +echo +echo "[√] Built .deb package:" +ls -la "$REPO_DIR/dist/"archivebox*.deb diff --git a/bin/release.sh b/bin/release.sh index 4170b0d2..a2fd719a 100755 --- a/bin/release.sh +++ b/bin/release.sh @@ -28,6 +28,8 @@ cd "$REPO_DIR" # ./bin/release_docs.sh ./bin/release_git.sh "$@" ./bin/release_pip.sh "$@" +./bin/release_deb.sh "$@" +./bin/release_brew.sh "$@" ./bin/release_docker.sh "$@" VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" diff --git a/bin/release_brew.sh b/bin/release_brew.sh new file mode 100755 index 00000000..19fbe90a --- /dev/null +++ b/bin/release_brew.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +cd "$REPO_DIR" + +VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" +FORMULA_FILE="$REPO_DIR/brew_dist/archivebox.rb" +TAP_REPO="ArchiveBox/homebrew-archivebox" + +if [ ! -f "$FORMULA_FILE" ]; then + echo "[!] Formula not found at $FORMULA_FILE" + echo " Run ./bin/build_brew.sh first to generate it." + exit 1 +fi + +echo "[+] Releasing Homebrew formula for archivebox==${VERSION} to ${TAP_REPO}..." + +# Clone the tap repo, update formula, commit, and push +TMPDIR="$(mktemp -d)" +trap "rm -rf $TMPDIR" EXIT + +git clone "https://github.com/${TAP_REPO}.git" "$TMPDIR/tap" +cp "$FORMULA_FILE" "$TMPDIR/tap/archivebox.rb" + +cd "$TMPDIR/tap" +git add archivebox.rb +if git diff --cached --quiet; then + echo "[i] No changes to formula, skipping release." + exit 0 +fi + +git commit -m "Update archivebox to v${VERSION}" +git push origin HEAD + +echo "[√] Homebrew formula pushed to ${TAP_REPO}" +echo " Users can install with:" +echo " brew tap archivebox/archivebox" +echo " brew install archivebox" diff --git a/bin/release_deb.sh b/bin/release_deb.sh new file mode 100755 index 00000000..45779f5c --- /dev/null +++ b/bin/release_deb.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +### Bash Environment Setup +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html +# set -o xtrace +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail +IFS=$'\n' + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" +cd "$REPO_DIR" + +VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" + +echo "[+] Releasing .deb package for archivebox==${VERSION}..." + +DEB_FILE="$(ls -1 "$REPO_DIR/dist/"archivebox*.deb 2>/dev/null | head -1)" +if [ -z "$DEB_FILE" ]; then + echo "[!] No .deb file found in dist/. Run ./bin/build_deb.sh first." + exit 1 +fi + +echo "[+] Uploading $DEB_FILE to GitHub Release v${VERSION}..." +gh release upload "v${VERSION}" "$DEB_FILE" --clobber 2>/dev/null || \ + gh release create "v${VERSION}" "$DEB_FILE" --title "v${VERSION}" --generate-notes + +echo "[√] .deb package uploaded to GitHub Release v${VERSION}" +echo " Users can install with:" +echo " curl -fsSL https://github.com/ArchiveBox/ArchiveBox/releases/download/v${VERSION}/archivebox_${VERSION}_amd64.deb -o /tmp/archivebox.deb" +echo " sudo apt install /tmp/archivebox.deb" diff --git a/bin/setup.sh b/bin/setup.sh index 5add55d4..d6a89fa7 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -122,32 +122,19 @@ echo # On Linux: if which apt-get > /dev/null; then - echo "[+] Adding ArchiveBox apt repo and signing key to sources..." - if ! (sudo apt install -y software-properties-common && sudo add-apt-repository -u ppa:archivebox/archivebox); then - echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list - echo "deb-src http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee -a /etc/apt/sources.list.d/archivebox.list - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C258F79DCC02E369 - sudo apt-get update -qq - fi - echo echo "[+] Installing ArchiveBox system dependencies using apt..." - sudo apt-get install -y git python3 python3-pip python3-distutils wget curl yt-dlp ffmpeg git nodejs npm ripgrep - sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb libgbm-dev || sudo apt-get install -y chromium || sudo apt-get install -y chromium-browser || true - sudo apt-get install -y archivebox - sudo apt-get --only-upgrade install -y archivebox + sudo apt-get update -qq + sudo apt-get install -y git python3 python3-pip python3-venv wget curl yt-dlp ffmpeg git nodejs npm ripgrep + sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libnss3 libxss1 libasound2 libxtst6 xauth xvfb libgbm-dev || sudo apt-get install -y chromium || sudo apt-get install -y chromium-browser || true echo echo "[+] Installing ArchiveBox python dependencies using pip3..." - sudo python3 -m pip install --upgrade --ignore-installed archivebox yt-dlp playwright + sudo python3 -m pip install --upgrade --ignore-installed archivebox yt-dlp # On Mac: elif which brew > /dev/null; then - echo "[+] Installing ArchiveBox system dependencies using brew..." + echo "[+] Installing ArchiveBox using Homebrew..." brew tap archivebox/archivebox brew update - brew install python3 node git wget curl yt-dlp ripgrep - brew install --fetch-HEAD -f archivebox - echo - echo "[+] Installing ArchiveBox python dependencies using pip3..." - python3 -m pip install --upgrade --ignore-installed archivebox yt-dlp playwright + brew install archivebox elif which pkg > /dev/null; then echo "[+] Installing ArchiveBox system dependencies using pkg and pip (python3.9)..." sudo pkg install -y python3 py39-pip py39-sqlite3 npm wget curl youtube_dl ffmpeg git ripgrep @@ -168,19 +155,28 @@ fi echo -if ! (python3 --version && python3 -m pip --version && python3 -m django --version); then - echo "[X] Python 3 pip was not found on your system!" - echo " You must first install Python >= 3.7 (and pip3):" - echo " https://www.python.org/downloads/" - echo " https://wiki.python.org/moin/BeginnersGuide/Download" - echo " After installing, run this script again." - exit 1 +if ! which archivebox > /dev/null 2>&1; then + # If archivebox isn't in PATH (e.g. pip install), check python modules directly + if ! (python3 --version && python3 -m pip --version && python3 -m django --version) 2>/dev/null; then + echo "[X] Python 3 pip was not found on your system!" + echo " You must first install Python >= 3.7 (and pip3):" + echo " https://www.python.org/downloads/" + echo " https://wiki.python.org/moin/BeginnersGuide/Download" + echo " After installing, run this script again." + exit 1 + fi + + if ! (python3 -m django --version && python3 -m pip show archivebox) 2>/dev/null; then + echo "[X] Django and ArchiveBox were not found after installing!" + echo " Check to see if a previous step failed." + echo + exit 1 + fi fi -if ! (python3 -m django --version && python3 -m pip show archivebox && which -a archivebox); then - echo "[X] Django and ArchiveBox were not found after installing!" +if ! which archivebox > /dev/null 2>&1; then + echo "[X] archivebox command was not found in PATH after installing!" echo " Check to see if a previous step failed." - echo exit 1 fi diff --git a/brew_dist/archivebox.rb b/brew_dist/archivebox.rb new file mode 100644 index 00000000..3e4d2f31 --- /dev/null +++ b/brew_dist/archivebox.rb @@ -0,0 +1,50 @@ +# Auto-generated by bin/build_brew.sh using homebrew-pypi-poet. +# Users install with: brew tap archivebox/archivebox && brew install archivebox + +class Archivebox < Formula + include Language::Python::Virtualenv + + desc "Self-hosted internet archiving solution" + homepage "https://github.com/ArchiveBox/ArchiveBox" + url "https://files.pythonhosted.org/packages/source/a/archivebox/archivebox-0.9.3.tar.gz" + sha256 "" # auto-filled by bin/build_brew.sh + license "MIT" + head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" + + depends_on "python@3.13" + # All other runtime deps (node, chrome, yt-dlp, etc.) are installed + # on-demand by `archivebox install` and should NOT be declared here. + + # Python dependency resource blocks auto-generated by homebrew-pypi-poet + # AUTOGENERATED_RESOURCES_START + # AUTOGENERATED_RESOURCES_END + + def install + virtualenv_install_with_resources + end + + def post_install + # Initialize ArchiveBox data in the Homebrew-managed var directory + data_dir = var/"archivebox" + data_dir.mkpath + ENV["DATA_DIR"] = data_dir.to_s + system bin/"archivebox", "init" + end + + def caveats + <<~EOS + ArchiveBox data is stored in: + #{var}/archivebox + + To start archiving, run: + cd #{var}/archivebox && archivebox add 'https://example.com' + + To start the web UI: + cd #{var}/archivebox && archivebox server 0.0.0.0:8000 + EOS + end + + test do + assert_match version.to_s, shell_output("#{bin}/archivebox version") + end +end diff --git a/pkg/debian/archivebox b/pkg/debian/archivebox new file mode 100755 index 00000000..0ebccabc --- /dev/null +++ b/pkg/debian/archivebox @@ -0,0 +1,15 @@ +#!/bin/bash +# /usr/bin/archivebox - wrapper script installed by the archivebox .deb package +# Activates the pip-installed virtualenv and runs archivebox CLI + +ARCHIVEBOX_VENV="/opt/archivebox/venv" + +if [ ! -f "$ARCHIVEBOX_VENV/bin/archivebox" ]; then + echo "Error: ArchiveBox is not installed in $ARCHIVEBOX_VENV" + echo "Try running: sudo /opt/archivebox/install.sh" + exit 1 +fi + +# Export venv bin to PATH so bundled console scripts (yt-dlp, etc.) are discoverable +export PATH="$ARCHIVEBOX_VENV/bin:$PATH" +exec "$ARCHIVEBOX_VENV/bin/archivebox" "$@" diff --git a/pkg/debian/archivebox.service b/pkg/debian/archivebox.service new file mode 100644 index 00000000..af8b4500 --- /dev/null +++ b/pkg/debian/archivebox.service @@ -0,0 +1,20 @@ +# The archivebox user/group and /var/lib/archivebox directory are created by +# postinstall.sh (which runs after dpkg unpacks the package contents). + +[Unit] +Description=ArchiveBox Web Archiving Server +After=network.target + +[Service] +Type=simple +User=archivebox +Group=archivebox +WorkingDirectory=/var/lib/archivebox +Environment="PATH=/opt/archivebox/venv/bin:/usr/local/bin:/usr/bin:/bin" +ExecStartPre=/usr/bin/archivebox init +ExecStart=/usr/bin/archivebox server 0.0.0.0:8000 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/pkg/debian/install.sh b/pkg/debian/install.sh new file mode 100755 index 00000000..1c72758e --- /dev/null +++ b/pkg/debian/install.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# /opt/archivebox/install.sh - installs/upgrades archivebox into its virtualenv +# Called by the postinstall script and can be run manually to upgrade + +set -e + +ARCHIVEBOX_VENV="/opt/archivebox/venv" +ARCHIVEBOX_VERSION="${ARCHIVEBOX_VERSION:-}" + +# ArchiveBox requires Python >= 3.13 (per pyproject.toml). +# Prefer python3.13 explicitly; fall back to python3 with a version check. +if command -v python3.13 >/dev/null 2>&1; then + PYTHON="python3.13" +elif command -v python3 >/dev/null 2>&1; then + PYTHON="python3" + PY_MAJOR="$("$PYTHON" -c 'import sys; print(sys.version_info.major)')" + PY_MINOR="$("$PYTHON" -c 'import sys; print(sys.version_info.minor)')" + if [ "$PY_MAJOR" -lt 3 ] || { [ "$PY_MAJOR" -eq 3 ] && [ "$PY_MINOR" -lt 13 ]; }; then + PY_VER="${PY_MAJOR}.${PY_MINOR}" + echo "[!] Error: ArchiveBox requires Python >= 3.13, but found Python $PY_VER" + echo " Install python3.13: sudo apt install python3.13 python3.13-venv" + exit 1 + fi +else + echo "[!] Error: python3 not found. Install python3.13: sudo apt install python3.13 python3.13-venv" + exit 1 +fi + +echo "[+] Setting up ArchiveBox virtualenv in $ARCHIVEBOX_VENV (using $PYTHON)..." + +# Create the virtualenv if it doesn't exist +if [ ! -d "$ARCHIVEBOX_VENV" ]; then + "$PYTHON" -m venv "$ARCHIVEBOX_VENV" +fi + +# Upgrade pip inside the virtualenv +"$ARCHIVEBOX_VENV/bin/python3" -m pip install --quiet --upgrade pip setuptools + +# Install or upgrade archivebox. +# ARCHIVEBOX_VERSION is set by postinstall.sh from the .deb package version. +# When run manually without it, install the latest release from PyPI. +if [ -n "$ARCHIVEBOX_VERSION" ]; then + echo "[+] Installing archivebox==$ARCHIVEBOX_VERSION..." + "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade "archivebox==$ARCHIVEBOX_VERSION" +else + echo "[+] Installing latest archivebox (no version pinned)..." + "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade archivebox +fi + +echo "[√] ArchiveBox installed successfully." +echo " Run 'archivebox version' to verify." diff --git a/pkg/debian/nfpm.yaml b/pkg/debian/nfpm.yaml new file mode 100644 index 00000000..fa3df26c --- /dev/null +++ b/pkg/debian/nfpm.yaml @@ -0,0 +1,69 @@ +# nFPM configuration for building ArchiveBox .deb packages +# Docs: https://nfpm.goreleaser.com/configuration/ +# Usage: nfpm package --config pkg/debian/nfpm.yaml --packager deb --target dist/ + +name: archivebox +arch: "${ARCH:-amd64}" +platform: linux +version: "${VERSION}" +version_schema: semver +maintainer: "Nick Sweeting " +description: | + Self-hosted internet archiving solution. + Save pages from the web including HTML, PDF, screenshots, media, and more. + Install with: sudo apt install archivebox && archivebox init --setup +vendor: "ArchiveBox" +homepage: "https://archivebox.io" +license: "MIT" +section: "web" +priority: "optional" + +depends: + # python3 >= 3.11 allows dpkg to install on more systems (e.g. Ubuntu 24.04). + # install.sh enforces the real >= 3.13 requirement at venv creation time, + # failing early with a clear error if only an older python3 is available. + - python3 (>= 3.11) + - python3-pip + - python3-venv + # All other runtime deps (node, chrome, yt-dlp, etc.) are installed on-demand + # by `archivebox install` and should NOT be declared as package dependencies. + +recommends: + # Common utilities used by archivebox extractors. Declared as recommends + # (not depends) so dpkg doesn't hard-fail if they're missing, but apt + # installs them by default so users have a working baseline out of the box. + - git + - curl + - wget + +contents: + # Wrapper script for /usr/bin/archivebox + - src: pkg/debian/archivebox + dst: /usr/bin/archivebox + file_info: + mode: 0755 + + # Install helper script + - src: pkg/debian/install.sh + dst: /opt/archivebox/install.sh + file_info: + mode: 0755 + + # Systemd service file + - src: pkg/debian/archivebox.service + dst: /usr/lib/systemd/system/archivebox.service + file_info: + mode: 0644 + + # Create data directory (unpacked as root; postinstall.sh chowns to archivebox user) + - dst: /var/lib/archivebox + type: dir + file_info: + mode: 0755 + +scripts: + postinstall: pkg/debian/scripts/postinstall.sh + preremove: pkg/debian/scripts/preremove.sh + +deb: + compression: zstd diff --git a/pkg/debian/scripts/postinstall.sh b/pkg/debian/scripts/postinstall.sh new file mode 100755 index 00000000..a175c1b5 --- /dev/null +++ b/pkg/debian/scripts/postinstall.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# postinstall script for archivebox .deb package +set -e + +# Create archivebox system user if it doesn't exist +if ! id -u archivebox >/dev/null 2>&1; then + useradd --system --shell /bin/bash --home-dir /var/lib/archivebox --create-home archivebox + echo "[+] Created archivebox system user" +fi + +# Ensure data directory exists and is owned by archivebox +mkdir -p /var/lib/archivebox +chown archivebox:archivebox /var/lib/archivebox + +# Run the virtualenv install script, pinning to the .deb package version +ARCHIVEBOX_VERSION="$(dpkg-query -W -f='${Version}' archivebox 2>/dev/null || echo '')" +export ARCHIVEBOX_VERSION +/opt/archivebox/install.sh + +# Reload systemd to pick up the service file (skip if systemd is not running) +if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + systemctl daemon-reload + + # On upgrade: restart the service if it was enabled (prerm stopped it) + if [ "$1" = "configure" ] && systemctl is-enabled archivebox >/dev/null 2>&1; then + systemctl start archivebox 2>/dev/null || true + echo "[+] Restarted archivebox service after upgrade" + else + echo "[i] To start ArchiveBox: sudo systemctl start archivebox" + echo "[i] To enable on boot: sudo systemctl enable archivebox" + fi +fi diff --git a/pkg/debian/scripts/preremove.sh b/pkg/debian/scripts/preremove.sh new file mode 100755 index 00000000..65fc7330 --- /dev/null +++ b/pkg/debian/scripts/preremove.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# preremove script for archivebox .deb package +set -e + +# dpkg passes "$1" as "remove", "purge", or "upgrade". + +# Always stop the service before removing or upgrading, because postinstall +# replaces the venv in-place — the running process would use stale binaries. +if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + systemctl stop archivebox 2>/dev/null || true +fi + +# Only disable + clean up on full removal, not during upgrade. +if [ "$1" = "remove" ] || [ "$1" = "purge" ]; then + if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + systemctl disable archivebox 2>/dev/null || true + fi + + echo "[+] Removing ArchiveBox virtualenv..." + rm -rf /opt/archivebox/venv + + echo "[i] Your ArchiveBox data in /var/lib/archivebox has NOT been removed." + echo " The 'archivebox' system user has NOT been removed." + echo " Remove them manually if you no longer need them:" + echo " sudo rm -rf /var/lib/archivebox" + echo " sudo userdel archivebox" +fi