release: archivebox 0.9.35rc34

.github/workflows/release-runner.yml

@@ -3,19 +3,21 @@ name: Release State
 on:
   push:
     branches:
-      - '**'
+      - dev
   workflow_dispatch:
 
 permissions:
+  actions: write
   contents: write
   id-token: write
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: false
 
 jobs:
   release-state:
+    if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'release:')
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -23,6 +25,7 @@ jobs:
           fetch-depth: 0
           submodules: true
           ref: ${{ github.ref_name }}
+          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
 
       - uses: actions/setup-python@v5
         with:
@@ -44,7 +47,14 @@ jobs:
 
       - name: Run release script
         env:
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-          GH_TOKEN: ${{ github.token }}
+          DEFAULT_BRANCH: dev
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
           PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
         run: ./bin/release.sh
+
+      - name: Dispatch image and tap updates
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
+        run: |
+          gh workflow run docker.yml --ref dev
+          gh workflow run update-homebrew-tap.yml --ref dev

Dockerfile

@@ -2,7 +2,7 @@
 
 # Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
 # abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
+# ArchiveBox owns sonic, supervisor, Django, and the app runtime.
 # Build abx-dl first, then point this file at it:
 #   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
 #       --build-context abxbus=../abxbus \
@@ -93,12 +93,6 @@ RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
     && which uv && uv self version \
     ) | tee -a /VERSION.txt
 
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends ripgrep \
-    && rm -rf /var/lib/apt/lists/*
-
 FROM archivebox-runtime-base AS archivebox-builder
 
 WORKDIR "$CODE_DIR"
@@ -255,7 +249,13 @@ RUN --mount=type=cache,target=/tmp/abxpkg-cache,sharing=locked,mode=1777 \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache archivebox version 2>&1 | tee -a /VERSION.txt \
+    && python3 -c 'from abx_dl.models import discover_plugins; [print(f"export {plugin.enabled_key}=True") for plugin in discover_plugins(runtime="archivebox").values() if plugin.enabled_key in plugin.config.properties]' > /tmp/archivebox-enable-plugins.env \
+    && sort /tmp/archivebox-enable-plugins.env | tee -a /VERSION.txt \
+    && source /tmp/archivebox-enable-plugins.env \
+    && python3 -c 'from abx_dl.config import get_derived_config, get_initial_env, get_required_binary_requests; from abx_dl.models import discover_plugins; user_env = get_initial_env(); derived_env = get_derived_config(user_env); seen = set(); names = []; [names.append(str(record["name"])) for plugin in discover_plugins(runtime="archivebox").values() for record in get_required_binary_requests(plugin, plugin.config.required_binaries, overrides=user_env, derived_overrides=derived_env, run_output_dir=None) if not (str(record["name"]) in seen or seen.add(str(record["name"])))] ; print(",".join(names))' > /tmp/archivebox-required-binaries.txt \
+    && cat /tmp/archivebox-required-binaries.txt | tee -a /VERSION.txt \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache ABXPKG_NO_CACHE=True ABXPKG_TMP_CACHE_DIR=/tmp/abxpkg-cache archivebox install \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache archivebox version --binaries "$(cat /tmp/archivebox-required-binaries.txt)" 2>&1 | tee -a /VERSION.txt \
     && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
 
 RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \

bin/release.sh

@@ -423,11 +423,6 @@ main() {
     slug="$(repo_slug)"
     branch="$(default_branch)"
 
-    if [[ "${GITHUB_EVENT_NAME:-}" == "push" ]]; then
-        validate_release_state "${slug}" "${branch}"
-        return 0
-    fi
-
     if [[ "$(git branch --show-current)" != "${branch}" ]]; then
         echo "Release must run from ${branch}, found $(git branch --show-current)" >&2
         return 1
@@ -449,16 +444,12 @@ main() {
         git add -A
         git commit -m "release: ${TAG_PREFIX}${version}"
         git push origin "${branch}"
-
-        wait_for_runs "${slug}" push "$(git rev-parse HEAD)" "push"
     elif [[ "${relation}" == "gt" ]]; then
         run_checks
         if [[ -n "$(git status --short)" ]]; then
             git add -A
             git commit -m "release: ${TAG_PREFIX}${version}"
             git push origin "${branch}"
-
-            wait_for_runs "${slug}" push "$(git rev-parse HEAD)" "push"
         fi
     else
         echo "Current version ${version} is behind latest GitHub release ${latest}" >&2

bin/watch_dev_image.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+IFS=$'\n\t'
+
+DEPLOY_PATH="${DEPLOY_PATH:-/opt/archivebox.demo}"
+DEPLOY_SERVICE="${DEPLOY_SERVICE:-archivebox}"
+DEPLOY_IMAGE="${DEPLOY_IMAGE:-archivebox/archivebox:dev}"
+DEPLOY_INTERVAL="${DEPLOY_INTERVAL:-60}"
+STATE_FILE="${STATE_FILE:-${DEPLOY_PATH}/.archivebox-dev-image.digest}"
+OVERRIDE_FILE="${OVERRIDE_FILE:-${DEPLOY_PATH}/.archivebox-deploy.override.yml}"
+
+cd "$DEPLOY_PATH"
+
+if [[ -f compose.yml ]]; then
+    COMPOSE_FILE=compose.yml
+elif [[ -f compose.yaml ]]; then
+    COMPOSE_FILE=compose.yaml
+elif [[ -f docker-compose.yml ]]; then
+    COMPOSE_FILE=docker-compose.yml
+else
+    echo "[X] No compose file found in $DEPLOY_PATH" >&2
+    exit 1
+fi
+
+cat > "$OVERRIDE_FILE" <<EOF
+services:
+  $DEPLOY_SERVICE:
+    image: $DEPLOY_IMAGE
+EOF
+
+COMPOSE=(docker compose -f "$COMPOSE_FILE" -f "$OVERRIDE_FILE")
+
+remote_digest() {
+    docker buildx imagetools inspect "$DEPLOY_IMAGE" | awk '/^Digest:/ {print $2; exit}'
+}
+
+deploy_digest() {
+    local digest="$1"
+
+    echo "[+] Deploying ${DEPLOY_IMAGE}@${digest}"
+    "${COMPOSE[@]}" pull "$DEPLOY_SERVICE"
+    "${COMPOSE[@]}" up -d "$DEPLOY_SERVICE"
+
+    if "${COMPOSE[@]}" config --services | grep -qx argo; then
+        "${COMPOSE[@]}" up -d argo
+    fi
+
+    "${COMPOSE[@]}" exec -T "$DEPLOY_SERVICE" archivebox version </dev/null | sed -n '1,40p'
+    "${COMPOSE[@]}" exec -T "$DEPLOY_SERVICE" curl -fsS --max-time 10 --connect-timeout 2 -H 'Host: admin.archivebox.io' http://127.0.0.1:8000/health/ </dev/null
+    printf '%s\n' "$digest" > "$STATE_FILE"
+}
+
+mkdir -p "$(dirname "$STATE_FILE")"
+touch "$STATE_FILE"
+
+while :; do
+    digest="$(remote_digest || true)"
+    last_digest="$(cat "$STATE_FILE" 2>/dev/null || true)"
+
+    if [[ -z "$digest" ]]; then
+        echo "[!] Could not resolve ${DEPLOY_IMAGE}; retrying in ${DEPLOY_INTERVAL}s" >&2
+    elif [[ "$digest" != "$last_digest" ]]; then
+        deploy_digest "$digest"
+    else
+        echo "[=] ${DEPLOY_IMAGE} already deployed at ${digest}"
+    fi
+
+    if [[ "${WATCH_ONCE:-0}" == "1" ]]; then
+        break
+    fi
+    sleep "$DEPLOY_INTERVAL"
+done

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.35rc33",
+  "version": "0.9.35rc34",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.35rc33"
+version = "0.9.35rc34"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,9 +79,9 @@ dependencies = [
     ### Extractor dependencies (optional binaries detected at runtime via shutil.which)
     ### Binary/Package Management
     "abxbus==2.5.10",         # EventBus API
-    "abxpkg>=1.11.214",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins>=1.11.218",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl>=1.11.218",       # shared ArchiveBox downloader package with blocking install preflight
+    "abxpkg>=1.11.216",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
+    "abx-plugins>=1.11.220",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl>=1.11.220",       # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]