Deduplicate ArchiveBox release CI

.github/workflows/docker.yml

@@ -2,12 +2,9 @@ name: Build Docker image
 
 on:
   workflow_dispatch:
-  workflow_call:
   push:
     branches:
-      - "**"
-    tags:
-      - 'v*'
+      - '**'
   # pull_request:
 
 env:
@@ -45,47 +42,6 @@ jobs:
         #   submodules: true
         #   fetch-depth: 1
 
-      - name: Wait for released ArchiveBox deps on PyPI
-        run: |
-          python3 - <<'PY'
-          import json
-          import re
-          import sys
-          import time
-          import tomllib
-          import urllib.request
-
-          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
-          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
-          required = {}
-          for dep in deps:
-              for name in watched:
-                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
-                  if match:
-                      required[name] = match.group(2)
-
-          deadline = time.monotonic() + 300
-          missing = required.copy()
-          while missing and time.monotonic() < deadline:
-              for name, version in list(missing.items()):
-                  try:
-                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
-                          available = True
-                  except Exception:
-                      available = False
-                  if available:
-                      print(f"{name} {version} is available on PyPI")
-                      missing.pop(name)
-                  else:
-                      print(f"{name} {version} is not available on PyPI yet")
-              if missing:
-                  time.sleep(10)
-
-          if missing:
-              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
-              sys.exit(1)
-          PY
-
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
@@ -99,7 +55,7 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
-      - name: Wait for published abx-dl image
+      - name: Verify published abx-dl image
         id: abx_dl_image
         shell: bash
         run: |
@@ -121,16 +77,6 @@ jobs:
           PY
           )"
           ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
-          deadline=$((SECONDS + 1800))
-          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
-            if (( SECONDS >= deadline )); then
-              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
-              exit 1
-            fi
-            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
-            sleep 30
-          done
-
           echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
           docker buildx imagetools inspect "${ABX_DL_IMAGE}"
 

.github/workflows/lint.yml

@@ -3,6 +3,7 @@ name: Run linters
 on:
   workflow_dispatch:
   push:
+    branches: ['**']
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/pip.yml

@@ -2,12 +2,9 @@ name: Build Pip package
 
 on:
   workflow_dispatch:
-  workflow_call:
   push:
     branches:
       - '**'
-    tags:
-      - 'v*'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -18,11 +15,7 @@ env:
 
 jobs:
   build:
-    permissions:
-      id-token: write
-
     runs-on: ubuntu-24.04
-    environment: pypi
     steps:
       - uses: actions/checkout@v4
 
@@ -44,89 +37,10 @@ jobs:
           packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
           version: 1.0
 
-      - name: Wait for released ArchiveBox deps on PyPI
-        run: |
-          python - <<'PY'
-          import re
-          import sys
-          import time
-          import tomllib
-          import urllib.request
-
-          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
-          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
-          required = {}
-          for dep in deps:
-              for name in watched:
-                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
-                  if match:
-                      required[name] = match.group(2)
-
-          deadline = time.monotonic() + 300
-          missing = required.copy()
-          while missing and time.monotonic() < deadline:
-              for name, version in list(missing.items()):
-                  try:
-                      with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20):
-                          available = True
-                  except Exception:
-                      available = False
-                  if available:
-                      print(f"{name} {version} is available on PyPI")
-                      missing.pop(name)
-                  else:
-                      print(f"{name} {version} is not available on PyPI yet")
-              if missing:
-                  time.sleep(10)
-
-          if missing:
-              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
-              sys.exit(1)
-          PY
-
-      - name: Prepare released dependency sync project
-        run: |
-          python - <<'PY'
-          from pathlib import Path
-          import json
-          import re
-          import urllib.request
-
-          source = Path("pyproject.toml")
-          target_dir = Path(".ci-uv-project")
-          target_dir.mkdir(exist_ok=True)
-          text = source.read_text()
-          text = text.replace(
-              'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
-              'environments = ["sys_platform == \'linux\'"]',
-          )
-
-          # CI runs immediately after publishing internal abx packages. PyPI's
-          # simple index can lag the version JSON endpoint, so sync against the
-          # exact wheel URLs from the version endpoint while leaving package
-          # metadata in pyproject.toml unchanged for the actual build.
-          for package in ("abxpkg", "abx-plugins", "abx-dl"):
-              match = re.search(rf'"{re.escape(package)}>=(?P<version>[^"]+)"', text)
-              if not match:
-                  continue
-              version = match.group("version")
-              with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
-                  data = json.load(response)
-              wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
-              text = re.sub(
-                  rf'"{re.escape(package)}>=[^"]+"',
-                  f'"{package} @ {wheel_url}"',
-                  text,
-                  count=1,
-              )
-
-          (target_dir / "pyproject.toml").write_text(text)
-          PY
-
       - name: UV install archivebox dev + run sub-dependencies
         env:
           UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
-        run: uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
 
       - name: UV build archivebox and archivebox/pkgs/* packages
         run: |
@@ -136,7 +50,7 @@ jobs:
         env:
           UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
         run: |
-          uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
           uv pip install --python .venv/bin/python --no-deps dist/archivebox-*.whl
 
       - name: Verify built package full install
@@ -146,7 +60,7 @@ jobs:
           set -Eeuo pipefail
           DATA_DIR="$(mktemp -d)"
           cd "$DATA_DIR"
-          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox init
-          timeout 30m uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox install
-          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox version
-          uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox status
+          uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox init
+          timeout 30m uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox install
+          uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox version
+          uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox status

.github/workflows/release-runner.yml

@@ -7,7 +7,6 @@ on:
   workflow_dispatch:
 
 permissions:
-  actions: write
   contents: write
   id-token: write
 
@@ -51,10 +50,3 @@ jobs:
           DEFAULT_BRANCH: dev
           GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
         run: ./bin/release.sh
-
-      - name: Dispatch image and tap updates
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-        run: |
-          gh workflow run docker.yml --ref dev
-          gh workflow run update-homebrew-tap.yml --ref dev

.github/workflows/release.yml

@@ -1,41 +0,0 @@
-name: Release
-
-# Orchestrates the full release pipeline:
-#   1. Build and publish pip package to PyPI
-#   2. Build Homebrew formula after pip
-#   3. Build Docker images in parallel
-#
-# Individual workflows also run on push for CI (see their own triggers).
-# Manual fallback for the hosted release pipeline. The normal prerelease loop
-# is run by bin/release.sh locally so published GitHub releases do not launch a
-# duplicate release job.
-
-on:
-  workflow_dispatch:
-
-permissions:
-  contents: write
-  packages: write
-  id-token: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  pip:
-    name: Publish to PyPI
-    uses: ./.github/workflows/pip.yml
-    secrets: inherit
-
-  homebrew:
-    name: Update Homebrew formula
-    needs: pip
-    uses: ./.github/workflows/homebrew.yml
-    secrets: inherit
-
-  docker:
-    name: Build Docker images
-    needs: pip
-    uses: ./.github/workflows/docker.yml
-    secrets: inherit

.github/workflows/test.yml

@@ -1,5 +1,8 @@
 name: Run tests
-on: [push]
+
+on:
+  push:
+    branches: ['**']
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -91,22 +94,9 @@ jobs:
           uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox version
           uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status
 
-      - name: Test built package with pytest
-        run: |
-          mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out
-
       - name: Run documentation code blocks
         run: uv run --no-sync --no-sources pytest -vv --tb=long README.md AGENTS.md skills docs
 
-      - name: Run plugin tests
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
-        run: |
-          uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage
-
   docker_tests:
     runs-on: ubuntu-latest
 

.github/workflows/update-homebrew-tap.yml

@@ -2,7 +2,7 @@ name: Update downstream packages
 
 on:
   push:
-    branches: [dev]
+    tags: ['v*']
   workflow_dispatch:
 
 permissions:

bin/release.sh

@@ -266,87 +266,6 @@ create_git_tag() {
     git push origin "refs/tags/${tag}"
 }
 
-wait_for_runs() {
-    local slug="$1"
-    local event="$2"
-    local sha="$3"
-    local label="$4"
-    local runs_json
-    local attempts=0
-
-    while :; do
-        runs_json="$(GH_FORCE_TTY=0 GH_PAGER=cat gh run list --repo "${slug}" --event "${event}" --commit "${sha}" --limit 20 --json databaseId,status,conclusion,workflowName)"
-        if [[ "$(jq 'length' <<<"${runs_json}")" -gt 0 ]]; then
-            break
-        fi
-        attempts=$((attempts + 1))
-        if [[ "${attempts}" -ge 30 ]]; then
-            echo "Timed out waiting for ${label} workflows to start" >&2
-            return 1
-        fi
-        sleep 10
-    done
-
-    while IFS=$'\t' read -r run_id workflow_name; do
-        workflow_name_lower="${workflow_name,,}"
-        if [[ "${workflow_name_lower}" == *"release state"* ]]; then
-            gh run watch "${run_id}" --repo "${slug}" --exit-status
-            continue
-        fi
-        if [[ "${workflow_name_lower}" != *"test"* ]]; then
-            echo "Skipping non-gating workflow: ${workflow_name}"
-            continue
-        fi
-
-        attempts=0
-        while :; do
-            precheck_state="$(
-                gh run view "${run_id}" --repo "${slug}" --json jobs --jq '
-                    [.jobs[] | select((.name | ascii_downcase) | test("precheck|pre-commit|prek"))][0]
-                    | if . == null then "missing:" else ((.status // "") + ":" + (.conclusion // "")) end
-                '
-            )"
-            case "${precheck_state}" in
-                missing:*)
-                    echo "Skipping test workflow without precheck job: ${workflow_name}"
-                    break
-                    ;;
-                completed:success|completed:skipped)
-                    break
-                    ;;
-                completed:failure|completed:cancelled|completed:timed_out)
-                    gh run view "${run_id}" --repo "${slug}"
-                    return 1
-                    ;;
-            esac
-            attempts=$((attempts + 1))
-            if [[ "${attempts}" -ge 120 ]]; then
-                echo "Timed out waiting for ${workflow_name} precheck job" >&2
-                return 1
-            fi
-            sleep 5
-        done
-    done < <(jq -r '.[] | [.databaseId, .workflowName] | @tsv' <<<"${runs_json}")
-}
-
-wait_for_pypi() {
-    local package_name="$1"
-    local expected_version="$2"
-    local attempts=0
-
-    while :; do
-        if curl -fsSL "https://pypi.org/pypi/${package_name}/json" | jq -e --arg version "${expected_version}" '.releases[$version] | length > 0' >/dev/null; then
-            return 0
-        fi
-        attempts=$((attempts + 1))
-        if [[ "${attempts}" -ge 30 ]]; then
-            echo "Timed out waiting for ${package_name}==${expected_version} on PyPI" >&2
-            return 1
-        fi
-        sleep 10
-    done
-}
-
 run_checks() {
     uv sync --extra sonic --extra debug --all-groups --no-cache --upgrade
     uv build --all
@@ -428,7 +347,6 @@ publish_artifacts() {
         uv publish --trusted-publishing always "${artifacts[@]}"
     fi
 
-    wait_for_pypi "${PYPI_PACKAGE}" "${version}"
 }
 
 main() {