From d4b35e52d3440f0ca5476e3fc8f940b5a9dc485c Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Fri, 28 Aug 2026 12:19:44 -0700 Subject: [PATCH] Add fast release candidate pipeline --- .github/workflows/ci.yml | 2 +- .github/workflows/docker.yml | 13 ++++-- .github/workflows/pip.yml | 40 +++++++++++++++++ .github/workflows/release-candidate.yml | 51 ++++++++++++++++++++++ .github/workflows/release.yml | 52 ++++++++++++----------- archivebox/tests/test_release_workflow.py | 20 +++++---- 6 files changed, 141 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/release-candidate.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04d4e672..f3ad14b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: name: Tested Docker images uses: ./.github/workflows/docker.yml with: - push_digests: ${{ github.event_name == 'push' }} + push_digests: false secrets: inherit required: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1da612a2..0e7bca5d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,6 +8,11 @@ on: required: false default: false type: boolean + full_tests: + description: Run storage, CLI, and size validation in addition to release smoke checks + required: false + default: true + type: boolean env: DOCKERHUB_IMAGE: archivebox/archivebox @@ -262,7 +267,7 @@ jobs: 'import json, pathlib, stat; cache_file = "/opt/archivebox/lib/env/derived.env"; raw = open(cache_file).read().strip(); records = json.loads(raw[len("ABXPKG_DERIVED_CACHE=\x27"):-1]); projections = [projection for record in records.values() for projection in record.get("request_exec_projections", {}).values()]; plans = [projection["validation"] for projection in projections]; fingerprints = [fingerprint for plan in plans for fingerprint in plan["fingerprint"]]; actual = lambda fingerprint: (lambda path, value: {"path": str(path.resolve(strict=False)), "size": value.st_size, "mtime_ns": value.st_mtime_ns, "mode": stat.S_IMODE(value.st_mode), "euid": value.st_uid})(pathlib.Path(fingerprint["path"]), pathlib.Path(fingerprint["path"]).stat()); mismatches = [(fingerprint, actual(fingerprint)) for fingerprint in fingerprints if fingerprint != actual(fingerprint)]; assert projections and {plan.get("euid") for plan in plans} == {911}; assert all(not record.get("script_exec_plans") for record in records.values()); assert "ABXPKG_TMP_CACHE_DIR" not in json.dumps(projections); assert not mismatches, mismatches[:3]' - name: Resolve real Docker CLI test dependencies through abxpkg - if: matrix.platform == 'linux/amd64' + if: inputs.full_tests && matrix.platform == 'linux/amd64' shell: bash run: | set -Eeuo pipefail @@ -284,7 +289,7 @@ jobs: done - name: Validate Docker UID and GID behavior on real NFS and SMB mounts - if: matrix.platform == 'linux/amd64' + if: inputs.full_tests && matrix.platform == 'linux/amd64' shell: bash run: | set -Eeuo pipefail @@ -340,7 +345,7 @@ jobs: "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only - name: Validate Docker CLI add/list and compose behavior - if: matrix.platform == 'linux/amd64' + if: inputs.full_tests && matrix.platform == 'linux/amd64' shell: bash env: ARCHIVEBOX_IMAGE: ${{ matrix.local_tag }} @@ -373,7 +378,7 @@ jobs: "$DOCKER_BINARY" compose down - name: Validate compressed image size - if: inputs.push_digests + if: inputs.full_tests && inputs.push_digests env: BUILD_METADATA: ${{ steps.docker_build.outputs.metadata }} shell: bash diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 2c05a9fe..a8c07767 100755 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -2,6 +2,12 @@ name: Build Pip package on: workflow_call: + inputs: + full_tests: + description: Run the full package compatibility matrix instead of one release smoke check + required: false + default: true + type: boolean env: UV_VERSION: "0.11.3" @@ -68,6 +74,7 @@ jobs: install-smoke: name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }} + if: inputs.full_tests needs: build runs-on: ${{ matrix.os }} strategy: @@ -219,3 +226,36 @@ jobs: fi sudo rm -rf "$root_tool_dir" fi + + release-smoke: + name: Release wheel import and CLI smoke + if: ${{ !inputs.full_tests }} + needs: build + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: ${{ env.UV_VERSION }} + enable-cache: false + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: python-distributions + path: ${{ runner.temp }}/python-distributions + + - name: Install the wheel and verify import and CLI version + shell: bash + run: | + set -Eeuo pipefail + shopt -s nullglob + wheels=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) + [[ "${#wheels[@]}" -eq 1 ]] + smoke_env="$(mktemp -d)/.venv" + uv venv --python 3.13 "$smoke_env" + uv pip install --no-cache --python "$smoke_env" "${wheels[0]}" + unset PYTHONPATH + cd "$(mktemp -d)" + VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync python -c 'import archivebox' + VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox version diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml new file mode 100644 index 00000000..80065e1f --- /dev/null +++ b/.github/workflows/release-candidate.yml @@ -0,0 +1,51 @@ +name: Release candidate + +on: + push: + branches: [dev] + +permissions: + actions: read + contents: write + packages: write + +concurrency: + group: release-candidate-dev + cancel-in-progress: false + +jobs: + prepare: + runs-on: ubuntu-24.04 + outputs: + action: ${{ steps.version.outputs.action }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + submodules: true + token: ${{ secrets.RELEASE_GH_TOKEN }} + + - id: version + uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@fcd68592e9300adacfc3fe1f12b86982413bd215 + with: + package: archivebox + branch: dev + tag-prefix: v + version-scheme: rc + + python-artifacts: + needs: prepare + if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved' + uses: ./.github/workflows/pip.yml + with: + full_tests: false + secrets: inherit + + docker-digests: + needs: prepare + if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved' + uses: ./.github/workflows/docker.yml + with: + push_digests: true + full_tests: false + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 52bbcc58..e1b82f45 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,7 @@ name: Release on: workflow_run: - workflows: [CI] + workflows: [CI, Release candidate] types: [completed] branches: [dev, main] @@ -20,9 +20,35 @@ env: CI_RUN_ID: ${{ github.event.workflow_run.id }} jobs: + candidate: + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + ((github.event.workflow_run.head_branch == 'dev' && github.event.workflow_run.name == 'Release candidate') || + (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.name == 'CI')) + runs-on: ubuntu-24.04 + outputs: + ready: ${{ steps.owner.outputs.ready }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + ref: ${{ env.RELEASE_SHA }} + - id: owner + shell: bash + run: | + if [[ "$RELEASE_BRANCH" == main ]]; then + echo 'ready=true' >> "$GITHUB_OUTPUT" + exit 0 + fi + VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)" + TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')" + [[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT" + python-release: name: Publish tested Python artifacts - if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' + needs: candidate + if: needs.candidate.outputs.ready == 'true' runs-on: ubuntu-24.04 environment: pypi permissions: @@ -133,28 +159,6 @@ jobs: echo "ready=false" >> "$GITHUB_OUTPUT" fi - - name: Verify published PyPI package installs and runs - if: steps.release-state.outputs.ready == 'true' - shell: bash - run: | - set -Eeuo pipefail - VERSION='${{ steps.version.outputs.version }}' - TOOL_ROOT="$(mktemp -d)" - export UV_TOOL_DIR="$TOOL_ROOT/tools" - export UV_TOOL_BIN_DIR="$TOOL_ROOT/bin" - for attempt in {1..40}; do - if "$UV_BINARY" tool install --no-cache --force "archivebox==$VERSION"; then - break - fi - if [[ "$attempt" == 40 ]]; then - echo "archivebox==$VERSION did not become installable from PyPI within 10 minutes" >&2 - exit 1 - fi - sleep 15 - done - INSTALLED_VERSION="$("$UV_TOOL_BIN_DIR/archivebox" --version)" - test "$INSTALLED_VERSION" = "$VERSION" - docker-release: name: Publish tested multiarch image if: >- diff --git a/archivebox/tests/test_release_workflow.py b/archivebox/tests/test_release_workflow.py index 8e925dcd..27552034 100644 --- a/archivebox/tests/test_release_workflow.py +++ b/archivebox/tests/test_release_workflow.py @@ -6,6 +6,8 @@ import yaml REPO_ROOT = Path(__file__).resolve().parents[2] RELEASE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "release.yml" +RELEASE_CANDIDATE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "release-candidate.yml" +PIP_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "pip.yml" def test_release_uses_registered_publisher_and_authorized_tag_credentials(): @@ -24,15 +26,17 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials(): assert "release_ready" not in docker_release["if"] assert jobs["cascade"]["if"] == "needs.python-release.outputs.release_ready == 'true'" - published_install = next( - step for step in python_release["steps"] if step.get("name") == "Verify published PyPI package installs and runs" + assert all(step.get("name") != "Verify published PyPI package installs and runs" for step in python_release["steps"]) + candidate = yaml.safe_load(RELEASE_CANDIDATE_WORKFLOW.read_text()) + assert candidate["jobs"]["python-artifacts"]["with"]["full_tests"] is False + pip_workflow = yaml.safe_load(PIP_WORKFLOW.read_text()) + release_smoke = pip_workflow["jobs"]["release-smoke"] + install_script = next( + step["run"] for step in release_smoke["steps"] if step.get("name") == "Install the wheel and verify import and CLI version" ) - install_script = published_install["run"] - assert "sleep 60" not in install_script - assert "for attempt in {1..40}" in install_script - assert 'tool install --no-cache --force "archivebox==$VERSION"' in install_script - assert "--prerelease" not in install_script - assert "did not become installable from PyPI within 10 minutes" in install_script + assert "uv pip install --no-cache" in install_script + assert "import archivebox" in install_script + assert "archivebox version" in install_script docker_meta = next(step for step in docker_release["steps"] if step.get("id") == "docker_meta") tag_script = docker_meta["run"]