From c36d3c15759842bfeeba8854ec4da74e3895618b Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 21 Jul 2026 19:03:04 -0700 Subject: [PATCH] Make release publication atomic and CI-only --- .github/workflows/docker.yml | 97 ++++++-- .github/workflows/release.yml | 7 - .github/workflows/update-homebrew-tap.yml | 12 +- AGENTS.md | 6 +- README.md | 16 -- bin/build_docker.sh | 2 +- bin/deploy_dev_demo.sh | 10 - bin/release.sh | 15 +- bin/release_dev_stack.sh | 270 ---------------------- bin/release_docker.sh | 148 ------------ bin/release_docs.sh | 25 -- bin/release_git.sh | 23 -- bin/release_pip.sh | 20 -- skills/archivebox/SKILL.md | 6 +- 14 files changed, 99 insertions(+), 558 deletions(-) delete mode 100755 bin/release_dev_stack.sh delete mode 100755 bin/release_docker.sh delete mode 100755 bin/release_docs.sh delete mode 100755 bin/release_git.sh delete mode 100755 bin/release_pip.sh diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a30a55e2..d5e2b8d3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -204,7 +204,10 @@ jobs: cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }} pull: true platforms: ${{ matrix.platform }} - outputs: type=image,push-by-digest=true,name-canonical=true,push=true + outputs: | + type=docker,name=${{ matrix.local_tag }} + type=oci,dest=${{ runner.temp }}/archivebox-image.tar + type=image,push-by-digest=true,name-canonical=true,push=true - name: Build pull request image if: ${{ !inputs.push_digests }} @@ -228,15 +231,11 @@ jobs: if: inputs.push_digests run: echo ${{ steps.docker_build.outputs.digest }} - - name: Validate pushed image version, commit, and size + - name: Validate exact built image version and commit shell: bash run: | set -Eeuo pipefail - if [[ "${{ inputs.push_digests }}" != "true" ]]; then - IMAGE="${{ matrix.local_tag }}" - else - IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}" - fi + IMAGE="${{ matrix.local_tag }}" SHORT_SHA="${GITHUB_SHA::7}" DATA_DIR="$(mktemp -d)" @@ -265,15 +264,87 @@ jobs: - name: Validate compressed image size if: inputs.push_digests + env: + BUILD_METADATA: ${{ steps.docker_build.outputs.metadata }} shell: bash run: | set -Eeuo pipefail - IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}" - LIMIT=$((780 * 1024 * 1024)) - TOTAL="$("$DOCKER_BINARY" manifest inspect "$IMAGE" | "$JQ_BINARY" '[.config.size, (.layers[]?.size)] | add')" - printf '%s compressed_size=%s MiB limit=%s MiB\n' \ - "$IMAGE" "$((TOTAL / 1024 / 1024))" "$((LIMIT / 1024 / 1024))" - [[ "$TOTAL" -le "$LIMIT" ]] || { echo "$IMAGE is over the compressed size limit" >&2; exit 1; } + LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')" + "$UV_BINARY" run --no-project python - \ + '${{ runner.temp }}/archivebox-image.tar' \ + '${{ steps.docker_build.outputs.digest }}' \ + "$LOCAL_IMAGE_ID" \ + "$((780 * 1024 * 1024))" <<'PY' + from hashlib import sha256 + import json + import os + import sys + import tarfile + + archive_path, pushed_digest, local_image_id, limit_text = sys.argv[1:] + limit = int(limit_text) + metadata = json.loads(os.environ["BUILD_METADATA"]) + if metadata["containerimage.digest"] != pushed_digest: + raise SystemExit("Build push transaction digest does not match the action output") + if not pushed_digest.startswith("sha256:") or len(pushed_digest) != 71: + raise SystemExit(f"Invalid pushed digest: {pushed_digest}") + + with tarfile.open(archive_path) as archive: + def read_blob(descriptor): + algorithm, digest = descriptor["digest"].split(":", 1) + if algorithm != "sha256": + raise SystemExit(f"Unsupported OCI digest: {descriptor['digest']}") + member = archive.extractfile(f"blobs/sha256/{digest}") + if member is None: + raise SystemExit(f"Missing OCI blob: {descriptor['digest']}") + blob = member.read() + if len(blob) != descriptor["size"]: + raise SystemExit(f"OCI size mismatch: {descriptor['digest']}") + if sha256(blob).hexdigest() != digest: + raise SystemExit(f"OCI digest mismatch: {descriptor['digest']}") + return blob + + index_member = archive.extractfile("index.json") + if index_member is None: + raise SystemExit("OCI export is missing index.json") + index = json.load(index_member) + + manifests = [] + + def collect(descriptor): + document = json.loads(read_blob(descriptor)) + media_type = descriptor["mediaType"] + if "image.index" in media_type or "manifest.list" in media_type: + for child in document["manifests"]: + collect(child) + elif "image.manifest" in media_type: + config_type = document["config"]["mediaType"] + if "image.config" in config_type or "container.image" in config_type: + manifests.append(document) + + for root in index["manifests"]: + collect(root) + if len(manifests) != 1: + raise SystemExit(f"Expected one runnable image manifest, found {len(manifests)}") + + manifest = manifests[0] + config_digest = manifest["config"]["digest"] + if metadata["containerimage.config.digest"] != config_digest: + raise SystemExit("OCI config digest does not match the pushed build result") + if local_image_id != config_digest: + raise SystemExit("Locally tested image does not match the OCI build result") + descriptors = [manifest["config"], *manifest["layers"]] + for descriptor in descriptors: + read_blob(descriptor) + total = sum(descriptor["size"] for descriptor in descriptors) + + print( + f"{pushed_digest} compressed_size={total // 1024 // 1024} MiB " + f"limit={limit // 1024 // 1024} MiB", + ) + if total > limit: + raise SystemExit(f"{pushed_digest} is over the compressed size limit") + PY - name: Export digest if: inputs.push_digests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bd70ab0a..fc68bd74 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -217,13 +217,6 @@ jobs: for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}" - - name: Inspect published images - shell: bash - run: | - set -Eeuo pipefail - while IFS= read -r tag; do [[ -n "$tag" ]] && $DOCKER_BINARY buildx imagetools inspect "$tag"; done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}' - while IFS= read -r tag; do [[ -n "$tag" ]] && $DOCKER_BINARY buildx imagetools inspect "$tag"; done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}' - - name: Update Docker Hub README uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4 with: diff --git a/.github/workflows/update-homebrew-tap.yml b/.github/workflows/update-homebrew-tap.yml index 79a345ab..b74807e8 100644 --- a/.github/workflows/update-homebrew-tap.yml +++ b/.github/workflows/update-homebrew-tap.yml @@ -48,7 +48,7 @@ jobs: with: version: "0.11.3" - - name: Resolve release and Docker binaries through abxpkg + - name: Resolve release binaries through abxpkg env: ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib shell: bash @@ -73,27 +73,23 @@ jobs: --install \ --lib="$ABXPKG_LIB_DIR" \ --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \ - --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" \ >/dev/null - for binary_name in uv gh git jq curl docker; do + for binary_name in uv git jq curl; do binary="$ABXPKG_LIB_DIR/env/bin/$binary_name" test -L "$binary" test -x "$binary" done { echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv" - echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh" echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git" echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq" echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl" - echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker" } >> "$GITHUB_ENV" - name: Verify the exact release and dispatch downstream builds env: DOWNSTREAM_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || secrets.HOMEBREW_TAP_TOKEN }} - GH_TOKEN: ${{ github.token }} INPUT_RELEASE_TAG: ${{ inputs.release_tag }} INPUT_RELEASE_SHA: ${{ inputs.release_sha }} shell: bash @@ -113,10 +109,6 @@ jobs: [[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] || { echo "${RELEASE_TAG} points to ${TAG_TARGET}, not ${RELEASE_SHA}" >&2; exit 1; } - "$CURL_BINARY" -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" >/dev/null - "$DOCKER_BINARY" manifest inspect "archivebox/archivebox:${VERSION}" >/dev/null - "$DOCKER_BINARY" manifest inspect "ghcr.io/archivebox/archivebox:${VERSION}" >/dev/null - PAYLOAD_FILTER="{event_type: \"archivebox-dev-updated\", client_payload: {ref: \$ref, sha: \$sha, version: \$version}}" PAYLOAD="$("$JQ_BINARY" -nc --arg ref "$RELEASE_TAG" --arg sha "$RELEASE_SHA" --arg version "$VERSION" "$PAYLOAD_FILTER")" for repo in homebrew-archivebox debian-archivebox; do diff --git a/AGENTS.md b/AGENTS.md index 3936b3d9..1134f683 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,8 +79,4 @@ uv run --project "$project_dir" --no-sync pytest "$project_dir/archivebox/tests/ (cd "$project_dir" && uv run --no-sync prek run --all-files) ``` -Use the full release/deploy loop only when requested: - -```console -./bin/release_dev_stack.sh -``` +Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases. diff --git a/README.md b/README.md index 705c62b4..c3dba53c 100644 --- a/README.md +++ b/README.md @@ -1609,22 +1609,6 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t -#### Roll a release - -
Click to expand... - -(Normally CI takes care of this, but these scripts can be run to do it manually) -```console -./bin/release.sh - -# or individually: -./bin/release_docs.sh -./bin/release_pip.sh -./bin/release_docker.sh -``` - -
- --- ## Further Reading diff --git a/bin/build_docker.sh b/bin/build_docker.sh index c22b8fc6..43dade5a 100755 --- a/bin/build_docker.sh +++ b/bin/build_docker.sh @@ -131,7 +131,7 @@ echo "[+] Building archivebox:$VERSION docker image..." mkdir -p "$HOME/.cache/docker/archivebox" "$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_IMAGE" if [[ "$SELECTED_PLATFORMS" == *,* ]]; then - echo "[X] --load only supports a single platform. Use bin/release_docker.sh or set DOCKER_PLATFORMS to one platform." >&2 + echo "[X] --load only supports a single platform. Set DOCKER_PLATFORMS to one platform." >&2 exit 1 fi "$DOCKER_BINARY" buildx build \ diff --git a/bin/deploy_dev_demo.sh b/bin/deploy_dev_demo.sh index 5510f1e5..bf6a2517 100755 --- a/bin/deploy_dev_demo.sh +++ b/bin/deploy_dev_demo.sh @@ -40,15 +40,10 @@ uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \ >/dev/null export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH" GIT_BINARY="$ABXPKG_LIB_DIR/env/bin/git" -PYTHON_BINARY="$ABXPKG_LIB_DIR/env/bin/python" SSH_BINARY="$ABXPKG_LIB_DIR/env/bin/ssh" test -x "$GIT_BINARY" -test -x "$PYTHON_BINARY" test -x "$SSH_BINARY" -VERSION="$("$PYTHON_BINARY" -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" -GIT_SHA="sha-$("$GIT_BINARY" rev-parse --short HEAD)" - if [[ "$("$GIT_BINARY" branch --show-current)" != "dev" ]]; then echo "[X] Run this from the dev branch." >&2 exit 1 @@ -63,11 +58,6 @@ fi echo "[+] Pushing dev to GitHub..." "$GIT_BINARY" push origin dev -if [[ "${SKIP_DOCKER:-0}" != "1" ]]; then - echo "[+] Publishing Docker image tags: dev ${VERSION} ${GIT_SHA}" - ./bin/release_docker.sh dev "$VERSION" "$GIT_SHA" -fi - if [[ "${SKIP_DEMO:-0}" == "1" ]]; then echo "[√] Skipped demo deploy." exit 0 diff --git a/bin/release.sh b/bin/release.sh index da4d324d..538b6532 100755 --- a/bin/release.sh +++ b/bin/release.sh @@ -125,15 +125,20 @@ if [[ ( "$PYPI_EXISTS" == true || "$GITHUB_EXISTS" == true ) && "$TAG_TARGET" != exit 1 fi -if [[ "$GITHUB_EXISTS" == false ]]; then - RELEASE_ARGS=() - [[ "$VERSION" == *rc* ]] && RELEASE_ARGS+=(--prerelease) - $GH_BINARY release create "$TAG" --repo "$SLUG" --target "$RELEASE_SHA" \ - --title "$TAG" --generate-notes "${RELEASE_ARGS[@]}" +if [[ -z "$TAG_TARGET" ]]; then + $GIT_BINARY tag "$TAG" "$RELEASE_SHA" + $GIT_BINARY push origin "refs/tags/${TAG}" fi if [[ "$PYPI_EXISTS" == false ]]; then $UV_BINARY publish --trusted-publishing always "${WHEELS[@]}" "${SDISTS[@]}" fi +if [[ "$GITHUB_EXISTS" == false ]]; then + RELEASE_ARGS=() + [[ "$VERSION" == *rc* ]] && RELEASE_ARGS+=(--prerelease) + $GH_BINARY release create "$TAG" --repo "$SLUG" --verify-tag \ + --title "$TAG" --generate-notes "${RELEASE_ARGS[@]}" +fi + echo "Released ${PYPI_PACKAGE} ${VERSION} from ${RELEASE_SHA} using CI run ${CI_RUN_ID:-unknown}" diff --git a/bin/release_dev_stack.sh b/bin/release_dev_stack.sh deleted file mode 100755 index 8655c0f1..00000000 --- a/bin/release_dev_stack.sh +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env bash - -set -Eeuo pipefail -IFS=$'\n\t' - -ARCHIVEBOX_REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -WORKSPACE_DIR="$(cd "${ARCHIVEBOX_REPO}/.." && pwd)" -DOCKER_IMAGE_REPOS="${DOCKER_IMAGE_REPOS:-archivebox/archivebox ghcr.io/archivebox/archivebox}" - -cd "${WORKSPACE_DIR}" - -repo_dir() { - local repo="$1" - printf '%s/%s\n' "${WORKSPACE_DIR}" "${repo}" -} - -ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-${LIB_DIR:-$HOME/.config/archivebox/lib}}" -locked_archivebox_abxpkg_version() { - local line package="" - while IFS= read -r line; do - case "$line" in - '[[package]]') package="" ;; - 'name = "abxpkg"') package="abxpkg" ;; - 'version = "'*'"') - [[ "$package" == "abxpkg" ]] || continue - line="${line#version = \"}" - printf '%s\n' "${line%\"}" - return 0 - ;; - esac - done < "$ARCHIVEBOX_REPO/uv.lock" - return 1 -} -BOOTSTRAP_ABXPKG_VERSION="$(locked_archivebox_abxpkg_version)" -mkdir -p "$ABXPKG_LIB_DIR/env/bin" -uv run --no-project --with "abxpkg==$BOOTSTRAP_ABXPKG_VERSION" abxpkg env \ - --install \ - --lib="$ABXPKG_LIB_DIR" \ - --deps-from="$ARCHIVEBOX_REPO/.github/configs/ci-tooling.json:release_binaries" \ - >/dev/null -GIT_BINARY="$ABXPKG_LIB_DIR/env/bin/git" -PYTHON_BINARY="$ABXPKG_LIB_DIR/env/bin/python" -UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv" -test -x "$GIT_BINARY" -test -x "$PYTHON_BINARY" -test -x "$UV_BINARY" - -current_version() { - local repo="$1" - "$PYTHON_BINARY" - "$repo" <<'PY' -from pathlib import Path -import re -import sys - -text = Path(sys.argv[1], "pyproject.toml").read_text() -match = re.search(r'^version = "([^"]+)"$', text, re.MULTILINE) -if not match: - raise SystemExit(f"Failed to find version in {sys.argv[1]}/pyproject.toml") -print(match.group(1)) -PY -} - -bump_patch_to() { - local repo="$1" - local version="$2" - "$PYTHON_BINARY" - "$repo" "$version" <<'PY' -from pathlib import Path -import re -import sys - -path = Path(sys.argv[1], "pyproject.toml") -version = sys.argv[2] -text = path.read_text() -path.write_text(re.sub(r'^version = "[^"]+"$', f'version = "{version}"', text, count=1, flags=re.MULTILINE)) -PY -} - -next_patch_version() { - "$PYTHON_BINARY" - "$@" <<'PY' -import re -import sys - -versions = sys.argv[1:] -parts = [] -for version in versions: - match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", version) - if not match: - raise SystemExit(f"Expected patch version, got {version}") - parts.append(tuple(int(part) for part in match.groups())) -major, minor, patch = max(parts) -print(f"{major}.{minor}.{patch + 1}") -PY -} - -bump_archivebox_rc() { - "$PYTHON_BINARY" - "${ARCHIVEBOX_REPO}" <<'PY' -from pathlib import Path -import json -import re -import sys - -repo = Path(sys.argv[1]) -pyproject_path = repo / "pyproject.toml" -package_path = repo / "etc" / "package.json" -pyproject_text = pyproject_path.read_text() -match = re.search(r'^version = "(\d+)\.(\d+)\.(\d+)(?:-?rc(\d+))?"$', pyproject_text, re.MULTILINE) -if not match: - raise SystemExit("Expected ArchiveBox version like 0.9.31rc15") - -major, minor, patch, rc = match.groups() -next_version = f"{major}.{minor}.{patch}rc{int(rc or 0) + 1}" -pyproject_path.write_text(re.sub(r'^version = "[^"]+"$', f'version = "{next_version}"', pyproject_text, count=1, flags=re.MULTILINE)) - -package_json = json.loads(package_path.read_text()) -package_json["version"] = next_version -package_path.write_text(json.dumps(package_json, indent=2) + "\n") -print(next_version) -PY -} - -set_dependency_version() { - local repo="$1" - local package="$2" - local version="$3" - "$PYTHON_BINARY" - "$repo" "$package" "$version" <<'PY' -from pathlib import Path -import re -import sys - -repo, package, version = sys.argv[1:] -path = Path(repo, "pyproject.toml") -text = path.read_text() -updated, count = re.subn(rf'("{re.escape(package)}>=)[^"]+(")', rf'\g<1>{version}\2', text) -if count: - path.write_text(updated) -PY -} - -assert_branch() { - local repo="$1" - local branch="$2" - local actual - actual="$("$GIT_BINARY" -C "$repo" branch --show-current)" - if [[ "$actual" != "$branch" ]]; then - echo "[X] Expected $(basename "$repo") on ${branch}, found ${actual}" >&2 - exit 1 - fi -} - -build_and_prek() { - local repo="$1" - ( - cd "$repo" - rm -rf dist .pdm-build - "$UV_BINARY" --no-cache build --out-dir dist - "$UV_BINARY" --no-cache run prek run --all-files - rm -rf dist .pdm-build - "$UV_BINARY" --no-cache build --out-dir dist - ) -} - -commit_push_publish() { - local repo="$1" - local branch="$2" - local package="$3" - local version="$4" - local tag="v${version}" - - ( - cd "$repo" - "$GIT_BINARY" add -u - while IFS= read -r path; do - "$GIT_BINARY" add -- "$path" - done < <("$GIT_BINARY" ls-files --others --exclude-standard) - if ! "$GIT_BINARY" diff --cached --quiet; then - "$GIT_BINARY" commit -m "release: ${package} ${version}" - else - echo "[*] No staged changes in ${package}; reusing existing commit." - fi - "$GIT_BINARY" push origin "$branch" - if "$GIT_BINARY" rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then - if [[ "$("$GIT_BINARY" rev-list -n1 "${tag}")" != "$("$GIT_BINARY" rev-parse HEAD)" ]]; then - echo "[X] Tag ${tag} already exists but does not point at HEAD in ${package}" >&2 - exit 1 - fi - else - "$GIT_BINARY" tag -a "${tag}" -m "release: ${package} ${version}" - fi - "$GIT_BINARY" push origin "refs/tags/${tag}" - if pypi_has_release "$package" "$version"; then - echo "[*] ${package}==${version} is already on PyPI; skipping upload." - else - "$UV_BINARY" --no-cache publish --trusted-publishing always dist/* - fi - ) -} - -pypi_has_release() { - local package="$1" - local version="$2" - - "$PYTHON_BINARY" - "$package" "$version" <<'PY' -import sys -import urllib.error -import urllib.request - -package, version = sys.argv[1:] -try: - with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=10): - raise SystemExit(0) -except urllib.error.HTTPError as err: - raise SystemExit(1 if err.code == 404 else 2) -PY -} - -release_python_repo() { - local repo_name="$1" - local branch="$2" - local package="$3" - local version="$4" - local repo - repo="$(repo_dir "$repo_name")" - - echo "[+] Releasing ${package} ${version} from ${repo_name}:${branch}" - assert_branch "$repo" "$branch" - build_and_prek "$repo" - commit_push_publish "$repo" "$branch" "$package" "$version" -} - -ABXPKG_VERSION="${ABXPKG_VERSION:-$(next_patch_version "$(current_version "$(repo_dir abxpkg)")")}" -ABX_SHARED_VERSION="${ABX_SHARED_VERSION:-$(next_patch_version "$(current_version "$(repo_dir abx-plugins)")" "$(current_version "$(repo_dir abx-dl)")")}" - -bump_patch_to "$(repo_dir abxpkg)" "$ABXPKG_VERSION" -release_python_repo abxpkg main abxpkg "$ABXPKG_VERSION" - -bump_patch_to "$(repo_dir abx-plugins)" "$ABX_SHARED_VERSION" -set_dependency_version "$(repo_dir abx-plugins)" abxpkg "$ABXPKG_VERSION" -release_python_repo abx-plugins main abx-plugins "$ABX_SHARED_VERSION" - -bump_patch_to "$(repo_dir abx-dl)" "$ABX_SHARED_VERSION" -set_dependency_version "$(repo_dir abx-dl)" abxpkg "$ABXPKG_VERSION" -set_dependency_version "$(repo_dir abx-dl)" abx-plugins "$ABX_SHARED_VERSION" -release_python_repo abx-dl main abx-dl "$ABX_SHARED_VERSION" - -ARCHIVEBOX_VERSION="$(bump_archivebox_rc)" -set_dependency_version "$ARCHIVEBOX_REPO" abxpkg "$ABXPKG_VERSION" -set_dependency_version "$ARCHIVEBOX_REPO" abx-plugins "$ABX_SHARED_VERSION" -set_dependency_version "$ARCHIVEBOX_REPO" abx-dl "$ABX_SHARED_VERSION" - -echo "[+] Releasing archivebox ${ARCHIVEBOX_VERSION} from archivebox:dev" -assert_branch "$ARCHIVEBOX_REPO" dev -build_and_prek "$ARCHIVEBOX_REPO" -commit_push_publish "$ARCHIVEBOX_REPO" dev archivebox "$ARCHIVEBOX_VERSION" - -( - cd "$ARCHIVEBOX_REPO" - ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-${LIB_DIR:-$HOME/.config/archivebox/lib}}" - mkdir -p "$ABXPKG_LIB_DIR/env/bin" - "$UV_BINARY" run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \ - --install \ - --lib="$ABXPKG_LIB_DIR" \ - --deps-from="$ARCHIVEBOX_REPO/.github/configs/ci-tooling.json:docker_binaries" \ - >/dev/null - DOCKER_BINARY="$ABXPKG_LIB_DIR/env/bin/docker" - test -x "$DOCKER_BINARY" - ./bin/release_docker.sh dev "$ARCHIVEBOX_VERSION" "sha-$("$GIT_BINARY" rev-parse --short HEAD)" - DEPLOY_IMAGE="${DOCKER_IMAGE_REPOS%% *}:dev" DEPLOY_EXPECT_VERSION="$ARCHIVEBOX_VERSION" SKIP_DOCKER=1 ./bin/deploy_dev_demo.sh -) - -echo "[√] Released abxpkg ${ABXPKG_VERSION}, abx-plugins/abx-dl ${ABX_SHARED_VERSION}, archivebox ${ARCHIVEBOX_VERSION}" diff --git a/bin/release_docker.sh b/bin/release_docker.sh deleted file mode 100755 index 51caf8b9..00000000 --- a/bin/release_docker.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash - -### Bash Environment Setup -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -set -o errexit -set -o errtrace -set -o nounset -set -o pipefail -IFS=$' ' - -REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" -cd "$REPO_DIR" - -ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-${LIB_DIR:-$HOME/.config/archivebox/lib}}" -locked_abxpkg_version() { - local line package="" - while IFS= read -r line; do - case "$line" in - '[[package]]') package="" ;; - 'name = "abxpkg"') package="abxpkg" ;; - 'version = "'*'"') - [[ "$package" == "abxpkg" ]] || continue - line="${line#version = \"}" - printf '%s\n' "${line%\"}" - return 0 - ;; - esac - done < "$REPO_DIR/uv.lock" - return 1 -} -ABXPKG_VERSION="$(locked_abxpkg_version)" -mkdir -p "$ABXPKG_LIB_DIR/env/bin" -uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \ - --install \ - --lib="$ABXPKG_LIB_DIR" \ - --deps-from="$REPO_DIR/.github/configs/ci-tooling.json:docker_binaries" \ - >/dev/null -DOCKER_BINARY="$ABXPKG_LIB_DIR/env/bin/docker" -GIT_BINARY="$ABXPKG_LIB_DIR/env/bin/git" -PYTHON_BINARY="$ABXPKG_LIB_DIR/env/bin/python" -test -x "$DOCKER_BINARY" -test -x "$GIT_BINARY" -test -x "$PYTHON_BINARY" - -declare -a TAG_NAMES=("$@") -BRANCH_NAME="${1:-$("$GIT_BINARY" rev-parse --abbrev-ref HEAD)}" -VERSION="$("$PYTHON_BINARY" -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" -GIT_SHA=sha-"$("$GIT_BINARY" rev-parse --short HEAD)" -SELECTED_PLATFORMS="${DOCKER_PLATFORMS:-${SELECTED_PLATFORMS:-linux/amd64,linux/arm64}}" -DOCKER_IMAGE_REPOS="${DOCKER_IMAGE_REPOS:-archivebox/archivebox ghcr.io/archivebox/archivebox}" -ABX_DL_VERSION="$("$PYTHON_BINARY" -c 'import tomllib; lock=tomllib.load(open("uv.lock", "rb")); print(next(pkg["version"] for pkg in lock["package"] if pkg["name"] == "abx-dl"))')" -test -n "$ABX_DL_VERSION" -ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}" - -contains_tag() { - local candidate="$1" tag - for tag in "${TAG_NAMES[@]}"; do - [[ "$tag" == "$candidate" ]] && return 0 - done - return 1 -} - -if ! contains_tag "$GIT_SHA"; then - TAG_NAMES+=("$GIT_SHA") -fi -if ! contains_tag "$BRANCH_NAME"; then - TAG_NAMES+=("$BRANCH_NAME") -fi -if ! contains_tag "$VERSION"; then - TAG_NAMES+=("$VERSION") -fi - -echo "[+] Building + releasing Docker image for $SELECTED_PLATFORMS: branch=$BRANCH_NAME version=$VERSION abx_dl_image=$ABX_DL_IMAGE tags=${TAG_NAMES[*]}" - -declare -a FULL_TAG_NAMES -for TAG_NAME in "${TAG_NAMES[@]}"; do - [[ "$TAG_NAME" == "" ]] && continue - for IMAGE_REPO in $DOCKER_IMAGE_REPOS; do - FULL_TAG_NAMES+=("-t" "$IMAGE_REPO:$TAG_NAME") - done -done -echo "${FULL_TAG_NAMES[@]}" - -function check_platforms() { - INSTALLED_PLATFORMS="$("$DOCKER_BINARY" buildx inspect)" - - for REQUIRED_PLATFORM in ${SELECTED_PLATFORMS//,/$IFS}; do - echo "[+] Checking for: $REQUIRED_PLATFORM..." - if [[ "$INSTALLED_PLATFORMS" != *"$REQUIRED_PLATFORM"* ]]; then - return 1 - fi - done - echo - return 0 -} - -function remove_builder() { - "$DOCKER_BINARY" buildx stop xbuilder - "$DOCKER_BINARY" buildx rm xbuilder -} - -function create_builder() { - "$DOCKER_BINARY" buildx use xbuilder && return 0 - echo "[+] Creating new xbuilder for: $SELECTED_PLATFORMS" - echo - "$DOCKER_BINARY" pull 'moby/buildkit:buildx-stable-1' - "$DOCKER_BINARY" buildx create --name xbuilder --driver docker-container --bootstrap --use --platform "$SELECTED_PLATFORMS" - "$DOCKER_BINARY" buildx inspect --bootstrap -} - -function recreate_builder() { - "$DOCKER_BINARY" run --privileged --rm 'tonistiigi/binfmt' --install all - - remove_builder - create_builder -} - -"$DOCKER_BINARY" buildx use xbuilder >/dev/null 2>&1 || create_builder -check_platforms || (recreate_builder && check_platforms) || exit 1 - -echo "[^] Uploading docker image" -mkdir -p "$HOME/.cache/docker/archivebox" -"$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_IMAGE" - -"$DOCKER_BINARY" buildx build \ - --platform "$SELECTED_PLATFORMS" \ - --pull \ - --build-arg "ABX_DL_IMAGE=$ABX_DL_IMAGE" \ - --cache-from type=local,src="$HOME/.cache/docker/archivebox" \ - --cache-to type=local,compression=zstd,mode=min,oci-mediatypes=true,dest="$HOME/.cache/docker/archivebox" \ - --push . "${FULL_TAG_NAMES[@]}" - -echo "[^] Verifying pushed Docker manifests include: $SELECTED_PLATFORMS" -for TAG_NAME in "${TAG_NAMES[@]}"; do - [[ "$TAG_NAME" == "" ]] && continue - for IMAGE_REPO in $DOCKER_IMAGE_REPOS; do - MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect "$IMAGE_REPO:$TAG_NAME")" - for REQUIRED_PLATFORM in ${SELECTED_PLATFORMS//,/$IFS}; do - if [[ "$MANIFEST" != *"Platform: $REQUIRED_PLATFORM"* ]]; then - echo "[X] $IMAGE_REPO:$TAG_NAME is missing platform: $REQUIRED_PLATFORM" >&2 - echo "$MANIFEST" >&2 - exit 1 - fi - done - done -done -echo "[√] Docker manifests include all requested platforms." diff --git a/bin/release_docs.sh b/bin/release_docs.sh deleted file mode 100755 index 61731242..00000000 --- a/bin/release_docs.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -### Bash Environment Setup -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -# set -o xtrace -set -o errexit -set -o errtrace -set -o nounset -set -o pipefail -IFS=$'\n' - -REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" -VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" -cd "$REPO_DIR" - - -echo "[^] Pushing docs to github" -cd docs/ -git add . -git commit -am "$VERSION release" -git push -git tag -a "v$VERSION" -m "v$VERSION" -git push origin -git push origin --tags diff --git a/bin/release_git.sh b/bin/release_git.sh deleted file mode 100755 index bf53542a..00000000 --- a/bin/release_git.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -### Bash Environment Setup -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -# set -o xtrace -set -o errexit -set -o errtrace -set -o nounset -set -o pipefail -IFS=$'\n' - -REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" -VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" -cd "$REPO_DIR" - - -# Push build to github -echo "[^] Pushing release commit + tag to Github" -git tag -f -a "v$VERSION" -m "v$VERSION" -git push origin -f --tags -echo " To finish publishing the release go here:" -echo " https://github.com/ArchiveBox/ArchiveBox/releases/new" diff --git a/bin/release_pip.sh b/bin/release_pip.sh deleted file mode 100755 index 6243bdf7..00000000 --- a/bin/release_pip.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -### Bash Environment Setup -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html -# set -o xtrace -set -o errexit -set -o errtrace -set -o nounset -set -o pipefail -IFS=$'\n' - -REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" -cd "$REPO_DIR" -source "$REPO_DIR/.venv/bin/activate" - -echo "[^] Publishing to PyPI..." -rm -Rf dist -uv build -uv publish --trusted-publishing always diff --git a/skills/archivebox/SKILL.md b/skills/archivebox/SKILL.md index 705f77e4..7956dae3 100644 --- a/skills/archivebox/SKILL.md +++ b/skills/archivebox/SKILL.md @@ -73,8 +73,4 @@ uv run --project "$project_dir" --no-sync pytest "$project_dir/archivebox/tests/ (cd "$project_dir" && uv run --no-sync prek run --all-files) ``` -Use the full release/deploy loop only when requested: - -```console -./bin/release_dev_stack.sh -``` +Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.