The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Make release publication atomic and CI-only

Browse Source
This commit is contained in:
Nick Sweeting 2026-07-21 19:03:04 -07:00
parent 6592af0d19
commit c36d3c1575
No known key found for this signature in database
14 changed files with 99 additions and 558 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
.github/workflows/docker.yml vendored
View File

@ -204,7 +204,10 @@ jobs:
cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
pull: true
platforms: ${{ matrix.platform }}
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
outputs: |
type=docker,name=${{ matrix.local_tag }}
type=oci,dest=${{ runner.temp }}/archivebox-image.tar
type=image,push-by-digest=true,name-canonical=true,push=true
- name: Build pull request image
if: ${{ !inputs.push_digests }}
@ -228,15 +231,11 @@ jobs:
if: inputs.push_digests
run: echo ${{ steps.docker_build.outputs.digest }}
- name: Validate pushed image version, commit, and size
- name: Validate exact built image version and commit
shell: bash
run: |
set -Eeuo pipefail
if [[ "${{ inputs.push_digests }}" != "true" ]]; then
IMAGE="${{ matrix.local_tag }}"
else
IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}"
fi
IMAGE="${{ matrix.local_tag }}"
SHORT_SHA="${GITHUB_SHA::7}"
DATA_DIR="$(mktemp -d)"
@ -265,15 +264,87 @@ jobs:
- name: Validate compressed image size
if: inputs.push_digests
env:
BUILD_METADATA: ${{ steps.docker_build.outputs.metadata }}
shell: bash
run: |
set -Eeuo pipefail
IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}"
LIMIT=$((780 * 1024 * 1024))
TOTAL="$("$DOCKER_BINARY" manifest inspect "$IMAGE" | "$JQ_BINARY" '[.config.size, (.layers[]?.size)] | add')"
printf '%s compressed_size=%s MiB limit=%s MiB\n' \
"$IMAGE" "$((TOTAL / 1024 / 1024))" "$((LIMIT / 1024 / 1024))"
[[ "$TOTAL" -le "$LIMIT" ]] || { echo "$IMAGE is over the compressed size limit" >&2; exit 1; }
LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')"
"$UV_BINARY" run --no-project python - \
'${{ runner.temp }}/archivebox-image.tar' \
'${{ steps.docker_build.outputs.digest }}' \
"$LOCAL_IMAGE_ID" \
"$((780 * 1024 * 1024))" <<'PY'
from hashlib import sha256
import json
import os
import sys
import tarfile
archive_path, pushed_digest, local_image_id, limit_text = sys.argv[1:]
limit = int(limit_text)
metadata = json.loads(os.environ["BUILD_METADATA"])
if metadata["containerimage.digest"] != pushed_digest:
raise SystemExit("Build push transaction digest does not match the action output")
if not pushed_digest.startswith("sha256:") or len(pushed_digest) != 71:
raise SystemExit(f"Invalid pushed digest: {pushed_digest}")
with tarfile.open(archive_path) as archive:
def read_blob(descriptor):
algorithm, digest = descriptor["digest"].split(":", 1)
if algorithm != "sha256":
raise SystemExit(f"Unsupported OCI digest: {descriptor['digest']}")
member = archive.extractfile(f"blobs/sha256/{digest}")
if member is None:
raise SystemExit(f"Missing OCI blob: {descriptor['digest']}")
blob = member.read()
if len(blob) != descriptor["size"]:
raise SystemExit(f"OCI size mismatch: {descriptor['digest']}")
if sha256(blob).hexdigest() != digest:
raise SystemExit(f"OCI digest mismatch: {descriptor['digest']}")
return blob
index_member = archive.extractfile("index.json")
if index_member is None:
raise SystemExit("OCI export is missing index.json")
index = json.load(index_member)
manifests = []
def collect(descriptor):
document = json.loads(read_blob(descriptor))
media_type = descriptor["mediaType"]
if "image.index" in media_type or "manifest.list" in media_type:
for child in document["manifests"]:
collect(child)
elif "image.manifest" in media_type:
config_type = document["config"]["mediaType"]
if "image.config" in config_type or "container.image" in config_type:
manifests.append(document)
for root in index["manifests"]:
collect(root)
if len(manifests) != 1:
raise SystemExit(f"Expected one runnable image manifest, found {len(manifests)}")
manifest = manifests[0]
config_digest = manifest["config"]["digest"]
if metadata["containerimage.config.digest"] != config_digest:
raise SystemExit("OCI config digest does not match the pushed build result")
if local_image_id != config_digest:
raise SystemExit("Locally tested image does not match the OCI build result")
descriptors = [manifest["config"], *manifest["layers"]]
for descriptor in descriptors:
read_blob(descriptor)
total = sum(descriptor["size"] for descriptor in descriptors)
print(
f"{pushed_digest} compressed_size={total // 1024 // 1024} MiB "
f"limit={limit // 1024 // 1024} MiB",
)
if total > limit:
raise SystemExit(f"{pushed_digest} is over the compressed size limit")
PY
- name: Export digest
if: inputs.push_digests

7
.github/workflows/release.yml vendored
View File

@ -217,13 +217,6 @@ jobs:
for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
$DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
- name: Inspect published images
shell: bash
run: |
set -Eeuo pipefail
while IFS= read -r tag; do [[ -n "$tag" ]] && $DOCKER_BINARY buildx imagetools inspect "$tag"; done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
while IFS= read -r tag; do [[ -n "$tag" ]] && $DOCKER_BINARY buildx imagetools inspect "$tag"; done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
- name: Update Docker Hub README
uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
with:

12
.github/workflows/update-homebrew-tap.yml vendored
View File

@ -48,7 +48,7 @@ jobs:
with:
version: "0.11.3"
- name: Resolve release and Docker binaries through abxpkg
- name: Resolve release binaries through abxpkg
env:
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
shell: bash
@ -73,27 +73,23 @@ jobs:
--install \
--lib="$ABXPKG_LIB_DIR" \
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" \
>/dev/null
for binary_name in uv gh git jq curl docker; do
for binary_name in uv git jq curl; do
binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
test -L "$binary"
test -x "$binary"
done
{
echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
} >> "$GITHUB_ENV"
- name: Verify the exact release and dispatch downstream builds
env:
DOWNSTREAM_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || secrets.HOMEBREW_TAP_TOKEN }}
GH_TOKEN: ${{ github.token }}
INPUT_RELEASE_TAG: ${{ inputs.release_tag }}
INPUT_RELEASE_SHA: ${{ inputs.release_sha }}
shell: bash
@ -113,10 +109,6 @@ jobs:
[[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
[[ "$TAG_TARGET" == "$RELEASE_SHA" ]] || { echo "${RELEASE_TAG} points to ${TAG_TARGET}, not ${RELEASE_SHA}" >&2; exit 1; }
"$CURL_BINARY" -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" >/dev/null
"$DOCKER_BINARY" manifest inspect "archivebox/archivebox:${VERSION}" >/dev/null
"$DOCKER_BINARY" manifest inspect "ghcr.io/archivebox/archivebox:${VERSION}" >/dev/null
PAYLOAD_FILTER="{event_type: \"archivebox-dev-updated\", client_payload: {ref: \$ref, sha: \$sha, version: \$version}}"
PAYLOAD="$("$JQ_BINARY" -nc --arg ref "$RELEASE_TAG" --arg sha "$RELEASE_SHA" --arg version "$VERSION" "$PAYLOAD_FILTER")"
for repo in homebrew-archivebox debian-archivebox; do

6
AGENTS.md
View File

@ -79,8 +79,4 @@ uv run --project "$project_dir" --no-sync pytest "$project_dir/archivebox/tests/
(cd "$project_dir" && uv run --no-sync prek run --all-files)
```
Use the full release/deploy loop only when requested:
```console
./bin/release_dev_stack.sh
```
Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

16
README.md
View File

@ -1609,22 +1609,6 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
</details>
#### Roll a release
<details><summary><i>Click to expand...</i></summary>
(Normally CI takes care of this, but these scripts can be run to do it manually)
```console
./bin/release.sh
# or individually:
./bin/release_docs.sh
./bin/release_pip.sh
./bin/release_docker.sh
```
</details>
---
## Further Reading

2
bin/build_docker.sh
View File

@ -131,7 +131,7 @@ echo "[+] Building archivebox:$VERSION docker image..."
mkdir -p "$HOME/.cache/docker/archivebox"
"$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_IMAGE"
if [[ "$SELECTED_PLATFORMS" == *,* ]]; then
echo "[X] --load only supports a single platform. Use bin/release_docker.sh or set DOCKER_PLATFORMS to one platform." >&2
echo "[X] --load only supports a single platform. Set DOCKER_PLATFORMS to one platform." >&2
exit 1
fi
"$DOCKER_BINARY" buildx build \

10
bin/deploy_dev_demo.sh
View File

@ -40,15 +40,10 @@ uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
>/dev/null
export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
GIT_BINARY="$ABXPKG_LIB_DIR/env/bin/git"
PYTHON_BINARY="$ABXPKG_LIB_DIR/env/bin/python"
SSH_BINARY="$ABXPKG_LIB_DIR/env/bin/ssh"
test -x "$GIT_BINARY"
test -x "$PYTHON_BINARY"
test -x "$SSH_BINARY"
VERSION="$("$PYTHON_BINARY" -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
GIT_SHA="sha-$("$GIT_BINARY" rev-parse --short HEAD)"
if [[ "$("$GIT_BINARY" branch --show-current)" != "dev" ]]; then
echo "[X] Run this from the dev branch." >&2
exit 1
@ -63,11 +58,6 @@ fi
echo "[+] Pushing dev to GitHub..."
"$GIT_BINARY" push origin dev
if [[ "${SKIP_DOCKER:-0}" != "1" ]]; then
echo "[+] Publishing Docker image tags: dev ${VERSION} ${GIT_SHA}"
./bin/release_docker.sh dev "$VERSION" "$GIT_SHA"
fi
if [[ "${SKIP_DEMO:-0}" == "1" ]]; then
echo "[√] Skipped demo deploy."
exit 0

15
bin/release.sh
View File

@ -125,15 +125,20 @@ if [[ ( "$PYPI_EXISTS" == true || "$GITHUB_EXISTS" == true ) && "$TAG_TARGET" !=
exit 1
fi
if [[ "$GITHUB_EXISTS" == false ]]; then
RELEASE_ARGS=()
[[ "$VERSION" == *rc* ]] && RELEASE_ARGS+=(--prerelease)
$GH_BINARY release create "$TAG" --repo "$SLUG" --target "$RELEASE_SHA" \
--title "$TAG" --generate-notes "${RELEASE_ARGS[@]}"
if [[ -z "$TAG_TARGET" ]]; then
$GIT_BINARY tag "$TAG" "$RELEASE_SHA"
$GIT_BINARY push origin "refs/tags/${TAG}"
fi
if [[ "$PYPI_EXISTS" == false ]]; then
$UV_BINARY publish --trusted-publishing always "${WHEELS[@]}" "${SDISTS[@]}"
fi
if [[ "$GITHUB_EXISTS" == false ]]; then
RELEASE_ARGS=()
[[ "$VERSION" == *rc* ]] && RELEASE_ARGS+=(--prerelease)
$GH_BINARY release create "$TAG" --repo "$SLUG" --verify-tag \
--title "$TAG" --generate-notes "${RELEASE_ARGS[@]}"
fi
echo "Released ${PYPI_PACKAGE} ${VERSION} from ${RELEASE_SHA} using CI run ${CI_RUN_ID:-unknown}"

270
bin/release_dev_stack.sh
View File

@ -1,270 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
IFS=$'\n\t'
ARCHIVEBOX_REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
WORKSPACE_DIR="$(cd "${ARCHIVEBOX_REPO}/.." && pwd)"
DOCKER_IMAGE_REPOS="${DOCKER_IMAGE_REPOS:-archivebox/archivebox ghcr.io/archivebox/archivebox}"
cd "${WORKSPACE_DIR}"
repo_dir() {
local repo="$1"
printf '%s/%s\n' "${WORKSPACE_DIR}" "${repo}"
}
ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-${LIB_DIR:-$HOME/.config/archivebox/lib}}"
locked_archivebox_abxpkg_version() {
local line package=""
while IFS= read -r line; do
case "$line" in
'[[package]]') package="" ;;
'name = "abxpkg"') package="abxpkg" ;;
'version = "'*'"')
[[ "$package" == "abxpkg" ]] || continue
line="${line#version = \"}"
printf '%s\n' "${line%\"}"
return 0
;;
esac
done < "$ARCHIVEBOX_REPO/uv.lock"
return 1
}
BOOTSTRAP_ABXPKG_VERSION="$(locked_archivebox_abxpkg_version)"
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
uv run --no-project --with "abxpkg==$BOOTSTRAP_ABXPKG_VERSION" abxpkg env \
--install \
--lib="$ABXPKG_LIB_DIR" \
--deps-from="$ARCHIVEBOX_REPO/.github/configs/ci-tooling.json:release_binaries" \
>/dev/null
GIT_BINARY="$ABXPKG_LIB_DIR/env/bin/git"
PYTHON_BINARY="$ABXPKG_LIB_DIR/env/bin/python"
UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv"
test -x "$GIT_BINARY"
test -x "$PYTHON_BINARY"
test -x "$UV_BINARY"
current_version() {
local repo="$1"
"$PYTHON_BINARY" - "$repo" <<'PY'
from pathlib import Path
import re
import sys
text = Path(sys.argv[1], "pyproject.toml").read_text()
match = re.search(r'^version = "([^"]+)"$', text, re.MULTILINE)
if not match:
raise SystemExit(f"Failed to find version in {sys.argv[1]}/pyproject.toml")
print(match.group(1))
PY
}
bump_patch_to() {
local repo="$1"
local version="$2"
"$PYTHON_BINARY" - "$repo" "$version" <<'PY'
from pathlib import Path
import re
import sys
path = Path(sys.argv[1], "pyproject.toml")
version = sys.argv[2]
text = path.read_text()
path.write_text(re.sub(r'^version = "[^"]+"$', f'version = "{version}"', text, count=1, flags=re.MULTILINE))
PY
}
next_patch_version() {
"$PYTHON_BINARY" - "$@" <<'PY'
import re
import sys
versions = sys.argv[1:]
parts = []
for version in versions:
match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", version)
if not match:
raise SystemExit(f"Expected patch version, got {version}")
parts.append(tuple(int(part) for part in match.groups()))
major, minor, patch = max(parts)
print(f"{major}.{minor}.{patch + 1}")
PY
}
bump_archivebox_rc() {
"$PYTHON_BINARY" - "${ARCHIVEBOX_REPO}" <<'PY'
from pathlib import Path
import json
import re
import sys
repo = Path(sys.argv[1])
pyproject_path = repo / "pyproject.toml"
package_path = repo / "etc" / "package.json"
pyproject_text = pyproject_path.read_text()
match = re.search(r'^version = "(\d+)\.(\d+)\.(\d+)(?:-?rc(\d+))?"$', pyproject_text, re.MULTILINE)
if not match:
raise SystemExit("Expected ArchiveBox version like 0.9.31rc15")
major, minor, patch, rc = match.groups()
next_version = f"{major}.{minor}.{patch}rc{int(rc or 0) + 1}"
pyproject_path.write_text(re.sub(r'^version = "[^"]+"$', f'version = "{next_version}"', pyproject_text, count=1, flags=re.MULTILINE))
package_json = json.loads(package_path.read_text())
package_json["version"] = next_version
package_path.write_text(json.dumps(package_json, indent=2) + "\n")
print(next_version)
PY
}
set_dependency_version() {
local repo="$1"
local package="$2"
local version="$3"
"$PYTHON_BINARY" - "$repo" "$package" "$version" <<'PY'
from pathlib import Path
import re
import sys
repo, package, version = sys.argv[1:]
path = Path(repo, "pyproject.toml")
text = path.read_text()
updated, count = re.subn(rf'("{re.escape(package)}>=)[^"]+(")', rf'\g<1>{version}\2', text)
if count:
path.write_text(updated)
PY
}
assert_branch() {
local repo="$1"
local branch="$2"
local actual
actual="$("$GIT_BINARY" -C "$repo" branch --show-current)"
if [[ "$actual" != "$branch" ]]; then
echo "[X] Expected $(basename "$repo") on ${branch}, found ${actual}" >&2
exit 1
fi
}
build_and_prek() {
local repo="$1"
(
cd "$repo"
rm -rf dist .pdm-build
"$UV_BINARY" --no-cache build --out-dir dist
"$UV_BINARY" --no-cache run prek run --all-files
rm -rf dist .pdm-build
"$UV_BINARY" --no-cache build --out-dir dist
)
}
commit_push_publish() {
local repo="$1"
local branch="$2"
local package="$3"
local version="$4"
local tag="v${version}"
(
cd "$repo"
"$GIT_BINARY" add -u
while IFS= read -r path; do
"$GIT_BINARY" add -- "$path"
done < <("$GIT_BINARY" ls-files --others --exclude-standard)
if ! "$GIT_BINARY" diff --cached --quiet; then
"$GIT_BINARY" commit -m "release: ${package} ${version}"
else
echo "[*] No staged changes in ${package}; reusing existing commit."
fi
"$GIT_BINARY" push origin "$branch"
if "$GIT_BINARY" rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then
if [[ "$("$GIT_BINARY" rev-list -n1 "${tag}")" != "$("$GIT_BINARY" rev-parse HEAD)" ]]; then
echo "[X] Tag ${tag} already exists but does not point at HEAD in ${package}" >&2
exit 1
fi
else
"$GIT_BINARY" tag -a "${tag}" -m "release: ${package} ${version}"
fi
"$GIT_BINARY" push origin "refs/tags/${tag}"
if pypi_has_release "$package" "$version"; then
echo "[*] ${package}==${version} is already on PyPI; skipping upload."
else
"$UV_BINARY" --no-cache publish --trusted-publishing always dist/*
fi
)
}
pypi_has_release() {
local package="$1"
local version="$2"
"$PYTHON_BINARY" - "$package" "$version" <<'PY'
import sys
import urllib.error
import urllib.request
package, version = sys.argv[1:]
try:
with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=10):
raise SystemExit(0)
except urllib.error.HTTPError as err:
raise SystemExit(1 if err.code == 404 else 2)
PY
}
release_python_repo() {
local repo_name="$1"
local branch="$2"
local package="$3"
local version="$4"
local repo
repo="$(repo_dir "$repo_name")"
echo "[+] Releasing ${package} ${version} from ${repo_name}:${branch}"
assert_branch "$repo" "$branch"
build_and_prek "$repo"
commit_push_publish "$repo" "$branch" "$package" "$version"
}
ABXPKG_VERSION="${ABXPKG_VERSION:-$(next_patch_version "$(current_version "$(repo_dir abxpkg)")")}"
ABX_SHARED_VERSION="${ABX_SHARED_VERSION:-$(next_patch_version "$(current_version "$(repo_dir abx-plugins)")" "$(current_version "$(repo_dir abx-dl)")")}"
bump_patch_to "$(repo_dir abxpkg)" "$ABXPKG_VERSION"
release_python_repo abxpkg main abxpkg "$ABXPKG_VERSION"
bump_patch_to "$(repo_dir abx-plugins)" "$ABX_SHARED_VERSION"
set_dependency_version "$(repo_dir abx-plugins)" abxpkg "$ABXPKG_VERSION"
release_python_repo abx-plugins main abx-plugins "$ABX_SHARED_VERSION"
bump_patch_to "$(repo_dir abx-dl)" "$ABX_SHARED_VERSION"
set_dependency_version "$(repo_dir abx-dl)" abxpkg "$ABXPKG_VERSION"
set_dependency_version "$(repo_dir abx-dl)" abx-plugins "$ABX_SHARED_VERSION"
release_python_repo abx-dl main abx-dl "$ABX_SHARED_VERSION"
ARCHIVEBOX_VERSION="$(bump_archivebox_rc)"
set_dependency_version "$ARCHIVEBOX_REPO" abxpkg "$ABXPKG_VERSION"
set_dependency_version "$ARCHIVEBOX_REPO" abx-plugins "$ABX_SHARED_VERSION"
set_dependency_version "$ARCHIVEBOX_REPO" abx-dl "$ABX_SHARED_VERSION"
echo "[+] Releasing archivebox ${ARCHIVEBOX_VERSION} from archivebox:dev"
assert_branch "$ARCHIVEBOX_REPO" dev
build_and_prek "$ARCHIVEBOX_REPO"
commit_push_publish "$ARCHIVEBOX_REPO" dev archivebox "$ARCHIVEBOX_VERSION"
(
cd "$ARCHIVEBOX_REPO"
ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-${LIB_DIR:-$HOME/.config/archivebox/lib}}"
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
"$UV_BINARY" run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
--install \
--lib="$ABXPKG_LIB_DIR" \
--deps-from="$ARCHIVEBOX_REPO/.github/configs/ci-tooling.json:docker_binaries" \
>/dev/null
DOCKER_BINARY="$ABXPKG_LIB_DIR/env/bin/docker"
test -x "$DOCKER_BINARY"
./bin/release_docker.sh dev "$ARCHIVEBOX_VERSION" "sha-$("$GIT_BINARY" rev-parse --short HEAD)"
DEPLOY_IMAGE="${DOCKER_IMAGE_REPOS%% *}:dev" DEPLOY_EXPECT_VERSION="$ARCHIVEBOX_VERSION" SKIP_DOCKER=1 ./bin/deploy_dev_demo.sh
)
echo "[√] Released abxpkg ${ABXPKG_VERSION}, abx-plugins/abx-dl ${ABX_SHARED_VERSION}, archivebox ${ARCHIVEBOX_VERSION}"

148
bin/release_docker.sh
View File

@ -1,148 +0,0 @@
#!/usr/bin/env bash
### Bash Environment Setup
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
set -o errexit
set -o errtrace
set -o nounset
set -o pipefail
IFS=$' '
REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
cd "$REPO_DIR"
ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-${LIB_DIR:-$HOME/.config/archivebox/lib}}"
locked_abxpkg_version() {
local line package=""
while IFS= read -r line; do
case "$line" in
'[[package]]') package="" ;;
'name = "abxpkg"') package="abxpkg" ;;
'version = "'*'"')
[[ "$package" == "abxpkg" ]] || continue
line="${line#version = \"}"
printf '%s\n' "${line%\"}"
return 0
;;
esac
done < "$REPO_DIR/uv.lock"
return 1
}
ABXPKG_VERSION="$(locked_abxpkg_version)"
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
--install \
--lib="$ABXPKG_LIB_DIR" \
--deps-from="$REPO_DIR/.github/configs/ci-tooling.json:docker_binaries" \
>/dev/null
DOCKER_BINARY="$ABXPKG_LIB_DIR/env/bin/docker"
GIT_BINARY="$ABXPKG_LIB_DIR/env/bin/git"
PYTHON_BINARY="$ABXPKG_LIB_DIR/env/bin/python"
test -x "$DOCKER_BINARY"
test -x "$GIT_BINARY"
test -x "$PYTHON_BINARY"
declare -a TAG_NAMES=("$@")
BRANCH_NAME="${1:-$("$GIT_BINARY" rev-parse --abbrev-ref HEAD)}"
VERSION="$("$PYTHON_BINARY" -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
GIT_SHA=sha-"$("$GIT_BINARY" rev-parse --short HEAD)"
SELECTED_PLATFORMS="${DOCKER_PLATFORMS:-${SELECTED_PLATFORMS:-linux/amd64,linux/arm64}}"
DOCKER_IMAGE_REPOS="${DOCKER_IMAGE_REPOS:-archivebox/archivebox ghcr.io/archivebox/archivebox}"
ABX_DL_VERSION="$("$PYTHON_BINARY" -c 'import tomllib; lock=tomllib.load(open("uv.lock", "rb")); print(next(pkg["version"] for pkg in lock["package"] if pkg["name"] == "abx-dl"))')"
test -n "$ABX_DL_VERSION"
ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
contains_tag() {
local candidate="$1" tag
for tag in "${TAG_NAMES[@]}"; do
[[ "$tag" == "$candidate" ]] && return 0
done
return 1
}
if ! contains_tag "$GIT_SHA"; then
TAG_NAMES+=("$GIT_SHA")
fi
if ! contains_tag "$BRANCH_NAME"; then
TAG_NAMES+=("$BRANCH_NAME")
fi
if ! contains_tag "$VERSION"; then
TAG_NAMES+=("$VERSION")
fi
echo "[+] Building + releasing Docker image for $SELECTED_PLATFORMS: branch=$BRANCH_NAME version=$VERSION abx_dl_image=$ABX_DL_IMAGE tags=${TAG_NAMES[*]}"
declare -a FULL_TAG_NAMES
for TAG_NAME in "${TAG_NAMES[@]}"; do
[[ "$TAG_NAME" == "" ]] && continue
for IMAGE_REPO in $DOCKER_IMAGE_REPOS; do
FULL_TAG_NAMES+=("-t" "$IMAGE_REPO:$TAG_NAME")
done
done
echo "${FULL_TAG_NAMES[@]}"
function check_platforms() {
INSTALLED_PLATFORMS="$("$DOCKER_BINARY" buildx inspect)"
for REQUIRED_PLATFORM in ${SELECTED_PLATFORMS//,/$IFS}; do
echo "[+] Checking for: $REQUIRED_PLATFORM..."
if [[ "$INSTALLED_PLATFORMS" != *"$REQUIRED_PLATFORM"* ]]; then
return 1
fi
done
echo
return 0
}
function remove_builder() {
"$DOCKER_BINARY" buildx stop xbuilder
"$DOCKER_BINARY" buildx rm xbuilder
}
function create_builder() {
"$DOCKER_BINARY" buildx use xbuilder && return 0
echo "[+] Creating new xbuilder for: $SELECTED_PLATFORMS"
echo
"$DOCKER_BINARY" pull 'moby/buildkit:buildx-stable-1'
"$DOCKER_BINARY" buildx create --name xbuilder --driver docker-container --bootstrap --use --platform "$SELECTED_PLATFORMS"
"$DOCKER_BINARY" buildx inspect --bootstrap
}
function recreate_builder() {
"$DOCKER_BINARY" run --privileged --rm 'tonistiigi/binfmt' --install all
remove_builder
create_builder
}
"$DOCKER_BINARY" buildx use xbuilder >/dev/null 2>&1 || create_builder
check_platforms || (recreate_builder && check_platforms) || exit 1
echo "[^] Uploading docker image"
mkdir -p "$HOME/.cache/docker/archivebox"
"$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_IMAGE"
"$DOCKER_BINARY" buildx build \
--platform "$SELECTED_PLATFORMS" \
--pull \
--build-arg "ABX_DL_IMAGE=$ABX_DL_IMAGE" \
--cache-from type=local,src="$HOME/.cache/docker/archivebox" \
--cache-to type=local,compression=zstd,mode=min,oci-mediatypes=true,dest="$HOME/.cache/docker/archivebox" \
--push . "${FULL_TAG_NAMES[@]}"
echo "[^] Verifying pushed Docker manifests include: $SELECTED_PLATFORMS"
for TAG_NAME in "${TAG_NAMES[@]}"; do
[[ "$TAG_NAME" == "" ]] && continue
for IMAGE_REPO in $DOCKER_IMAGE_REPOS; do
MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect "$IMAGE_REPO:$TAG_NAME")"
for REQUIRED_PLATFORM in ${SELECTED_PLATFORMS//,/$IFS}; do
if [[ "$MANIFEST" != *"Platform: $REQUIRED_PLATFORM"* ]]; then
echo "[X] $IMAGE_REPO:$TAG_NAME is missing platform: $REQUIRED_PLATFORM" >&2
echo "$MANIFEST" >&2
exit 1
fi
done
done
done
echo "[√] Docker manifests include all requested platforms."

25
bin/release_docs.sh
View File

@ -1,25 +0,0 @@
#!/usr/bin/env bash
### Bash Environment Setup
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
# set -o xtrace
set -o errexit
set -o errtrace
set -o nounset
set -o pipefail
IFS=$'\n'
REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')"
cd "$REPO_DIR"
echo "[^] Pushing docs to github"
cd docs/
git add .
git commit -am "$VERSION release"
git push
git tag -a "v$VERSION" -m "v$VERSION"
git push origin
git push origin --tags

23
bin/release_git.sh
View File

@ -1,23 +0,0 @@
#!/usr/bin/env bash
### Bash Environment Setup
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
# set -o xtrace
set -o errexit
set -o errtrace
set -o nounset
set -o pipefail
IFS=$'\n'
REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')"
cd "$REPO_DIR"
# Push build to github
echo "[^] Pushing release commit + tag to Github"
git tag -f -a "v$VERSION" -m "v$VERSION"
git push origin -f --tags
echo " To finish publishing the release go here:"
echo " https://github.com/ArchiveBox/ArchiveBox/releases/new"

20
bin/release_pip.sh
View File

@ -1,20 +0,0 @@
#!/usr/bin/env bash
### Bash Environment Setup
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
# set -o xtrace
set -o errexit
set -o errtrace
set -o nounset
set -o pipefail
IFS=$'\n'
REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
cd "$REPO_DIR"
source "$REPO_DIR/.venv/bin/activate"
echo "[^] Publishing to PyPI..."
rm -Rf dist
uv build
uv publish --trusted-publishing always

6
skills/archivebox/SKILL.md
View File

@ -73,8 +73,4 @@ uv run --project "$project_dir" --no-sync pytest "$project_dir/archivebox/tests/
(cd "$project_dir" && uv run --no-sync prek run --all-files)
```
Use the full release/deploy loop only when requested:
```console
./bin/release_dev_stack.sh
```
Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.