diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ecee6a9b..c7517ad4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -13,7 +13,6 @@ on: env: DOCKERHUB_IMAGE: archivebox/archivebox GHCR_IMAGE: ghcr.io/archivebox/archivebox - ABX_DL_IMAGE: archivebox/abx-dl:latest permissions: contents: read @@ -105,6 +104,23 @@ jobs: shell: bash run: | set -Eeuo pipefail + ABX_DL_VERSION="$(python3 - <<'PY' + import re + import tomllib + + with open("pyproject.toml", "rb") as f: + deps = tomllib.load(f)["project"]["dependencies"] + + for dep in deps: + match = re.match(r"abx-dl\s*(?:==|>=)\s*([^,;\s]+)", dep) + if match: + print(match.group(1)) + break + else: + raise SystemExit("Missing abx-dl dependency in pyproject.toml") + PY + )" + ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}" deadline=$((SECONDS + 1800)) until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do if (( SECONDS >= deadline )); then @@ -173,6 +189,7 @@ jobs: ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }} cache-from: type=gha,scope=${{ matrix.cache_scope }} cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }} + pull: true platforms: ${{ matrix.platform }} outputs: type=image,push-by-digest=true,name-canonical=true,push=true diff --git a/bin/build_docker.sh b/bin/build_docker.sh index cbc4606f..07ead00a 100755 --- a/bin/build_docker.sh +++ b/bin/build_docker.sh @@ -22,6 +22,23 @@ declare -a TAG_NAMES=("$@") BRANCH_NAME="${1:-$(git rev-parse --abbrev-ref HEAD)}" VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $2}')" GIT_SHA=sha-"$(git rev-parse --short HEAD)" +ABX_DL_VERSION="$(python3 - <<'PY' +import re +import tomllib + +with open("pyproject.toml", "rb") as f: + deps = tomllib.load(f)["project"]["dependencies"] + +for dep in deps: + match = re.match(r"abx-dl\s*(?:==|>=)\s*([^,;\s]+)", dep) + if match: + print(match.group(1)) + break +else: + raise SystemExit("Missing abx-dl dependency in pyproject.toml") +PY +)" +ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}" NATIVE_ARCH="$(docker info --format '{{.Architecture}}' 2>/dev/null || uname -m)" case "$NATIVE_ARCH" in x86_64|amd64) NATIVE_PLATFORM="linux/amd64" ;; @@ -41,7 +58,7 @@ if ! echo "${TAG_NAMES[@]}" | grep -q "$VERSION"; then TAG_NAMES+=("$VERSION") fi -echo "[+] Building Docker image for $SELECTED_PLATFORMS: branch=$BRANCH_NAME version=$VERSION tags=${TAG_NAMES[*]}" +echo "[+] Building Docker image for $SELECTED_PLATFORMS: branch=$BRANCH_NAME version=$VERSION abx_dl_image=$ABX_DL_IMAGE tags=${TAG_NAMES[*]}" declare -a FULL_TAG_NAMES # for each tag in TAG_NAMES, add archivebox/archivebox:tag and its mirrors to FULL_TAG_NAMES @@ -97,6 +114,7 @@ check_platforms || (recreate_builder && check_platforms) || exit 1 echo "[+] Building archivebox:$VERSION docker image..." mkdir -p "$HOME/.cache/docker/archivebox" +docker buildx imagetools inspect "$ABX_DL_IMAGE" # docker builder prune # docker build . --no-cache -t archivebox-dev \ # replace --load with --push to deploy @@ -107,6 +125,8 @@ if [[ "$SELECTED_PLATFORMS" == *,* ]]; then fi docker buildx build \ --platform "$SELECTED_PLATFORMS" \ + --pull \ + --build-arg "ABX_DL_IMAGE=$ABX_DL_IMAGE" \ --cache-from type=local,src="$HOME/.cache/docker/archivebox" \ --cache-to type=local,compression=zstd,mode=min,oci-mediatypes=true,dest="$HOME/.cache/docker/archivebox" \ --load . ${FULL_TAG_NAMES[@]} diff --git a/bin/release_docker.sh b/bin/release_docker.sh index ad67aefd..58e74ae7 100755 --- a/bin/release_docker.sh +++ b/bin/release_docker.sh @@ -19,6 +19,23 @@ VERSION="$(grep '^version = ' "${REPO_DIR}/pyproject.toml" | awk -F'"' '{print $ GIT_SHA=sha-"$(git rev-parse --short HEAD)" SELECTED_PLATFORMS="${DOCKER_PLATFORMS:-${SELECTED_PLATFORMS:-linux/amd64,linux/arm64}}" DOCKER_IMAGE_REPOS="${DOCKER_IMAGE_REPOS:-archivebox/archivebox ghcr.io/archivebox/archivebox}" +ABX_DL_VERSION="$(python3 - <<'PY' +import re +import tomllib + +with open("pyproject.toml", "rb") as f: + deps = tomllib.load(f)["project"]["dependencies"] + +for dep in deps: + match = re.match(r"abx-dl\s*(?:==|>=)\s*([^,;\s]+)", dep) + if match: + print(match.group(1)) + break +else: + raise SystemExit("Missing abx-dl dependency in pyproject.toml") +PY +)" +ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}" # if not already in TAG_NAMES, add GIT_SHA and BRANCH_NAME if ! echo "${TAG_NAMES[@]}" | grep -q "$GIT_SHA"; then @@ -31,7 +48,7 @@ if ! echo "${TAG_NAMES[@]}" | grep -q "$VERSION"; then TAG_NAMES+=("$VERSION") fi -echo "[+] Building + releasing Docker image for $SELECTED_PLATFORMS: branch=$BRANCH_NAME version=$VERSION tags=${TAG_NAMES[*]}" +echo "[+] Building + releasing Docker image for $SELECTED_PLATFORMS: branch=$BRANCH_NAME version=$VERSION abx_dl_image=$ABX_DL_IMAGE tags=${TAG_NAMES[*]}" declare -a FULL_TAG_NAMES for TAG_NAME in "${TAG_NAMES[@]}"; do @@ -85,11 +102,14 @@ check_platforms || (recreate_builder && check_platforms) || exit 1 echo "[^] Uploading docker image" mkdir -p "$HOME/.cache/docker/archivebox" +docker buildx imagetools inspect "$ABX_DL_IMAGE" # https://docs.docker.com/build/cache/backends/ # shellcheck disable=SC2068 docker buildx build \ --platform "$SELECTED_PLATFORMS" \ + --pull \ + --build-arg "ABX_DL_IMAGE=$ABX_DL_IMAGE" \ --cache-from type=local,src="$HOME/.cache/docker/archivebox" \ --cache-to type=local,compression=zstd,mode=min,oci-mediatypes=true,dest="$HOME/.cache/docker/archivebox" \ --push . ${FULL_TAG_NAMES[@]}