From bb598b97105e68ea4e938999410d4be4285072ce Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 11 Jun 2026 00:41:34 -0700 Subject: [PATCH] release: archivebox 0.9.35rc23 --- .github/workflows/docker.yml | 9 ++-- .github/workflows/pip.yml | 65 +++++++++++++++++++++++---- Dockerfile | 87 ++++++++++++++++++++++++++---------- bin/release_dev_stack.sh | 11 ----- etc/package.json | 2 +- pyproject.toml | 2 +- 6 files changed, 129 insertions(+), 47 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c6a51d8b..ecee6a9b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -69,9 +69,12 @@ jobs: missing = required.copy() while missing and time.monotonic() < deadline: for name, version in list(missing.items()): - with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp: - releases = json.load(resp)["releases"] - if version in releases: + try: + with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20): + available = True + except Exception: + available = False + if available: print(f"{name} {version} is available on PyPI") missing.pop(name) else: diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 8954235b..5baeae33 100755 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -47,7 +47,6 @@ jobs: - name: Wait for released ArchiveBox deps on PyPI run: | python - <<'PY' - import json import re import sys import time @@ -67,9 +66,12 @@ jobs: missing = required.copy() while missing and time.monotonic() < deadline: for name, version in list(missing.items()): - with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp: - releases = json.load(resp)["releases"] - if version in releases: + try: + with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20): + available = True + except Exception: + available = False + if available: print(f"{name} {version} is available on PyPI") missing.pop(name) else: @@ -82,8 +84,49 @@ jobs: sys.exit(1) PY + - name: Prepare released dependency sync project + run: | + python - <<'PY' + from pathlib import Path + import json + import re + import urllib.request + + source = Path("pyproject.toml") + target_dir = Path(".ci-uv-project") + target_dir.mkdir(exist_ok=True) + text = source.read_text() + text = text.replace( + 'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]', + 'environments = ["sys_platform == \'linux\'"]', + ) + + # CI runs immediately after publishing internal abx packages. PyPI's + # simple index can lag the version JSON endpoint, so sync against the + # exact wheel URLs from the version endpoint while leaving package + # metadata in pyproject.toml unchanged for the actual build. + for package in ("abxpkg", "abx-plugins", "abx-dl"): + match = re.search(rf'"{re.escape(package)}>=(?P[^"]+)"', text) + if not match: + continue + version = match.group("version") + with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response: + data = json.load(response) + wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl")) + text = re.sub( + rf'"{re.escape(package)}>=[^"]+"', + f'"{package} @ {wheel_url}"', + text, + count=1, + ) + + (target_dir / "pyproject.toml").write_text(text) + PY + - name: UV install archivebox dev + run sub-dependencies - run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache + env: + UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv + run: uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache - name: UV build archivebox and archivebox/pkgs/* packages run: | @@ -97,13 +140,19 @@ jobs: password: ${{ secrets.PYPI_PAT_SECRET }} - name: UV install archivebox and archivebox/pkgs/* locally for tests - run: uv sync --all-extras --no-sources --no-cache + env: + UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv + run: | + uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache + uv pip install --python .venv/bin/python --no-deps dist/archivebox-*.whl - name: UV run archivebox init + archivebox version + env: + UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv run: | mkdir -p data && cd data - uv run --no-sync --no-sources archivebox init \ - && uv run --no-sync --no-sources archivebox version + uv run --project ../.ci-uv-project --no-sync --no-sources archivebox init \ + && uv run --project ../.ci-uv-project --no-sync --no-sources archivebox version # && uv run archivebox add 'https://example.com' \ # && uv run archivebox status \ # || (echo "UV Failed to run archivebox!" && exit 1) diff --git a/Dockerfile b/Dockerfile index 6f66a845..feb2c489 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,29 +105,70 @@ WORKDIR "$CODE_DIR" RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ - echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \ - && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \ - && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \ - && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \ - && rm -f /etc/apt/apt.conf.d/docker-clean \ - && apt-get update -qq \ - && apt-get install -qq -y --no-install-recommends \ - build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ - && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \ - && /usr/bin/uv pip install setuptools pip wheel \ - && /usr/bin/uv sync \ - --refresh \ - --python-platform linux \ - --no-dev \ - --inexact \ - --no-install-project \ - --no-install-workspace \ - --no-sources \ - && (find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true) \ - && rm -f /venv/bin/uv /venv/bin/uvx \ - && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* + <<'EOF' +echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." +echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache +echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends +echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests +rm -f /etc/apt/apt.conf.d/docker-clean +apt-get update -qq +apt-get install -qq -y --no-install-recommends \ + build-essential gcc libldap2-dev libsasl2-dev libssl-dev +/usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" +/usr/bin/uv pip install setuptools pip wheel + +mkdir -p /tmp/archivebox-uv-project +/venv/bin/python - <<'PY' +from pathlib import Path +import json +import re +import urllib.request + +source = Path("/app/pyproject.toml") +target = Path("/tmp/archivebox-uv-project/pyproject.toml") +text = source.read_text() +text = text.replace( + 'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]', + 'environments = ["sys_platform == \'linux\'"]', +) + +# Docker builds need the just-published internal abx wheels immediately, but +# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a +# Docker-only dependency view from the version JSON so the published package +# metadata stays normal while image builds remain resumable after a release. +for package in ("abxpkg", "abx-plugins", "abx-dl"): + match = re.search(rf'"{re.escape(package)}>=(?P[^"]+)"', text) + if not match: + continue + version = match.group("version") + with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response: + data = json.load(response) + wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl")) + text = re.sub( + rf'"{re.escape(package)}>=[^"]+"', + f'"{package} @ {wheel_url}"', + text, + count=1, + ) + +target.write_text(text) +PY + +/usr/bin/uv sync \ + --project /tmp/archivebox-uv-project \ + --refresh \ + --python-platform linux \ + --no-dev \ + --inexact \ + --no-install-project \ + --no-install-workspace \ + --no-sources +(find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true) +rm -f /venv/bin/uv /venv/bin/uvx +apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev +apt-get autoremove -y +rm -rf /var/lib/apt/lists/* +EOF COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ diff --git a/bin/release_dev_stack.sh b/bin/release_dev_stack.sh index 3394ed25..1190a9d2 100755 --- a/bin/release_dev_stack.sh +++ b/bin/release_dev_stack.sh @@ -211,17 +211,6 @@ wait_for_pypi() { fi sleep 10 done - - attempts=0 - until uv --no-cache pip install --dry-run --no-deps "${package}==${version}" >/dev/null - do - attempts=$((attempts + 1)) - if [[ "$attempts" -ge "$PYPI_WAIT_ATTEMPTS" ]]; then - echo "[X] Timed out waiting for uv to resolve ${package}==${version} from PyPI" >&2 - exit 1 - fi - sleep 10 - done } release_python_repo() { diff --git a/etc/package.json b/etc/package.json index dd042a44..7f977286 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.35rc22", + "version": "0.9.35rc23", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index 6dc986e5..862d40d5 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc22" +version = "0.9.35rc23" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]