diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 21e4ffef..e8fdef2a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,12 +2,9 @@ name: Build Docker image on: workflow_dispatch: - workflow_call: push: branches: - - "**" - tags: - - 'v*' + - '**' # pull_request: env: @@ -45,47 +42,6 @@ jobs: # submodules: true # fetch-depth: 1 - - name: Wait for released ArchiveBox deps on PyPI - run: | - python3 - <<'PY' - import json - import re - import sys - import time - import tomllib - import urllib.request - - watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"} - deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"] - required = {} - for dep in deps: - for name in watched: - match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep) - if match: - required[name] = match.group(2) - - deadline = time.monotonic() + 300 - missing = required.copy() - while missing and time.monotonic() < deadline: - for name, version in list(missing.items()): - try: - with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20): - available = True - except Exception: - available = False - if available: - print(f"{name} {version} is available on PyPI") - missing.pop(name) - else: - print(f"{name} {version} is not available on PyPI yet") - if missing: - time.sleep(10) - - if missing: - print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr) - sys.exit(1) - PY - - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -99,7 +55,7 @@ jobs: - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} - - name: Wait for published abx-dl image + - name: Verify published abx-dl image id: abx_dl_image shell: bash run: | @@ -121,16 +77,6 @@ jobs: PY )" ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}" - deadline=$((SECONDS + 1800)) - until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do - if (( SECONDS >= deadline )); then - echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2 - exit 1 - fi - echo "${ABX_DL_IMAGE} is not published yet; waiting..." - sleep 30 - done - echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT" docker buildx imagetools inspect "${ABX_DL_IMAGE}" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f0682d00..0066dd52 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,6 +3,7 @@ name: Run linters on: workflow_dispatch: push: + branches: ['**'] concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index f7e1cc9d..cb1e02a1 100755 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -2,12 +2,9 @@ name: Build Pip package on: workflow_dispatch: - workflow_call: push: branches: - '**' - tags: - - 'v*' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -18,11 +15,7 @@ env: jobs: build: - permissions: - id-token: write - runs-on: ubuntu-24.04 - environment: pypi steps: - uses: actions/checkout@v4 @@ -44,89 +37,10 @@ jobs: packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps version: 1.0 - - name: Wait for released ArchiveBox deps on PyPI - run: | - python - <<'PY' - import re - import sys - import time - import tomllib - import urllib.request - - watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"} - deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"] - required = {} - for dep in deps: - for name in watched: - match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep) - if match: - required[name] = match.group(2) - - deadline = time.monotonic() + 300 - missing = required.copy() - while missing and time.monotonic() < deadline: - for name, version in list(missing.items()): - try: - with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=20): - available = True - except Exception: - available = False - if available: - print(f"{name} {version} is available on PyPI") - missing.pop(name) - else: - print(f"{name} {version} is not available on PyPI yet") - if missing: - time.sleep(10) - - if missing: - print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr) - sys.exit(1) - PY - - - name: Prepare released dependency sync project - run: | - python - <<'PY' - from pathlib import Path - import json - import re - import urllib.request - - source = Path("pyproject.toml") - target_dir = Path(".ci-uv-project") - target_dir.mkdir(exist_ok=True) - text = source.read_text() - text = text.replace( - 'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]', - 'environments = ["sys_platform == \'linux\'"]', - ) - - # CI runs immediately after publishing internal abx packages. PyPI's - # simple index can lag the version JSON endpoint, so sync against the - # exact wheel URLs from the version endpoint while leaving package - # metadata in pyproject.toml unchanged for the actual build. - for package in ("abxpkg", "abx-plugins", "abx-dl"): - match = re.search(rf'"{re.escape(package)}>=(?P[^"]+)"', text) - if not match: - continue - version = match.group("version") - with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response: - data = json.load(response) - wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl")) - text = re.sub( - rf'"{re.escape(package)}>=[^"]+"', - f'"{package} @ {wheel_url}"', - text, - count=1, - ) - - (target_dir / "pyproject.toml").write_text(text) - PY - - name: UV install archivebox dev + run sub-dependencies env: UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv - run: uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache + run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache - name: UV build archivebox and archivebox/pkgs/* packages run: | @@ -136,7 +50,7 @@ jobs: env: UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv run: | - uv sync --project .ci-uv-project --all-extras --no-install-project --no-install-workspace --no-sources --no-cache + uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache uv pip install --python .venv/bin/python --no-deps dist/archivebox-*.whl - name: Verify built package full install @@ -146,7 +60,7 @@ jobs: set -Eeuo pipefail DATA_DIR="$(mktemp -d)" cd "$DATA_DIR" - uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox init - timeout 30m uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox install - uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox version - uv run --project "$GITHUB_WORKSPACE/.ci-uv-project" --no-sync --no-sources archivebox status + uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox init + timeout 30m uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox install + uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox version + uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox status diff --git a/.github/workflows/release-runner.yml b/.github/workflows/release-runner.yml index 7158b59d..2f43db98 100644 --- a/.github/workflows/release-runner.yml +++ b/.github/workflows/release-runner.yml @@ -7,7 +7,6 @@ on: workflow_dispatch: permissions: - actions: write contents: write id-token: write @@ -51,10 +50,3 @@ jobs: DEFAULT_BRANCH: dev GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }} run: ./bin/release.sh - - - name: Dispatch image and tap updates - env: - GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }} - run: | - gh workflow run docker.yml --ref dev - gh workflow run update-homebrew-tap.yml --ref dev diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 41436c48..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Release - -# Orchestrates the full release pipeline: -# 1. Build and publish pip package to PyPI -# 2. Build Homebrew formula after pip -# 3. Build Docker images in parallel -# -# Individual workflows also run on push for CI (see their own triggers). -# Manual fallback for the hosted release pipeline. The normal prerelease loop -# is run by bin/release.sh locally so published GitHub releases do not launch a -# duplicate release job. - -on: - workflow_dispatch: - -permissions: - contents: write - packages: write - id-token: write - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - pip: - name: Publish to PyPI - uses: ./.github/workflows/pip.yml - secrets: inherit - - homebrew: - name: Update Homebrew formula - needs: pip - uses: ./.github/workflows/homebrew.yml - secrets: inherit - - docker: - name: Build Docker images - needs: pip - uses: ./.github/workflows/docker.yml - secrets: inherit diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a4b8a858..f4b58b83 100755 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,8 @@ name: Run tests -on: [push] + +on: + push: + branches: ['**'] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -91,22 +94,9 @@ jobs: uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox version uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status - - name: Test built package with pytest - run: | - mkdir -p tests/out - uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out - - name: Run documentation code blocks run: uv run --no-sync --no-sources pytest -vv --tb=long README.md AGENTS.md skills docs - - name: Run plugin tests - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }} - API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }} - run: | - uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage - docker_tests: runs-on: ubuntu-latest diff --git a/.github/workflows/update-homebrew-tap.yml b/.github/workflows/update-homebrew-tap.yml index 550449ac..803368af 100644 --- a/.github/workflows/update-homebrew-tap.yml +++ b/.github/workflows/update-homebrew-tap.yml @@ -2,7 +2,7 @@ name: Update downstream packages on: push: - branches: [dev] + tags: ['v*'] workflow_dispatch: permissions: diff --git a/bin/release.sh b/bin/release.sh index 62f74df8..65794ced 100755 --- a/bin/release.sh +++ b/bin/release.sh @@ -266,87 +266,6 @@ create_git_tag() { git push origin "refs/tags/${tag}" } -wait_for_runs() { - local slug="$1" - local event="$2" - local sha="$3" - local label="$4" - local runs_json - local attempts=0 - - while :; do - runs_json="$(GH_FORCE_TTY=0 GH_PAGER=cat gh run list --repo "${slug}" --event "${event}" --commit "${sha}" --limit 20 --json databaseId,status,conclusion,workflowName)" - if [[ "$(jq 'length' <<<"${runs_json}")" -gt 0 ]]; then - break - fi - attempts=$((attempts + 1)) - if [[ "${attempts}" -ge 30 ]]; then - echo "Timed out waiting for ${label} workflows to start" >&2 - return 1 - fi - sleep 10 - done - - while IFS=$'\t' read -r run_id workflow_name; do - workflow_name_lower="${workflow_name,,}" - if [[ "${workflow_name_lower}" == *"release state"* ]]; then - gh run watch "${run_id}" --repo "${slug}" --exit-status - continue - fi - if [[ "${workflow_name_lower}" != *"test"* ]]; then - echo "Skipping non-gating workflow: ${workflow_name}" - continue - fi - - attempts=0 - while :; do - precheck_state="$( - gh run view "${run_id}" --repo "${slug}" --json jobs --jq ' - [.jobs[] | select((.name | ascii_downcase) | test("precheck|pre-commit|prek"))][0] - | if . == null then "missing:" else ((.status // "") + ":" + (.conclusion // "")) end - ' - )" - case "${precheck_state}" in - missing:*) - echo "Skipping test workflow without precheck job: ${workflow_name}" - break - ;; - completed:success|completed:skipped) - break - ;; - completed:failure|completed:cancelled|completed:timed_out) - gh run view "${run_id}" --repo "${slug}" - return 1 - ;; - esac - attempts=$((attempts + 1)) - if [[ "${attempts}" -ge 120 ]]; then - echo "Timed out waiting for ${workflow_name} precheck job" >&2 - return 1 - fi - sleep 5 - done - done < <(jq -r '.[] | [.databaseId, .workflowName] | @tsv' <<<"${runs_json}") -} - -wait_for_pypi() { - local package_name="$1" - local expected_version="$2" - local attempts=0 - - while :; do - if curl -fsSL "https://pypi.org/pypi/${package_name}/json" | jq -e --arg version "${expected_version}" '.releases[$version] | length > 0' >/dev/null; then - return 0 - fi - attempts=$((attempts + 1)) - if [[ "${attempts}" -ge 30 ]]; then - echo "Timed out waiting for ${package_name}==${expected_version} on PyPI" >&2 - return 1 - fi - sleep 10 - done -} - run_checks() { uv sync --extra sonic --extra debug --all-groups --no-cache --upgrade uv build --all @@ -428,7 +347,6 @@ publish_artifacts() { uv publish --trusted-publishing always "${artifacts[@]}" fi - wait_for_pypi "${PYPI_PACKAGE}" "${version}" } main() {