ArchiveBox/.github/workflows/test-parallel.yml

name: Parallel Tests

on:
  workflow_call:

env:
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
  UV_NO_SOURCES: "1"
  CI_PYTHON_VERSION: "3.13.14"

jobs:
  discover-tests:
    name: Discover every test
    runs-on: ubuntu-24.04
    outputs:
      tests: ${{ steps.discover.outputs.tests }}

    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
        with:
          submodules: true
          fetch-depth: 1

      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: ${{ env.CI_PYTHON_VERSION }}

      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
        with:
          version: "0.11.3"
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Resolve discovery tools through abxpkg
        env:
          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
        run: |
          set -Eeuo pipefail
          ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
          test -n "$ABXPKG_VERSION"
          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
          export PYTHON_BINARY="$pythonLocation/bin/python"
          test -x "$PYTHON_BINARY"
          uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
            --install \
            --lib="$ABXPKG_LIB_DIR" \
            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
          {
            echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
          } >> "$GITHUB_ENV"
          test -L "$ABXPKG_LIB_DIR/env/bin/python"
          test -x "$ABXPKG_LIB_DIR/env/bin/python"
          test -L "$ABXPKG_LIB_DIR/env/bin/jq"
          test -x "$ABXPKG_LIB_DIR/env/bin/jq"          

      - name: Discover every test exactly once
        id: discover
        run: |
          set -Eeuo pipefail
          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
          test "${#output[@]}" -eq 2
          echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
          echo "${output[0]}"
          "$JQ_BINARY" -e 'length > 0 and length <= 32 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"          

  tests:
    name: ${{ matrix.test.name }}
    needs: discover-tests
    runs-on: ubuntu-24.04
    env:
      ABXPKG_LIB_DIR: /tmp/archivebox-lib
      CHROME_HEADLESS: "true"
      DATA_DIR: /tmp/archivebox-test-data
      PERSONAS_DIR: /tmp/abx-personas
      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile

    strategy:
      fail-fast: false
      matrix:
        test: ${{ fromJson(needs.discover-tests.outputs.tests) }}

    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
        with:
          submodules: true
          fetch-depth: 1

      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: ${{ env.CI_PYTHON_VERSION }}
          architecture: x64

      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
        with:
          version: "0.11.3"
          enable-cache: true
          cache-dependency-glob: uv.lock

      - name: Resolve optional build dependencies through abxpkg
        if: matrix.test.extra != ''
        run: |
          set -Eeuo pipefail
          ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
          test -n "$ABXPKG_VERSION"
          uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
            --install \
            --no-cache \
            --json \
            --lib="$ABXPKG_LIB_DIR" \
            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries"          

      - name: Install test dependencies
        env:
          TEST_EXTRA: ${{ matrix.test.extra }}
        run: |
          set -Eeuo pipefail
          extra_args=()
          if [[ -n "$TEST_EXTRA" ]]; then
            extra_args=(--extra "$TEST_EXTRA")
          fi
          uv sync --locked --dev "${extra_args[@]}"          

      - name: Install PostgreSQL server binaries
        if: contains(matrix.test.paths_arg, 'test_postgres_backend')
        run: |
          set -Eeuo pipefail
          if ! ls /usr/lib/postgresql/*/bin/initdb >/dev/null 2>&1 && ! command -v initdb >/dev/null 2>&1; then
            sudo apt-get update && sudo apt-get install -y postgresql
          fi          

      - name: Run ${{ matrix.test.name }}
        env:
          TEST_PATHS: ${{ matrix.test.paths_arg }}
        run: |
          set -Eeuo pipefail
          read -r -a test_paths <<< "$TEST_PATHS"
          for test_path in "${test_paths[@]}"; do
            test -f "$test_path"
          done
          mkdir -p "$DATA_DIR"
          uv run --no-sync --no-sources pytest -vs "${test_paths[@]}" \
            --basetemp="$RUNNER_TEMP/pytest"