diff --git a/.github/scripts/discover_test_matrix.py b/.github/scripts/discover_test_matrix.py index 3cb6723b..67412ae9 100755 --- a/.github/scripts/discover_test_matrix.py +++ b/.github/scripts/discover_test_matrix.py @@ -1,28 +1,9 @@ #!/usr/bin/env python3 -"""Discover every ArchiveBox test file and pack it into one CI matrix.""" +"""Discover every ArchiveBox test file and expose it as one CI job.""" import json from pathlib import Path -TARGET_SHARDS = 18 - -SLOW_TEST_WEIGHTS = { - "test_cli_archiveresult.py": 200, - "test_opencode_agent.py": 180, - "test_cli_piping.py": 135, - "test_api_v1_crawls_crawl_crawl_id.py": 130, - "test_binary_service.py": 80, - "test_api_v1_workflow_core_token_auth_side_effects.py": 55, - "test_api_v1_cli_schedule.py": 50, - "test_api_v1_cli_remove.py": 40, - "test_api_v1_workflow_frozen_crawl_config_sources.py": 35, - "test_cli_help.py": 35, -} - - -def test_weight(path: Path) -> int: - return SLOW_TEST_WEIGHTS.get(path.name, 25) - def main() -> None: root = Path.cwd().resolve() @@ -31,26 +12,16 @@ def main() -> None: if not archivebox_tests: raise SystemExit("No ArchiveBox tests discovered") - shard_count = min(TARGET_SHARDS, len(archivebox_tests)) - shards: list[list[Path]] = [[] for _ in range(shard_count)] - shard_weights = [0 for _ in range(shard_count)] - - for path in sorted(archivebox_tests, key=lambda item: (-test_weight(item), item.as_posix())): - shard_index = min(range(shard_count), key=lambda index: (shard_weights[index], index)) - shards[shard_index].append(path) - shard_weights[shard_index] += test_weight(path) - matrix: list[dict[str, object]] = [] - for index, shard in enumerate(shards, start=1): - shard_paths = sorted(path.relative_to(root).as_posix() for path in shard) - paths_arg = " ".join(shard_paths) + for path in archivebox_tests: + test_path = path.relative_to(root).as_posix() matrix.append( { - "name": f"main/shard-{index:02d}", - "paths": shard_paths, - "paths_arg": paths_arg, - "extra": "ldap" if any(path.endswith("/test_auth_ldap.py") for path in shard_paths) else "", - "count": len(shard_paths), + "name": f"main/{path.stem}", + "paths": [test_path], + "paths_arg": test_path, + "extra": "ldap" if test_path.endswith("/test_auth_ldap.py") else "", + "count": 1, }, ) @@ -60,7 +31,7 @@ def main() -> None: if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]: raise SystemExit("Discovered test shard coverage does not match test files") - print(f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} balanced shards") + print(f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs") print(json.dumps(matrix, separators=(",", ":"))) diff --git a/.github/workflows/test-parallel.yml b/.github/workflows/test-parallel.yml index 3d0493cf..f10757f5 100644 --- a/.github/workflows/test-parallel.yml +++ b/.github/workflows/test-parallel.yml @@ -64,7 +64,7 @@ jobs: test "${#output[@]}" -eq 2 echo "tests=${output[1]}" >> "$GITHUB_OUTPUT" echo "${output[0]}" - "$JQ_BINARY" -e 'length > 0 and length <= 32 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}" + "$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}" tests: name: ${{ matrix.test.name }} diff --git a/pyproject.toml b/pyproject.toml index 7022bc85..47e2b614 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc141" +version = "0.9.35rc142" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -348,7 +348,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations" [tool.bumpver] -current_version = "v0.9.35rc141" +current_version = "v0.9.35rc142" version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" tag_message = "{new_version}" diff --git a/uv.lock b/uv.lock index 4d1bd364..c87c5d5f 100644 --- a/uv.lock +++ b/uv.lock @@ -123,7 +123,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.9.35rc141" +version = "0.9.35rc142" source = { editable = "." } dependencies = [ { name = "abx-dl" },