From a3c5e13ff8ae800aa948e7d9e4a6bb13254aa497 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 23 Jul 2026 07:00:05 -0700 Subject: [PATCH] Release ArchiveBox through complete CI coverage --- .github/configs/ci-tooling.json | 4 + .github/workflows/ci.yml | 24 +++- .github/workflows/codeql.yml | 4 + .github/workflows/docs.yml | 4 +- .github/workflows/lint.yml | 3 + .github/workflows/pip.yml | 9 +- .github/workflows/release.yml | 2 +- .github/workflows/test-parallel.yml | 2 +- .github/workflows/test.yml | 2 +- Dockerfile | 28 ++-- archivebox/cli/archivebox_run.py | 16 +-- archivebox/machine/detect.py | 10 +- archivebox/services/runner.py | 5 + archivebox/tests/conftest.py | 11 +- .../tests/test_archive_result_service.py | 11 +- archivebox/tests/test_auth_ldap.py | 39 +++-- archivebox/tests/test_cli_oneshot.py | 28 ++-- archivebox/tests/test_cli_run.py | 18 ++- archivebox/tests/test_opencode_agent.py | 11 +- .../tests/test_server_security_browser.py | 17 ++- archivebox/tests/test_version_metadata.py | 1 + bin/bumpver_update_lock.sh | 10 ++ bin/release.sh | 135 ++++++++++++++++-- bin/setup.sh | 2 +- etc/package.json | 2 +- pyproject.toml | 17 ++- uv.lock | 54 ++++--- 27 files changed, 365 insertions(+), 104 deletions(-) create mode 100755 bin/bumpver_update_lock.sh diff --git a/.github/configs/ci-tooling.json b/.github/configs/ci-tooling.json index ec58ef27..7fc7c474 100644 --- a/.github/configs/ci-tooling.json +++ b/.github/configs/ci-tooling.json @@ -263,6 +263,10 @@ { "name": "{CURL_BINARY}", "binproviders": "env,apt,brew" + }, + { + "name": "{SUDO_BINARY}", + "binproviders": "env" } ], "git_binaries": [ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a52c138..df0bb818 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,7 @@ jobs: required: name: All required CI lanes + if: ${{ always() }} needs: - lint - complete-tests @@ -67,4 +68,25 @@ jobs: - docker-digests runs-on: ubuntu-24.04 steps: - - run: echo 'All required CI lanes and tested release artifacts succeeded.' + - name: Verify every required lane succeeded + env: + LINT_RESULT: ${{ needs.lint.result }} + COMPLETE_TESTS_RESULT: ${{ needs.complete-tests.result }} + SHARDED_PLUGIN_TESTS_RESULT: ${{ needs.sharded-plugin-tests.result }} + DOCUMENTATION_RESULT: ${{ needs.documentation.result }} + CODEQL_RESULT: ${{ needs.codeql.result }} + PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }} + DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }} + run: | + for result in \ + "$LINT_RESULT" \ + "$COMPLETE_TESTS_RESULT" \ + "$SHARDED_PLUGIN_TESTS_RESULT" \ + "$DOCUMENTATION_RESULT" \ + "$CODEQL_RESULT" \ + "$PYTHON_ARTIFACTS_RESULT" \ + "$DOCKER_DIGESTS_RESULT" + do + [[ "$result" == success ]] + done + echo 'All required CI lanes and tested release artifacts succeeded.' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1b683f84..a6c520e4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -5,6 +5,10 @@ on: schedule: - cron: '33 17 * * 6' +concurrency: + group: codeql-${{ github.repository }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + jobs: analyze: name: Analyze (${{ matrix.language }}) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7c24d8a0..71daa961 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -78,7 +78,7 @@ jobs: with: version: "0.11.3" - name: Install ArchiveBox - run: uv sync --dev --all-extras + run: uv sync --locked --dev --all-extras - name: Prepare abxpkg environment shell: bash run: | @@ -259,7 +259,7 @@ jobs: print(*(str(path) for path in sorted(docs_dir.rglob("*.md")) if not path.is_symlink()), sep="\n") PY ) - sudo --preserve-env=PATH,ABXPKG_LIB_DIR,ARCHIVEBOX_DOCS_URL_ONE,ARCHIVEBOX_DOCS_URL_TWO,ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT,ARCHIVEBOX_DOCS_STATIC_PORT,ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD "$uv_bin" run --project "$GITHUB_WORKSPACE" --no-sync pytest -vv --tb=long "$GITHUB_WORKSPACE/README.md" "$GITHUB_WORKSPACE/AGENTS.md" "$GITHUB_WORKSPACE/skills" "${docs_paths[@]}" --docs-environment=${{ matrix.environment }} + "$SUDO_BINARY" --preserve-env=PATH,ABXPKG_LIB_DIR,ARCHIVEBOX_PROJECT_DIR,ARCHIVEBOX_DOCS_URL_ONE,ARCHIVEBOX_DOCS_URL_TWO,ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT,ARCHIVEBOX_DOCS_STATIC_PORT,ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD "$uv_bin" run --project "$GITHUB_WORKSPACE" --no-sync pytest -vv --tb=long "$GITHUB_WORKSPACE/README.md" "$GITHUB_WORKSPACE/AGENTS.md" "$GITHUB_WORKSPACE/skills" "${docs_paths[@]}" --docs-environment=${{ matrix.environment }} - name: Stop local documentation site if: always() run: kill "$ARCHIVEBOX_DOCS_SERVER_PID" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c8d94af9..f8244738 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -28,6 +28,9 @@ jobs: version: "0.11.3" enable-cache: false + - name: Verify checked dependency lock + run: uv lock --check + - name: Clone abxpkg run: bash .github/scripts/clone_abx_repo.sh abxpkg diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 06e81610..0f9547c4 100755 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -42,11 +42,14 @@ jobs: import hashlib from pathlib import Path - artifacts = sorted(Path("dist").iterdir()) + root = Path("dist") + wheels = list(root.glob("archivebox-*.whl")) + sdists = list(root.glob("archivebox-*.tar.gz")) + if len(wheels) != 1 or len(sdists) != 1: + raise SystemExit("Expected exactly one ArchiveBox wheel and one sdist") + artifacts = [root / "COMMIT_SHA", wheels[0], sdists[0]] checksum_lines = [] for artifact in artifacts: - if artifact.name == "SHA256SUMS": - continue digest = hashlib.sha256(artifact.read_bytes()).hexdigest() line = f"{digest} {artifact.name}" checksum_lines.append(line) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc68bd74..adb05d7d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ permissions: packages: write concurrency: - group: release-dev + group: release-${{ github.repository }}-${{ github.event.workflow_run.head_branch }} cancel-in-progress: false env: diff --git a/.github/workflows/test-parallel.yml b/.github/workflows/test-parallel.yml index 40eb9d98..8ad7c5e0 100644 --- a/.github/workflows/test-parallel.yml +++ b/.github/workflows/test-parallel.yml @@ -325,7 +325,7 @@ jobs: mapfile -t test_paths < <("$JQ_BINARY" -r '.[]' <<<"$TEST_PATHS_JSON") [[ "${#test_paths[@]}" -gt 0 ]] mkdir -p tests/out - uv run --no-sync --no-sources pytest -vs "${test_paths[@]}" --basetemp=tests/out + uv run --no-sync --no-sources pytest -vs "${test_paths[@]}" --basetemp="$GITHUB_WORKSPACE/tests/out" plugin-tests: name: ${{ matrix.plugin.name }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 862cf28c..e0fcb26c 100755 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -209,7 +209,7 @@ jobs: if: matrix.os_name == 'macOS' || matrix.python == '3.14' run: | mkdir -p tests/out - uv run --no-sync --no-sources pytest -q archivebox/tests --basetemp="tests/out/${{ matrix.os_name }}-python-${{ matrix.python }}" + uv run --no-sync --no-sources pytest -q archivebox/tests --basetemp="$GITHUB_WORKSPACE/tests/out/${{ matrix.os_name }}-python-${{ matrix.python }}" docker_tests: runs-on: ubuntu-24.04 diff --git a/Dockerfile b/Dockerfile index f43062dd..429130d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,10 @@ # --build-context abx-plugins=../abx-plugins \ # -t archivebox/abx-dl:dev # docker buildx build . -f Dockerfile \ -# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.11.263 \ +# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.11.268 \ # -t archivebox:multistage -ARG ABX_DL_IMAGE=archivebox/abx-dl:1.11.263 +ARG ABX_DL_IMAGE=archivebox/abx-dl:1.11.268 FROM archivebox/sonic:1.4.9 AS sonic FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base @@ -126,8 +126,11 @@ text = text.replace( # PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a # Docker-only dependency view from the version JSON so the published package # metadata stays normal while image builds remain resumable after a release. -for package in ("abxpkg", "abx-plugins", "abx-dl"): - match = re.search(rf'"{re.escape(package)}>=(?P[^"]+)"', text) +for package in ("abxbus", "abxpkg", "abx-plugins", "abx-dl"): + match = re.search( + rf'"{re.escape(package)}(?P==|>=)(?P[^"]+)"', + text, + ) if not match: continue version = match.group("version") @@ -135,7 +138,7 @@ for package in ("abxpkg", "abx-plugins", "abx-dl"): data = json.load(response) wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl")) text = re.sub( - rf'"{re.escape(package)}>=[^"]+"', + rf'"{re.escape(package)}(?:==|>=)[^"]+"', f'"{package} @ {wheel_url}"', text, count=1, @@ -152,10 +155,19 @@ PY --no-install-project \ --no-install-workspace \ --no-sources -(find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true) +ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$ABXPKG_LIB_DIR" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null +"$ABXPKG_LIB_DIR/env/bin/find" /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries +while IFS= read -r -d '' native_library; do + magic='' + if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then + "$ABXPKG_LIB_DIR/env/bin/strip" --strip-unneeded "$native_library" || exit $? + fi +done < /tmp/archivebox-native-libraries +rm -f /tmp/archivebox-native-libraries rm -f /venv/bin/uv /venv/bin/uvx -apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev -apt-get autoremove -y +abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev +abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get autoremove -y +"$ABXPKG_LIB_DIR/env/bin/find" "$ABXPKG_LIB_DIR/env/bin" -maxdepth 1 -type l -name strip -delete rm -rf /var/lib/apt/lists/* EOF diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py index ce40ce7d..8dec58f9 100644 --- a/archivebox/cli/archivebox_run.py +++ b/archivebox/cli/archivebox_run.py @@ -294,17 +294,11 @@ def run_runner( crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first() now = timezone.now() - # Only re-lease when the row is unscheduled (retry_at IS NULL) or its - # existing lease has already expired. A future retry_at means another - # worker is already scheduled — don't clobber. - if crawl is not None and (crawl.retry_at is None or crawl.retry_at <= now): - # extra_filter pins the read-time retry_at so a concurrent worker - # that grabbed the lease between our SELECT and UPDATE wins. - crawl.safe_update( - {"retry_at": now}, - refresh=False, - extra_filter={"retry_at": crawl.retry_at}, - ) + # Winning the single-runner gate terminates and waits for every older + # runner. Requeue the explicitly requested crawl so an abandoned + # future ownership lease cannot hide it from the unified scheduler. + if crawl is not None: + crawl.update_and_requeue(retry_at=now, refresh=False) # Only a foreground `archivebox add` gets the interactive "abort current # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned # orchestrators should shut down immediately and cleanly on the first signal. diff --git a/archivebox/machine/detect.py b/archivebox/machine/detect.py index d73cf9a6..212c1372 100644 --- a/archivebox/machine/detect.py +++ b/archivebox/machine/detect.py @@ -20,9 +20,10 @@ DATA_DIR = Path(os.getcwd()).resolve() def _run_abxpkg_host_binary(name: str, *args: str) -> subprocess.CompletedProcess[str]: from abxpkg import EnvProvider - from archivebox.config.constants import CONSTANTS + from archivebox.config.common import get_config - provider = EnvProvider(install_root=CONSTANTS.DEFAULT_ABXPKG_LIB_DIR / "env", PATH=os.environ["PATH"]) + env_root = get_config(include_machine=False).ABXPKG_LIB_DIR / "env" + provider = EnvProvider(install_root=env_root, PATH=os.environ["PATH"]) if name == "system_profiler": provider = provider.get_provider_with_overrides( overrides={name: {"version": platform.mac_ver()[0] or "0.0.0"}}, @@ -31,8 +32,9 @@ def _run_abxpkg_host_binary(name: str, *args: str) -> subprocess.CompletedProces if loaded is None or loaded.loaded_abspath is None: raise RuntimeError(f"abxpkg could not resolve {name}") projection = Path(loaded.loaded_abspath) - if not projection.is_symlink() or not os.access(projection, os.X_OK): - raise RuntimeError(f"abxpkg did not project {name} into {projection}") + expected_projection = env_root / "bin" / name + if projection != expected_projection or not projection.is_symlink() or not os.access(projection, os.X_OK): + raise RuntimeError(f"abxpkg did not project {name} into {expected_projection}") return subprocess.run([str(projection), *args], capture_output=True, text=True, check=True) diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 02729bae..b7e2d05b 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -21,6 +21,7 @@ from rich.console import Console from rich.text import Text from abxpkg.binary_service import BinaryCacheService, BinaryRequestEvent, BinaryService +from abx_dl.config import GlobalConfig, RuntimeConfig from abx_dl.events import ( CrawlAbortEvent, CrawlCleanupEvent, @@ -1177,6 +1178,10 @@ class CrawlRunner: snapshot=abx_snapshot, output_dir=output_dir, plugins=plugins, + config=RuntimeConfig( + user=GlobalConfig(**{**config, "ABX_RUNTIME": "archivebox"}), + derived=derived_config, + ), snapshot_phase_timeout=snapshot_phase_timeout, snapshot_cleanup_enabled=True, snapshot_cleanup_phase_timeout=snapshot_phase_timeout, diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py index 4d885078..17e8bad2 100644 --- a/archivebox/tests/conftest.py +++ b/archivebox/tests/conftest.py @@ -911,10 +911,17 @@ def wait_for_worker_pid_from_log(log_path: Path, worker_name: str, *, timeout: f def pgrep_data_dir(data_dir: Path) -> list[str]: lines: list[str] = [] seen_pids: set[int] = set() - for process in psutil.process_iter(["pid", "ppid", "cmdline"]): + resolved_data_dir = data_dir.resolve() + for process in psutil.process_iter(["pid", "ppid", "cmdline", "cwd"]): try: + if process.pid == os.getpid(): + continue command = " ".join(process.info.get("cmdline") or []) - if str(data_dir) not in command: + process_cwd = process.info.get("cwd") + cwd_matches = bool(process_cwd and Path(process_cwd).resolve() == resolved_data_dir) + env_data_dir = None if cwd_matches else process.environ().get("DATA_DIR") + env_matches = bool(env_data_dir and Path(env_data_dir).resolve() == resolved_data_dir) + if str(data_dir) not in command and not cwd_matches and not env_matches: continue pid = int(process.info["pid"]) seen_pids.add(pid) diff --git a/archivebox/tests/test_archive_result_service.py b/archivebox/tests/test_archive_result_service.py index 8ded5435..50618c58 100644 --- a/archivebox/tests/test_archive_result_service.py +++ b/archivebox/tests/test_archive_result_service.py @@ -512,7 +512,15 @@ def test_process_started_hydrates_binary_and_iface_from_existing_binary_records( install=False, ) mercury_path = Path(mercury_env["MERCURY_BINARY"]) - assert Path(binary.abspath).resolve() == mercury_path.resolve() + provider_path = Path(binary.abspath) + assert binary.binprovider == "pnpm" + assert provider_path == lib_dir / "pnpm" / "packages" / "mercury" / "node_modules" / ".bin" / "postlight-parser" + assert provider_path.is_file() + assert os.access(provider_path, os.X_OK) + assert mercury_path == lib_dir / "env" / "bin" / "postlight-parser" + assert mercury_path.is_symlink() + assert mercury_path.resolve().is_file() + assert os.access(mercury_path, os.X_OK) hook_path = Path(str(files("abx_plugins.plugins.mercury").joinpath("on_Snapshot__57_mercury.py"))) output_dir = tmp_path / "mercury" @@ -534,7 +542,6 @@ def test_process_started_hydrates_binary_and_iface_from_existing_binary_records( env={ **mercury_env, "ABXPKG_LIB_DIR": str(lib_dir), - "MERCURY_BINARY": binary.abspath, "SNAP_DIR": str(tmp_path), }, timeout=60, diff --git a/archivebox/tests/test_auth_ldap.py b/archivebox/tests/test_auth_ldap.py index dc143e6d..25d3eb79 100644 --- a/archivebox/tests/test_auth_ldap.py +++ b/archivebox/tests/test_auth_ldap.py @@ -97,17 +97,38 @@ class TestLDAPIntegration: ldap_backends = [b for b in settings.AUTHENTICATION_BACKENDS if "ldap" in b.lower()] assert len(ldap_backends) == 0, "LDAP backend should not be present when LDAP_ENABLED=False" - def test_django_settings_with_ldap_library_check(self): - """Test that Django settings check for LDAP libraries when enabled.""" - ldap_available = find_spec("django_auth_ldap") is not None and find_spec("ldap") is not None + def test_django_settings_with_ldap_enabled(self, initialized_archive): + """Test the real enabled LDAP settings path with installed libraries.""" + assert find_spec("django_auth_ldap") is not None + assert find_spec("ldap") is not None - # If LDAP libraries are not available, settings should handle gracefully - if not ldap_available: - # Settings should have loaded without LDAP backend - from django.conf import settings + result = run_archivebox_cmd( + [ + "manage", + "shell", + "-c", + ( + "from django.conf import settings; " + "print('LDAP_BACKENDS=' + ','.join(settings.AUTHENTICATION_BACKENDS)); " + "print('LDAP_SERVER_URI=' + settings.AUTH_LDAP_SERVER_URI)" + ), + ], + cwd=initialized_archive, + timeout=45, + env={ + "LDAP_ENABLED": "True", + "LDAP_SERVER_URI": "ldap://ldap-test.localhost:389", + "LDAP_BIND_DN": "cn=admin,dc=example,dc=com", + "LDAP_BIND_PASSWORD": "password", + "LDAP_USER_BASE": "ou=users,dc=example,dc=com", + }, + default_cli_env=True, + disable_extractors=True, + ) - ldap_backends = [b for b in settings.AUTHENTICATION_BACKENDS if "ldap" in b.lower()] - assert len(ldap_backends) == 0, "LDAP backend should not be present when libraries unavailable" + assert result.returncode == 0, result.stderr or result.stdout + assert "LDAP_BACKENDS=archivebox.ldap.auth.ArchiveBoxLDAPBackend," in result.stdout + assert "LDAP_SERVER_URI=ldap://ldap-test.localhost:389" in result.stdout class TestLDAPAuthBackend: diff --git a/archivebox/tests/test_cli_oneshot.py b/archivebox/tests/test_cli_oneshot.py index 84c1376f..a69144ee 100644 --- a/archivebox/tests/test_cli_oneshot.py +++ b/archivebox/tests/test_cli_oneshot.py @@ -1,19 +1,23 @@ +import tempfile from importlib.metadata import version +from pathlib import Path from archivebox.tests.conftest import run_archivebox_cmd -def test_oneshot_runs_abx_dl_through_abxpkg_env_projection(tmp_path): - lib_dir = tmp_path / "lib" +def test_oneshot_runs_abx_dl_through_abxpkg_env_projection(): + with tempfile.TemporaryDirectory(prefix="archivebox-oneshot-") as tmp_dir: + work_dir = Path(tmp_dir) + lib_dir = work_dir / "lib" - result = run_archivebox_cmd( - ["oneshot", "--version"], - cwd=tmp_path, - env={"ABXPKG_LIB_DIR": str(lib_dir)}, - ) + result = run_archivebox_cmd( + ["oneshot", "--version"], + cwd=work_dir, + env={"ABXPKG_LIB_DIR": str(lib_dir)}, + ) - abx_dl_projection = lib_dir / "env" / "bin" / "abx-dl" - assert result.returncode == 0, result.stderr - assert version("abx-dl") in result.stdout - assert abx_dl_projection.is_symlink() - assert abx_dl_projection.resolve().is_file() + abx_dl_projection = lib_dir / "env" / "bin" / "abx-dl" + assert result.returncode == 0, result.stderr + assert version("abx-dl") in result.stdout + assert abx_dl_projection.is_symlink() + assert abx_dl_projection.resolve().is_file() diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index 08258d6b..9534f15a 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -526,17 +526,31 @@ class TestRunEmpty: class TestRunDaemonMode: @pytest.mark.parametrize("stdin_kind", ["malformed", "valid-snapshot"]) - def test_run_daemon_ignores_piped_stdin_and_starts_real_runner(self, initialized_archive, db, stdin_kind): + def test_run_daemon_ignores_piped_stdin_and_starts_real_runner( + self, + initialized_archive, + tmp_path_factory, + db, + stdin_kind, + ): from archivebox.machine.models import Process from archivebox.core.models import Snapshot from archivebox.tests.test_orm_helpers import use_archivebox_db snapshot_url = None if stdin_kind == "valid-snapshot": + piped_source_archive = tmp_path_factory.mktemp("daemon-piped-source") + init_result = run_archivebox_cmd( + ["init", "--quick"], + cwd=piped_source_archive, + default_cli_env=True, + disable_extractors=True, + ) + assert init_result.returncode == 0, init_result.stderr snapshot_url = create_test_url() snapshot_result = run_archivebox_cmd( ["snapshot", "create", snapshot_url], - cwd=initialized_archive, + cwd=piped_source_archive, default_cli_env=True, disable_extractors=True, ) diff --git a/archivebox/tests/test_opencode_agent.py b/archivebox/tests/test_opencode_agent.py index 2876841f..667bd0ab 100644 --- a/archivebox/tests/test_opencode_agent.py +++ b/archivebox/tests/test_opencode_agent.py @@ -1,6 +1,5 @@ import os import socket -import subprocess from pathlib import Path from types import SimpleNamespace from urllib.parse import quote @@ -92,12 +91,10 @@ def live_opencode(opencode_archive_config): settings["archivebox_base_url"] = "http://admin.archivebox.localhost:8000" settings["archivebox_admin_url"] = "http://admin.archivebox.localhost:8000/admin" settings["archivebox_api_url"] = "http://admin.archivebox.localhost:8000/api/" - binary, binary_env = views._resolve_binary(settings["binary"], settings["config"]) - version = subprocess.run( - [binary, "--version"], + binary, _, binary_env = views._resolve_binary(settings["binary"], settings["config"]) + version = binary.exec( + cmd=("--version",), env={**os.environ, **binary_env}, - text=True, - capture_output=True, timeout=120, ) assert version.returncode == 0, version.stderr or version.stdout @@ -234,10 +231,10 @@ def test_opencode_starts_with_isolated_state(live_opencode): assert Path(live_opencode.settings["workdir"]).resolve() == Path(workdir) assert Path(str(project.json()["worktree"])).resolve() == Path(workdir) assert live_opencode.process.poll() is None + assert (live_opencode.config.data_dir / ".git").is_dir() assert (state_dir / "data" / "opencode" / "opencode.db").is_file() assert (state_dir / "SKILL.md").is_file() assert (state_dir / "config" / "opencode" / "skills" / "archivebox" / "SKILL.md").resolve() == state_dir / "SKILL.md" - assert not (live_opencode.config.data_dir / ".git" / "not-a-git").exists() def test_opencode_state_dir_is_separate_from_workdir(tmp_path): diff --git a/archivebox/tests/test_server_security_browser.py b/archivebox/tests/test_server_security_browser.py index 3b907fb8..7bb1e042 100644 --- a/archivebox/tests/test_server_security_browser.py +++ b/archivebox/tests/test_server_security_browser.py @@ -199,6 +199,17 @@ async function readPreviewText(page, expectedText) { return {matched: false, frames: frameState}; } +async function waitForPreviewReady(frame) { + return await frame.evaluate(() => { + if (window.archiveboxReplayError) throw new Error(window.archiveboxReplayError); + if (window.archiveboxReplayReady) return window.archiveboxReplayReady; + return new Promise((resolve, reject) => { + window.addEventListener("archivebox-replay-ready", (event) => resolve(event.detail), {once: true}); + window.addEventListener("archivebox-replay-error", (event) => reject(new Error(event.detail)), {once: true}); + }); + }); +} + async function main() { const config = JSON.parse(fs.readFileSync(0, "utf8")); const browser = await puppeteer.launch({ @@ -231,7 +242,11 @@ async function main() { waitUntil: "domcontentloaded", timeout: 30000, }); - await page.waitForNetworkIdle({idleTime: 500, timeout: 60000}); + const previewFrame = await page.waitForFrame((frame) => { + const url = frame.url(); + return url.includes("/archivewebpage/archivewebpage.wacz") && url.includes("preview=1"); + }); + await waitForPreviewReady(previewFrame); const previewResult = await readPreviewText(page, config.expectedText); console.log(JSON.stringify({ diff --git a/archivebox/tests/test_version_metadata.py b/archivebox/tests/test_version_metadata.py index cf8f22cd..ffeb8a99 100644 --- a/archivebox/tests/test_version_metadata.py +++ b/archivebox/tests/test_version_metadata.py @@ -41,6 +41,7 @@ def _create_git_repo(tmp_path: Path) -> tuple[Path, Path, str]: _git(git, repo, "init", "--initial-branch=dev") _git(git, repo, "config", "user.name", "ArchiveBox Tests") _git(git, repo, "config", "user.email", "tests@archivebox.io") + _git(git, repo, "config", "commit.gpgsign", "false") copied_version = repo / "archivebox" / "config" / "version.py" copied_version.parent.mkdir(parents=True) shutil.copyfile(version.__file__, copied_version) diff --git a/bin/bumpver_update_lock.sh b/bin/bumpver_update_lock.sh new file mode 100755 index 00000000..d3516d1e --- /dev/null +++ b/bin/bumpver_update_lock.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +[[ -z "$(git status --short -- uv.lock)" ]] +uv lock +if git diff --quiet -- uv.lock; then + echo "uv lock did not record the bumped ArchiveBox version" >&2 + exit 1 +fi +git add uv.lock diff --git a/bin/release.sh b/bin/release.sh index 538b6532..6216e756 100755 --- a/bin/release.sh +++ b/bin/release.sh @@ -82,16 +82,40 @@ $GIT_BINARY fetch --quiet --no-tags origin "+refs/heads/${RELEASE_BRANCH:-dev}:r $GIT_BINARY merge-base --is-ancestor "$RELEASE_SHA" "refs/remotes/origin/${RELEASE_BRANCH:-dev}" [[ "$(<"$RELEASE_DISTRIBUTIONS_DIR/COMMIT_SHA")" == "$RELEASE_SHA" ]] -$UV_BINARY run --no-project python - "$RELEASE_DISTRIBUTIONS_DIR" <<'PY' +$UV_BINARY run --no-project python - "$RELEASE_DISTRIBUTIONS_DIR" "$VERSION" <<'PY' from hashlib import sha256 from pathlib import Path +import re import sys root = Path(sys.argv[1]) +version = sys.argv[2] +expected_names = { + "COMMIT_SHA", + f"archivebox-{version}-py3-none-any.whl", + f"archivebox-{version}.tar.gz", +} +files = {path.name for path in root.iterdir() if path.is_file()} +if files != expected_names | {"SHA256SUMS"}: + raise SystemExit(f"Unexpected tested distribution set: {sorted(files)}") + +manifest = {} for line in (root / 'SHA256SUMS').read_text().splitlines(): expected, separator, filename = line.partition(' ') - if not separator or len(expected) != 64: + if ( + not separator + or not re.fullmatch(r'[0-9a-f]{64}', expected) + or Path(filename).name != filename + ): raise SystemExit(f'Invalid SHA256SUMS line: {line!r}') + if filename in manifest: + raise SystemExit(f'Duplicate SHA256SUMS entry: {filename}') + manifest[filename] = expected + +if set(manifest) != expected_names: + raise SystemExit(f'SHA256SUMS names the wrong files: {sorted(manifest)}') + +for filename, expected in manifest.items(): artifact = root / filename actual = sha256(artifact.read_bytes()).hexdigest() if actual != expected: @@ -102,7 +126,7 @@ WHEELS=("$RELEASE_DISTRIBUTIONS_DIR"/archivebox-*.whl) SDISTS=("$RELEASE_DISTRIBUTIONS_DIR"/archivebox-*.tar.gz) [[ "${#WHEELS[@]}" -eq 1 ]] [[ "${#SDISTS[@]}" -eq 1 ]] -[[ "${WHEELS[0]##*/}" == archivebox-${VERSION}-*.whl ]] +[[ "${WHEELS[0]##*/}" == "archivebox-${VERSION}-py3-none-any.whl" ]] [[ "${SDISTS[0]##*/}" == archivebox-${VERSION}.tar.gz ]] TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/${TAG}^{}")" @@ -116,11 +140,55 @@ fi exit 1 } -PYPI_EXISTS=false +PYPI_URLS="$($CURL_BINARY -fsSL "https://pypi.org/pypi/${PYPI_PACKAGE}/json" | $JQ_BINARY -c --arg version "$VERSION" ".releases[\$version] // []")" +PYPI_STATUS_OUTPUT="$(PYPI_URLS="$PYPI_URLS" RELEASE_DISTRIBUTIONS_DIR="$RELEASE_DISTRIBUTIONS_DIR" VERSION="$VERSION" $UV_BINARY run --no-project python - <<'PY' +import json +import os +from pathlib import Path + +root = Path(os.environ["RELEASE_DISTRIBUTIONS_DIR"]) +version = os.environ["VERSION"] +expected_names = { + f"archivebox-{version}-py3-none-any.whl", + f"archivebox-{version}.tar.gz", +} +manifest = {} +for line in (root / "SHA256SUMS").read_text().splitlines(): + digest, filename = line.split(maxsplit=1) + if filename in expected_names: + manifest[filename] = digest +if set(manifest) != expected_names: + raise SystemExit("Tested manifest does not contain the exact PyPI artifacts") + +published_files = json.loads(os.environ["PYPI_URLS"]) +if not published_files: + print("absent") + for filename in sorted(expected_names): + print(filename) + raise SystemExit(0) +published = {item["filename"]: item["digests"]["sha256"] for item in published_files} +if len(published) != len(published_files) or not set(published).issubset(expected_names): + raise SystemExit("PyPI release contains duplicate or unexpected distributions") +for filename, digest in published.items(): + if manifest[filename] != digest: + raise SystemExit(f"PyPI digest mismatch for {filename}") + +missing = sorted(expected_names - set(published)) +print("partial" if missing else "complete") +for filename in missing: + print(filename) +PY +)" +mapfile -t PYPI_STATUS_LINES <<< "$PYPI_STATUS_OUTPUT" +PYPI_STATE="${PYPI_STATUS_LINES[0]}" +PYPI_MISSING=("${PYPI_STATUS_LINES[@]:1}") +[[ "$PYPI_STATE" == absent || "$PYPI_STATE" == partial || "$PYPI_STATE" == complete ]] + GITHUB_EXISTS=false -$CURL_BINARY -fsSL "https://pypi.org/pypi/${PYPI_PACKAGE}/${VERSION}/json" >/dev/null 2>&1 && PYPI_EXISTS=true -$GH_BINARY release view "$TAG" --repo "$SLUG" >/dev/null 2>&1 && GITHUB_EXISTS=true -if [[ ( "$PYPI_EXISTS" == true || "$GITHUB_EXISTS" == true ) && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then +if $GH_BINARY release view "$TAG" --repo "$SLUG" >/dev/null 2>&1; then + GITHUB_EXISTS=true +fi +if [[ ( "$PYPI_STATE" != absent || "$GITHUB_EXISTS" == true ) && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then echo "Cannot recover partial release $VERSION without an exact-SHA tag" >&2 exit 1 fi @@ -130,8 +198,14 @@ if [[ -z "$TAG_TARGET" ]]; then $GIT_BINARY push origin "refs/tags/${TAG}" fi -if [[ "$PYPI_EXISTS" == false ]]; then - $UV_BINARY publish --trusted-publishing always "${WHEELS[@]}" "${SDISTS[@]}" +if [[ "$PYPI_STATE" != complete ]]; then + PYPI_ARTIFACTS=() + for filename in "${PYPI_MISSING[@]}"; do + [[ "$filename" == "${filename##*/}" && -f "$RELEASE_DISTRIBUTIONS_DIR/$filename" ]] + PYPI_ARTIFACTS+=("$RELEASE_DISTRIBUTIONS_DIR/$filename") + done + [[ "${#PYPI_ARTIFACTS[@]}" -gt 0 ]] + $UV_BINARY publish --trusted-publishing always "${PYPI_ARTIFACTS[@]}" fi if [[ "$GITHUB_EXISTS" == false ]]; then @@ -141,4 +215,47 @@ if [[ "$GITHUB_EXISTS" == false ]]; then --title "$TAG" --generate-notes "${RELEASE_ARGS[@]}" fi +$GH_BINARY release upload "$TAG" --repo "$SLUG" \ + "$RELEASE_DISTRIBUTIONS_DIR/COMMIT_SHA" \ + "${WHEELS[0]}" \ + "${SDISTS[0]}" \ + "$RELEASE_DISTRIBUTIONS_DIR/SHA256SUMS" \ + --clobber + +RELEASE_JSON="$($GH_BINARY release view "$TAG" --repo "$SLUG" --json assets,tagName)" +RELEASE_JSON="$RELEASE_JSON" RELEASE_DISTRIBUTIONS_DIR="$RELEASE_DISTRIBUTIONS_DIR" VERSION="$VERSION" TAG="$TAG" $UV_BINARY run --no-project python - <<'PY' +from hashlib import sha256 +from pathlib import Path +import json +import os + +root = Path(os.environ["RELEASE_DISTRIBUTIONS_DIR"]) +version = os.environ["VERSION"] +release = json.loads(os.environ["RELEASE_JSON"]) +expected_names = { + "COMMIT_SHA", + "SHA256SUMS", + f"archivebox-{version}-py3-none-any.whl", + f"archivebox-{version}.tar.gz", +} +if release["tagName"] != os.environ["TAG"]: + raise SystemExit("GitHub release tag does not match the source version") + +assets = release["assets"] +published = {asset["name"]: asset.get("digest", "") for asset in assets} +if len(published) != len(assets) or set(published) != expected_names: + raise SystemExit("GitHub release asset set is incomplete or contains extras") + +manifest = {} +for line in (root / "SHA256SUMS").read_text().splitlines(): + digest, filename = line.split(maxsplit=1) + manifest[filename] = digest +for filename in expected_names: + local_digest = sha256((root / filename).read_bytes()).hexdigest() + if filename != "SHA256SUMS" and manifest.get(filename) != local_digest: + raise SystemExit(f"Local manifest mismatch for {filename}") + if published[filename] != f"sha256:{local_digest}": + raise SystemExit(f"GitHub release digest mismatch for {filename}") +PY + echo "Released ${PYPI_PACKAGE} ${VERSION} from ${RELEASE_SHA} using CI run ${CI_RUN_ID:-unknown}" diff --git a/bin/setup.sh b/bin/setup.sh index 34fa2fcd..4689e6da 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -25,7 +25,7 @@ ARCHIVEBOX_PYTHON="${ARCHIVEBOX_PYTHON:-3.13}" ARCHIVEBOX_PACKAGE="${ARCHIVEBOX_PACKAGE:-git+https://github.com/ArchiveBox/ArchiveBox.git@${ARCHIVEBOX_BRANCH}}" ARCHIVEBOX_PLATFORM="${ARCHIVEBOX_PLATFORM:-}" ARCHIVEBOX_COMPOSE_URL="${ARCHIVEBOX_COMPOSE_URL:-https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/${ARCHIVEBOX_BRANCH}/docker-compose.yml}" -ABXPKG_PACKAGE="${ABXPKG_PACKAGE:-abxpkg==1.11.280}" +ABXPKG_PACKAGE="${ABXPKG_PACKAGE:-abxpkg==1.11.288}" ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-$HOME/.cache/archivebox/setup-abxpkg}" BOOTSTRAP_UV_BINARY="" UV_BINARY="" diff --git a/etc/package.json b/etc/package.json index 26bb7543..8bd8e0f1 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.35rc137", + "version": "0.9.35rc138", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index 99a77a54..bcca18a7 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc137" +version = "0.9.35rc138" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -78,10 +78,10 @@ dependencies = [ "w3lib>=2.2.1", # used for parsing content-type encoding from http response headers & html tags ### Extractor dependencies (runtime binaries resolved through abxpkg) ### Binary/Package Management - "abxbus==2.5.39", # EventBus API - "abxpkg==1.11.284", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins==1.11.296", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl==1.11.263", # shared ArchiveBox downloader package with blocking install preflight + "abxbus==2.5.40", # EventBus API + "abxpkg==1.11.288", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins==1.11.311", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl==1.11.268", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ] @@ -351,12 +351,12 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations" [tool.bumpver] -current_version = "v0.8.5rc53" +current_version = "v0.9.35rc138" version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" tag_message = "{new_version}" tag_scope = "default" -pre_commit_hook = "" +pre_commit_hook = "bin/bumpver_update_lock.sh" post_commit_hook = "" commit = true tag = true @@ -367,3 +367,6 @@ push = true 'current_version = "{version}"', 'version = "{pep440_version}"', ] +"etc/package.json" = [ + '"version": "{pep440_version}"', +] diff --git a/uv.lock b/uv.lock index fac66594..59a779b9 100644 --- a/uv.lock +++ b/uv.lock @@ -13,18 +13,18 @@ supported-markers = [ ] [options] -exclude-newer = "2026-07-15T23:42:23.736593744Z" +exclude-newer = "2026-07-18T13:51:28.491653Z" exclude-newer-span = "P5D" [options.exclude-newer-package] -abxbus = { timestamp = "2026-07-20T23:42:22.736604335Z", span = "PT1S" } +abxbus = { timestamp = "2026-07-23T13:51:27.491669Z", span = "PT1S" } abx-plugins = "2100-01-01T00:00:00Z" abx-dl = "2100-01-01T00:00:00Z" abxpkg = "2100-01-01T00:00:00Z" [[package]] name = "abx-dl" -version = "1.11.262" +version = "1.11.268" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -39,14 +39,14 @@ dependencies = [ { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a0/70/82a0e059b3288860314ce79e2517150b9741100eb053df95c7779719138a/abx_dl-1.11.262.tar.gz", hash = "sha256:f813c17743f827efed9bf2969884808c913be2069e91deaa5ea5ccb4dbdea800", size = 86982, upload-time = "2026-07-20T22:25:00.279Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/b3/ba83c367c76c1fa82e8e94f468f093d092f52f13a31c468a56436f388442/abx_dl-1.11.268.tar.gz", hash = "sha256:e8cb9ce008f2d0dfdc36adbd54c692caf505b78f5ab8510fe77eda7c4a44699b", size = 86729, upload-time = "2026-07-23T12:12:52.806Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/0f/239548a876f7dad576777a358e4e4f5a23cb98cc126d42ae5d1ccb5e1409/abx_dl-1.11.262-py3-none-any.whl", hash = "sha256:eb621cf0d93fd521db13f7983cef12d20302505c31fa01d182791e9dbf0894aa", size = 90765, upload-time = "2026-07-20T22:24:58.91Z" }, + { url = "https://files.pythonhosted.org/packages/51/e3/fff903a6e9e3515bee7d8f06cc9d561e8f95881ab3bbdad7033b36008c22/abx_dl-1.11.268-py3-none-any.whl", hash = "sha256:429e9afa258dd714b3dc1e3b1cc0ada4f25b395d19db194752a2bf1e1bdc54c6", size = 90472, upload-time = "2026-07-23T12:12:53.915Z" }, ] [[package]] name = "abx-plugins" -version = "1.11.290" +version = "1.11.311" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -57,28 +57,28 @@ dependencies = [ { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4e/67/53a2c443194da95669ec6967907f342f202a06d2ebf7a0884e1d1832ff13/abx_plugins-1.11.290.tar.gz", hash = "sha256:893038dfe1a3685c2697f3890a0e0f97bd5389f10c9ddcd38b4032d7547f3bdc", size = 257626, upload-time = "2026-07-20T22:24:27.917Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/f8/ad69bed9751ccbdc892ac3fd6a63b9eace0bd7b0bbb254b855fd8351bc58/abx_plugins-1.11.311.tar.gz", hash = "sha256:acf3ad909f63f610b8981cf9796ff758d713585a380dd9404f42c8419f1ef020", size = 266481, upload-time = "2026-07-23T11:32:48.583Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/84/049fdc270f5de46f1845be27859124e452794160da8868ee6e532738fd99/abx_plugins-1.11.290-py3-none-any.whl", hash = "sha256:e645491d4be069c9343adecdc5431ecebcc2b8e16c61232e8006a4f266bb8f9a", size = 406639, upload-time = "2026-07-20T22:24:26.435Z" }, + { url = "https://files.pythonhosted.org/packages/31/43/27f1582e2d6800cec0dd9f98957d5a02088072b0e48932c3405277c91f29/abx_plugins-1.11.311-py3-none-any.whl", hash = "sha256:67b289c4f2ed4eced783c5a86cf8065f517cbfe42202f54554f33b24411e53c9", size = 415643, upload-time = "2026-07-23T11:32:49.84Z" }, ] [[package]] name = "abxbus" -version = "2.5.28" +version = "2.5.40" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "uuid7", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4a/96/b59c6d0f0e17af0f89cb1c2c6c183095411e4a3fcceb783deeb5cc33031e/abxbus-2.5.28.tar.gz", hash = "sha256:bf0c8bb07596339d9f3a86c01459b02b7575c51b34ee5610312709914cc3640d", size = 133433, upload-time = "2026-07-20T20:58:13.479Z" } +sdist = { url = "https://files.pythonhosted.org/packages/32/c4/bf7415d0e158cc76860264b3e66a409872541544059f0833e057ae4dcfaf/abxbus-2.5.40.tar.gz", hash = "sha256:f9660128149f9079d18cba1ddbd0daa1941246ac0d19bbfff05491ba417e6447", size = 133584, upload-time = "2026-07-22T03:15:20.732Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/0a/d289b192f3d16debf4037aa6dd3a8d2a07b32e11a7d79920df9f259e52c0/abxbus-2.5.28-py3-none-any.whl", hash = "sha256:fc57de323ee931278c89ad835294b9cdd0b842b586dca5f2591949abdf1b70f7", size = 127219, upload-time = "2026-07-20T20:58:12.262Z" }, + { url = "https://files.pythonhosted.org/packages/57/c6/75ced4b6c24c121f3e364656f18c9d3533af416b4c9ac3cb9281dfdcd1f7/abxbus-2.5.40-py3-none-any.whl", hash = "sha256:43787d333df878847939c3e7d02b8ec7584b18aaf5dbaf9ffd203336bbda7d4a", size = 127347, upload-time = "2026-07-22T03:15:22.086Z" }, ] [[package]] name = "abxpkg" -version = "1.11.274" +version = "1.11.288" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pip", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -87,9 +87,9 @@ dependencies = [ { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b9/a3/6e9b847d3c1f75624691942c88db32d156beb183725e78daa9a45a73d2c2/abxpkg-1.11.274.tar.gz", hash = "sha256:4a7b38bdfffee199b4d2239ac494d54f6e58f7ecfc4aae42fe28f2a1b1222230", size = 209831, upload-time = "2026-07-20T23:26:10.776Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/e0/f98aab6246c6d328ee289ced80e95befd46d8046abbbfc317ccf696582b1/abxpkg-1.11.288.tar.gz", hash = "sha256:d802c34ee655cc17930572b07a4ed9f62c18022fe98f380b38fb043dc7ba26b1", size = 212909, upload-time = "2026-07-23T03:30:24.528Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/1b/abfa5349bee1e08d5fd76bd3dc43f8dcd3718e04e2d1b66d2b5cab332f51/abxpkg-1.11.274-py3-none-any.whl", hash = "sha256:99178a53723ffd92687ac4b24d3466a87162a435adf50a0ec13180c7de92284f", size = 225316, upload-time = "2026-07-20T23:26:09.427Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c5/21478d9a246cacd51dc7d700bd0bce3ea08ac14b34ce09496640dea27d71/abxpkg-1.11.288-py3-none-any.whl", hash = "sha256:de7b0ed66a4d7ef06764634a1dae1b28f27d17c40f824159df4c2d2d89e3efa9", size = 227472, upload-time = "2026-07-23T03:30:22.856Z" }, ] [[package]] @@ -124,7 +124,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.9.35rc137" +version = "0.9.35rc138" source = { editable = "." } dependencies = [ { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -215,6 +215,7 @@ dev = [ { name = "sphinx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "sphinx-autodoc2", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "sphinx-rtd-theme", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "sphinxcontrib-mermaid", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "ty", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "viztracer", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -222,10 +223,10 @@ dev = [ [package.metadata] requires-dist = [ - { name = "abx-dl", specifier = ">=1.11.259" }, - { name = "abx-plugins", specifier = ">=1.11.288" }, - { name = "abxbus", specifier = "==2.5.28" }, - { name = "abxpkg", specifier = ">=1.11.271" }, + { name = "abx-dl", specifier = "==1.11.268" }, + { name = "abx-plugins", specifier = "==1.11.311" }, + { name = "abxbus", specifier = "==2.5.40" }, + { name = "abxpkg", specifier = "==1.11.288" }, { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" }, { name = "atomicwrites", specifier = "==1.4.1" }, { name = "base32-crockford", specifier = ">=0.3.0" }, @@ -297,6 +298,7 @@ dev = [ { name = "sphinx", specifier = ">=8.1.3" }, { name = "sphinx-autodoc2", specifier = ">=0.5.0" }, { name = "sphinx-rtd-theme", specifier = ">=2.0.0" }, + { name = "sphinxcontrib-mermaid", specifier = ">=1.0.0" }, { name = "ty", specifier = ">=0.0.1a19" }, { name = "uv", specifier = ">=0.11.3" }, { name = "viztracer", specifier = ">=0.17.0" }, @@ -2478,6 +2480,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, ] +[[package]] +name = "sphinxcontrib-mermaid" +version = "2.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "pyyaml", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "sphinx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/29/54cf1f7e03630ca4859caba25bc192698954d7c630377c62d1e264715c37/sphinxcontrib_mermaid-2.0.3.tar.gz", hash = "sha256:a6865ef6b65b225c5403a3170de63a04a07227cada11a4a71a6b87b4f9ed185a", size = 20764, upload-time = "2026-07-08T00:30:44.216Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/57/b39c7a69b70a3ce999732bdb57fcaac78fb3cc8e2843a3daf1c1f821bc9d/sphinxcontrib_mermaid-2.0.3-py3-none-any.whl", hash = "sha256:f001ed36a55c108f6221a2d656a441c487ee30651b54db72b7c752a20c7a66e8", size = 15401, upload-time = "2026-07-08T00:30:42.877Z" }, +] + [[package]] name = "sphinxcontrib-qthelp" version = "2.0.0"