diff --git a/.github/configs/ci-linux-build.json b/.github/configs/ci-linux-build.json new file mode 100644 index 00000000..5ceea642 --- /dev/null +++ b/.github/configs/ci-linux-build.json @@ -0,0 +1,51 @@ +{ + "properties": { + "CC_BINARY": { + "default": "cc" + }, + "LDAPSEARCH_BINARY": { + "default": "ldapsearch" + } + }, + "required_binaries": [ + { + "name": "{CC_BINARY}", + "binproviders": "env,apt", + "overrides": { + "apt": { + "install_args": [ + "build-essential" + ] + } + } + }, + { + "name": "{LDAPSEARCH_BINARY}", + "binproviders": "env,apt", + "overrides": { + "env": { + "version": [ + "ldapsearch", + "-VV" + ] + }, + "apt": { + "install_args": [ + "ldap-utils", + "python3-dev", + "python3-setuptools", + "libssl-dev", + "libldap2-dev", + "libsasl2-dev", + "zlib1g-dev", + "libatomic1" + ], + "version": [ + "ldapsearch", + "-VV" + ] + } + } + } + ] +} diff --git a/.github/configs/ci-linux-env.json b/.github/configs/ci-linux-env.json new file mode 100644 index 00000000..b4aa592a --- /dev/null +++ b/.github/configs/ci-linux-env.json @@ -0,0 +1,28 @@ +{ + "properties": { + "CC_BINARY": { + "default": "cc" + }, + "LDAPSEARCH_BINARY": { + "default": "ldapsearch" + } + }, + "required_binaries": [ + { + "name": "{CC_BINARY}", + "binproviders": "env" + }, + { + "name": "{LDAPSEARCH_BINARY}", + "binproviders": "env", + "overrides": { + "env": { + "version": [ + "ldapsearch", + "-VV" + ] + } + } + } + ] +} diff --git a/.github/configs/ci-macos-brew.json b/.github/configs/ci-macos-brew.json new file mode 100644 index 00000000..bb41dace --- /dev/null +++ b/.github/configs/ci-macos-brew.json @@ -0,0 +1,13 @@ +{ + "properties": { + "BREW_BINARY": { + "default": "brew" + } + }, + "required_binaries": [ + { + "name": "{BREW_BINARY}", + "binproviders": "env" + } + ] +} diff --git a/.github/configs/ci-macos-build.json b/.github/configs/ci-macos-build.json new file mode 100644 index 00000000..fbaf66bb --- /dev/null +++ b/.github/configs/ci-macos-build.json @@ -0,0 +1,44 @@ +{ + "properties": { + "CLANG_BINARY": { + "default": "clang" + }, + "LDAPVC_BINARY": { + "default": "ldapvc" + } + }, + "required_binaries": [ + { + "name": "{CLANG_BINARY}", + "binproviders": "env,brew", + "overrides": { + "brew": { + "install_args": [ + "llvm" + ] + } + } + }, + { + "name": "{LDAPVC_BINARY}", + "binproviders": "env,brew", + "overrides": { + "env": { + "version": [ + "ldapvc", + "-VV" + ] + }, + "brew": { + "install_args": [ + "openldap" + ], + "version": [ + "ldapvc", + "-VV" + ] + } + } + } + ] +} diff --git a/.github/configs/ci-macos-openldap-env.json b/.github/configs/ci-macos-openldap-env.json new file mode 100644 index 00000000..37ff2bae --- /dev/null +++ b/.github/configs/ci-macos-openldap-env.json @@ -0,0 +1,21 @@ +{ + "properties": { + "LDAPVC_BINARY": { + "default": "ldapvc" + } + }, + "required_binaries": [ + { + "name": "{LDAPVC_BINARY}", + "binproviders": "env", + "overrides": { + "env": { + "version": [ + "ldapvc", + "-VV" + ] + } + } + } + ] +} diff --git a/.github/configs/ci-tooling.json b/.github/configs/ci-tooling.json new file mode 100644 index 00000000..ec58ef27 --- /dev/null +++ b/.github/configs/ci-tooling.json @@ -0,0 +1,321 @@ +{ + "properties": { + "CURL_BINARY": { + "default": "curl" + }, + "BASH_BINARY": { + "default": "bash" + }, + "CHMOD_BINARY": { + "default": "chmod" + }, + "DOCKER_BINARY": { + "default": "docker" + }, + "GIT_BINARY": { + "default": "git" + }, + "GREP_BINARY": { + "default": "grep" + }, + "GH_BINARY": { + "default": "gh" + }, + "EXPORTFS_BINARY": { + "default": "exportfs" + }, + "JQ_BINARY": { + "default": "jq" + }, + "MKDIR_BINARY": { + "default": "mkdir" + }, + "MOUNT_CIFS_BINARY": { + "default": "mount.cifs" + }, + "MOUNT_NFS_BINARY": { + "default": "mount.nfs" + }, + "NODE_BINARY": { + "default": "node" + }, + "PS_BINARY": { + "default": "ps" + }, + "PYTHON_BINARY": { + "default": "python" + }, + "PV_BINARY": { + "default": "pv" + }, + "SSH_BINARY": { + "default": "ssh" + }, + "SED_BINARY": { + "default": "sed" + }, + "SMBD_BINARY": { + "default": "smbd" + }, + "SUDO_BINARY": { + "default": "sudo" + }, + "SYSTEMCTL_BINARY": { + "default": "systemctl" + }, + "TAR_BINARY": { + "default": "tar" + }, + "TREE_BINARY": { + "default": "tree" + }, + "TR_BINARY": { + "default": "tr" + }, + "RM_BINARY": { + "default": "rm" + }, + "UMOUNT_BINARY": { + "default": "umount" + }, + "UV_BINARY": { + "default": "uv" + }, + "UNAME_BINARY": { + "default": "uname" + } + }, + "ci_binaries": [ + { + "name": "{PYTHON_BINARY}", + "binproviders": "env" + }, + { + "name": "{JQ_BINARY}", + "binproviders": "env,apt,brew" + }, + { + "name": "{UV_BINARY}", + "binproviders": "env" + } + ], + "docker_binaries": [ + { + "name": "{DOCKER_BINARY}", + "binproviders": "env" + }, + { + "name": "{GIT_BINARY}", + "binproviders": "env,apt,brew" + }, + { + "name": "{PYTHON_BINARY}", + "binproviders": "env" + }, + { + "name": "{UNAME_BINARY}", + "binproviders": "env" + } + ], + "deploy_binaries": [ + { + "name": "{GIT_BINARY}", + "binproviders": "env,apt,brew" + }, + { + "name": "{PYTHON_BINARY}", + "binproviders": "env" + }, + { + "name": "{SSH_BINARY}", + "binproviders": "env" + } + ], + "docker_debug_binaries": [ + { + "name": "{DOCKER_BINARY}", + "binproviders": "env" + }, + { + "name": "{PV_BINARY}", + "binproviders": "env,apt,brew" + }, + { + "name": "{TAR_BINARY}", + "binproviders": "env,apt,brew" + }, + { + "name": "{TREE_BINARY}", + "binproviders": "env,apt,brew" + } + ], + "docker_validation_binaries": [ + { + "name": "{DOCKER_BINARY}", + "binproviders": "env" + }, + { + "name": "{GREP_BINARY}", + "binproviders": "env" + }, + { + "name": "{MKDIR_BINARY}", + "binproviders": "env" + }, + { + "name": "{RM_BINARY}", + "binproviders": "env" + }, + { + "name": "{SED_BINARY}", + "binproviders": "env" + }, + { + "name": "{SSH_BINARY}", + "binproviders": "env" + }, + { + "name": "{TR_BINARY}", + "binproviders": "env" + }, + { + "name": "{UNAME_BINARY}", + "binproviders": "env" + } + ], + "mount_validation_binaries": [ + { + "name": "{BASH_BINARY}", + "binproviders": "env" + }, + { + "name": "{CHMOD_BINARY}", + "binproviders": "env" + }, + { + "name": "{EXPORTFS_BINARY}", + "binproviders": "env,apt", + "overrides": { + "apt": { + "install_args": [ + "nfs-kernel-server" + ] + } + } + }, + { + "name": "{MKDIR_BINARY}", + "binproviders": "env" + }, + { + "name": "{MOUNT_CIFS_BINARY}", + "binproviders": "env,apt", + "overrides": { + "apt": { + "install_args": [ + "cifs-utils" + ] + } + } + }, + { + "name": "{MOUNT_NFS_BINARY}", + "binproviders": "env,apt", + "overrides": { + "apt": { + "install_args": [ + "nfs-common" + ] + } + } + }, + { + "name": "{SMBD_BINARY}", + "binproviders": "env,apt", + "overrides": { + "apt": { + "install_args": [ + "samba" + ] + } + } + }, + { + "name": "{SUDO_BINARY}", + "binproviders": "env" + }, + { + "name": "{SYSTEMCTL_BINARY}", + "binproviders": "env" + }, + { + "name": "{UMOUNT_BINARY}", + "binproviders": "env" + } + ], + "process_binaries": [ + { + "name": "{PS_BINARY}", + "binproviders": "env" + } + ], + "docs_binaries": [ + { + "name": "{CURL_BINARY}", + "binproviders": "env,apt,brew" + } + ], + "git_binaries": [ + { + "name": "{GIT_BINARY}", + "binproviders": "env,apt,brew" + } + ], + "github_binaries": [ + { + "name": "{GH_BINARY}", + "binproviders": "env,apt,brew" + } + ], + "node_binaries": [ + { + "name": "{NODE_BINARY}", + "binproviders": "env,npm", + "min_version": "22.23.1", + "overrides": { + "npm": { + "install_args": [ + "node@22.23.1" + ], + "postinstall_scripts": true + } + } + } + ], + "release_binaries": [ + { + "name": "{UV_BINARY}", + "binproviders": "env,pip,brew" + }, + { + "name": "{PYTHON_BINARY}", + "binproviders": "env" + }, + { + "name": "{GH_BINARY}", + "binproviders": "env,apt,brew" + }, + { + "name": "{GIT_BINARY}", + "binproviders": "env,apt,brew" + }, + { + "name": "{JQ_BINARY}", + "binproviders": "env,apt,brew" + }, + { + "name": "{CURL_BINARY}", + "binproviders": "env,apt,brew" + } + ] +} diff --git a/.github/scripts/clone_abx_repo.sh b/.github/scripts/clone_abx_repo.sh index 01908e03..ea4fa2c1 100755 --- a/.github/scripts/clone_abx_repo.sh +++ b/.github/scripts/clone_abx_repo.sh @@ -3,25 +3,44 @@ set -Eeuo pipefail repo_name="$1" target_dir="${2:-$repo_name}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +lock_file="$repo_root/uv.lock" +tooling_config="$repo_root/.github/configs/ci-tooling.json" -version="$( -python3 - "$repo_name" <<'PY' -import re -import sys -from pathlib import Path +locked_version() { + local wanted="$1" line package="" + while IFS= read -r line; do + case "$line" in + '[[package]]') package="" ;; + "name = \"${wanted}\"") package="$wanted" ;; + 'version = "'*'"') + if [[ "$package" == "$wanted" ]]; then + line="${line#version = \"}" + printf '%s\n' "${line%\"}" + return 0 + fi + ;; + esac + done < "$lock_file" + return 1 +} -repo_name = sys.argv[1] -lock_text = Path("uv.lock").read_text() -match = re.search( - rf'^\[\[package\]\]\s*\nname = "{re.escape(repo_name)}"\s*\nversion = "([^"]+)"', - lock_text, - re.MULTILINE, -) -if not match: - raise SystemExit(f"Could not find {repo_name} in uv.lock") -print(match.group(1)) -PY -)" +version="$(locked_version "$repo_name")" +[[ -n "$version" ]] || { echo "Could not find ${repo_name} in uv.lock" >&2; exit 1; } + +abxpkg_version="$(locked_version abxpkg)" +[[ -n "$abxpkg_version" ]] || { echo "Could not find abxpkg in uv.lock" >&2; exit 1; } + +ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-${RUNNER_TEMP:-/tmp}/archivebox-clone-abxpkg}" +mkdir -p "$ABXPKG_LIB_DIR/env/bin" +uv run --no-project --with "abxpkg==$abxpkg_version" abxpkg env \ + --install \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$tooling_config:git_binaries" \ + >/dev/null +git_binary="$ABXPKG_LIB_DIR/env/bin/git" +[[ -L "$git_binary" ]] +[[ -x "$git_binary" ]] echo "Cloning ArchiveBox/${repo_name}@v${version} into ${target_dir}" -git clone --depth=1 --branch "v${version}" "https://github.com/ArchiveBox/${repo_name}.git" "$target_dir" +"$git_binary" clone --depth=1 --branch "v${version}" "https://github.com/ArchiveBox/${repo_name}.git" "$target_dir" diff --git a/.github/scripts/discover_test_matrix.py b/.github/scripts/discover_test_matrix.py new file mode 100755 index 00000000..f2c2f03b --- /dev/null +++ b/.github/scripts/discover_test_matrix.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Build deterministic GitHub Actions matrices from every discovered test file.""" + +import argparse +import json +import re +from pathlib import Path + + +CHROMIUM_PATTERN = re.compile( + rb"chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue", + re.IGNORECASE, +) +SONIC_PATTERN = re.compile( + rb"""shutil\.which\(["']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic""", + re.IGNORECASE, +) + + +def contains(pattern: re.Pattern[bytes], paths: list[Path]) -> bool: + return any(pattern.search(path.read_bytes()) for path in paths) + + +def archivebox_matrix(root: Path) -> list[dict[str, object]]: + tests = sorted((root / "archivebox/tests").glob("test_*.py")) + if not tests: + raise SystemExit("No ArchiveBox tests discovered") + + shard_count = min(16, len(tests)) + matrix = [] + assigned: list[Path] = [] + for shard in range(shard_count): + shard_tests = tests[shard::shard_count] + assigned.extend(shard_tests) + matrix.append( + { + "name": f"main/shard-{shard + 1}", + "paths": [path.relative_to(root).as_posix() for path in shard_tests], + "needs_chromium": contains(CHROMIUM_PATTERN, shard_tests), + "needs_sonic": contains(SONIC_PATTERN, shard_tests), + }, + ) + + if sorted(assigned) != tests or len(assigned) != len(set(assigned)): + raise SystemExit("ArchiveBox tests were not assigned exactly once") + print(f"Assigned {len(tests)} test files exactly once across {shard_count} shards") + return matrix + + +def plugin_matrix(root: Path) -> list[dict[str, object]]: + plugins_root = root / "abx-plugins/abx_plugins/plugins" + suite_dirs = sorted(path for path in plugins_root.glob("*/tests") if path.is_dir()) + root_tests = sorted((root / "abx-plugins/tests").glob("test_*.py")) + if not suite_dirs or not root_tests: + raise SystemExit("Plugin suites or root tests were not discovered") + + matrix: list[dict[str, object]] = [] + expected = list(root_tests) + for suite_dir in suite_dirs: + suite_tests = sorted(suite_dir.rglob("test_*.py")) + if not suite_tests: + raise SystemExit(f"No tests found in {suite_dir}") + expected.extend(suite_tests) + plugin = suite_dir.parent.name + matrix.append( + { + "plugin": plugin, + "name": f"plugin/{plugin}", + "test_path": suite_dir.relative_to(root).as_posix(), + "config_path": (suite_dir.parent / "config.json").relative_to(root).as_posix(), + "needs_chromium": contains(re.compile(rb"chrom", re.IGNORECASE), suite_tests), + "needs_sonic": plugin == "search_backend_sonic", + }, + ) + + matrix.append( + { + "plugin": "root", + "name": "plugin/root", + "test_path": "abx-plugins/tests", + "config_path": "abx-plugins/abx_plugins/plugins/base/config.json", + "needs_chromium": contains(re.compile(rb"chrom|archivewebpage", re.IGNORECASE), root_tests), + "needs_sonic": False, + }, + ) + + assigned = [] + for entry in matrix: + assigned.extend(sorted((root / str(entry["test_path"])).rglob("test_*.py"))) + if sorted(assigned) != sorted(expected) or len(assigned) != len(set(assigned)): + raise SystemExit("Plugin tests were not assigned exactly once") + print( + f"Assigned {len(suite_dirs)} plugin suites and {len(root_tests)} root test files exactly once", + ) + return matrix + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("scope", choices=("archivebox", "plugins")) + parser.add_argument("--workspace", type=Path, default=Path.cwd()) + args = parser.parse_args() + root = args.workspace.resolve() + matrix = archivebox_matrix(root) if args.scope == "archivebox" else plugin_matrix(root) + print(json.dumps(matrix, separators=(",", ":"))) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/docs_http_server.py b/.github/scripts/docs_http_server.py new file mode 100644 index 00000000..2f7029f6 --- /dev/null +++ b/.github/scripts/docs_http_server.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import argparse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + + +class DocsRequestHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = f"ArchiveBox docs fixture

{self.path}

\n".encode() + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--ready-fifo", type=Path, required=True) + args = parser.parse_args() + + with ThreadingHTTPServer(("127.0.0.1", 0), DocsRequestHandler) as server: + host, port = server.server_address + with args.ready_fifo.open("w") as ready_fifo: + ready_fifo.write(f"http://{host}:{port}\n") + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..7a52c138 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,70 @@ +name: CI + +on: + pull_request: + branches: [dev] + push: + branches: [dev] + workflow_dispatch: + +permissions: + actions: read + contents: read + packages: write + security-events: write + +concurrency: + group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }} + cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }} + +jobs: + lint: + name: Linters + uses: ./.github/workflows/lint.yml + secrets: inherit + + complete-tests: + name: Complete test suite + uses: ./.github/workflows/test.yml + secrets: inherit + + sharded-plugin-tests: + name: Sharded and plugin tests + uses: ./.github/workflows/test-parallel.yml + secrets: inherit + + documentation: + name: Documentation and root tests + uses: ./.github/workflows/docs.yml + secrets: inherit + + codeql: + name: CodeQL + uses: ./.github/workflows/codeql.yml + secrets: inherit + + python-artifacts: + name: Tested Python artifacts + uses: ./.github/workflows/pip.yml + secrets: inherit + + docker-digests: + name: Tested Docker images + uses: ./.github/workflows/docker.yml + with: + push_digests: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' }} + secrets: inherit + + required: + name: All required CI lanes + needs: + - lint + - complete-tests + - sharded-plugin-tests + - documentation + - codeql + - python-artifacts + - docker-digests + runs-on: ubuntu-24.04 + steps: + - run: echo 'All required CI lanes and tested release artifacts succeeded.' diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 2feee2e3..447aec19 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -17,33 +17,24 @@ jobs: (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read pull-requests: write issues: write id-token: write - actions: read # Required for Claude to read CI results on PRs + actions: read steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 1 - name: Run Claude Code id: claude - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@b76a0776ae74036e77cd11018083743453d7ad35 # v1 with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - - # This is an optional setting that allows Claude to read CI results on PRs additional_permissions: | actions: read - - # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. - # prompt: 'Update the pull request description to include a summary of changes.' - - # Optional: Add claude_args to customize behavior and configuration - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://code.claude.com/docs/en/cli-reference for available options claude_args: '--allowed-tools Bash(gh pr:*)' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 78bff057..1b683f84 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,46 +1,17 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# name: "CodeQL" on: - push: - branches: [ "dev" ] - pull_request: - branches: [ "dev" ] + workflow_call: schedule: - cron: '33 17 * * 6' -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - jobs: analyze: name: Analyze (${{ matrix.language }}) - # Runner size impacts CodeQL analysis time. To learn more, please see: - # - https://gh.io/recommended-hardware-resources-for-running-codeql - # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners (GitHub.com only) - # Consider using larger runners or machines with greater resources for possible analysis time improvements. - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + runs-on: ubuntu-24.04 permissions: - # required for all workflows security-events: write - - # required to fetch internal or private CodeQL packs packages: read - - # only required for workflows in private repositories actions: read contents: read @@ -50,47 +21,18 @@ jobs: include: - language: python build-mode: none - # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' - # Use `c-cpp` to analyze code written in C, C++ or both - # Use 'java-kotlin' to analyze code written in Java, Kotlin or both - # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both - # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, - # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. - # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how - # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + - language: javascript-typescript + build-mode: none steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - # If the analyze step fails for one of the languages you are analyzing with - # "We were unable to automatically build your code", modify the matrix above - # to set the build mode to "manual" for that language. Then modify this step - # to build your code. - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - if: matrix.build-mode == 'manual' - run: | - echo 'If you are using a "manual" build mode for one or more of the' \ - 'languages you are analyzing, replace this with the commands to build' \ - 'your code, for example:' - echo ' make bootstrap' - echo ' make release' - exit 1 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/deploy-publicsite.yml b/.github/workflows/deploy-publicsite.yml index daaf5a21..c42ded47 100644 --- a/.github/workflows/deploy-publicsite.yml +++ b/.github/workflows/deploy-publicsite.yml @@ -15,29 +15,29 @@ permissions: id-token: write concurrency: - group: github-pages-publicsite - cancel-in-progress: true + group: ${{ github.event_name == 'workflow_dispatch' && format('github-pages-publicsite-manual-{0}', github.run_id) || 'github-pages-publicsite' }} + cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }} jobs: deploy: environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 1 - name: Setup Pages - uses: actions/configure-pages@v5 + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3 with: path: ./publicsite - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e8fdef2a..a30a55e2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,24 +1,23 @@ name: Build Docker image on: - workflow_dispatch: - push: - branches: - - '**' - # pull_request: + workflow_call: + inputs: + push_digests: + description: Push validated per-platform images by digest for the release workflow + required: false + default: false + type: boolean env: DOCKERHUB_IMAGE: archivebox/archivebox GHCR_IMAGE: ghcr.io/archivebox/archivebox permissions: + actions: read contents: read packages: write -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - jobs: build: name: build ${{ matrix.platform }} @@ -31,22 +30,78 @@ jobs: runner: ubuntu-24.04 cache_scope: docker-amd64 artifact_name: digest-linux-amd64 + local_tag: archivebox-ci:amd64 - platform: linux/arm64 runner: ubuntu-24.04-arm cache_scope: docker-arm64 artifact_name: digest-linux-arm64 + local_tag: archivebox-ci:arm64 steps: - name: Checkout - uses: actions/checkout@v4 - # with: - # submodules: true - # fetch-depth: 1 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + + - name: Resolve Docker through abxpkg + env: + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + shell: bash + run: | + set -Eeuo pipefail + ABXPKG_VERSION="$(uv run --no-project python - <<'PY' + import tomllib + from pathlib import Path + + packages = tomllib.loads(Path("uv.lock").read_text())["package"] + matches = [package["version"] for package in packages if package["name"] == "abxpkg"] + if len(matches) != 1: + raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}") + print(matches[0]) + PY + )" + test -n "$ABXPKG_VERSION" + mkdir -p "$ABXPKG_LIB_DIR/env/bin" + export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV" + echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH" + + docker_env="$( + uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \ + --install \ + --json \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" + )" + JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq" + "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + <<< "$docker_env" >> "$GITHUB_ENV" + + docker_binary="$ABXPKG_LIB_DIR/env/bin/docker" + UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv" + { + echo "DOCKER_BINARY=$docker_binary" + echo "JQ_BINARY=$JQ_BINARY" + echo "UV_BINARY=$UV_BINARY" + } >> "$GITHUB_ENV" + for binary in "$docker_binary" "$JQ_BINARY" "$UV_BINARY"; do + test -L "$binary" + test -x "$binary" + done + "$docker_binary" --version - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 with: - version: latest install: true - name: Builder instance name @@ -60,35 +115,45 @@ jobs: shell: bash run: | set -Eeuo pipefail - ABX_DL_VERSION="$(python3 - <<'PY' - import re + ABX_DL_VERSION="$("$UV_BINARY" run --no-project python - <<'PY' import tomllib + from pathlib import Path - with open("pyproject.toml", "rb") as f: - deps = tomllib.load(f)["project"]["dependencies"] - - for dep in deps: - match = re.match(r"abx-dl\s*(?:==|>=)\s*([^,;\s]+)", dep) - if match: - print(match.group(1)) - break - else: - raise SystemExit("Missing abx-dl dependency in pyproject.toml") + packages = tomllib.loads(Path("uv.lock").read_text())["package"] + matches = [package["version"] for package in packages if package["name"] == "abx-dl"] + if len(matches) != 1: + raise SystemExit(f"Expected one locked abx-dl package, found {len(matches)}") + print(matches[0]) PY )" - ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}" + test -n "$ABX_DL_VERSION" + ABX_DL_REFERENCE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}" + INSPECT_OUTPUT="$("$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_REFERENCE")" + printf '%s\n' "$INSPECT_OUTPUT" + ABX_DL_DIGEST="$(INSPECT_OUTPUT="$INSPECT_OUTPUT" "$UV_BINARY" run --no-project python - <<'PY' + import os + + digest = next( + (fields[1] for line in os.environ["INSPECT_OUTPUT"].splitlines() if (fields := line.split()) and fields[0] == "Digest:"), + "", + ) + print(digest) + PY + )" + [[ "$ABX_DL_DIGEST" == sha256:* ]] + ABX_DL_IMAGE="${ABX_DL_REFERENCE%%@*}@${ABX_DL_DIGEST}" echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT" - docker buildx imagetools inspect "${ABX_DL_IMAGE}" - name: Login to Docker Hub - uses: docker/login-action@v3 - if: github.event_name != 'pull_request' + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + if: inputs.push_digests with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + if: inputs.push_digests with: registry: ghcr.io username: ${{ github.actor }} @@ -99,13 +164,14 @@ jobs: shell: bash run: | set -Eeuo pipefail - VERSION="$(python3 - <<'PY' + VERSION="$("$UV_BINARY" run --no-project python - <<'PY' import tomllib + from pathlib import Path - with open("pyproject.toml", "rb") as f: - print(tomllib.load(f)["project"]["version"]) + print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]) PY )" + test -n "$VERSION" { echo 'labels< /tmp/archivebox-image-version.txt - grep -F "ArchiveBox v${{ steps.docker_meta.outputs.version }}" /tmp/archivebox-version.out - grep -F "COMMIT_HASH=${SHORT_SHA}" /tmp/archivebox-version.out - grep -Fx "COMMIT_HASH=${GITHUB_SHA}" /tmp/archivebox-image-version.txt - - python3 - "$IMAGE" <<'PY' - import json - import subprocess + "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init + ARCHIVEBOX_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version)" + printf '%s\n' "$ARCHIVEBOX_VERSION_OUTPUT" + "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status + IMAGE_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt)" + ARCHIVEBOX_VERSION_OUTPUT="$ARCHIVEBOX_VERSION_OUTPUT" IMAGE_VERSION_OUTPUT="$IMAGE_VERSION_OUTPUT" \ + "$UV_BINARY" run --no-project python - \ + "ArchiveBox v${{ steps.docker_meta.outputs.version }}" \ + "COMMIT_HASH=${SHORT_SHA}" \ + "COMMIT_HASH=${GITHUB_SHA}" <<'PY' + import os import sys - image = sys.argv[1] - limit = 780 * 1024 * 1024 - manifest = json.loads(subprocess.check_output(["docker", "manifest", "inspect", image])) - total = manifest.get("config", {}).get("size", 0) + sum(layer.get("size", 0) for layer in manifest.get("layers", [])) - print(f"{image} compressed_size={total / 1024 / 1024:.2f} MiB limit={limit / 1024 / 1024:.2f} MiB") - if total > limit: - raise SystemExit(f"{image} is over the compressed size limit") + version_output = os.environ["ARCHIVEBOX_VERSION_OUTPUT"] + image_version_lines = os.environ["IMAGE_VERSION_OUTPUT"].splitlines() + if sys.argv[1] not in version_output: + raise SystemExit(f"Missing version marker: {sys.argv[1]}") + if sys.argv[2] not in version_output: + raise SystemExit(f"Missing short commit marker: {sys.argv[2]}") + if sys.argv[3] not in image_version_lines: + raise SystemExit(f"Missing exact commit marker: {sys.argv[3]}") PY + - name: Validate compressed image size + if: inputs.push_digests + shell: bash + run: | + set -Eeuo pipefail + IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}" + LIMIT=$((780 * 1024 * 1024)) + TOTAL="$("$DOCKER_BINARY" manifest inspect "$IMAGE" | "$JQ_BINARY" '[.config.size, (.layers[]?.size)] | add')" + printf '%s compressed_size=%s MiB limit=%s MiB\n' \ + "$IMAGE" "$((TOTAL / 1024 / 1024))" "$((LIMIT / 1024 / 1024))" + [[ "$TOTAL" -le "$LIMIT" ]] || { echo "$IMAGE is over the compressed size limit" >&2; exit 1; } + - name: Export digest + if: inputs.push_digests shell: bash run: | set -Eeuo pipefail @@ -182,151 +285,10 @@ jobs: touch "/tmp/digests/${digest#sha256:}" - name: Upload digest - uses: actions/upload-artifact@v4 + if: inputs.push_digests + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: ${{ matrix.artifact_name }} path: /tmp/digests/* if-no-files-found: error retention-days: 1 - - publish: - name: publish multiarch tags - runs-on: ubuntu-24.04 - needs: - - build - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: latest - install: true - - - name: Login to Docker Hub - uses: docker/login-action@v3 - if: github.event_name != 'pull_request' - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digest-* - merge-multiple: true - - - name: Collect Docker tags - id: docker_meta - shell: bash - run: | - set -Eeuo pipefail - VERSION="$(python3 - <<'PY' - import tomllib - - with open("pyproject.toml", "rb") as f: - print(tomllib.load(f)["project"]["version"]) - PY - )" - BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)" - SHORT_SHA="${GITHUB_SHA::12}" - test -n "$BRANCH_TAG" - test -n "$SHORT_SHA" - - { - echo 'dockerhub_tags<> "$GITHUB_OUTPUT" - - echo "[+] Publishing Docker Hub tags:" - if [[ "${GITHUB_REF_NAME}" == "main" ]]; then - printf '%s\n' "${DOCKERHUB_IMAGE}:latest" - fi - printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" - echo "[+] Publishing GHCR tags:" - if [[ "${GITHUB_REF_NAME}" == "main" ]]; then - printf '%s\n' "${GHCR_IMAGE}:latest" - fi - printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}" - - - name: Create Docker Hub manifest - shell: bash - run: | - set -Eeuo pipefail - mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort) - [[ "${#DIGESTS[@]}" -gt 0 ]] - - TAG_ARGS=() - while IFS= read -r tag; do - [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag") - done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}" - - REFS=() - for digest in "${DIGESTS[@]}"; do - REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}") - done - - docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}" - - - name: Create GHCR manifest - shell: bash - run: | - set -Eeuo pipefail - mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort) - [[ "${#DIGESTS[@]}" -gt 0 ]] - - TAG_ARGS=() - while IFS= read -r tag; do - [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag") - done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}" - - REFS=() - for digest in "${DIGESTS[@]}"; do - REFS+=("${GHCR_IMAGE}@sha256:${digest}") - done - - docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}" - - - name: Inspect published images - shell: bash - run: | - set -Eeuo pipefail - while IFS= read -r tag; do - [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag" - done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}" - while IFS= read -r tag; do - [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag" - done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}" - - - name: Update README - uses: peter-evans/dockerhub-description@v4 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - repository: archivebox/archivebox diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..7c24d8a0 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,289 @@ +name: Documentation tests + +on: + workflow_call: + +env: + PYTHONIOENCODING: utf-8 + USE_COLOR: "False" + SHOW_PROGRESS: "False" + ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD: "archivebox-docs-ci-only" + +jobs: + docs-matrix: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + - id: matrix + run: | + uv run --no-project python - <<'PY' >> "$GITHUB_OUTPUT" + import json + import tomllib + + with open("docs/codeblocks.toml", "rb") as manifest_file: + ci = tomllib.load(manifest_file)["ci"] + + include = [] + for environment, runner in ci["standard"].items(): + if environment == "core": + include.extend( + { + "environment": environment, + "runner": runner, + "core_shard": shard, + "job_name": f"core/{shard}", + "validate_manifest": shard == "metadata", + } + for shard in ci["core_shards"] + ) + else: + include.append( + { + "environment": environment, + "runner": runner, + "core_shard": "", + "job_name": environment, + "validate_manifest": False, + }, + ) + + print("matrix=" + json.dumps({"include": include}, separators=(",", ":"))) + PY + + docs-standard: + name: docs/${{ matrix.job_name }} + needs: docs-matrix + runs-on: ${{ matrix.runner }} + timeout-minutes: 5 + env: + DOCS_CORE_SHARD: ${{ matrix.core_shard }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.docs-matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + submodules: true + fetch-depth: 1 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + - name: Install ArchiveBox + run: uv sync --dev --all-extras + - name: Prepare abxpkg environment + shell: bash + run: | + { + echo "ABXPKG_LIB_DIR=${{ runner.temp }}/archivebox-docs-lib" + echo "ARCHIVEBOX_PROJECT_DIR=$GITHUB_WORKSPACE" + } >> "$GITHUB_ENV" + echo "${{ runner.temp }}/archivebox-docs-lib/env/bin" >> "$GITHUB_PATH" + - name: Resolve Node.js through abxpkg + shell: bash + run: | + set -Eeuo pipefail + mkdir -p "$ABXPKG_LIB_DIR/env/bin" + export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH" + tooling_env="$( + uv run --no-sync abxpkg env \ + --install \ + --json \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries" + )" + JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq" + "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + <<< "$tooling_env" >> "$GITHUB_ENV" + + node_binary="$ABXPKG_LIB_DIR/env/bin/node" + { + echo "JQ_BINARY=$JQ_BINARY" + echo "NODE_BINARY=$node_binary" + } >> "$GITHUB_ENV" + test -L "$node_binary" + test -x "$node_binary" + "$node_binary" --version + - name: Resolve documentation shell tools through abxpkg + shell: bash + run: | + set -Eeuo pipefail + shell_env="$( + uv run --no-sync abxpkg env \ + --install \ + --json \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries" + )" + "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + <<< "$shell_env" >> "$GITHUB_ENV" + + curl_binary="$ABXPKG_LIB_DIR/env/bin/curl" + echo "CURL_BINARY=$curl_binary" >> "$GITHUB_ENV" + test -L "$curl_binary" + test -x "$curl_binary" + - name: Resolve Docker through abxpkg + if: matrix.environment == 'docker' + shell: bash + run: | + set -Eeuo pipefail + docker_env="$( + uv run --no-sync abxpkg env \ + --install \ + --json \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" + )" + "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + <<< "$docker_env" >> "$GITHUB_ENV" + + docker_binary="$ABXPKG_LIB_DIR/env/bin/docker" + echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV" + test -L "$docker_binary" + test -x "$docker_binary" + "$docker_binary" --version + - name: Start bound local documentation site + shell: bash + run: | + ready_fifo="${{ runner.temp }}/archivebox-docs-http-ready" + mkfifo "$ready_fifo" + uv run --no-sync python .github/scripts/docs_http_server.py --ready-fifo "$ready_fifo" & + server_pid=$! + IFS= read -r docs_url < "$ready_fifo" + { + echo "ARCHIVEBOX_DOCS_SERVER_PID=$server_pid" + echo "ARCHIVEBOX_DOCS_URL_ONE=$docs_url/collection-one" + echo "ARCHIVEBOX_DOCS_URL_TWO=$docs_url/collection-two" + echo "ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT=18000" + echo "ARCHIVEBOX_DOCS_STATIC_PORT=18001" + } >> "$GITHUB_ENV" + - name: Resolve documentation validators through abxpkg + if: matrix.validate_manifest + run: | + validators_env="$( + uv run abxpkg env \ + --install \ + --json \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \ + --deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries" + )" + "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + <<< "$validators_env" >> "$GITHUB_ENV" + - name: Resolve merge tools through abxpkg + if: matrix.environment == 'merge' + run: | + uv run abxpkg install rsync --lib "$ABXPKG_LIB_DIR" --binproviders env,apt,brew + uv run abxpkg install sqlite3 --lib "$ABXPKG_LIB_DIR" --binproviders env,apt,brew + - name: Validate documentation inventory and structured fences + if: matrix.validate_manifest + run: uv run --no-sync pytest -q docs/test_codeblocks_manifest.py + - name: Initialize documentation collection + if: contains(fromJSON('["core", "macos", "root"]'), matrix.environment) + shell: bash + run: | + docs_data_dir="${{ runner.temp }}/archivebox-docs-data" + mkdir -p "$docs_data_dir" + (cd "$docs_data_dir" && uv run --project "$GITHUB_WORKSPACE" --no-sync archivebox init) + echo "ARCHIVEBOX_DOCS_DATA_DIR=$docs_data_dir" >> "$GITHUB_ENV" + - name: Build local documentation image + if: matrix.environment == 'docker' + run: '"$DOCKER_BINARY" build --tag archivebox-docs-ci .' + - name: Run documentation code blocks + if: matrix.environment != 'root' && !contains(fromJSON('["core", "macos"]'), matrix.environment) + run: | + mapfile -t docs_paths < <( + uv run --no-sync python - <<'PY' + from pathlib import Path + + print(*(str(path) for path in sorted(Path("docs").rglob("*.md")) if not path.is_symlink()), sep="\n") + PY + ) + uv run --no-sync pytest -vv --tb=long README.md AGENTS.md skills "${docs_paths[@]}" --docs-environment=${{ matrix.environment }} + - name: Run collection documentation code blocks + if: contains(fromJSON('["core", "macos"]'), matrix.environment) + working-directory: ${{ env.ARCHIVEBOX_DOCS_DATA_DIR }} + run: | + docs_paths=() + if [[ -n "$DOCS_CORE_SHARD" ]]; then + mapfile -t docs_paths < <( + uv run --project "$GITHUB_WORKSPACE" --no-sync python - "$DOCS_CORE_SHARD" <<'PY' + import os + from pathlib import Path + import sys + import tomllib + + workspace = Path(os.environ["GITHUB_WORKSPACE"]) + with (workspace / "docs" / "codeblocks.toml").open("rb") as manifest_file: + shard_paths = tomllib.load(manifest_file)["ci"]["core_shards"][sys.argv[1]] + print(*(workspace / path for path in shard_paths), sep="\n") + PY + ) + else + while IFS= read -r docs_path; do + docs_paths+=("$docs_path") + done < <( + uv run --project "$GITHUB_WORKSPACE" --no-sync python - <<'PY' + import os + from pathlib import Path + + docs_dir = Path(os.environ["GITHUB_WORKSPACE"]) / "docs" + print(*(str(path) for path in sorted(docs_dir.rglob("*.md")) if not path.is_symlink()), sep="\n") + PY + ) + docs_paths=("$GITHUB_WORKSPACE/README.md" "$GITHUB_WORKSPACE/AGENTS.md" "$GITHUB_WORKSPACE/skills" "${docs_paths[@]}") + fi + uv run --project "$GITHUB_WORKSPACE" --no-sync pytest -vv --tb=long "${docs_paths[@]}" --docs-environment=${{ matrix.environment }} + - name: Run root documentation code blocks + if: matrix.environment == 'root' + working-directory: ${{ env.ARCHIVEBOX_DOCS_DATA_DIR }} + run: | + uv_bin="$ABXPKG_LIB_DIR/env/bin/uv" + test -L "$uv_bin" + test -x "$uv_bin" + mapfile -t docs_paths < <( + uv run --project "$GITHUB_WORKSPACE" --no-sync python - <<'PY' + import os + from pathlib import Path + + docs_dir = Path(os.environ["GITHUB_WORKSPACE"]) / "docs" + print(*(str(path) for path in sorted(docs_dir.rglob("*.md")) if not path.is_symlink()), sep="\n") + PY + ) + sudo --preserve-env=PATH,ABXPKG_LIB_DIR,ARCHIVEBOX_DOCS_URL_ONE,ARCHIVEBOX_DOCS_URL_TWO,ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT,ARCHIVEBOX_DOCS_STATIC_PORT,ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD "$uv_bin" run --project "$GITHUB_WORKSPACE" --no-sync pytest -vv --tb=long "$GITHUB_WORKSPACE/README.md" "$GITHUB_WORKSPACE/AGENTS.md" "$GITHUB_WORKSPACE/skills" "${docs_paths[@]}" --docs-environment=${{ matrix.environment }} + - name: Stop local documentation site + if: always() + run: kill "$ARCHIVEBOX_DOCS_SERVER_PID" + + docs-freebsd: + name: docs/freebsd + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: vmactions/freebsd-vm@77ed28d336d03fe19a3f4f7266c1d2c4714dd79d # v1.5.2 + with: + usesh: true + prepare: pkg install -y py313-uv + run: uv run --no-project --with pytest --with pytest-codeblocks pytest -o addopts=--codeblocks -vv --tb=long README.md AGENTS.md skills $(uv run --no-project python -c 'from pathlib import Path; print(*(str(path) for path in sorted(Path("docs").rglob("*.md")) if not path.is_symlink()))') --docs-environment=freebsd + + docs-openbsd: + name: docs/openbsd + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: vmactions/openbsd-vm@c941015845c0f0c429676840963dc63b226d4f69 # v1.4.5 + with: + usesh: true + prepare: pkg_add uv + run: uv run --no-project --with pytest --with pytest-codeblocks pytest -o addopts=--codeblocks -vv --tb=long README.md AGENTS.md skills $(uv run --no-project python -c 'from pathlib import Path; print(*(str(path) for path in sorted(Path("docs").rglob("*.md")) if not path.is_symlink()))') --docs-environment=openbsd diff --git a/.github/workflows/duplicate-issue-detection.yml b/.github/workflows/duplicate-issue-detection.yml index 98dcd839..4d1df6c3 100644 --- a/.github/workflows/duplicate-issue-detection.yml +++ b/.github/workflows/duplicate-issue-detection.yml @@ -6,18 +6,55 @@ on: jobs: check-duplicates: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read issues: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 1 - - name: Install opencode - run: curl -fsSL https://opencode.ai/install | bash + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + enable-cache: false + + - name: Resolve OpenCode and workflow tools through abxpkg + env: + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + run: | + set -Eeuo pipefail + uv sync --locked --no-sources --no-install-project + OPENCODE_CONFIG="$(uv run --no-sync --no-sources python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "opencode" / "config.json")')" + mkdir -p "$ABXPKG_LIB_DIR/env/bin" + export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV" + echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH" + opencode_env="$( + uv run --no-sync --no-sources abxpkg env \ + --install \ + --json \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:github_binaries" \ + --deps-from="$OPENCODE_CONFIG:required_binaries" + )" + jq_binary="$ABXPKG_LIB_DIR/env/bin/jq" + "$jq_binary" -r 'to_entries[] | "\(.key)=\(.value)"' \ + <<< "$opencode_env" >> "$GITHUB_ENV" + + opencode_binary="$ABXPKG_LIB_DIR/env/bin/opencode" + echo "OPENCODE_BINARY=$opencode_binary" >> "$GITHUB_ENV" + test -L "$opencode_binary" + test -x "$opencode_binary" - name: Check for duplicate issues env: @@ -33,7 +70,7 @@ jobs: "webfetch": "allow" } run: | - opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created: + "$OPENCODE_BINARY" run -m anthropic/claude-haiku-4-5 "A new issue has been created: Issue number: ${{ github.event.issue.number }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b3d64672..c8d94af9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,13 +1,7 @@ name: Run linters on: - workflow_dispatch: - push: - branches: ['**'] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + workflow_call: env: UV_NO_SOURCES: "1" @@ -17,11 +11,23 @@ jobs: lint: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: submodules: true fetch-depth: 1 + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + architecture: x64 + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + enable-cache: false + - name: Clone abxpkg run: bash .github/scripts/clone_abx_repo.sh abxpkg @@ -31,18 +37,6 @@ jobs: - name: Clone abx-dl run: bash .github/scripts/clone_abx_repo.sh abx-dl - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.13" - architecture: x64 - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - version: "0.10.6" - enable-cache: false - - name: Install dependencies with uv run: | uv venv diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index cb1e02a1..06e81610 100755 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -1,66 +1,134 @@ name: Build Pip package on: - workflow_dispatch: - push: - branches: - - '**' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + workflow_call: env: - PYTHON_VERSION: "3.13" + UV_VERSION: "0.11.3" jobs: build: + name: build distributions runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - name: Install uv - uses: astral-sh/setup-uv@v6 + - name: Set up Python 3.13 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - version: "0.10.6" - enable-cache: false - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: "3.13" architecture: x64 - - name: APT install archivebox dev + run dependencies - uses: awalsh128/cache-apt-pkgs-action@latest + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 with: - packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps - version: 1.0 + version: ${{ env.UV_VERSION }} + enable-cache: false - - name: UV install archivebox dev + run sub-dependencies - env: - UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv - run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache - - - name: UV build archivebox and archivebox/pkgs/* packages - run: | - uv build --all - - - name: UV install archivebox and archivebox/pkgs/* locally for tests - env: - UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv - run: | - uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache - uv pip install --python .venv/bin/python --no-deps dist/archivebox-*.whl - - - name: Verify built package full install - env: - UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv + - name: Build every distribution run: | set -Eeuo pipefail - DATA_DIR="$(mktemp -d)" - cd "$DATA_DIR" - uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox init - timeout 30m uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox install - uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox version - uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox status + uv build --all-packages --no-sources --out-dir dist --clear + + shopt -s nullglob + artifacts=(dist/*) + wheels=(dist/archivebox-*.whl) + sdists=(dist/archivebox-*.tar.gz) + [[ "${#artifacts[@]}" -gt 0 ]] + [[ "${#wheels[@]}" -eq 1 ]] + [[ "${#sdists[@]}" -eq 1 ]] + printf '%s\n' "$GITHUB_SHA" > dist/COMMIT_SHA + uv run --no-project python - <<'PY' + import hashlib + from pathlib import Path + + artifacts = sorted(Path("dist").iterdir()) + checksum_lines = [] + for artifact in artifacts: + if artifact.name == "SHA256SUMS": + continue + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + line = f"{digest} {artifact.name}" + checksum_lines.append(line) + print(line) + Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n") + PY + + - name: Upload every distribution + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: python-distributions + path: dist/* + if-no-files-found: error + retention-days: 7 + + install-smoke: + name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }} + needs: build + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + python: ["3.13", "3.14"] + artifact: [wheel, sdist] + + steps: + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ matrix.python }} + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: ${{ env.UV_VERSION }} + enable-cache: false + + - name: Download distributions + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: python-distributions + path: ${{ runner.temp }}/python-distributions + + - name: Install and smoke-test the built ${{ matrix.artifact }} + shell: bash + run: | + set -Eeuo pipefail + shopt -s nullglob + + case "${{ matrix.artifact }}" in + wheel) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) ;; + sdist) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.tar.gz) ;; + *) exit 2 ;; + esac + [[ "${#candidates[@]}" -eq 1 ]] + artifact="${candidates[0]}" + + smoke_root="$(mktemp -d "$RUNNER_TEMP/archivebox-package-smoke.XXXXXX")" + trap 'rm -rf "$smoke_root"' EXIT + smoke_env="$smoke_root/env" + data_dir="$smoke_root/data" + mkdir -p "$data_dir" + + uv venv --python "${{ matrix.python }}" "$smoke_env" + uv pip install --python "$smoke_env" --no-cache "$artifact" + + unset PYTHONPATH + cd "$data_dir" + VIRTUAL_ENV="$smoke_env" uv run --active --no-project --no-sync python - <<'PY' + import os + from pathlib import Path + + import archivebox + + installed_path = Path(archivebox.__file__).resolve() + environment_path = Path(os.environ["VIRTUAL_ENV"]).resolve() + workspace_path = Path(os.environ["GITHUB_WORKSPACE"]).resolve() + assert installed_path.is_relative_to(environment_path), (installed_path, environment_path) + assert not installed_path.is_relative_to(workspace_path), (installed_path, workspace_path) + print(installed_path) + PY + VIRTUAL_ENV="$smoke_env" uv run --active --no-project --no-sync archivebox version + VIRTUAL_ENV="$smoke_env" uv run --active --no-project --no-sync archivebox init + VIRTUAL_ENV="$smoke_env" uv run --active --no-project --no-sync archivebox status diff --git a/.github/workflows/release-runner.yml b/.github/workflows/release-runner.yml deleted file mode 100644 index 2f43db98..00000000 --- a/.github/workflows/release-runner.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Release State - -on: - push: - branches: - - dev - workflow_dispatch: - -permissions: - contents: write - id-token: write - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -jobs: - release-state: - if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'release:') - runs-on: ubuntu-latest - environment: pypi - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: true - ref: ${{ github.ref_name }} - token: ${{ secrets.RELEASE_GH_TOKEN || github.token }} - - - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - - uses: astral-sh/setup-uv@v6 - with: - version: "0.10.6" - enable-cache: false - - - uses: actions/setup-node@v4 - with: - node-version: 22 - - - name: Configure git identity - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - - name: Run release script - env: - DEFAULT_BRANCH: dev - GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }} - run: ./bin/release.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..bd70ab0a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,243 @@ +name: Release + +on: + workflow_run: + workflows: [CI] + types: [completed] + branches: [dev] + +permissions: + actions: read + contents: write + id-token: write + packages: write + +concurrency: + group: release-dev + cancel-in-progress: false + +env: + DOCKERHUB_IMAGE: archivebox/archivebox + GHCR_IMAGE: ghcr.io/archivebox/archivebox + RELEASE_BRANCH: dev + RELEASE_SHA: ${{ github.event.workflow_run.head_sha }} + CI_RUN_ID: ${{ github.event.workflow_run.id }} + +jobs: + python-release: + name: Publish tested Python artifacts + if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' + runs-on: ubuntu-24.04 + environment: pypi + outputs: + version: ${{ steps.version.outputs.version }} + tag: ${{ steps.version.outputs.tag }} + steps: + - name: Checkout the tested commit + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + submodules: true + ref: ${{ env.RELEASE_SHA }} + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + enable-cache: false + + - name: Resolve release binaries through abxpkg + env: + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + shell: bash + run: | + set -Eeuo pipefail + ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')" + mkdir -p "$ABXPKG_LIB_DIR/env/bin" + uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env --install \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null + for name in uv gh git jq curl; do + test -L "$ABXPKG_LIB_DIR/env/bin/$name" + test -x "$ABXPKG_LIB_DIR/env/bin/$name" + done + { + echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv" + echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh" + echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git" + echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq" + echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl" + } >> "$GITHUB_ENV" + + - name: Download exact tested distributions + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: python-distributions + path: ${{ runner.temp }}/python-distributions + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ env.CI_RUN_ID }} + + - name: Record release version + id: version + shell: bash + run: | + set -Eeuo pipefail + VERSION="$($UV_BINARY run --no-project python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "tag=v$VERSION" >> "$GITHUB_OUTPUT" + + - name: Publish the exact tested distributions + env: + GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }} + RELEASE_DISTRIBUTIONS_DIR: ${{ runner.temp }}/python-distributions + run: ./bin/release.sh + + docker-release: + name: Publish tested multiarch image + if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' + needs: python-release + runs-on: ubuntu-24.04 + steps: + - name: Checkout the tested commit + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 1 + ref: ${{ env.RELEASE_SHA }} + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + enable-cache: false + + - name: Resolve release and Docker binaries through abxpkg + env: + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + shell: bash + run: | + set -Eeuo pipefail + ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')" + mkdir -p "$ABXPKG_LIB_DIR/env/bin" + uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env --install \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null + for name in uv git jq docker; do + test -L "$ABXPKG_LIB_DIR/env/bin/$name" + test -x "$ABXPKG_LIB_DIR/env/bin/$name" + done + { + echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv" + echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git" + echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq" + echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker" + } >> "$GITHUB_ENV" + + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + with: + install: true + + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Download exact tested image digests + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + path: /tmp/digests + pattern: digest-* + merge-multiple: true + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ env.CI_RUN_ID }} + + - name: Verify digest set and collect tags + id: docker_meta + shell: bash + run: | + set -Eeuo pipefail + mapfile -t DIGESTS < <($UV_BINARY run --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))') + [[ "${#DIGESTS[@]}" -eq 2 ]] + for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done + + VERSION='${{ needs.python-release.outputs.version }}' + SHORT_SHA="${RELEASE_SHA::12}" + TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}^{}")" + TAG_TARGET="${TAG_TARGET%%[[:space:]]*}" + if [[ -z "$TAG_TARGET" ]]; then + TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")" + TAG_TARGET="${TAG_TARGET%%[[:space:]]*}" + fi + { + echo 'dockerhub_tags<> "$GITHUB_OUTPUT" + + - name: Create Docker Hub manifest + shell: bash + run: | + set -Eeuo pipefail + mapfile -t DIGESTS < <($UV_BINARY run --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))') + TAG_ARGS=() + while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}' + REFS=() + for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done + $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}" + + - name: Create GHCR manifest + shell: bash + run: | + set -Eeuo pipefail + mapfile -t DIGESTS < <($UV_BINARY run --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))') + TAG_ARGS=() + while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}' + REFS=() + for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done + $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}" + + - name: Inspect published images + shell: bash + run: | + set -Eeuo pipefail + while IFS= read -r tag; do [[ -n "$tag" ]] && $DOCKER_BINARY buildx imagetools inspect "$tag"; done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}' + while IFS= read -r tag; do [[ -n "$tag" ]] && $DOCKER_BINARY buildx imagetools inspect "$tag"; done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}' + + - name: Update Docker Hub README + uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: archivebox/archivebox + + downstream-packages: + name: Dispatch downstream packages + needs: + - python-release + - docker-release + uses: ./.github/workflows/update-homebrew-tap.yml + with: + release_tag: ${{ needs.python-release.outputs.tag }} + release_sha: ${{ github.event.workflow_run.head_sha }} + secrets: inherit diff --git a/.github/workflows/test-parallel.yml b/.github/workflows/test-parallel.yml index 7939cfb6..40eb9d98 100644 --- a/.github/workflows/test-parallel.yml +++ b/.github/workflows/test-parallel.yml @@ -1,22 +1,15 @@ name: Parallel Tests on: - workflow_dispatch: - pull_request: - branches: [dev, main, master] - push: - branches: [dev] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + workflow_call: env: PYTHONIOENCODING: utf-8 PYTHONLEGACYWINDOWSSTDIO: utf-8 USE_COLOR: False UV_NO_SOURCES: "1" - ABXPKG_MIN_RELEASE_AGE: "0" + CI_PYTHON_VERSION: "3.13.14" + UV_CACHE_DIR: ${{ github.workspace }}/.uv-cache jobs: discover-tests: @@ -27,84 +20,150 @@ jobs: plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: submodules: true fetch-depth: 1 + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ env.CI_PYTHON_VERSION }} + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + enable-cache: false + + - name: Resolve matrix tools through abxpkg + env: + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + shell: bash + run: | + set -Eeuo pipefail + ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')" + test -n "$ABXPKG_VERSION" + mkdir -p "$ABXPKG_LIB_DIR/env/bin" + uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \ + --install \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \ + >/dev/null + JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq" + PYTHON_BINARY="$ABXPKG_LIB_DIR/env/bin/python" + test -L "$JQ_BINARY" + test -x "$JQ_BINARY" + test -L "$PYTHON_BINARY" + test -x "$PYTHON_BINARY" + { + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" + echo "JQ_BINARY=$JQ_BINARY" + echo "PYTHON_BINARY=$PYTHON_BINARY" + } >> "$GITHUB_ENV" + echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH" + - name: Discover test files id: set-matrix + shell: bash run: | - # Find all main test files - all_tests=$(find archivebox/tests -maxdepth 1 -name "test_*.py" -type f | sort) - - # Create JSON array with test file info - json_array="[" - first=true - for test_file in $all_tests; do - if [ "$first" = true ]; then - first=false - else - json_array+="," - fi - - # Extract a display name for the test - name="main/$(basename $test_file .py | sed 's/^test_//')" - needs_chromium=false - if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then - needs_chromium=true - fi - needs_sonic=false - if grep -Eiq "shutil\\.which\\([\"']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic" "$test_file"; then - needs_sonic=true - fi - - json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}" - done - json_array+="]" - - echo "test-files=$json_array" >> $GITHUB_OUTPUT - echo "Found $(echo $all_tests | wc -w) test files" - echo "$json_array" | jq '.' + set -Eeuo pipefail + mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py archivebox) + test "${#output[@]}" -eq 2 + json_array="${output[1]}" + echo "test-files=$json_array" >> "$GITHUB_OUTPUT" + echo "${output[0]}" + echo "$json_array" | "$JQ_BINARY" '.' - name: Clone abx-plugins run: bash .github/scripts/clone_abx_repo.sh abx-plugins - name: Discover plugin tests id: set-plugin-matrix + shell: bash run: | - plugin_tests=$(find abx-plugins/abx_plugins/plugins -maxdepth 2 -type d -name tests | sed 's#abx-plugins/abx_plugins/plugins/##; s#/tests##' | sort) + set -Eeuo pipefail + mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py plugins) + test "${#output[@]}" -eq 2 + json_array="${output[1]}" + echo "plugin-tests=$json_array" >> "$GITHUB_OUTPUT" + echo "${output[0]}" + echo "$json_array" | "$JQ_BINARY" '.' - json_array="[" - first=true - for plugin_name in $plugin_tests; do - if [ "$first" = true ]; then - first=false - else - json_array+="," - fi + prepare-python-dependencies: + name: Prepare Python 3.13.14 dependency cache + runs-on: ubuntu-24.04 + env: + PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl + LIB_DIR: /tmp/abx-lib + ABXPKG_LIB_DIR: /tmp/abx-lib - needs_chromium=false - if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then - needs_chromium=true - fi - needs_sonic=false - if [ "$plugin_name" = "search_backend_sonic" ]; then - needs_sonic=true - fi + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + submodules: true + fetch-depth: 1 - json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}" - done - json_array+="]" + - name: Set up Python ${{ env.CI_PYTHON_VERSION }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ env.CI_PYTHON_VERSION }} + architecture: x64 - echo "plugin-tests=$json_array" >> $GITHUB_OUTPUT - echo "Found $(echo $plugin_tests | wc -w) plugin test suites" - echo "$json_array" | jq '.' + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + enable-cache: false + + - name: Clone abxpkg + run: bash .github/scripts/clone_abx_repo.sh abxpkg + + - name: Clone abx-plugins + run: bash .github/scripts/clone_abx_repo.sh abx-plugins + + - name: Clone abx-dl + run: bash .github/scripts/clone_abx_repo.sh abx-dl + + - name: Restore or create the SHA-specific uv cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ${{ env.UV_CACHE_DIR }} + key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }} + + - name: Bootstrap local abxpkg + run: | + set -Eeuo pipefail + uv venv + uv pip install -e ./abxpkg + + - name: Resolve Linux build dependencies through abxpkg + run: | + set -Eeuo pipefail + mkdir -p "$LIB_DIR/env/bin" + echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH" + uv run --no-sync --no-sources abxpkg env \ + --install \ + --no-cache \ + --json \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries" + uv run --no-sync --no-sources abxpkg env \ + --install \ + --json \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-env.json:required_binaries" + + - name: Populate the full editable dependency cache + run: | + set -Eeuo pipefail + uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]" + uv run --no-sync --no-sources python -c 'import ldap; print(ldap.__version__)' run-tests: name: ${{ matrix.test.name }} runs-on: ubuntu-24.04 - needs: discover-tests + needs: [discover-tests, prepare-python-dependencies] env: PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl CHROME_HEADLESS: "true" @@ -118,11 +177,23 @@ jobs: python: ["3.13"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: submodules: true fetch-depth: 1 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ matrix.python }} + architecture: x64 + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + enable-cache: false + - name: Clone abxpkg run: bash .github/scripts/clone_abx_repo.sh abxpkg @@ -132,56 +203,76 @@ jobs: - name: Clone abx-dl run: bash .github/scripts/clone_abx_repo.sh abx-dl - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - architecture: x64 - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - version: "0.10.6" - - - name: Set up Node JS - uses: actions/setup-node@v4 - with: - node-version: 22 - - name: Cache uv - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: - path: ~/.cache/uv - key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.python }}-uv- - - - uses: awalsh128/cache-apt-pkgs-action@latest - with: - packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps - version: 1.1 + path: ${{ env.UV_CACHE_DIR }} + key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }} + fail-on-cache-miss: true - name: Install dependencies with uv run: | + set -Eeuo pipefail uv venv - uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]" + uv pip install --offline --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]" + + - name: Resolve Node.js through abxpkg + env: + LIB_DIR: ${{ runner.temp }}/abx-lib + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + run: | + set -Eeuo pipefail + mkdir -p "$LIB_DIR/env/bin" + export PATH="$LIB_DIR/env/bin:$PATH" + { + echo "LIB_DIR=$LIB_DIR" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" + } >> "$GITHUB_ENV" + echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH" + + tooling_env="$( + uv run --no-sync --no-sources abxpkg env \ + --install \ + --json \ + --lib="$LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries" + )" + JQ_BINARY="$LIB_DIR/env/bin/jq" + "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + <<< "$tooling_env" >> "$GITHUB_ENV" + + node_binary="$LIB_DIR/env/bin/node" + { + echo "JQ_BINARY=$JQ_BINARY" + echo "NODE_BINARY=$node_binary" + } >> "$GITHUB_ENV" + test -L "$JQ_BINARY" + test -x "$JQ_BINARY" + test -L "$node_binary" + test -x "$node_binary" + "$node_binary" --version - name: Resolve core test binaries through abxpkg env: LIB_DIR: ${{ runner.temp }}/abx-lib ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib - ABXPKG_INSTALL_TIMEOUT: "900" run: | set -euo pipefail - mkdir -p "$LIB_DIR" - echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV" - echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV" + mkdir -p "$LIB_DIR/env/bin" + { + echo "LIB_DIR=$LIB_DIR" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" + } >> "$GITHUB_ENV" + echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH" uv run --no-sync --no-sources abxpkg env \ --install \ + --json \ --lib="$LIB_DIR" \ --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/wget/config.json:required_binaries" \ --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/git/config.json:required_binaries" \ --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_ripgrep/config.json:required_binaries" \ + | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ >> "$GITHUB_ENV" - name: Resolve Chrome through abxpkg @@ -189,17 +280,20 @@ jobs: env: LIB_DIR: ${{ runner.temp }}/abx-lib ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib - ABXPKG_INSTALL_TIMEOUT: "900" run: | set -euo pipefail mkdir -p "$LIB_DIR" - echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV" - echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV" + { + echo "LIB_DIR=$LIB_DIR" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" + } >> "$GITHUB_ENV" uv run --no-sync --no-sources abxpkg env \ --install \ + --json \ --lib="$LIB_DIR" \ --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \ + | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ >> "$GITHUB_ENV" - name: Resolve Sonic through abxpkg @@ -207,28 +301,36 @@ jobs: env: LIB_DIR: ${{ runner.temp }}/abx-lib ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib - ABXPKG_INSTALL_TIMEOUT: "900" run: | set -euo pipefail mkdir -p "$LIB_DIR" - echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV" - echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV" + { + echo "LIB_DIR=$LIB_DIR" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" + } >> "$GITHUB_ENV" uv run --no-sync --no-sources abxpkg env \ --install \ + --json \ --lib="$LIB_DIR" \ --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \ + | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ >> "$GITHUB_ENV" - name: Run test - ${{ matrix.test.name }} + env: + TEST_PATHS_JSON: ${{ toJson(matrix.test.paths) }} run: | + set -Eeuo pipefail + mapfile -t test_paths < <("$JQ_BINARY" -r '.[]' <<<"$TEST_PATHS_JSON") + [[ "${#test_paths[@]}" -gt 0 ]] mkdir -p tests/out - uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out + uv run --no-sync --no-sources pytest -vs "${test_paths[@]}" --basetemp=tests/out plugin-tests: name: ${{ matrix.plugin.name }} runs-on: ubuntu-24.04 - needs: discover-tests + needs: [discover-tests, prepare-python-dependencies] env: PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl CHROME_HEADLESS: "true" @@ -241,11 +343,23 @@ jobs: plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: submodules: true fetch-depth: 1 + - name: Set up Python ${{ env.CI_PYTHON_VERSION }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ env.CI_PYTHON_VERSION }} + architecture: x64 + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + enable-cache: false + - name: Clone abxpkg run: bash .github/scripts/clone_abx_repo.sh abxpkg @@ -255,54 +369,74 @@ jobs: - name: Clone abx-dl run: bash .github/scripts/clone_abx_repo.sh abx-dl - - name: Set up Python 3.13 - uses: actions/setup-python@v5 - with: - python-version: "3.13" - architecture: x64 - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - version: "0.10.6" - - - name: Set up Node JS - uses: actions/setup-node@v4 - with: - node-version: 22 - - name: Cache uv - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: - path: ~/.cache/uv - key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-3.13-uv- - - - uses: awalsh128/cache-apt-pkgs-action@latest - with: - packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps - version: 1.1 + path: ${{ env.UV_CACHE_DIR }} + key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }} + fail-on-cache-miss: true - name: Install dependencies with uv run: | + set -Eeuo pipefail uv venv - uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]" + uv pip install --offline --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]" + + - name: Resolve Node.js through abxpkg + env: + LIB_DIR: ${{ runner.temp }}/abx-lib + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + run: | + set -Eeuo pipefail + mkdir -p "$LIB_DIR/env/bin" + export PATH="$LIB_DIR/env/bin:$PATH" + { + echo "LIB_DIR=$LIB_DIR" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" + } >> "$GITHUB_ENV" + echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH" + + tooling_env="$( + uv run --no-sync --no-sources abxpkg env \ + --install \ + --json \ + --lib="$LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries" + )" + JQ_BINARY="$LIB_DIR/env/bin/jq" + "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + <<< "$tooling_env" >> "$GITHUB_ENV" + + node_binary="$LIB_DIR/env/bin/node" + { + echo "JQ_BINARY=$JQ_BINARY" + echo "NODE_BINARY=$node_binary" + } >> "$GITHUB_ENV" + test -L "$JQ_BINARY" + test -x "$JQ_BINARY" + test -L "$node_binary" + test -x "$node_binary" + "$node_binary" --version - name: Resolve plugin dependencies through abxpkg env: LIB_DIR: ${{ runner.temp }}/abx-lib ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib - ABXPKG_INSTALL_TIMEOUT: "900" run: | set -euo pipefail - mkdir -p "$LIB_DIR" - echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV" - echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV" + mkdir -p "$LIB_DIR/env/bin" + { + echo "LIB_DIR=$LIB_DIR" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" + } >> "$GITHUB_ENV" + echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH" uv run --no-sync --no-sources abxpkg env \ --install \ + --json \ --lib="$LIB_DIR" \ - --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/${{ matrix.plugin.plugin }}/config.json:required_binaries" \ + --deps-from="$GITHUB_WORKSPACE/${{ matrix.plugin.config_path }}:required_binaries" \ + | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ >> "$GITHUB_ENV" - name: Resolve Chrome through abxpkg @@ -310,17 +444,20 @@ jobs: env: LIB_DIR: ${{ runner.temp }}/abx-lib ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib - ABXPKG_INSTALL_TIMEOUT: "900" run: | set -euo pipefail mkdir -p "$LIB_DIR" - echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV" - echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV" + { + echo "LIB_DIR=$LIB_DIR" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" + } >> "$GITHUB_ENV" uv run --no-sync --no-sources abxpkg env \ --install \ + --json \ --lib="$LIB_DIR" \ --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \ + | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ >> "$GITHUB_ENV" - name: Resolve Sonic through abxpkg @@ -328,23 +465,37 @@ jobs: env: LIB_DIR: ${{ runner.temp }}/abx-lib ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib - ABXPKG_INSTALL_TIMEOUT: "900" run: | set -euo pipefail mkdir -p "$LIB_DIR" - echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV" - echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV" + { + echo "LIB_DIR=$LIB_DIR" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" + } >> "$GITHUB_ENV" uv run --no-sync --no-sources abxpkg env \ --install \ + --json \ --lib="$LIB_DIR" \ --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \ + | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ >> "$GITHUB_ENV" - name: Run plugin tests - ${{ matrix.plugin.name }} env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }} API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }} run: | - uv run --no-sync --no-sources bash ./bin/test_plugins.sh "${{ matrix.plugin.plugin }}" --no-coverage + set -Eeuo pipefail + test_path="$GITHUB_WORKSPACE/${{ matrix.plugin.test_path }}" + test -d "$test_path" + test_count="$("$LIB_DIR/env/bin/python" -c 'import pathlib, sys; print(sum(1 for _ in pathlib.Path(sys.argv[1]).rglob("test_*.py")))' "$test_path")" + test "$test_count" -gt 0 + DATA_DIR="$(mktemp -d -t archivebox_plugin_tests.XXXXXX)" + export DATA_DIR + plugin_tmpdir="$(mktemp -d -t archivebox_plugin_run.XXXXXX)" + cd "$plugin_tmpdir" + uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources python -m pytest \ + "$test_path" -p no:django -v --tb=short diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3de484de..70f1e2ef 100755 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,12 +1,7 @@ -name: Run tests +name: Integration Tests on: - push: - branches: ['**'] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + workflow_call: env: DOCKER_IMAGE: archivebox-ci @@ -23,16 +18,36 @@ jobs: PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl strategy: + fail-fast: false matrix: - os: [ubuntu-24.04] - python: ["3.13"] + include: + - os: ubuntu-24.04 + os_name: linux + python: "3.14" + - os: macos-15 + os_name: macOS + python: "3.13" + - os: macos-15 + os_name: macOS + python: "3.14" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: submodules: true fetch-depth: 1 + ### Set up Python + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ matrix.python }} + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.11.3" + - name: Clone abxpkg run: bash .github/scripts/clone_abx_repo.sh abxpkg @@ -42,121 +57,329 @@ jobs: - name: Clone abx-dl run: bash .github/scripts/clone_abx_repo.sh abx-dl - ### Setup Python & JS Languages - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - architecture: x64 - - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - version: "0.10.6" - - - name: Set up Node JS - uses: actions/setup-node@v4 - with: - node-version: 22 - - ### Install Python & JS Dependencies + ### Install dependencies - name: Cache uv - uses: actions/cache@v3 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: ~/.cache/uv key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }} restore-keys: | ${{ runner.os }}-${{ matrix.python }}-uv- - - uses: awalsh128/cache-apt-pkgs-action@latest - with: - packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps - version: 1.0 + - name: Bootstrap local abxpkg + run: | + set -Eeuo pipefail + uv venv + uv pip install -e ./abxpkg + + - name: Prepare abxpkg environment + env: + LIB_DIR: ${{ runner.temp }}/abx-lib + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + run: | + set -Eeuo pipefail + mkdir -p "$LIB_DIR/env/bin" + { + echo "LIB_DIR=$LIB_DIR" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" + } >> "$GITHUB_ENV" + echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH" + + - name: Resolve Node.js through abxpkg + env: + LIB_DIR: ${{ runner.temp }}/abx-lib + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + run: | + set -Eeuo pipefail + export PATH="$LIB_DIR/env/bin:$PATH" + tooling_env="$( + uv run --no-sync --no-sources abxpkg env \ + --install \ + --json \ + --lib="$LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries" + )" + JQ_BINARY="$LIB_DIR/env/bin/jq" + "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + <<< "$tooling_env" >> "$GITHUB_ENV" + + node_binary="$LIB_DIR/env/bin/node" + { + echo "JQ_BINARY=$JQ_BINARY" + echo "NODE_BINARY=$node_binary" + } >> "$GITHUB_ENV" + test -L "$node_binary" + test -x "$node_binary" + "$node_binary" --version + + - name: Resolve Linux build dependencies through abxpkg + if: runner.os == 'Linux' + env: + LIB_DIR: ${{ runner.temp }}/abx-lib + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + run: | + set -Eeuo pipefail + uv run --no-sync --no-sources abxpkg env \ + --install \ + --no-cache \ + --json \ + --lib="$LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries" + uv run --no-sync --no-sources abxpkg env \ + --install \ + --json \ + --lib="$LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-env.json:required_binaries" \ + | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + >> "$GITHUB_ENV" + + - name: Resolve host Homebrew through abxpkg + if: runner.os == 'macOS' + env: + LIB_DIR: ${{ runner.temp }}/abx-lib + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + run: | + set -Eeuo pipefail + brew_env="$( + uv run --no-sync --no-sources abxpkg env \ + --install \ + --json \ + --lib="$LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries" + )" + brew_binary="$("$JQ_BINARY" -r '.BREW_BINARY' <<< "$brew_env")" + test -L "$brew_binary" + echo "BREW_BINARY=$brew_binary" >> "$GITHUB_ENV" + + - name: Resolve macOS build dependencies through abxpkg + if: runner.os == 'macOS' + env: + LIB_DIR: ${{ runner.temp }}/abx-lib + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + run: | + set -Eeuo pipefail + brew_target="$(readlink "$BREW_BINARY")" + test -x "$brew_target" + brew_root="$(dirname "$(dirname "$brew_target")")" + export ABXPKG_BREW_ROOT="$brew_root" + + uv run --no-sync --no-sources abxpkg env \ + --install \ + --json \ + --lib="$LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-build.json:required_binaries" + + PATH="$brew_root/opt/openldap/bin:$PATH" \ + uv run --no-sync --no-sources abxpkg env \ + --install \ + --json \ + --lib="$LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-openldap-env.json:required_binaries" \ + | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + >> "$GITHUB_ENV" + + ldapvc_target="$(readlink "$LIB_DIR/env/bin/ldapvc")" + test -x "$ldapvc_target" + openldap_prefix="$(dirname "$(dirname "$ldapvc_target")")" + test -f "$openldap_prefix/include/ldap.h" + test -f "$openldap_prefix/lib/libldap.dylib" + { + echo "ABXPKG_BREW_ROOT=$brew_root" + echo "CPPFLAGS=-I$openldap_prefix/include${CPPFLAGS:+ $CPPFLAGS}" + echo "LDFLAGS=-L$openldap_prefix/lib${LDFLAGS:+ $LDFLAGS}" + echo "PKG_CONFIG_PATH=$openldap_prefix/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" + } >> "$GITHUB_ENV" - name: Install dependencies with uv run: | - uv venv uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]" - ### Run the tests - - name: Directory listing for debugging - run: | - pwd - ls - - name: ArchiveBox full install check run: | set -Eeuo pipefail DATA_DIR="$PWD/tests/out/data" mkdir -p "$DATA_DIR" uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox init - timeout 30m uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox install + uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox install uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox version uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status - - name: Test built package with pytest - if: github.ref != 'refs/heads/dev' + - name: Run consolidated core suite + if: matrix.os_name == 'macOS' || matrix.python == '3.14.6' run: | mkdir -p tests/out - uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out - - - name: Run documentation code blocks - run: uv run --no-sync --no-sources pytest -vv --tb=long README.md AGENTS.md skills docs - - - name: Run plugin tests - if: github.ref != 'refs/heads/dev' - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }} - API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }} - run: | - uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage + uv run --no-sync --no-sources pytest -q archivebox/tests --basetemp="tests/out/${{ matrix.os_name }}-python-${{ matrix.python }}" docker_tests: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: submodules: true fetch-depth: 1 + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + + - name: Resolve Docker and curl through abxpkg + env: + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + shell: bash + run: | + set -Eeuo pipefail + ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')" + test -n "$ABXPKG_VERSION" + mkdir -p "$ABXPKG_LIB_DIR/env/bin" + export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV" + echo "ABXPKG_VERSION=$ABXPKG_VERSION" >> "$GITHUB_ENV" + echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH" + + tooling_env="$( + uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \ + --install \ + --json \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries" + )" + JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq" + "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + <<< "$tooling_env" >> "$GITHUB_ENV" + + for binary in "$ABXPKG_LIB_DIR/env/bin/docker" "$ABXPKG_LIB_DIR/env/bin/curl"; do + test -L "$binary" + test -x "$binary" + done + { + echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker" + echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl" + echo "JQ_BINARY=$JQ_BINARY" + } >> "$GITHUB_ENV" + - name: Build image run: | - docker build . -t "$DOCKER_IMAGE" + "$DOCKER_BINARY" build . -t "$DOCKER_IMAGE" + + - name: Resolve real mount test dependencies through abxpkg + run: | + set -Eeuo pipefail + mount_env="$( + uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \ + --install \ + --json \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries" + )" + "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \ + <<< "$mount_env" >> "$GITHUB_ENV" + + for binary_name in bash chmod exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do + binary="$ABXPKG_LIB_DIR/env/bin/$binary_name" + test -L "$binary" + test -x "$binary" + done + + - name: Validate Docker UID and GID behavior on real NFS and SMB mounts + shell: bash + run: | + set -Eeuo pipefail + NFS_SOURCE="/srv/archivebox-ci-nfs" + SMB_SOURCE="/srv/archivebox-ci-smb" + NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount" + SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount" + mounted_nfs=0 + mounted_smb=0 + + cleanup() { + local original_status=$? cleanup_status=0 + trap - EXIT + if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then + cleanup_status=1 + fi + if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then + cleanup_status=1 + fi + if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then + cleanup_status=1 + fi + if [[ "$original_status" != "0" ]]; then + exit "$original_status" + fi + exit "$cleanup_status" + } + trap cleanup EXIT + + "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT" + "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT" + "$SUDO_BINARY" "$BASH_BINARY" -c \ + "printf '%s *(rw,sync,no_subtree_check,no_root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports" + "$SUDO_BINARY" "$BASH_BINARY" -c \ + "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf" + + "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service + "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra + "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service + "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service + "$SYSTEMCTL_BINARY" is-active --quiet smbd.service + + "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT" + mounted_nfs=1 + "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \ + -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix" + mounted_smb=1 + + NFS_TEST_DIR="$NFS_MOUNT" \ + SMB_TEST_DIR="$SMB_MOUNT" \ + IMAGE="$DOCKER_IMAGE" \ + "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only - name: Init data dir run: | - mkdir "${{ github.workspace }}/data" - docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init + "$MKDIR_BINARY" "${{ github.workspace }}/data" + "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init - name: Run test server run: | - sudo bash -c 'echo "127.0.0.1 www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts' - docker run --name www-nginx -p 80:80 -d nginx + "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1 www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts' + "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e - name: Add link run: | - docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local + "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local - name: Add stdin link run: | - echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add + echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add - name: List links run: | - docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; } - docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; } + archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$DOCKER_IMAGE" list)" + [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; } + [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; } - name: Start docker-compose stack run: | - docker compose run --rm archivebox init - docker compose up -d --wait - curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox' - curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django' + "$DOCKER_BINARY" compose run --rm archivebox init + "$DOCKER_BINARY" compose up -d --wait + index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')" + admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js')" + [[ "$index_html" == *ArchiveBox* ]] + [[ "$admin_js" == *window.django* ]] - name: Check added urls show up in index run: | - docker compose run --rm archivebox add --plugins=parse_txt_urls 'http://example.com/#test_docker' - curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker' - docker compose down + "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://example.com/#test_docker' + index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')" + [[ "$index_html" == *'http://example.com/#test_docker'* ]] + "$DOCKER_BINARY" compose down diff --git a/.github/workflows/update-homebrew-tap.yml b/.github/workflows/update-homebrew-tap.yml index 803368af..79a345ab 100644 --- a/.github/workflows/update-homebrew-tap.yml +++ b/.github/workflows/update-homebrew-tap.yml @@ -1,37 +1,130 @@ name: Update downstream packages on: - push: - tags: ['v*'] + workflow_call: + inputs: + release_tag: + required: true + type: string + release_sha: + required: true + type: string workflow_dispatch: + inputs: + release_tag: + description: Exact ArchiveBox release tag + required: true + type: string + release_sha: + description: Full commit SHA owned by the release tag + required: true + type: string permissions: + actions: read contents: read concurrency: - group: update-downstream-packages-${{ github.ref }} - cancel-in-progress: true + group: update-downstream-packages-${{ inputs.release_tag || github.ref_name }} + cancel-in-progress: false jobs: dispatch: - runs-on: ubuntu-latest - + runs-on: ubuntu-24.04 steps: - - name: Dispatch downstream package updates + - name: Checkout release source + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ inputs.release_sha || github.sha }} + fetch-depth: 1 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + with: + version: "0.11.3" + + - name: Resolve release and Docker binaries through abxpkg + env: + ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib + shell: bash + run: | + set -Eeuo pipefail + ABXPKG_VERSION="$(uv run --no-project python - <<'PY' + import tomllib + from pathlib import Path + + packages = tomllib.loads(Path("uv.lock").read_text())["package"] + matches = [package["version"] for package in packages if package["name"] == "abxpkg"] + if len(matches) != 1: + raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}") + print(matches[0]) + PY + )" + test -n "$ABXPKG_VERSION" + mkdir -p "$ABXPKG_LIB_DIR/env/bin" + echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV" + + uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \ + --install \ + --lib="$ABXPKG_LIB_DIR" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \ + --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" \ + >/dev/null + + for binary_name in uv gh git jq curl docker; do + binary="$ABXPKG_LIB_DIR/env/bin/$binary_name" + test -L "$binary" + test -x "$binary" + done + { + echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv" + echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh" + echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git" + echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq" + echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl" + echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker" + } >> "$GITHUB_ENV" + + - name: Verify the exact release and dispatch downstream builds env: DOWNSTREAM_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || secrets.HOMEBREW_TAP_TOKEN }} + GH_TOKEN: ${{ github.token }} + INPUT_RELEASE_TAG: ${{ inputs.release_tag }} + INPUT_RELEASE_SHA: ${{ inputs.release_sha }} + shell: bash run: | - if [ -z "$DOWNSTREAM_TOKEN" ]; then - echo "RELEASE_GH_TOKEN/HOMEBREW_TAP_TOKEN is not configured; downstream schedules will update packages." - exit 0 - fi + set -Eeuo pipefail + [[ -n "$DOWNSTREAM_TOKEN" ]] || { echo "RELEASE_GH_TOKEN/HOMEBREW_TAP_TOKEN is not configured" >&2; exit 1; } + RELEASE_TAG="${INPUT_RELEASE_TAG:-${GITHUB_REF_NAME}}" + [[ "$RELEASE_TAG" == v* ]] + VERSION="${RELEASE_TAG#v}" + + TAG_TARGET="$("$GIT_BINARY" ls-remote "https://github.com/${GITHUB_REPOSITORY}.git" "refs/tags/${RELEASE_TAG}^{}" | "$UV_BINARY" run --no-project python -c 'import sys; print(next((line.split()[0] for line in sys.stdin if line.strip()), ""))')" + if [[ -z "$TAG_TARGET" ]]; then + TAG_TARGET="$("$GIT_BINARY" ls-remote "https://github.com/${GITHUB_REPOSITORY}.git" "refs/tags/${RELEASE_TAG}" | "$UV_BINARY" run --no-project python -c 'import sys; print(next((line.split()[0] for line in sys.stdin if line.strip()), ""))')" + fi + RELEASE_SHA="${INPUT_RELEASE_SHA:-${TAG_TARGET}}" + [[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] || { echo "${RELEASE_TAG} points to ${TAG_TARGET}, not ${RELEASE_SHA}" >&2; exit 1; } + + "$CURL_BINARY" -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" >/dev/null + "$DOCKER_BINARY" manifest inspect "archivebox/archivebox:${VERSION}" >/dev/null + "$DOCKER_BINARY" manifest inspect "ghcr.io/archivebox/archivebox:${VERSION}" >/dev/null + + PAYLOAD_FILTER="{event_type: \"archivebox-dev-updated\", client_payload: {ref: \$ref, sha: \$sha, version: \$version}}" + PAYLOAD="$("$JQ_BINARY" -nc --arg ref "$RELEASE_TAG" --arg sha "$RELEASE_SHA" --arg version "$VERSION" "$PAYLOAD_FILTER")" for repo in homebrew-archivebox debian-archivebox; do - curl --fail-with-body \ + "$CURL_BINARY" --fail-with-body \ -X POST \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer ${DOWNSTREAM_TOKEN}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/ArchiveBox/${repo}/dispatches" \ - -d '{"event_type":"archivebox-dev-updated","client_payload":{"ref":"dev"}}' + --data "$PAYLOAD" done diff --git a/AGENTS.md b/AGENTS.md index b38e54f4..3936b3d9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,19 +18,19 @@ ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev ## Development Setup ```bash -uv sync --dev --all-extras -mkdir -p data -cd data -uv run --project .. archivebox init --install +project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}" +archivebox_data="$(mktemp -d)" +uv sync --project "$project_dir" --dev --all-extras +cd "$archivebox_data" && uv run --project "$project_dir" --no-sync archivebox init --install ``` Run collection commands from inside an initialized data directory: ```bash -cd data -uv run --project .. archivebox status -uv run --project .. archivebox add 'https://example.com' -uv run --project .. archivebox run +project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}" +archivebox_data="$(mktemp -d)" +cd "$archivebox_data" +uv run --project "$project_dir" --no-sync archivebox init --install && uv run --project "$project_dir" --no-sync archivebox status && uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}" && uv run --project "$project_dir" --no-sync archivebox run ``` ## User-Facing Setup @@ -38,12 +38,12 @@ uv run --project .. archivebox run Recommended CLI install: ```bash -uv tool install --force . +project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}" +tool_root="$(mktemp -d)"; export UV_TOOL_DIR="$tool_root/tools" UV_TOOL_BIN_DIR="$tool_root/bin" +uv tool install --force "$project_dir" export PLUGINS=parse_txt_urls archivebox_data="$(mktemp -d)" -cd "$archivebox_data" -archivebox init --install -archivebox add --plugins=parse_txt_urls 'https://example.com' +cd "$archivebox_data" && "$UV_TOOL_BIN_DIR/archivebox" init --install && "$UV_TOOL_BIN_DIR/archivebox" add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}" ``` Alternative install methods: @@ -57,16 +57,16 @@ Alternative install methods: ```bash -archivebox version -archivebox help -archivebox status -archivebox install -archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage' -archivebox list --json --with-headers -archivebox search 'example' -archivebox update --filter-type=domain example.com -archivebox remove --filter-type=exact 'https://example.com' -archivebox run +project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}" +archivebox_data="$(mktemp -d)" +cd "$archivebox_data" && uv run --project "$project_dir" --no-sync archivebox init --install +uv run --project "$project_dir" --no-sync archivebox version && uv run --project "$project_dir" --no-sync archivebox help && uv run --project "$project_dir" --no-sync archivebox status +uv run --project "$project_dir" --no-sync archivebox install +uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/docs-basic-usage}" +uv run --project "$project_dir" --no-sync archivebox list --json --with-headers +uv run --project "$project_dir" --no-sync archivebox search 'example' && uv run --project "$project_dir" --no-sync archivebox update --filter-type=domain example.com +uv run --project "$project_dir" --no-sync archivebox remove --yes --delete --filter-type=exact "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/docs-basic-usage}" +uv run --project "$project_dir" --no-sync archivebox run ``` ## Verification @@ -74,8 +74,9 @@ archivebox run Use targeted tests for focused work: ```bash -uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q -uv run prek run --all-files +project_dir="${ARCHIVEBOX_PROJECT_DIR:-.}" +uv run --project "$project_dir" --no-sync pytest "$project_dir/archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options" -q +(cd "$project_dir" && uv run --no-sync prek run --all-files) ``` Use the full release/deploy loop only when requested: diff --git a/Dockerfile b/Dockerfile index c49f7e66..f43062dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,10 @@ # --build-context abx-plugins=../abx-plugins \ # -t archivebox/abx-dl:dev # docker buildx build . -f Dockerfile \ -# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \ +# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.11.263 \ # -t archivebox:multistage -ARG ABX_DL_IMAGE=archivebox/abx-dl:latest +ARG ABX_DL_IMAGE=archivebox/abx-dl:1.11.263 FROM archivebox/sonic:1.4.9 AS sonic FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base @@ -66,9 +66,6 @@ ENV TMP_DIR=/tmp/archivebox \ ENV HOME=/home/archivebox \ XDG_CONFIG_HOME=/home/archivebox/.config \ XDG_CACHE_HOME=/opt/archivebox/lib/cache \ - ABXPKG_INSTALL_TIMEOUT=600 \ - ABXPKG_POSTINSTALL_SCRIPTS=True \ - ABXPKG_MIN_RELEASE_AGE=0 \ TIMEOUT=600 ENV UV_COMPILE_BYTECODE=false \ diff --git a/README.md b/README.md index 00b95245..705c62b4 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,8 @@
- -     - -

@@ -171,7 +165,7 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur # Quickstart -**đŸ–Ĩ  [Supported OSs](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#supported-systems):** Linux/BSD, macOS, Windows (Docker)   **👾  CPUs:** `amd64` (`x86_64`), `arm64`, `arm7` (raspi>=3)
+**đŸ–Ĩ  [Supported OSs](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#supported-systems):** Linux/BSD, macOS, Windows (Docker)   **👾  CPUs:** `amd64` (`x86_64`), `arm64`, `arm7`

@@ -1291,7 +1285,6 @@ ArchiveBox is neither the highest fidelity nor the simplest tool available for s
- ## Internet Archiving Ecosystem diff --git a/archivebox/__init__.py b/archivebox/__init__.py index d8d69109..898e7932 100755 --- a/archivebox/__init__.py +++ b/archivebox/__init__.py @@ -38,11 +38,6 @@ ASCII_LOGO = """ PACKAGE_DIR = Path(__file__).resolve().parent -# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models -# # Migrations reference models like 'machine.Binary' which need to be importable -# if str(PACKAGE_DIR) not in sys.path: -# sys.path.append(str(PACKAGE_DIR)) - os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings" os.environ["TZ"] = "UTC" diff --git a/archivebox/api/urls.py b/archivebox/api/urls.py index ced60588..0d460098 100644 --- a/archivebox/api/urls.py +++ b/archivebox/api/urls.py @@ -23,7 +23,4 @@ urlpatterns = [ path("v1/", RedirectView.as_view(url="/api/v1/docs")), path("v1/", v1_api_urls), path("v1", RedirectView.as_view(url="/api/v1/docs")), - # ... v2 can be added here ... - # path("v2/", v2_api_urls), - # path("v2", RedirectView.as_view(url='/api/v2/docs')), ] diff --git a/archivebox/api/v1_api.py b/archivebox/api/v1_api.py index a6784e6d..8925b4e3 100644 --- a/archivebox/api/v1_api.py +++ b/archivebox/api/v1_api.py @@ -78,9 +78,6 @@ class NinjaAPIWithIOCapture(NinjaAPI): response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None" response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None" - # import ipdb; ipdb.set_trace() - # print('RESPONDING NOW', response) - return response @@ -91,8 +88,6 @@ api = NinjaAPIWithIOCapture( auth=API_AUTH_METHODS, urls_namespace="api-1", docs=Swagger(settings={"persistAuthorization": True}), - # docs_decorator=login_required, - # renderer=ORJSONRenderer(), ) api = register_urls(api) urls = api.urls @@ -113,24 +108,7 @@ def generic_exception_handler(request, err): "message": f"{err.__class__.__name__}: {err}", "errors": [ "".join(format_exception(err)), - # or send simpler parent-only traceback: - # *([str(err.__context__)] if getattr(err, '__context__', None) else []), ], }, status=status, ) - - -# import orjson -# from ninja.renderers import BaseRenderer -# class ORJSONRenderer(BaseRenderer): -# media_type = "application/json" -# def render(self, request, data, *, response_status): -# return { -# "success": True, -# "errors": [], -# "result": data, -# "stdout": ansi_to_html(stdout.getvalue().strip()), -# "stderr": ansi_to_html(stderr.getvalue().strip()), -# } -# return orjson.dumps(data) diff --git a/archivebox/api/v1_cli.py b/archivebox/api/v1_cli.py index 7f898164..297838d1 100644 --- a/archivebox/api/v1_cli.py +++ b/archivebox/api/v1_cli.py @@ -15,8 +15,6 @@ from archivebox.misc.util import ansi_to_html from archivebox.core.models import SnapshotQuerySet -# from .auth import API_AUTH_METHODS - # router for API that exposes archivebox cli subcommands as REST endpoints router = Router(tags=["ArchiveBox CLI Sub-Commands"]) diff --git a/archivebox/api/v1_crawls.py b/archivebox/api/v1_crawls.py index e0bcbcf9..5f31d3b5 100644 --- a/archivebox/api/v1_crawls.py +++ b/archivebox/api/v1_crawls.py @@ -49,8 +49,6 @@ class CrawlSchema(Schema): tags_str: str config: dict - # snapshots: List[SnapshotSchema] - @staticmethod def resolve_created_by_id(obj): return str(obj.created_by_id) diff --git a/archivebox/base_models/apps.py b/archivebox/base_models/apps.py index 82bd72f8..e69de29b 100644 --- a/archivebox/base_models/apps.py +++ b/archivebox/base_models/apps.py @@ -1,7 +0,0 @@ -# from django.apps import AppConfig - - -# class BaseModelsConfig(AppConfig): -# default_auto_field = 'django.db.models.BigAutoField' - -# name = 'base_models' diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index 031af38f..12624d2e 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -128,16 +128,12 @@ class ArchiveBoxGroup(click.Group): import_path = cmd_name_or_path modname, funcname = import_path.rsplit(".", 1) - # print(f'LAZY LOADING {import_path}') mod = import_module(modname) func = vars(mod)[funcname] if func.__doc__ is None: raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method") - # if not isinstance(cmd, click.BaseCommand): - # raise ValueError(f'lazy loading of {import_path} failed - not a click command') - return func @@ -158,7 +154,6 @@ def cli(ctx, help=False): # then we need to set up the django environment and check that we're in a valid data folder wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:]) if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands): - # print('SETUP DJANGO AND CHECK DATA FOLDER') try: if subcommand == "server": run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes") diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index b6fe5817..9e0ea425 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -216,7 +216,7 @@ def add( ) from archivebox.services.runner import ensure_background_runner - ensure_background_runner(allow_under_pytest=True) + ensure_background_runner() else: # Foreground mode: run full crawl runner until all work is done print("[green]\\[*] Starting crawl runner to process crawl...[/green]") diff --git a/archivebox/cli/archivebox_config.py b/archivebox/cli/archivebox_config.py index 5dc50528..54317816 100644 --- a/archivebox/cli/archivebox_config.py +++ b/archivebox/cli/archivebox_config.py @@ -59,28 +59,20 @@ def config( matching_config = {} if search: if config_options: - config_options = [ - core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options - ] - matching_config = {key: readable_config[key] for key in config_options if key in readable_config} - for config_section in CONFIGS.values(): - aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias} + search_terms = [key.strip().lower() for key in config_options] - for search_key in config_options: - # search all aliases in the section - for alias_key, key in aliases.items(): - if key in readable_config and search_key.lower() in alias_key.lower(): - matching_config[key] = dict(config_section)[key] + for existing_key, value in readable_config.items(): + if any(term in existing_key.lower() or term in str(value).lower() for term in search_terms): + matching_config[existing_key] = value - # search all keys and values in the section - for existing_key, value in dict(config_section).items(): - if existing_key in readable_config and ( - search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower() - ): - matching_config[existing_key] = value - for existing_key, value in readonly_config.items(): - if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower(): - matching_config[existing_key] = value + for alias, key in core_config_aliases.items(): + if key in readable_config and any(term in alias.lower() for term in search_terms): + matching_config[key] = readable_config[key] + + for schema in plugin_schemas.values(): + for key, metadata in schema.items(): + if key in readable_config and any(term in key.lower() or term in str(metadata).lower() for term in search_terms): + matching_config[key] = readable_config[key] print(printable_config(matching_config)) raise SystemExit(not matching_config) diff --git a/archivebox/cli/archivebox_init.py b/archivebox/cli/archivebox_init.py index 4826cf43..f20d3d16 100755 --- a/archivebox/cli/archivebox_init.py +++ b/archivebox/cli/archivebox_init.py @@ -33,10 +33,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non config = get_config() - # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK): - # print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr) - # print("[red] You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr) - is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR) existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE) if is_empty and not existing_index: @@ -111,9 +107,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non print() print(f" √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}") - # from django.contrib.auth.models import User - # call_command("createsuperuser", interactive=True) - print() print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]") diff --git a/archivebox/cli/archivebox_oneshot.py b/archivebox/cli/archivebox_oneshot.py index 560ba7e8..419f2aca 100644 --- a/archivebox/cli/archivebox_oneshot.py +++ b/archivebox/cli/archivebox_oneshot.py @@ -4,11 +4,13 @@ __package__ = "archivebox.cli" __command__ = "archivebox oneshot" import subprocess +import sys from pathlib import Path import rich_click as click from archivebox.config import CONSTANTS +from archivebox.config.common import get_config @click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True)) @@ -20,7 +22,23 @@ def main(args: tuple[str, ...] = ()) -> None: raise click.ClickException( "Refusing to run `archivebox oneshot` inside an ArchiveBox DATA_DIR. Use `archivebox add` here, or run oneshot from another directory.", ) - raise SystemExit(subprocess.run(["abx-dl", *args]).returncode) + abxpkg_binary = Path(sys.executable).with_name("abxpkg") + if not abxpkg_binary.is_file(): + raise click.ClickException(f"abxpkg executable is missing from the ArchiveBox environment: {abxpkg_binary}") + abxpkg_lib_dir = get_config(include_machine=False).ABXPKG_LIB_DIR + raise SystemExit( + subprocess.run( + [ + str(abxpkg_binary), + f"--lib={abxpkg_lib_dir}", + "--binproviders=env", + "--install", + "run", + "abx-dl", + *args, + ], + ).returncode, + ) if __name__ == "__main__": diff --git a/archivebox/cli/archivebox_remove.py b/archivebox/cli/archivebox_remove.py index bd38f8e1..8942e0c9 100644 --- a/archivebox/cli/archivebox_remove.py +++ b/archivebox/cli/archivebox_remove.py @@ -9,7 +9,6 @@ from collections.abc import Iterable import rich_click as click -from django.db import OperationalError from django.db.models import QuerySet from archivebox.config import CONSTANTS @@ -92,12 +91,6 @@ def remove( # in ``base_models/models.py`` and runs AFTER its row's tx commits — so # rmtree doesn't hold the lock either. # - # The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock - # errors when called inside an atomic block (because it can't safely - # release+reacquire a transaction), so we wrap each row's delete in our - # own retry loop at this outer (non-atomic) level. Each attempt is a - # fresh atomic; an exception cleanly rolls it back before we sleep. - retry_interval = 1.0 deleted_snapshot_pks = [] timed_out = False timeout_error = "" @@ -106,23 +99,9 @@ def remove( timed_out = True timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining." break - while True: - try: - deleted_count, _ = Snapshot.objects.filter(pk=pk).delete() - if deleted_count: - deleted_snapshot_pks.append(pk) - break - except OperationalError as err: - if "database is locked" not in str(err): - raise - remaining_time = deadline - time.monotonic() if deadline is not None else None - if remaining_time is not None and remaining_time <= 0: - timed_out = True - timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock." - break - time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval) - if timed_out: - break + deleted_count, _ = Snapshot.objects.filter(pk=pk).delete() + if deleted_count: + deleted_snapshot_pks.append(pk) all_snapshots = Snapshot.objects.all() remaining_count = all_snapshots.count() diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index 16c64d22..db8255b0 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -267,11 +267,13 @@ def server( host, port = _parse_and_validate_bind_spec(bind_spec) if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1": + from archivebox.workers.supervisord_util import resolve_env_binary + log_path = CONSTANTS.LOGS_DIR / "server.log" log_path.parent.mkdir(parents=True, exist_ok=True) daemon_env = os.environ.copy() daemon_env["ARCHIVEBOX_SERVER_DAEMON_CHILD"] = "1" - daemon_cmd = [sys.executable, "-m", "archivebox", "server"] + daemon_cmd = [str(resolve_env_binary("archivebox")), "server"] if debug: daemon_cmd.append("--debug") if reload: diff --git a/archivebox/cli/archivebox_snapshot.py b/archivebox/cli/archivebox_snapshot.py index c1509e77..49321f09 100644 --- a/archivebox/cli/archivebox_snapshot.py +++ b/archivebox/cli/archivebox_snapshot.py @@ -31,14 +31,62 @@ __package__ = "archivebox.cli" __command__ = "archivebox snapshot" import sys -from collections.abc import Iterable +from collections.abc import Iterable, Iterator +from itertools import islice import rich_click as click from rich import print as rprint from django.db.models import QuerySet SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp") -SNAPSHOT_LIST_CHUNK_SIZE = 100 +SNAPSHOT_LIST_CHUNK_SIZE = 5000 + + +def iter_snapshot_json(queryset: QuerySet) -> Iterator[dict[str, object]]: + from archivebox.config import VERSION + from archivebox.core.models import SnapshotTag + + fields = ( + "id", + "crawl_id", + "url", + "title", + "bookmarked_at", + "created_at", + "timestamp", + "depth", + "status", + "fs_version", + "output_size", + ) + rows = queryset.values(*fields).iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE) + while batch := list(islice(rows, SNAPSHOT_LIST_CHUNK_SIZE)): + tags_by_snapshot = {row["id"]: [] for row in batch} + tag_rows = ( + SnapshotTag.objects.filter(snapshot_id__in=tags_by_snapshot).order_by("tag__name").values_list("snapshot_id", "tag__name") + ) + for snapshot_id, tag_name in tag_rows: + tags_by_snapshot[snapshot_id].append(tag_name) + + for row in batch: + archive_size = int(row["output_size"] or 0) + yield { + "type": "Snapshot", + "schema_version": VERSION, + "id": str(row["id"]), + "crawl_id": str(row["crawl_id"]), + "url": row["url"], + "title": row["title"], + "tags": ",".join(sorted(tags_by_snapshot[row["id"]])), + "bookmarked_at": row["bookmarked_at"].isoformat() if row["bookmarked_at"] else None, + "created_at": row["created_at"].isoformat() if row["created_at"] else None, + "timestamp": row["timestamp"], + "depth": row["depth"], + "status": row["status"], + "fs_version": row["fs_version"], + "archive_size": archive_size, + "output_size": archive_size, + } # ============================================================================= @@ -296,8 +344,8 @@ def list_snapshots( return 0 if not is_tty: - for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE): - write_record(snapshot.to_json()) + for snapshot_json in iter_snapshot_json(queryset): + write_record(snapshot_json) count += 1 rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr) return 0 diff --git a/archivebox/config/constants.py b/archivebox/config/constants.py index 53f3a780..3ba2bb3d 100644 --- a/archivebox/config/constants.py +++ b/archivebox/config/constants.py @@ -293,5 +293,3 @@ CONSTANTS = ConstantsDict CONSTANTS_CONFIG = AttrDict({key: value for key, value in CONSTANTS.__dict__.items() if key.isupper() and not key.startswith("_")}) # add all key: values to globals() for easier importing, e.g.: -# from archivebox.config.constants import IS_ROOT, PERSONAS_DIR, ... -# globals().update(CONSTANTS) diff --git a/archivebox/config/django.py b/archivebox/config/django.py index 538a9e7d..9f326522 100644 --- a/archivebox/config/django.py +++ b/archivebox/config/django.py @@ -2,7 +2,6 @@ __package__ = "archivebox.config" import os import sys -import subprocess from datetime import datetime, timezone @@ -37,7 +36,6 @@ def setup_django(check_db=False, in_memory_db=False) -> None: global DJANGO_SET_UP if DJANGO_SET_UP: - # raise Exception('django is already set up!') # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes return @@ -59,10 +57,10 @@ def setup_django(check_db=False, in_memory_db=False) -> None: with SudoPermission(uid=0): # running as root is a special case where it's ok to be a bit slower # make sure data dir is always owned by the correct user - subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(CONSTANTS.DATA_DIR)], stderr=subprocess.DEVNULL) + os.chown(CONSTANTS.DATA_DIR, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP) if CONSTANTS.DATA_DIR.exists(): for child in CONSTANTS.DATA_DIR.iterdir(): - subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(child)], stderr=subprocess.DEVNULL) + os.chown(child, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP) # Suppress the "database access during app initialization" warning # This warning can be triggered during django.setup() but is safe to ignore @@ -156,17 +154,6 @@ def setup_django(check_db=False, in_memory_db=False) -> None: f"No database file {sql_index_path} found in: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)" ) - # https://docs.pydantic.dev/logfire/integrations/django/ Logfire Debugging - # if settings.DEBUG_LOGFIRE: - # from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor - # SQLite3Instrumentor().instrument() - - # import logfire - - # logfire.configure() - # logfire.instrument_django(is_sql_commentor_enabled=True) - # logfire.info(f'Started ArchiveBox v{CONSTANTS.VERSION}', argv=sys.argv) - except KeyboardInterrupt: raise diff --git a/archivebox/config/paths.py b/archivebox/config/paths.py index be256688..036cc550 100644 --- a/archivebox/config/paths.py +++ b/archivebox/config/paths.py @@ -5,7 +5,6 @@ import socket import hashlib import tempfile import platform -import subprocess from pathlib import Path from functools import cache from datetime import datetime @@ -69,9 +68,9 @@ def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str: if IS_ROOT: with SudoPermission(uid=0): if ARCHIVEBOX_USER == 0: - subprocess.run(["chmod", "777", str(collection_id_file)]) + collection_id_file.chmod(0o777) else: - subprocess.run(["chown", str(ARCHIVEBOX_USER), str(collection_id_file)]) + os.chown(collection_id_file, ARCHIVEBOX_USER, -1) except (OSError, FileNotFoundError, PermissionError): pass return collection_id @@ -129,7 +128,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No if chown: # try fixing it using sudo permissions with SudoPermission(uid=uid, fallback=fallback): - subprocess.run(["chown", f"{uid}:{gid}", str(dir_path)], stderr=subprocess.DEVNULL) + os.chown(dir_path, uid, gid) return dir_is_writable(dir_path, uid=uid, gid=gid, fallback=fallback, chown=False) return False @@ -348,11 +347,6 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs and os.access(tmp_dir, os.W_OK) and tmp_dir_socket_path_is_short_enough(tmp_dir), }, - # "CACHE_DIR": { - # "path": CACHE_DIR.resolve(), - # "enabled": True, - # "is_valid": os.access(CACHE_DIR, os.R_OK) and os.access(CACHE_DIR, os.W_OK), # read + write - # }, }, ) diff --git a/archivebox/config/permissions.py b/archivebox/config/permissions.py index 8868a812..0a9edd48 100644 --- a/archivebox/config/permissions.py +++ b/archivebox/config/permissions.py @@ -37,8 +37,6 @@ HOSTNAME: str = cast(str, max([socket.gethostname(), platform.node()], key=len)) IS_ROOT = RUNNING_AS_UID == 0 IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes") -# IN_DOCKER_COMPOSE = # TODO: figure out a way to detect if running in docker compose - FALLBACK_UID = RUNNING_AS_UID or SUDO_UID FALLBACK_GID = RUNNING_AS_GID or SUDO_GID diff --git a/archivebox/config/version.py b/archivebox/config/version.py index eeb526cd..55aca3c2 100644 --- a/archivebox/config/version.py +++ b/archivebox/config/version.py @@ -36,7 +36,6 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR): # building docs, pyproject.toml is not available pass - # raise Exception('Failed to detect installed archivebox version!') return "dev" @@ -115,53 +114,4 @@ def get_BUILD_TIME() -> str: return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime("%Y-%m-%d %H:%M:%S %s") -# def get_versions_available_on_github(config): -# """ -# returns a dictionary containing the ArchiveBox GitHub release info for -# the recommended upgrade version and the currently installed version -# """ - -# # we only want to perform the (relatively expensive) check for new versions -# # when its most relevant, e.g. when the user runs a long-running command -# subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help' -# long_running_commands = ('add', 'schedule', 'update', 'status', 'server') -# if subcommand_run_by_user not in long_running_commands: -# return None - -# github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases" -# response = requests.get(github_releases_api) -# if response.status_code != 200: -# stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config) -# return None -# all_releases = response.json() - -# installed_version = parse_version_string(config['VERSION']) - -# # find current version or nearest older version (to link to) -# current_version = None -# for idx, release in enumerate(all_releases): -# release_version = parse_version_string(release['tag_name']) -# if release_version <= installed_version: -# current_version = release -# break - -# current_version = current_version or all_releases[-1] - -# # recommended version is whatever comes after current_version in the release list -# # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest) -# try: -# recommended_version = all_releases[idx+1] -# except IndexError: -# recommended_version = None - -# return {'recommended_version': recommended_version, 'current_version': current_version} - -# def can_upgrade(config): -# if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']: -# recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name']) -# current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name']) -# return recommended_version > current_version -# return False - - VERSION: str = detect_installed_version() diff --git a/archivebox/core/admin_archiveresults.py b/archivebox/core/admin_archiveresults.py index db569ceb..c4b153b6 100644 --- a/archivebox/core/admin_archiveresults.py +++ b/archivebox/core/admin_archiveresults.py @@ -328,15 +328,12 @@ class ArchiveResultInline(admin.TabularInline): name = "Archive Results Log" model = ArchiveResult parent_model = Snapshot - # fk_name = 'snapshot' extra = 0 sort_fields = ("end_ts", "plugin", "output_str", "status", "cmd_version") readonly_fields = ("id", "result_id", "completed", "command", "version") fields = ("start_ts", "end_ts", *readonly_fields, "plugin", "cmd", "cmd_version", "pwd", "status", "output_str") - # exclude = ('id',) ordering = ("end_ts",) show_change_link = True - # # classes = ['collapse'] def get_parent_object_from_request(self, request): resolved = resolve(request.path_info) @@ -371,7 +368,6 @@ class ArchiveResultInline(admin.TabularInline): base_fields = formset.form.base_fields snapshot_output_dir = str(snapshot.output_dir) if snapshot else "" - # import ipdb; ipdb.set_trace() # formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget() # default values for new entries @@ -817,8 +813,6 @@ class ArchiveResultAdmin(BaseModelAdmin): else: root_dir = str(snapshot_dir) - # print(root_dir, str(list(os.walk(root_dir)))) - for root, dirs, files in os.walk(root_dir): depth = root.replace(root_dir, "").count(os.sep) + 1 if depth > 2: diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index 693b8dcf..6abb2cee 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -43,7 +43,6 @@ from archivebox.core.permissions import ( from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget -# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False} GLOBAL_CONTEXT = {} SNAPSHOT_PERMISSION_META = PERMISSIONS_META @@ -72,13 +71,6 @@ class SnapshotActionForm(ActionForm): return [name.strip() for name in tags_str.split(",") if name.strip()] - # TODO: allow selecting actions for specific extractor plugins? is this useful? - # plugin = forms.ChoiceField( - # choices=ArchiveResult.PLUGIN_CHOICES, - # required=False, - # widget=forms.MultileChoiceField(attrs={'class': "form-control"}) - # ) - class TagNameListFilter(admin.SimpleListFilter): title = "By tag name" @@ -748,31 +740,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): html = Template("""{{bookmarked_date}} ({{timestamp}})""") return mark_safe(html.render(context)) - # pretty_time = obj.bookmarked.strftime('%Y-%m-%d %H:%M:%S') - # return f'{pretty_time} ({obj.timestamp})' - - # TODO: figure out a different way to do this, you cant nest forms so this doenst work - # def action(self, obj): - # # csrfmiddlewaretoken: Wa8UcQ4fD3FJibzxqHN3IYrrjLo4VguWynmbzzcPYoebfVUnDovon7GEMYFRgsh0 - # # action: update_snapshots - # # select_across: 0 - # # _selected_action: 76d29b26-2a88-439e-877c-a7cca1b72bb3 - # return format_html( - # ''' - #
- # - # - # - # - # - # - # - #
- # ''', - # csrf.get_token(self.request), - # obj.pk, - # ) - @admin.display(description="") def admin_actions(self, obj): summary_url = self.get_snapshot_view_url(obj) @@ -1149,9 +1116,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): mark_safe("".join(output)), ) - @admin.display( - # ordering='archiveresult_count' - ) + @admin.display() def size(self, obj): request = self.request config = request.archivebox_config @@ -1401,11 +1366,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): extra_context["snapshot_is_grid_view"] = True return self.changelist_view(request, extra_context=extra_context) - # for debugging, uncomment this to print all requests: - # def changelist_view(self, request, extra_context=None): - # print('[*] Got request', request.method, request.POST) - # return super().changelist_view(request, extra_context=None) - @admin.action( description="🔁 Redo Failed", ) diff --git a/archivebox/core/asgi.py b/archivebox/core/asgi.py index 7f79ffd4..2d52aee9 100644 --- a/archivebox/core/asgi.py +++ b/archivebox/core/asgi.py @@ -73,16 +73,3 @@ _patch_thread_sensitive_context_shutdown() # Standard Django ASGI application (no websockets/channels needed) application = get_asgi_application() - -# If websocket support is needed later, install channels and use: -# from channels.routing import ProtocolTypeRouter, URLRouter -# from channels.auth import AuthMiddlewareStack -# from channels.security.websocket import AllowedHostsOriginValidator -# from archivebox.core.routing import websocket_urlpatterns -# -# application = ProtocolTypeRouter({ -# "http": get_asgi_application(), -# "websocket": AllowedHostsOriginValidator( -# AuthMiddlewareStack(URLRouter(websocket_urlpatterns)) -# ), -# }) diff --git a/archivebox/core/middleware.py b/archivebox/core/middleware.py index 6a99cea5..2ed6260f 100644 --- a/archivebox/core/middleware.py +++ b/archivebox/core/middleware.py @@ -73,7 +73,6 @@ def detect_timezone(request, activate: bool = True): tz = timezone.get_fixed_timezone(int(gmt_offset)) if activate: timezone.activate(tz) - # print('GMT_OFFSET', gmt_offset, tz) return tz @@ -153,7 +152,6 @@ def CacheControlMiddleware(get_response): request.archivebox_config = config policy = "private" if config.PERMISSIONS == "private" else "public" response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300" - # print('Set Cache-Control header to', response['Cache-Control']) return response return middleware diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 6585a026..04fd18e4 100755 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -1032,21 +1032,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW transaction.on_commit(lambda: self._cleanup_old_migration_dir(old_dir, new_dir)) delattr(self, "_pending_fs_migration_cleanup") - # if is_new: - # from archivebox.misc.logging_util import log_worker_event - # log_worker_event( - # worker_type='DB', - # event='Created Snapshot', - # indent_level=2, - # url=self.url, - # metadata={ - # 'id': str(self.id), - # 'crawl_id': str(self.crawl_id), - # 'depth': self.depth, - # 'status': self.status, - # }, - # ) - # ========================================================================= # Filesystem Migration Methods # ========================================================================= @@ -4063,21 +4048,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes): if is_new or update_fields is None or "status" in update_fields or "snapshot" in update_fields or "snapshot_id" in update_fields: transaction.on_commit(type(self).clear_majority_status_cache) - # if is_new: - # from archivebox.misc.logging_util import log_worker_event - # log_worker_event( - # worker_type='DB', - # event='Created ArchiveResult', - # indent_level=3, - # plugin=self.plugin, - # metadata={ - # 'id': str(self.id), - # 'snapshot_id': str(self.snapshot_id), - # 'snapshot_url': str(self.snapshot.url)[:64], - # 'status': self.status, - # }, - # ) - def delete(self, *args, **kwargs): snapshot_id = self.snapshot_id deleted = super().delete(*args, **kwargs) diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index d4e8f877..f3e0370f 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -18,7 +18,7 @@ from .settings_logging import SETTINGS_LOGGING IS_MIGRATING = "makemigrations" in sys.argv[:3] or "migrate" in sys.argv[:3] -IS_TESTING = "test" in sys.argv[:3] or "PYTEST_CURRENT_TEST" in os.environ +IS_TESTING = "test" in sys.argv[:3] IS_SHELL = "shell" in sys.argv[:3] or "shell_plus" in sys.argv[:3] IS_GETTING_VERSION_OR_HELP = "version" in sys.argv or "help" in sys.argv or "--version" in sys.argv or "--help" in sys.argv CONFIG = get_config() @@ -103,7 +103,6 @@ MIDDLEWARE = [ ### Authentication Settings ################################################################################ -# AUTH_USER_MODEL = 'auth.User' # cannot be easily changed unfortunately AUTHENTICATION_BACKENDS = [ "django.contrib.auth.backends.RemoteUserBackend", @@ -181,23 +180,11 @@ TEMPLATES_DIR_NAME = "templates" CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK) STATICFILES_DIRS = [ *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []), - # *[ - # str(plugin_dir / 'static') - # for plugin_dir in PLUGIN_DIRS.values() - # if (plugin_dir / 'static').is_dir() - # ], - # Additional static file dirs from plugins str(PACKAGE_DIR / TEMPLATES_DIR_NAME / "static"), ] TEMPLATE_DIRS = [ *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []), - # *[ - # str(plugin_dir / 'templates') - # for plugin_dir in PLUGIN_DIRS.values() - # if (plugin_dir / 'templates').is_dir() - # ], - # Additional template dirs from plugins str(PACKAGE_DIR / TEMPLATES_DIR_NAME / "core"), str(PACKAGE_DIR / TEMPLATES_DIR_NAME / "admin"), str(PACKAGE_DIR / TEMPLATES_DIR_NAME), @@ -225,10 +212,6 @@ TEMPLATES = [ ### External Service Settings ################################################################################ -# CACHE_DB_FILENAME = 'cache.sqlite3' -# CACHE_DB_PATH = CONSTANTS.CACHE_DIR / CACHE_DB_FILENAME -# CACHE_DB_TABLE = 'django_cache' - DATABASE_NAME = CONFIG.DATABASE_NAME SQLITE_JOURNAL_MODE = CONFIG.SQLITE_JOURNAL_MODE SQLITE_MMAP_SIZE = CONFIG.SQLITE_MMAP_SIZE @@ -267,14 +250,6 @@ DATABASES = { "NAME": DATABASE_NAME, **SQLITE_CONNECTION_OPTIONS, }, - # "filestore": { - # "NAME": CONSTANTS.FILESTORE_DATABASE_FILE, - # **SQLITE_CONNECTION_OPTIONS, - # }, - # 'cache': { - # 'NAME': CACHE_DB_PATH, - # **SQLITE_CONNECTION_OPTIONS, - # }, } MIGRATION_MODULES = {"signal_webhooks": None} @@ -285,43 +260,10 @@ MIGRATION_MODULES = {"signal_webhooks": None} DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" -# class FilestoreDBRouter: -# """ -# A router to store all the File models in the filestore.sqlite3 database. -# This data just mirrors what is in the file system, so we want to keep it in a separate database -# from the main index database to avoid contention. -# """ - -# route_app_labels = {"filestore"} -# db_name = "filestore" - -# def db_for_read(self, model, **hints): -# if model._meta.app_label in self.route_app_labels: -# return self.db_name -# return 'default' - -# def db_for_write(self, model, **hints): -# if model._meta.app_label in self.route_app_labels: -# return self.db_name -# return 'default' - -# def allow_relation(self, obj1, obj2, **hints): -# if obj1._meta.app_label in self.route_app_labels or obj2._meta.app_label in self.route_app_labels: -# return obj1._meta.app_label == obj2._meta.app_label -# return None - -# def allow_migrate(self, db, app_label, model_name=None, **hints): -# if app_label in self.route_app_labels: -# return db == self.db_name -# return db == "default" - DATABASE_ROUTERS = [] CACHES = { "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, - # 'sqlite': {'BACKEND': 'django.core.cache.backends.db.DatabaseCache', 'LOCATION': 'cache'}, - # 'dummy': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}, - # 'filebased': {"BACKEND": "django.core.cache.backends.filebased.FileBasedCache", "LOCATION": CACHE_DIR / 'cache_filebased'}, } EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" @@ -666,16 +608,3 @@ if DEBUG_REQUESTS_TRACKER: r"^SELECT .* FROM django_migrations WHERE app = 'auth'", ), } - -# # https://docs.pydantic.dev/logfire/integrations/django/ (similar to DataDog / NewRelic / etc.) -# DEBUG_LOGFIRE = False -# DEBUG_LOGFIRE = DEBUG_LOGFIRE and os.access(DATA_DIR / '.logfire', os.W_OK) and (DATA_DIR / '.logfire').is_dir() - - -# For usage with https://www.jetadmin.io/integrations/django -# INSTALLED_APPS += ['jet_django'] -# JET_PROJECT = 'archivebox' -# JET_TOKEN = 'some-api-token-here' - - -# import ipdb; ipdb.set_trace() diff --git a/archivebox/core/settings_logging.py b/archivebox/core/settings_logging.py index 4f591db8..100b3c20 100644 --- a/archivebox/core/settings_logging.py +++ b/archivebox/core/settings_logging.py @@ -130,8 +130,6 @@ if os.access(LOGS_DIR, os.W_OK) and LOGS_DIR.is_dir(): ERROR_LOG = LOGS_DIR / "errors.log" else: # historically too many edge cases here around creating log dir w/ correct permissions early on - # if there's an issue on startup, we trash the log and let user figure it out via stdout/stderr - # print(f'[!] WARNING: data/logs dir does not exist. Logging to temp file: {ERROR_LOG}') pass LOG_LEVEL_DATABASE = "WARNING" # change to DEBUG to log all SQL queries diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py index 2b51baad..1462b7dd 100644 --- a/archivebox/core/urls.py +++ b/archivebox/core/urls.py @@ -30,12 +30,6 @@ from archivebox.progressmonitor.views import live_progress_view from archivebox.search.views import public_snapshot_search_stream_view from abx_plugins.plugins.opencode.views import opencode_proxy_view - -# GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306 -# from archivebox.config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE -# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE} - - CONFIG = get_config() DEBUG = CONFIG.DEBUG or ("--debug" in sys.argv) @@ -86,7 +80,6 @@ urlpatterns = [ path("api/", include("archivebox.api.urls"), name="api"), path("health/", HealthCheckView.as_view(), name="healthcheck"), path("error/", lambda request: _raise_test_error(request)), - # path('jet_api/', include('jet_django.urls')), Enable to use https://www.jetadmin.io/integrations/django path("index.html", RedirectView.as_view(url="/")), path("", HomepageView.as_view(), name="Home"), ] @@ -101,36 +94,3 @@ if getattr(settings, "DEBUG_TOOLBAR", False): if getattr(settings, "DEBUG_REQUESTS_TRACKER", False) and find_spec("requests_tracker"): urlpatterns += [path("__requests_tracker__/", include("requests_tracker.urls"))] - - -# # Proposed FUTURE URLs spec -# path('', HomepageView) -# path('/add', AddView) -# path('/public', PublicIndexView) -# path('/snapshot/:slug', SnapshotView) - -# path('/admin', admin.site.urls) -# path('/accounts', django.contrib.auth.urls) - -# # Proposed REST API spec -# # :slugs can be uuid, short_uuid, or any of the unique index_fields -# path('api/v1/'), -# path('api/v1/core/' [GET]) -# path('api/v1/core/snapshot/', [GET, POST, PUT]), -# path('api/v1/core/snapshot/:slug', [GET, PATCH, DELETE]), -# path('api/v1/core/archiveresult', [GET, POST, PUT]), -# path('api/v1/core/archiveresult/:slug', [GET, PATCH, DELETE]), -# path('api/v1/core/tag/', [GET, POST, PUT]), -# path('api/v1/core/tag/:slug', [GET, PATCH, DELETE]), - -# path('api/v1/cli/', [GET]) -# path('api/v1/cli/{add,list,config,...}', [POST]), # pass query as kwargs directly to `run_subcommand` and return stdout, stderr, exitcode - -# path('api/v1/extractors/', [GET]) -# path('api/v1/extractors/:extractor/', [GET]), -# path('api/v1/extractors/:extractor/:func', [GET, POST]), # pass query as args directly to chosen function - -# future, just an idea: -# path('api/v1/scheduler/', [GET]) -# path('api/v1/scheduler/task/', [GET, POST, PUT]), -# path('api/v1/scheduler/task/:slug', [GET, PATCH, DELETE]), diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 8e3299c1..d9f4bfe9 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -1746,8 +1746,6 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext: find_config_default(key) or "See here...", ), ) - # rows['Documentation'].append(mark_safe(f'Wiki: {key}')) - # rows['Aliases'].append(', '.join(find_config_aliases(key))) section = "CONSTANT" for key in CONSTANTS_CONFIG.keys(): @@ -1763,8 +1761,6 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext: find_config_default(key) or "See here...", ), ) - # rows['Documentation'].append(mark_safe(f'Wiki: {key}')) - # rows['Aliases'].append('') return TableContext( title="Computed Configuration Values", @@ -1820,7 +1816,6 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont ((color, source, value) for source, value, color in sources_info), ) - # aliases = USER_CONFIG.get(key, {}).get("aliases", []) aliases = [] if key in CONSTANTS_CONFIG: diff --git a/archivebox/crawls/models.py b/archivebox/crawls/models.py index c8461f54..fe262135 100755 --- a/archivebox/crawls/models.py +++ b/archivebox/crawls/models.py @@ -209,7 +209,13 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith return resolve_delete_after_config_value(self.config) def pause(self, *, save: bool = True) -> bool: - return super().pause(save=save) + paused = super().pause(save=save) + if paused and save and self.pk: + from archivebox.core.models import Snapshot + + for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).iterator(): + snapshot.pause() + return paused def resume(self, *, when=None, save: bool = True) -> bool: resumed = super().resume(when=when, save=save) @@ -338,20 +344,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith removed_tag_names=removed_tag_names, ), ) - # if is_new: - # from archivebox.misc.logging_util import log_worker_event - # first_url = self.get_urls_list()[0] if self.get_urls_list() else '' - # log_worker_event( - # worker_type='DB', - # event='Created Crawl', - # indent_level=1, - # metadata={ - # 'id': str(self.id), - # 'first_url': first_url[:64], - # 'max_depth': self.max_depth, - # 'status': self.status, - # }, - # ) def update_child_snapshot_permissions(self, old_permissions: str | None, new_permissions: str | None) -> int: from archivebox.core.models import Snapshot diff --git a/archivebox/machine/detect.py b/archivebox/machine/detect.py index cf635e46..d73cf9a6 100644 --- a/archivebox/machine/detect.py +++ b/archivebox/machine/detect.py @@ -18,6 +18,24 @@ PACKAGE_DIR = Path(__file__).parent DATA_DIR = Path(os.getcwd()).resolve() +def _run_abxpkg_host_binary(name: str, *args: str) -> subprocess.CompletedProcess[str]: + from abxpkg import EnvProvider + from archivebox.config.constants import CONSTANTS + + provider = EnvProvider(install_root=CONSTANTS.DEFAULT_ABXPKG_LIB_DIR / "env", PATH=os.environ["PATH"]) + if name == "system_profiler": + provider = provider.get_provider_with_overrides( + overrides={name: {"version": platform.mac_ver()[0] or "0.0.0"}}, + ) + loaded = provider.load(name) + if loaded is None or loaded.loaded_abspath is None: + raise RuntimeError(f"abxpkg could not resolve {name}") + projection = Path(loaded.loaded_abspath) + if not projection.is_symlink() or not os.access(projection, os.X_OK): + raise RuntimeError(f"abxpkg did not project {name} into {projection}") + return subprocess.run([str(projection), *args], capture_output=True, text=True, check=True) + + def get_vm_info(): hw_in_docker = bool(os.getenv("IN_DOCKER", False) in ("1", "true", "True", "TRUE")) hw_in_vm = False @@ -49,7 +67,7 @@ def get_vm_info(): # Serial Number (system): M230YYTD77 # Hardware UUID: 39A12B50-1972-5910-8BEE-235AD20C8EE3 # ... - result = subprocess.run(["system_profiler", "SPHardwareDataType"], capture_output=True, text=True, check=True) + result = _run_abxpkg_host_binary("system_profiler", "SPHardwareDataType") for line in result.stdout.split("\n"): if "Model Name:" in line: hw_product = line.split(":", 1)[-1].strip() @@ -73,7 +91,7 @@ def get_vm_info(): # UUID: fb65f41c-ec24-4539-beaf-f941903bdb2c # ... # Family: DigitalOcean_Droplet - dmidecode = subprocess.run(["dmidecode", "-t", "system"], capture_output=True, text=True, check=True) + dmidecode = _run_abxpkg_host_binary("dmidecode", "-t", "system") for line in dmidecode.stdout.split("\n"): if "Manufacturer:" in line: hw_manufacturer = line.split(":", 1)[-1].strip() @@ -90,7 +108,7 @@ def get_vm_info(): # Check for QEMU explicitly in pmap output try: - result = subprocess.run(["pmap", "1"], capture_output=True, text=True, check=True) + result = _run_abxpkg_host_binary("pmap", "1") if "qemu" in result.stdout.lower(): hw_in_vm = True except Exception: @@ -174,7 +192,7 @@ def get_isp_info(ip=None): # Get system DNS resolver servers dns_server = None try: - result = subprocess.run(["dig", "example.com", "A"], capture_output=True, text=True, check=True).stdout + result = _run_abxpkg_host_binary("dig", "example.com", "A").stdout dns_server = result.split(";; SERVER: ", 1)[-1].split("\n")[0].split("#")[0].strip() except Exception: try: @@ -183,10 +201,6 @@ def get_isp_info(ip=None): dns_server = "127.0.0.1" print(f"[red]:warning: WARNING: Could not determine DNS server, using {dns_server}[/red]") - # Get DNS resolver's ISP name - # url = f'https://ipapi.co/{dns_server}/json/' - # dns_isp = json.loads(urllib.request.urlopen(url).read().decode()).get('org', 'Unknown') - return { "isp": isp, "city": city, @@ -225,7 +239,7 @@ def get_os_info() -> dict[str, Any]: os_release = "macOS " + platform.mac_ver()[0] else: try: - os_release = subprocess.run(["lsb_release", "-ds"], capture_output=True, text=True, check=True).stdout.strip() + os_release = _run_abxpkg_host_binary("lsb_release", "-ds").stdout.strip() except Exception: pass diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py index 253dfc2e..5d5b6ec2 100755 --- a/archivebox/machine/models.py +++ b/archivebox/machine/models.py @@ -400,9 +400,6 @@ class NetworkInterface(ModelWithHealthStats): city = models.CharField(max_length=63, default="", null=False) region = models.CharField(max_length=63, default="", null=False) country = models.CharField(max_length=63, default="", null=False) - # num_uses_failed = models.PositiveIntegerField(default=0) # from ModelWithHealthStats - # num_uses_succeeded = models.PositiveIntegerField(default=0) # from ModelWithHealthStats - objects = NetworkInterfaceManager() # pyright: ignore[reportIncompatibleVariableOverride] machine_id: uuid.UUID @@ -1493,15 +1490,11 @@ class Process(ModelWithDeleteAfter, models.Model): ppid = os.getppid() machine = machine or Machine.current() - # Debug logging - # print(f"DEBUG _find_parent_process: my_pid={os.getpid()}, ppid={ppid}", file=sys.stderr) - # Get parent process start time from OS try: os_parent = psutil.Process(ppid) os_parent_start = os_parent.create_time() except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): - # print(f"DEBUG _find_parent_process: Parent process {ppid} not accessible", file=sys.stderr) return None # Parent process doesn't exist # Find matching Process record @@ -1512,18 +1505,13 @@ class Process(ModelWithDeleteAfter, models.Model): started_at__gte=timezone.now() - PID_REUSE_WINDOW, ).order_by("-started_at") - # print(f"DEBUG _find_parent_process: Found {candidates.count()} candidates for ppid={ppid}", file=sys.stderr) - for candidate in candidates: if candidate.started_at: db_start_time = candidate.started_at.timestamp() time_diff = abs(db_start_time - os_parent_start) - # print(f"DEBUG _find_parent_process: Checking candidate id={candidate.id} time_diff={time_diff:.2f}s tolerance={START_TIME_TOLERANCE}s", file=sys.stderr) if time_diff < START_TIME_TOLERANCE: - # print(f"DEBUG _find_parent_process: MATCH! Returning parent id={candidate.id} pid={candidate.pid}", file=sys.stderr) return candidate - # print(f"DEBUG _find_parent_process: No matching parent found for ppid={ppid}", file=sys.stderr) return None # No matching ArchiveBox parent process @classmethod @@ -2142,12 +2130,8 @@ class Process(ModelWithDeleteAfter, models.Model): # Process exited - read output and copy to DB if self.stdout_file and self.stdout_file.exists(): self.stdout = self.stdout_file.read_text(errors="replace") - # TODO: Uncomment to cleanup (keeping for debugging for now) - # self.stdout_file.unlink(missing_ok=True) if self.stderr_file and self.stderr_file.exists(): self.stderr = self.stderr_file.read_text(errors="replace") - # TODO: Uncomment to cleanup (keeping for debugging for now) - # self.stderr_file.unlink(missing_ok=True) self.exit_code = self.exit_code if self.exit_code is not None else _default_exit_code_for_unowned_process(self.process_type) if self.exit_code == -1: @@ -2462,6 +2446,15 @@ class Process(ModelWithDeleteAfter, models.Model): if not chrome_utils.exists(): return 0 + node_binary = cast(BinaryManager, Binary.objects).get_valid_binary("node") + if node_binary is None or not node_binary.is_valid: + return 0 + from archivebox.config.common import get_config + + node_projection = get_config().ABXPKG_LIB_DIR / "env" / "bin" / "node" + if not node_projection.is_symlink() or not os.access(node_projection, os.X_OK): + return 0 + crawl_roots = [ crawls_dir for user_dir in CONSTANTS.USERS_DIR.iterdir() @@ -2476,7 +2469,7 @@ class Process(ModelWithDeleteAfter, models.Model): try: for crawl_root in crawl_roots: result = subprocess.run( - ["node", str(chrome_utils), "killZombieChrome", str(crawl_root)], + [str(node_projection), str(chrome_utils), "killZombieChrome", str(crawl_root)], capture_output=True, timeout=30, text=True, diff --git a/archivebox/misc/checks.py b/archivebox/misc/checks.py index b4a32f52..3234569e 100644 --- a/archivebox/misc/checks.py +++ b/archivebox/misc/checks.py @@ -85,7 +85,6 @@ def check_data_folder(config=None, **config_kwargs) -> None: create_and_chown_dir(CONSTANTS.USERS_DIR) create_and_chown_dir(CONSTANTS.PERSONAS_DIR / "Default") create_and_chown_dir(CONSTANTS.LOGS_DIR) - # create_and_chown_dir(CONSTANTS.CACHE_DIR) # Create /tmp and /lib dirs if they don't exist get_or_create_working_tmp_dir(autofix=True, quiet=False, config=config) @@ -182,18 +181,6 @@ def check_io_encoding(): print(' python3 -c "import sys; print(sys.stdout.encoding)" # should output UTF-8', file=sys.stderr) raise SystemExit(2) - # # hard errors: check python version - # if sys.version_info[:3] < (3, 10, 0): - # print('[red][X] Python version is not new enough: {sys.version} (>3.10 is required)[/red]', file=sys.stderr) - # print(' See https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#python for help upgrading your Python installation.', file=sys.stderr) - # raise SystemExit(2) - - # # hard errors: check django version - # if int(django.VERSION[0]) < 5: - # print('[red][X] Django version is not new enough: {django.VERSION[:3]} (>=5.0 is required)[/red]', file=sys.stderr) - # print(' Upgrade django using pip or your system package manager: pip3 install --upgrade django', file=sys.stderr) - # raise SystemExit(2) - def check_not_root(): from archivebox.config.permissions import IS_ROOT @@ -227,7 +214,6 @@ def check_data_dir_permissions(config=None, **config_kwargs): data_dir_uid, data_dir_gid = data_dir_stat.st_uid, data_dir_stat.st_gid data_owned_by_root = data_dir_uid == 0 - # data_owned_by_default_user = data_dir_uid == DEFAULT_UID or data_dir_gid == DEFAULT_GID data_owner_doesnt_match = (data_dir_uid != ARCHIVEBOX_USER and data_dir_gid != ARCHIVEBOX_GROUP) if not IS_ROOT else False data_not_writable = not (os.path.isdir(DATA_DIR) and os.access(DATA_DIR, os.W_OK)) if data_not_writable: diff --git a/archivebox/misc/logging_util.py b/archivebox/misc/logging_util.py index ac4b2718..a14ac3b5 100644 --- a/archivebox/misc/logging_util.py +++ b/archivebox/misc/logging_util.py @@ -132,10 +132,6 @@ def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI, co ), ) sys.stdout.flush() - # uncomment to have it disappear when it hits 100% instead of staying full red: - # time.sleep(0.5) - # sys.stdout.write('\r{}{}\r'.format((' ' * get_config().TERM_WIDTH), ANSI['reset'])) - # sys.stdout.flush() except (KeyboardInterrupt, BrokenPipeError): print() @@ -148,9 +144,6 @@ def log_cli_command(subcommand: str, subcommand_args: Iterable[str] = (), stdin: subcommand=subcommand, args=args, ) - # stderr() - # stderr('[bright_black] > {pwd}[/]'.format(pwd=pwd, **ANSI)) - # stderr() print(Panel(version_msg), file=sys.stderr) diff --git a/archivebox/misc/monkey_patches.py b/archivebox/misc/monkey_patches.py index a2f99e32..89d858ef 100644 --- a/archivebox/misc/monkey_patches.py +++ b/archivebox/misc/monkey_patches.py @@ -15,17 +15,6 @@ django_stubs_ext.monkeypatch() # monkey patch django timezone to add back utc (it was removed in Django 5.0) setattr(timezone, "utc", datetime.UTC) -# monkey patch django-signals-webhooks to change how it shows up in Admin UI -# from signal_webhooks.apps import DjangoSignalWebhooksConfig -# DjangoSignalWebhooksConfig.verbose_name = 'API' - - -# Rich traceback handler disabled - it adds frames/boxes that wrap weirdly in log files -# Standard Python tracebacks are used instead (full width, no frames) -# from rich.traceback import install -# install(show_locals=True, word_wrap=False, ...) - - # Hide site-packages/sonic/client.py:115: SyntaxWarning # https://github.com/xmonader/python-sonic-client/pull/18 warnings.filterwarnings("ignore", category=SyntaxWarning, module="sonic") diff --git a/archivebox/misc/system.py b/archivebox/misc/system.py index 13e937db..b02ec4c5 100644 --- a/archivebox/misc/system.py +++ b/archivebox/misc/system.py @@ -22,7 +22,6 @@ def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool mode = "wb+" if isinstance(contents, bytes) else "w" encoding = None if isinstance(contents, bytes) else "utf-8" # enforce utf-8 on all text writes - # print('\n> Atomic Write:', mode, path, len(contents), f'overwrite={overwrite}') try: with lib_atomic_write(path, mode=mode, overwrite=overwrite, encoding=encoding) as f: if isinstance(contents, dict): diff --git a/archivebox/misc/toml_util.py b/archivebox/misc/toml_util.py index 33daaa40..457db420 100644 --- a/archivebox/misc/toml_util.py +++ b/archivebox/misc/toml_util.py @@ -90,9 +90,6 @@ class JSONSchemaWithLambdas(GenerateJsonSchema): serialize_unknown=True, ) - # for computed_field properties render them like this instead: - # inspect.getsource(field.wrapped_property.fget).split('def ', 1)[-1].split('\n', 1)[-1].strip().strip('return '), - def better_toml_dump_str(val: Any) -> str: try: diff --git a/archivebox/personas/importers.py b/archivebox/personas/importers.py index b55a3d89..2b172493 100644 --- a/archivebox/personas/importers.py +++ b/archivebox/personas/importers.py @@ -634,7 +634,12 @@ def export_browser_state( return False, None, "abxpkg returned an invalid browser dependency environment." if not isinstance(resolved_env, dict): return False, None, "abxpkg returned an invalid browser dependency environment." + abxpkg_lib_dir = get_config().ABXPKG_LIB_DIR + node_projection = abxpkg_lib_dir / "env" / "bin" / "node" + if not node_projection.is_symlink() or not os.access(node_projection, os.X_OK): + return False, None, f"abxpkg did not resolve Node.js into {node_projection}." env.update({str(key): str(value) for key, value in resolved_env.items()}) + env["NODE_BINARY"] = str(node_projection) env["ARCHIVEBOX_ABX_PLUGINS_DIR"] = str(chrome_plugin_dir) if user_data_dir: @@ -683,7 +688,7 @@ def export_browser_state( try: result = subprocess.run( - ["node", str(state_script)], + [str(node_projection), str(state_script)], env=env, capture_output=True, text=True, diff --git a/archivebox/plugins/hooks.py b/archivebox/plugins/hooks.py index f75d5b68..eeeaa6d2 100644 --- a/archivebox/plugins/hooks.py +++ b/archivebox/plugins/hooks.py @@ -323,21 +323,8 @@ def run_hook( # If Process.current() fails (e.g., not in a worker context), leave parent as None pass - if not script.exists(): - # Create a failed Process record for hooks that don't exist - process = Process.objects.create( - machine=machine, - iface=iface, - parent=parent, - process_type=Process.TypeChoices.HOOK, - pwd=str(output_dir), - cmd=["echo", f"Hook script not found: {script}"], - timeout=timeout, - status=Process.StatusChoices.EXITED, - exit_code=1, - stderr=f"Hook script not found: {script}", - ) - return process + if not script.is_file(): + raise FileNotFoundError(f"Hook script not found: {script}") # Python hooks carry their runtime contract in the shebang # (usually `abxpkg run --script python3`), so execute them directly. @@ -345,11 +332,18 @@ def run_hook( # interpreter because those hooks do not need per-script Python env setup. ext = script.suffix.lower() if ext == ".sh": - cmd = ["bash", str(script)] + bash_projection = Path(hook_config["ABXPKG_LIB_DIR"]).expanduser() / "env" / "bin" / "bash" + if not bash_projection.is_symlink() or not os.access(bash_projection, os.X_OK): + raise RuntimeError(f"Bash must be resolved by abxpkg into {bash_projection}") + cmd = [str(bash_projection), str(script)] elif ext == ".py": cmd = [str(script)] elif ext == ".js": - cmd = ["node", str(script)] + node_projection = Path(hook_config["ABXPKG_LIB_DIR"]).expanduser() / "env" / "bin" / "node" + if not node_projection.is_symlink() or not os.access(node_projection, os.X_OK): + raise RuntimeError(f"Node.js must be resolved by abxpkg into {node_projection}") + hook_config["NODE_BINARY"] = str(node_projection) + cmd = [str(node_projection), str(script)] else: # Try to execute directly (assumes shebang) cmd = [str(script)] @@ -510,10 +504,12 @@ def extract_records_from_process(process: "Process") -> list[dict[str, Any]]: if not records: return [] - # Extract plugin metadata from process.pwd and process.cmd + # Extract plugin metadata from process.pwd and the shipped hook path in cmd. + # Python hooks execute directly through their shebang, while JS and shell + # hooks have an interpreter in cmd[0]. plugin_name = Path(process.pwd).name if process.pwd else "unknown" - hook_name = Path(process.cmd[1]).name if len(process.cmd) > 1 else "unknown" - plugin_hook = process.cmd[1] if len(process.cmd) > 1 else "" + plugin_hook = next((str(arg) for arg in process.cmd if Path(str(arg)).name.startswith("on_")), "") + hook_name = Path(plugin_hook).name if plugin_hook else "unknown" for record in records: # Add plugin metadata to record diff --git a/archivebox/progressmonitor/views.py b/archivebox/progressmonitor/views.py index daba7a2d..d1d4c0ef 100644 --- a/archivebox/progressmonitor/views.py +++ b/archivebox/progressmonitor/views.py @@ -133,9 +133,12 @@ def live_progress_view(request): def process_label(cmd: list[str] | None) -> tuple[str, str, str, str]: hook_path = "" if isinstance(cmd, list) and cmd: - first = cmd[0] - if isinstance(first, str): - hook_path = first + hook_path = next( + (arg for arg in cmd if isinstance(arg, str) and Path(arg).name.startswith("on_") and "__" in Path(arg).name), + "", + ) + if not hook_path and isinstance(cmd[0], str): + hook_path = cmd[0] if not hook_path: return ("", "setup", "unknown", "") diff --git a/archivebox/services/archive_result_service.py b/archivebox/services/archive_result_service.py index cec3be70..0ed9ff31 100644 --- a/archivebox/services/archive_result_service.py +++ b/archivebox/services/archive_result_service.py @@ -381,6 +381,40 @@ def _save_archiveresult_event_to_db( snapshot.title = next_title snapshot.save(update_fields=["title", "modified_at"]) + # Parser output becomes durable when its ArchiveResult reaches a final + # state. Project at that same lifecycle boundary so every completed parser + # can enrich an already-discovered URL before Snapshot/Crawl completion. + # create_discovered_snapshots() owns cross-parser dedupe and metadata merge. + if ( + result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS) + and (plugin_dir / "urls.jsonl").exists() + ): + from .snapshot_service import project_discovered_snapshots + + with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.project_discovered_snapshots"): + project_discovered_snapshots(str(snapshot.id)) + + +def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str, process_id: str) -> None: + """Advance an existing queued hook row after its OS process is persisted.""" + from archivebox.core.models import ArchiveResult + + started_at = parse_event_datetime(event.start_ts) + if started_at is None: + raise ValueError("ProcessStartedEvent.start_ts is required") + ArchiveResult.objects.filter( + snapshot_id=snapshot_id, + plugin=event.plugin_name, + hook_name=event.hook_name, + status=ArchiveResult.StatusChoices.QUEUED, + ).update( + status=ArchiveResult.StatusChoices.STARTED, + start_ts=started_at, + end_ts=None, + process_id=process_id, + modified_at=timezone.now(), + ) + class ArchiveResultService(BaseService): LISTENS_TO = [ArchiveResultEvent, ProcessCompletedEvent] diff --git a/archivebox/services/process_service.py b/archivebox/services/process_service.py index 17353c00..9be179d5 100644 --- a/archivebox/services/process_service.py +++ b/archivebox/services/process_service.py @@ -8,7 +8,7 @@ from asgiref.sync import sync_to_async from django.utils import timezone from abxbus import BaseEvent -from abx_dl.events import CrawlCleanupEvent, CrawlCompletedEvent, ProcessCompletedEvent, ProcessStartedEvent +from abx_dl.events import CrawlCleanupEvent, CrawlCompletedEvent, ProcessCompletedEvent, ProcessStartedEvent, SnapshotEvent from abx_dl.services.base import BaseService @@ -147,6 +147,21 @@ class ProcessService(BaseService): binary_id=process.binary_id, modified_at=timezone.now(), ) + if event.hook_name.startswith("on_Snapshot"): + snapshot_event = await self.bus.find( + SnapshotEvent, + past=True, + future=False, + where=lambda candidate: self.bus.event_is_child_of(event, candidate), + ) + if snapshot_event is not None: + from .archive_result_service import mark_archiveresult_started + + await sync_to_async(mark_archiveresult_started, thread_sensitive=True)( + event, + snapshot_id=snapshot_event.snapshot_id, + process_id=str(process.id), + ) async def _completed_worker_loop(self) -> None: while True: diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 13e68b69..02729bae 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -180,10 +180,7 @@ async def _run_event_now(event, timeout: float | None = None): return event -def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool: - if os.environ.get("PYTEST_CURRENT_TEST") and not allow_under_pytest: - return False - +def ensure_background_runner() -> bool: from archivebox.machine.models import Machine, Process from archivebox.workers.supervisord_util import RUNNER_WORKER, get_existing_supervisord_process, get_worker, start_worker @@ -192,7 +189,7 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool: if runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"): return False if supervisor is not None: - start_worker(supervisor, RUNNER_WORKER) + start_worker(supervisor, RUNNER_WORKER()) return True machine = Machine.current() @@ -238,13 +235,9 @@ class CrawlRunner: self.interactive_interrupts = interactive_interrupts self.config_overrides = dict(config_overrides or {}) - async def ignore_snapshot(_snapshot_id: str) -> None: - return None - SnapshotService( self.bus, crawl_id=str(crawl.id), - schedule_snapshot=self.enqueue_snapshot if process_discovered_snapshots_inline else ignore_snapshot, ) ArchiveResultService(self.bus) self.selected_plugins = selected_plugins diff --git a/archivebox/services/snapshot_service.py b/archivebox/services/snapshot_service.py index 1934596d..5f3148ce 100644 --- a/archivebox/services/snapshot_service.py +++ b/archivebox/services/snapshot_service.py @@ -98,9 +98,8 @@ class SnapshotService(BaseService): LISTENS_TO = [SnapshotEvent, SnapshotCompletedEvent] EMITS = [] - def __init__(self, bus, *, crawl_id: str, schedule_snapshot): + def __init__(self, bus, *, crawl_id: str): self.crawl_id = crawl_id - self.schedule_snapshot = schedule_snapshot super().__init__(bus) self.bus.on(SnapshotEvent, self.on_SnapshotEvent) self.bus.on(SnapshotCompletedEvent, self.on_SnapshotCompletedEvent) diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py index 413a0f90..4d885078 100644 --- a/archivebox/tests/conftest.py +++ b/archivebox/tests/conftest.py @@ -5,6 +5,7 @@ import json import re import secrets import signal +import select import socket import subprocess import sys @@ -12,10 +13,11 @@ import tempfile import textwrap import time import shutil +import ctypes from datetime import timedelta from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from threading import Thread +from threading import Event, Thread from types import SimpleNamespace from typing import Any from collections.abc import Callable @@ -41,6 +43,8 @@ os.environ.pop("USERS_DIR", None) os.environ.pop("CRAWL_DIR", None) os.environ.pop("SNAP_DIR", None) +_RUNTIME_GUARD_ACTIVE = False + def _is_repo_path(path: Path) -> bool: resolved = path.expanduser().resolve(strict=False) @@ -120,6 +124,25 @@ def _sync_archivebox_test_data_dir(data_dir: Path) -> None: ) +def _archivebox_test_audit_hook(event: str, args: tuple[Any, ...]) -> None: + """Enforce runtime path isolation through Python's native audit surface.""" + if not _RUNTIME_GUARD_ACTIVE: + return + if event == "os.chdir": + path = Path(args[0]) + _assert_not_repo_path(path, label="cwd") + _sync_archivebox_test_data_dir(path) + elif event == "subprocess.Popen": + cwd = args[2] + env = args[3] + if cwd is not None: + _assert_not_repo_path(Path(cwd), label="cwd") + _assert_safe_runtime_paths(cwd=Path(cwd) if cwd is not None else None, env=env) + + +sys.addaudithook(_archivebox_test_audit_hook) + + # ============================================================================= # CLI Helpers (defined before fixtures that use them) # ============================================================================= @@ -171,9 +194,6 @@ class ArchiveBoxCmdResult: def terminate(self) -> None: self._process.terminate() - def kill(self) -> None: - self._process.kill() - def send_signal(self, sig: int) -> None: self._process.send_signal(sig) @@ -263,8 +283,8 @@ def run_archivebox_cmd( try: result.communicate(input=input, timeout=timeout) except subprocess.TimeoutExpired: - process.kill() - result.communicate() + process.terminate() + process.wait(timeout=5) raise if check and result.returncode: raise subprocess.CalledProcessError( @@ -305,7 +325,7 @@ def pytest_configure(): @pytest.fixture(autouse=True) -def isolate_test_runtime(tmp_path, monkeypatch): +def isolate_test_runtime(tmp_path): """ Run each pytest test from an isolated temp cwd and restore env mutations. @@ -317,11 +337,11 @@ def isolate_test_runtime(tmp_path, monkeypatch): ArchiveBox derives DATA_DIR from cwd, so subprocess helpers pass the target collection as cwd instead of using DATA_DIR as an override. """ + global _RUNTIME_GUARD_ACTIVE + _assert_not_repo_path(tmp_path, label="tmp_path") original_cwd = Path.cwd() original_env = os.environ.copy() - original_chdir = os.chdir - original_popen = subprocess.Popen os.chdir(tmp_path) _sync_archivebox_test_data_dir(tmp_path) os.environ.pop("DATA_DIR", None) @@ -334,28 +354,15 @@ def isolate_test_runtime(tmp_path, monkeypatch): machine_models._CURRENT_PROCESS = None machine_models._CURRENT_BINARIES.clear() - def guarded_chdir(path: os.PathLike[str] | str) -> None: - _assert_not_repo_path(Path(path), label="cwd") - original_chdir(path) - _sync_archivebox_test_data_dir(Path(path)) - - def guarded_popen(*args: Any, **kwargs: Any): - cwd = kwargs.get("cwd") - env = kwargs.get("env") - if cwd is not None: - _assert_not_repo_path(Path(cwd), label="cwd") - _assert_safe_runtime_paths(cwd=Path(cwd) if cwd is not None else None, env=env) - return original_popen(*args, **kwargs) - - monkeypatch.setattr(os, "chdir", guarded_chdir) - monkeypatch.setattr(subprocess, "Popen", guarded_popen) reset_machine_model_caches() + _RUNTIME_GUARD_ACTIVE = True try: _assert_safe_runtime_paths(cwd=Path.cwd(), env=os.environ) yield finally: + _RUNTIME_GUARD_ACTIVE = False reset_machine_model_caches() - original_chdir(original_cwd) + os.chdir(original_cwd) _sync_archivebox_test_data_dir(original_cwd) os.environ.clear() os.environ.update(original_env) @@ -378,7 +385,7 @@ def isolated_data_dir(tmp_path): @pytest.fixture -def hermetic_lib_dir(tmp_path, monkeypatch): +def hermetic_lib_dir(tmp_path): """ Point ABXPKG_LIB_DIR at a temporary directory for isolated abxpkg resolution. @@ -390,10 +397,27 @@ def hermetic_lib_dir(tmp_path, monkeypatch): lib_dir = tmp_path / "lib" lib_dir.mkdir(parents=True, exist_ok=True) - monkeypatch.setenv("ABXPKG_LIB_DIR", str(lib_dir)) + original_lib_dir = os.environ.get("ABXPKG_LIB_DIR") + os.environ["ABXPKG_LIB_DIR"] = str(lib_dir) machine_models._CURRENT_MACHINE = None machine_models._CURRENT_PROCESS = None - return lib_dir + try: + yield lib_dir + finally: + if original_lib_dir is None: + os.environ.pop("ABXPKG_LIB_DIR", None) + else: + os.environ["ABXPKG_LIB_DIR"] = original_lib_dir + machine_models._CURRENT_MACHINE = None + machine_models._CURRENT_PROCESS = None + + +@pytest.fixture +def cached_abxpkg_lib_dir(): + """Reuse the configured abxpkg cache when a test is not validating LIB_DIR isolation.""" + from archivebox.config.common import get_config + + return get_config().ABXPKG_LIB_DIR @pytest.fixture @@ -509,8 +533,43 @@ def recursive_test_site(): } finally: server.shutdown() - server.server_close() - thread.join(timeout=5) + server.server_close() + thread.join() + + +@pytest.fixture +def blocking_http_server(): + """Serve one real request behind explicit start/release synchronization.""" + + request_started = Event() + release_response = Event() + + class BlockingHandler(BaseHTTPRequestHandler): + def do_GET(self): + request_started.set() + release_response.wait() + body = b"Barrierreleased" + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), BlockingHandler) + thread = Thread(target=server.serve_forever, name="archivebox-test-http-barrier") + thread.start() + yield SimpleNamespace( + url=f"http://127.0.0.1:{server.server_port}/", + request_started=request_started, + release_response=release_response, + ) + release_response.set() + server.shutdown() + server.server_close() + thread.join() @pytest.fixture @@ -555,21 +614,18 @@ def archivebox_daemon_server(initialized_archive, free_tcp_port_factory): _stop_archivebox_supervisord(cwd, env) -def wait_for_process(predicate: Callable[[psutil.Process, str], bool], *, timeout: float = 20.0) -> psutil.Process: - deadline = time.time() + timeout +def find_process(predicate: Callable[[psutil.Process, str], bool]) -> psutil.Process: + """Locate a process after its native readiness signal has fired.""" last_seen: list[str] = [] - while time.time() < deadline: - last_seen = [] - for proc in psutil.process_iter(["pid", "ppid", "cmdline"]): - try: - cmdline = proc.info.get("cmdline") or [] - command = " ".join(cmdline) - last_seen.append(f"{proc.info.get('pid')} {proc.info.get('ppid')} {command}") - if predicate(proc, command): - return proc - except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): - continue - time.sleep(0.2) + for proc in psutil.process_iter(["pid", "ppid", "cmdline"]): + try: + cmdline = proc.info.get("cmdline") or [] + command = " ".join(cmdline) + last_seen.append(f"{proc.info.get('pid')} {proc.info.get('ppid')} {command}") + if predicate(proc, command): + return proc + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + continue raise AssertionError("No matching live process found. Last seen:\n" + "\n".join(last_seen[-50:])) @@ -583,31 +639,37 @@ def pid_is_alive(pid: int) -> bool: def wait_for_pid_to_disappear(pid: int, *, timeout: float = 20.0) -> None: - deadline = time.time() + timeout - while time.time() < deadline: - if not pid_is_alive(pid): - return - time.sleep(0.1) - raise AssertionError(f"PID {pid} is still running") + try: + psutil.Process(pid).wait(timeout=timeout) + except psutil.NoSuchProcess: + return + except psutil.TimeoutExpired as exc: + raise AssertionError(f"PID {pid} is still running") from exc def cleanup_process_group(group_pid: int | None, *child_pids: int | None) -> None: + processes: list[psutil.Process] = [] if group_pid and pid_is_alive(group_pid): try: - os.killpg(group_pid, signal.SIGKILL) + os.killpg(group_pid, signal.SIGTERM) except ProcessLookupError: pass except OSError: try: - os.kill(group_pid, signal.SIGKILL) + os.kill(group_pid, signal.SIGTERM) except ProcessLookupError: pass + processes.append(psutil.Process(group_pid)) for pid in child_pids: if pid and pid_is_alive(pid): try: - os.kill(pid, signal.SIGKILL) + os.kill(pid, signal.SIGTERM) except ProcessLookupError: pass + else: + processes.append(psutil.Process(pid)) + _gone, alive = psutil.wait_procs(processes, timeout=10) + assert not alive, f"processes did not stop after SIGTERM: {[proc.pid for proc in alive]}" def cli_env( @@ -685,52 +747,142 @@ def cli_env( return env -def wait_for_port_open(host: str, port: int, *, timeout: float = 30.0) -> None: - deadline = time.time() + timeout - while time.time() < deadline: - try: - with socket.create_connection((host, port), timeout=0.25): - return - except OSError: - time.sleep(0.1) - raise AssertionError(f"server did not listen on {host}:{port}") +def assert_port_open(host: str, port: int, *, timeout: float = 30.0) -> None: + """Verify a listening socket after the server emitted its readiness event.""" + with socket.create_connection((host, port), timeout=timeout) as connection: + assert connection.getpeername() == (host, port) + + +def _wait_for_log_match(log_path: Path, pattern: str, *, fixed: bool, count: int, timeout: float) -> str: + """Block on native filesystem notifications until the requested log event exists.""" + log_path.parent.mkdir(parents=True, exist_ok=True) + deadline = time.monotonic() + timeout + + if sys.platform == "darwin": + parent_fd = os.open(log_path.parent, os.O_RDONLY) + watched_fd: int | None = None + event_queue = select.kqueue() + event_queue.control( + [ + select.kevent( + parent_fd, + filter=select.KQ_FILTER_VNODE, + flags=select.KQ_EV_ADD | select.KQ_EV_CLEAR, + fflags=select.KQ_NOTE_WRITE | select.KQ_NOTE_RENAME | select.KQ_NOTE_DELETE, + ), + ], + 0, + 0, + ) + + def refresh_file_watcher() -> None: + nonlocal watched_fd + if watched_fd is not None: + try: + same_file = os.fstat(watched_fd).st_ino == log_path.stat().st_ino + except (FileNotFoundError, OSError): + same_file = False + if same_file: + return + try: + event_queue.control( + [ + select.kevent( + watched_fd, + filter=select.KQ_FILTER_VNODE, + flags=select.KQ_EV_DELETE, + ), + ], + 0, + 0, + ) + except OSError: + pass + os.close(watched_fd) + watched_fd = None + if log_path.exists(): + try: + watched_fd = os.open(log_path, os.O_RDONLY) + except FileNotFoundError: + return + event_queue.control( + [ + select.kevent( + watched_fd, + filter=select.KQ_FILTER_VNODE, + flags=select.KQ_EV_ADD | select.KQ_EV_CLEAR, + fflags=( + select.KQ_NOTE_WRITE + | select.KQ_NOTE_EXTEND + | select.KQ_NOTE_ATTRIB + | select.KQ_NOTE_RENAME + | select.KQ_NOTE_DELETE + ), + ), + ], + 0, + 0, + ) + + def wait_for_change(remaining: float) -> None: + events = event_queue.control(None, 1, remaining) + assert events, f"timed out waiting for filesystem event on {log_path}" + refresh_file_watcher() + + def close_watcher() -> None: + event_queue.close() + if watched_fd is not None: + os.close(watched_fd) + os.close(parent_fd) + + refresh_file_watcher() + + else: + libc = ctypes.CDLL(None, use_errno=True) + inotify_fd = libc.inotify_init1(os.O_CLOEXEC) + assert inotify_fd >= 0, os.strerror(ctypes.get_errno()) + watch_mask = 0x00000002 | 0x00000008 | 0x00000080 | 0x00000100 + watch_descriptor = libc.inotify_add_watch(inotify_fd, os.fsencode(log_path.parent), watch_mask) + assert watch_descriptor >= 0, os.strerror(ctypes.get_errno()) + + def wait_for_change(remaining: float) -> None: + readable, _writable, _errors = select.select([inotify_fd], [], [], remaining) + assert readable, f"timed out waiting for filesystem event on {log_path}" + os.read(inotify_fd, 65536) + + def close_watcher() -> None: + os.close(inotify_fd) + + try: + while True: + content = log_path.read_text(encoding="utf-8", errors="replace") if log_path.exists() else "" + matches = content.count(pattern) if fixed else len(re.findall(pattern, content)) + if matches >= count: + return content + remaining = deadline - time.monotonic() + assert remaining > 0, f"timed out waiting for {pattern!r} in {log_path}:\n{content}" + wait_for_change(remaining) + finally: + close_watcher() def wait_for_log(log_path: Path, text: str, *, timeout: float = 30.0) -> str: - deadline = time.time() + timeout - content = "" - while time.time() < deadline: - if log_path.exists(): - content = log_path.read_text(encoding="utf-8", errors="replace") - if text in content: - return content - time.sleep(0.1) - raise AssertionError(f"timed out waiting for {text!r} in {log_path}:\n{content}") + content = _wait_for_log_match(log_path, text, fixed=True, count=1, timeout=timeout) + assert text in content, content + return content def wait_for_log_count(log_path: Path, text: str, count: int, *, timeout: float = 30.0) -> str: - deadline = time.time() + timeout - content = "" - while time.time() < deadline: - if log_path.exists(): - content = log_path.read_text(encoding="utf-8", errors="replace") - if content.count(text) >= count: - return content - time.sleep(0.1) - raise AssertionError(f"timed out waiting for {count} occurrences of {text!r} in {log_path}:\n{content}") + content = _wait_for_log_match(log_path, text, fixed=True, count=count, timeout=timeout) + assert content.count(text) >= count, content + return content def wait_for_log_pattern(log_path: Path, pattern: str, *, timeout: float = 30.0) -> re.Match[str]: - deadline = time.time() + timeout - content = "" - while time.time() < deadline: - if log_path.exists(): - content = log_path.read_text(encoding="utf-8", errors="replace") - match = re.search(pattern, content) - if match: - return match - time.sleep(0.1) - raise AssertionError(f"timed out waiting for pattern {pattern!r} in {log_path}:\n{content}") + content = _wait_for_log_match(log_path, pattern, fixed=False, count=1, timeout=timeout) + match = re.search(pattern, content) + assert match is not None, content + return match def supervisor_pid_from_log(log_path: Path) -> int: @@ -748,20 +900,27 @@ def worker_pid_from_log(log_path: Path, worker_name: str) -> int: def wait_for_worker_pid_from_log(log_path: Path, worker_name: str, *, timeout: float = 45.0) -> int: - deadline = time.time() + timeout - last_error = "" - while time.time() < deadline: - try: - return worker_pid_from_log(log_path, worker_name) - except AssertionError as err: - last_error = str(err) - time.sleep(0.1) - raise AssertionError(last_error or f"timed out waiting for worker {worker_name!r} in {log_path}") + wait_for_log_pattern( + log_path, + rf"Worker {re.escape(worker_name)}: started RUNNING \(pid [0-9]+,", + timeout=timeout, + ) + return worker_pid_from_log(log_path, worker_name) def pgrep_data_dir(data_dir: Path) -> list[str]: - result = subprocess.run(["pgrep", "-af", str(data_dir)], capture_output=True, text=True, timeout=5) - lines = [line for line in result.stdout.splitlines() if "pgrep -af" not in line] + lines: list[str] = [] + seen_pids: set[int] = set() + for process in psutil.process_iter(["pid", "ppid", "cmdline"]): + try: + command = " ".join(process.info.get("cmdline") or []) + if str(data_dir) not in command: + continue + pid = int(process.info["pid"]) + seen_pids.add(pid) + lines.append(f"{pid} {process.info.get('ppid') or 0} {command}") + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + continue for runtime_root in (Path("/tmp/archivebox"), data_dir / "tmp"): for config_path in runtime_root.glob("*/supervisord.conf"): @@ -778,30 +937,34 @@ def pgrep_data_dir(data_dir: Path) -> list[str]: continue if not pid_is_alive(pid): continue - ps_line = subprocess.run( - ["ps", "-p", str(pid), "-o", "pid=,ppid=,command="], - capture_output=True, - text=True, - timeout=5, - ).stdout.strip() - if ps_line: - lines.append(ps_line) + if pid in seen_pids: + continue + try: + process = psutil.Process(pid) + lines.append(f"{pid} {process.ppid()} {' '.join(process.cmdline())}") + seen_pids.add(pid) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + continue return sorted(set(lines)) def assert_no_processes_for_data_dir(data_dir: Path, *, timeout: float = 10.0) -> None: - deadline = time.time() + timeout - remaining: list[str] = [] - while time.time() < deadline: - remaining = pgrep_data_dir(data_dir) - if not remaining: - return - time.sleep(0.25) - raise AssertionError("processes still reference test DATA_DIR:\n" + "\n".join(remaining)) + remaining = pgrep_data_dir(data_dir) + processes = [] + for line in remaining: + pid = int(line.split(None, 1)[0]) + try: + processes.append(psutil.Process(pid)) + except psutil.NoSuchProcess: + continue + _gone, alive = psutil.wait_procs(processes, timeout=timeout) + final_remaining = pgrep_data_dir(data_dir) + assert not alive and not final_remaining, "processes still reference test DATA_DIR:\n" + "\n".join(final_remaining) def kill_processes_for_data_dir(data_dir: Path) -> None: + processes: list[psutil.Process] = [] for line in pgrep_data_dir(data_dir): try: pid = int(line.split(None, 1)[0]) @@ -809,9 +972,13 @@ def kill_processes_for_data_dir(data_dir: Path) -> None: continue if pid != os.getpid(): try: - os.kill(pid, signal.SIGKILL) + os.kill(pid, signal.SIGTERM) except ProcessLookupError: pass + else: + processes.append(psutil.Process(pid)) + _gone, alive = psutil.wait_procs(processes, timeout=10) + assert not alive, f"processes did not stop after SIGTERM: {[proc.pid for proc in alive]}" def start_archivebox_server( @@ -821,7 +988,7 @@ def start_archivebox_server( env: dict[str, str] | None = None, daemonize: bool | None = None, log_name: str | None = None, - wait_for_log_text: str | None = "Tailing worker logs", + wait_for_log_text: str | None = "Listening on TCP", ): if daemonize is None: daemonize = log_name is None @@ -848,34 +1015,22 @@ def start_archivebox_server( if daemonize: assert proc.returncode == 0, proc.stderr or proc.stdout return proc - wait_for_port_open("127.0.0.1", port) if log_path is not None and wait_for_log_text is not None: wait_for_log(log_path, wait_for_log_text, timeout=30.0) + assert_port_open("127.0.0.1", port) return proc def stop_archivebox_process(proc: subprocess.Popen[str], sig=signal.SIGTERM, *, timeout: float = 15.0) -> str: - if proc.poll() is None: - try: - os.killpg(proc.pid, sig) - except (ProcessLookupError, OSError): - try: - os.kill(proc.pid, sig) - except ProcessLookupError: - pass try: - stdout, _stderr = proc.communicate(timeout=timeout) - return stdout or "" - except subprocess.TimeoutExpired: + os.killpg(proc.pid, sig) + except (ProcessLookupError, OSError): try: - os.killpg(proc.pid, signal.SIGKILL) - except (ProcessLookupError, OSError): - try: - os.kill(proc.pid, signal.SIGKILL) - except ProcessLookupError: - pass - stdout, _stderr = proc.communicate(timeout=5) - return stdout or "" + os.kill(proc.pid, sig) + except ProcessLookupError: + pass + stdout, _stderr = proc.communicate(timeout=timeout) + return stdout or "" def run_queued_crawls(cwd: Path, env: dict[str, str] | None = None, timeout: int = 180) -> None: @@ -933,24 +1088,24 @@ def _stop_archivebox_supervisord(cwd: Path, env: dict[str, str]) -> None: def _wait_for_archivebox_workers(cwd: Path, env: dict[str, str], names: tuple[str, ...] | list[str], timeout: int = 45) -> dict[str, Any]: - deadline = time.time() + timeout - state: dict[str, Any] = {} - while time.time() < deadline: - state = _archivebox_worker_state(cwd, env) - if all(isinstance(state.get(name), dict) and state[name].get("statename") == "RUNNING" for name in names): - return state - time.sleep(1) + supervisord_log = cwd / "logs" / "supervisord.log" + deadline = time.monotonic() + timeout + for name in names: + remaining = deadline - time.monotonic() + assert remaining > 0, f"timed out waiting for workers {names} in {supervisord_log}" + wait_for_log_pattern( + supervisord_log, + rf"success: {re.escape(name)} entered RUNNING state,", + timeout=remaining, + ) + state = _archivebox_worker_state(cwd, env) + assert all(isinstance(state.get(name), dict) and state[name].get("statename") == "RUNNING" for name in names), state return state def stop_process(proc: subprocess.Popen[str]) -> tuple[str, str]: - if proc.poll() is None: - proc.terminate() - try: - return proc.communicate(timeout=5) - except subprocess.TimeoutExpired: - proc.kill() - return proc.communicate() + proc.terminate() + return proc.communicate(timeout=5) def run_python_cwd( @@ -1082,7 +1237,7 @@ def api_auth_headers(api_token: str, *, django_client: bool = False, port: int | def wait_for_live_api(port: int, *, path: str = "/api/v1/docs"): - return wait_for_http(port, host=f"api.archivebox.localhost:{port}", path=path) + return get_http_response(port, host=f"api.archivebox.localhost:{port}", path=path) def live_api_request(port: int, method: str, path: str, *, api_token: str, timeout: int = 30, **kwargs): @@ -1146,32 +1301,25 @@ def stop_server(cwd: Path) -> None: run_python_cwd(script, cwd=cwd, timeout=30) -def wait_for_http( +def get_http_response( port: int, host: str, path: str = "/", timeout: float = 30.0, process: subprocess.Popen[str] | None = None, ) -> requests.Response: - deadline = time.time() + timeout - last_exc = None - while time.time() < deadline: - if process is not None and process.poll() is not None: - raise AssertionError(f"Server exited before becoming ready with code {process.returncode}") - try: - response = requests.get( - f"http://127.0.0.1:{port}{path}", - headers={"Host": host}, - timeout=2, - allow_redirects=False, - ) - if response.status_code < 500: - return response - last_exc = f"HTTP {response.status_code}" - except requests.RequestException as exc: - last_exc = exc - time.sleep(0.5) - raise AssertionError(f"Timed out waiting for HTTP on {host}: {last_exc}") + """Perform one blocking HTTP exchange after native server readiness.""" + if process is not None: + returncode = process.poll() + assert returncode is None, f"Server exited before becoming ready with code {returncode}" + response = requests.get( + f"http://127.0.0.1:{port}{path}", + headers={"Host": host}, + timeout=timeout, + allow_redirects=False, + ) + assert response.status_code < 500, f"HTTP {response.status_code} from {host}{path}" + return response def make_latest_schedule_due(cwd: Path) -> None: @@ -1244,15 +1392,19 @@ def get_snapshot_file_text(cwd: Path, url: str) -> str: def wait_for_snapshot_capture(cwd: Path, url: str, timeout: int = 180) -> str: - deadline = time.time() + timeout - last_error = None - while time.time() < deadline: - try: - return get_snapshot_file_text(cwd, url) - except AssertionError as err: - last_error = err - time.sleep(2) - raise AssertionError(f"timed out waiting for captured content for {url}: {last_error}") + script = textwrap.dedent( + f""" + from archivebox.core.models import Snapshot + snapshot = Snapshot.objects.filter(url={url!r}).order_by('-created_at').first() + assert snapshot is not None + print(snapshot.output_dir / 'index.jsonl') + """, + ) + result = run_archivebox_cmd(["manage", "shell", "-c", script], cwd=cwd, timeout=30) + assert result.returncode == 0, result.stderr or result.stdout + index_path = Path(result.stdout.strip().splitlines()[-1]) + _wait_for_log_match(index_path, ".", fixed=False, count=1, timeout=timeout) + return get_snapshot_file_text(cwd, url) def get_counts(cwd: Path, scheduled_url: str, one_shot_url: str) -> tuple[int, int, int]: @@ -1358,8 +1510,8 @@ def wait_for_archive_outputs( cwd: Path, url: str, timeout: int = 120, - interval: float = 1.0, ) -> bool: + wait_for_snapshot_capture(cwd, url, timeout=timeout) script = textwrap.dedent( f"""\ from pathlib import Path @@ -1412,13 +1564,8 @@ def wait_for_archive_outputs( """, ) - deadline = time.time() + timeout - while time.time() < deadline: - stdout, _stderr, returncode = run_python_cwd(script, cwd=cwd, timeout=30) - if returncode == 0 and "READY" in stdout: - return True - time.sleep(interval) - return False + stdout, _stderr, returncode = run_python_cwd(script, cwd=cwd, timeout=30) + return returncode == 0 and "READY" in stdout def _get_machine_type() -> str: @@ -1439,8 +1586,7 @@ def resolve_abxpkg_binary_env( install: bool = True, ) -> dict[str, str]: """Resolve real test dependencies through abxpkg and return its exported env.""" - command_env = os.environ.copy() - command_env.update(env or {}) + command_env = dict(env) if env is not None else os.environ.copy() command_env["ABXPKG_LIB_DIR"] = str(lib_dir) command = [ str(Path(sys.executable).with_name("abxpkg")), @@ -1481,6 +1627,34 @@ def resolve_abxpkg_chrome_env(lib_dir: Path, env: dict[str, str] | None = None) return payload +def install_real_binary( + name: str, + *, + machine=None, + binproviders: str = "env", + overrides: dict[str, dict[str, Any]] | None = None, +): + """Install and persist a real binary through the normal Binary state machine.""" + from archivebox.machine.models import Binary, Machine + + binary = Binary.objects.create( + machine=machine or Machine.current(refresh=True), + name=name, + binproviders=binproviders, + overrides=overrides or {}, + status=Binary.StatusChoices.QUEUED, + ) + assert binary.tick_claimed(lock_seconds=600) + binary.refresh_from_db() + assert binary.status == Binary.StatusChoices.INSTALLED + assert binary.retry_at is None + assert binary.binprovider in binary.binproviders.split(",") + assert binary.version + assert binary.abspath + assert Path(binary.abspath).exists() + return binary + + @pytest.fixture(scope="class") def real_archive_with_example(tmp_path_factory, request): """ diff --git a/archivebox/tests/test_api_v1_cli_add.py b/archivebox/tests/test_api_v1_cli_add.py index 486f6ab5..28485a0c 100644 --- a/archivebox/tests/test_api_v1_cli_add.py +++ b/archivebox/tests/test_api_v1_cli_add.py @@ -1,6 +1,6 @@ import pytest import json -import time +import subprocess from pathlib import Path from concurrent.futures import ThreadPoolExecutor from threading import Event @@ -12,8 +12,13 @@ from .conftest import ( get_free_port, init_archive, live_api_request, + run_archivebox_cmd, + run_queued_crawls, start_archivebox_server, + stop_archivebox_process, stop_server, + get_http_response, + wait_for_log, ) from archivebox.core.models import Snapshot, SnapshotTag from archivebox.crawls.models import Crawl @@ -147,44 +152,54 @@ IMPORT_FORMAT_ENV = { } -def wait_for_expected_import_snapshots( +def start_api_server_without_runner(cwd: Path, env: dict[str, str], port: int): + log_path = cwd / "api-server.log" + log = log_path.open("w", encoding="utf-8") + process = run_archivebox_cmd( + ["manage", "runserver", f"127.0.0.1:{port}", "--noreload"], + cwd=cwd, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + wait=False, + start_new_session=True, + ) + log.close() + wait_for_log(log_path, "Listening on TCP", timeout=30) + get_http_response(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") + return process + + +def assert_expected_import_snapshots( cwd: Path, expected_urls: set[str], *, - timeout: float = 180.0, expected_tags: set[str] | None = None, ) -> None: - import time - allowed_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED} - deadline = time.time() + timeout - while time.time() < deadline: - with use_archivebox_db(cwd): - snapshots = list(Snapshot.objects.filter(url__in=expected_urls).values("id", "url", "status")) - tag_names_by_snapshot_id = {} - if expected_tags and snapshots: - for snapshot_id, tag_name in SnapshotTag.objects.filter( - snapshot_id__in=[snapshot["id"] for snapshot in snapshots], - ).values_list("snapshot_id", "tag__name"): - tag_names_by_snapshot_id.setdefault(snapshot_id, set()).add(tag_name) - counts = {url: 0 for url in expected_urls} - bad_statuses = [] - missing_tags = {} - for snapshot in snapshots: - counts[snapshot["url"]] += 1 - if snapshot["status"] not in allowed_statuses: - bad_statuses.append((snapshot["url"], snapshot["status"])) - if expected_tags: - tag_names = tag_names_by_snapshot_id.get(snapshot["id"], set()) - missing = expected_tags - tag_names - if missing: - missing_tags[snapshot["url"]] = missing - if all(count == 1 for count in counts.values()) and not bad_statuses and not missing_tags: - return - time.sleep(1) - raise AssertionError( - f"timed out waiting for one queued/started/sealed snapshot per URL, got counts={counts}, bad_statuses={bad_statuses}, missing_tags={missing_tags}", - ) + with use_archivebox_db(cwd): + snapshots = list(Snapshot.objects.filter(url__in=expected_urls).values("id", "url", "status")) + tag_names_by_snapshot_id = {} + if expected_tags and snapshots: + for snapshot_id, tag_name in SnapshotTag.objects.filter( + snapshot_id__in=[snapshot["id"] for snapshot in snapshots], + ).values_list("snapshot_id", "tag__name"): + tag_names_by_snapshot_id.setdefault(snapshot_id, set()).add(tag_name) + counts = {url: 0 for url in expected_urls} + bad_statuses = [] + missing_tags = {} + for snapshot in snapshots: + counts[snapshot["url"]] += 1 + if snapshot["status"] not in allowed_statuses: + bad_statuses.append((snapshot["url"], snapshot["status"])) + if expected_tags: + tag_names = tag_names_by_snapshot_id.get(snapshot["id"], set()) + missing = expected_tags - tag_names + if missing: + missing_tags[snapshot["url"]] = missing + assert all(count == 1 for count in counts.values()), counts + assert not bad_statuses, bad_statuses + assert not missing_tags, missing_tags def malicious_add_inputs(tmp_path: Path, *, safe_url: str) -> tuple[list[str], Path]: @@ -323,8 +338,8 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls( env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV) api_token = create_admin_and_token(tmp_path) + api_server = start_api_server_without_runner(tmp_path, env, port) try: - start_archivebox_server(tmp_path, env=env, port=port) for import_name, import_path in import_files.items(): response = live_api_request( port, @@ -348,25 +363,21 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls( source_text = import_path.read_text(encoding="utf-8") assert crawl.urls == source_text - deadline = time.time() + 240 - root_counts = {} - while time.time() < deadline: - with use_archivebox_db(tmp_path): - root_counts = { - str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all() - } - if root_counts and all(count == 1 for count in root_counts.values()): - break - time.sleep(1) + stop_archivebox_process(api_server) + api_server = None + run_queued_crawls(tmp_path, env=env, timeout=240) + with use_archivebox_db(tmp_path): + root_counts = { + str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all() + } assert root_counts and all(count == 1 for count in root_counts.values()), root_counts with use_archivebox_db(tmp_path): for crawl in Crawl.objects.all(): root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL) root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") assert root_input == crawl.urls - stop_server(tmp_path) - start_archivebox_server(tmp_path, env=env, port=port) - wait_for_expected_import_snapshots(tmp_path, expected_urls) + api_server = start_api_server_without_runner(tmp_path, env, port) + assert_expected_import_snapshots(tmp_path, expected_urls) for import_name, expected in IMPORT_FORMAT_EXPECTATIONS.items(): with use_archivebox_db(tmp_path): @@ -383,7 +394,8 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls( assert snapshot_response.status_code == 200, snapshot_response.text assert snapshot_response.json()["url"] == expected["url"] finally: - stop_server(tmp_path) + if api_server is not None: + stop_archivebox_process(api_server) with use_archivebox_db(tmp_path): crawls = list(Crawl.objects.order_by("created_at")) @@ -418,8 +430,8 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path): env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV) api_token = create_admin_and_token(tmp_path) + api_server = start_api_server_without_runner(tmp_path, env, port) try: - start_archivebox_server(tmp_path, env=env, port=port) response = live_api_request( port, "post", @@ -436,11 +448,14 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path): assert response.status_code == 200, response.text assert response.json()["success"] is True - wait_for_expected_import_snapshots(tmp_path, {safe_url}, timeout=120) + stop_archivebox_process(api_server) + api_server = None + run_queued_crawls(tmp_path, env=env, timeout=120) finally: - stop_server(tmp_path) + if api_server is not None: + stop_archivebox_process(api_server) - wait_for_expected_import_snapshots(tmp_path, {safe_url}, timeout=30, expected_tags={"api-security"}) + assert_expected_import_snapshots(tmp_path, {safe_url}, expected_tags={"api-security"}) assert_no_file_or_shell_payload_snapshots(tmp_path, canary=canary) with use_archivebox_db(tmp_path): snapshot = Snapshot.objects.get(url=safe_url) diff --git a/archivebox/tests/test_api_v1_cli_remove.py b/archivebox/tests/test_api_v1_cli_remove.py index f1eabc31..8948c9d4 100644 --- a/archivebox/tests/test_api_v1_cli_remove.py +++ b/archivebox/tests/test_api_v1_cli_remove.py @@ -43,7 +43,7 @@ def _touch_output(snapshot: Snapshot) -> Path: return output_dir -def _bulk_timeout_snapshots(crawl: Crawl, *, count: int = 30000) -> tuple[list[Snapshot], dict[str, Path]]: +def _bulk_timeout_snapshots(crawl: Crawl, *, count: int = 3) -> tuple[list[Snapshot], dict[str, Path]]: base = timezone.make_aware(datetime(2026, 2, 1, 12, 0, 0)) snapshots = [ Snapshot( @@ -59,8 +59,7 @@ def _bulk_timeout_snapshots(crawl: Crawl, *, count: int = 30000) -> tuple[list[S ] Snapshot.objects.bulk_create(snapshots, batch_size=1000) - sample = [*snapshots[-200:], *snapshots[:200]] - return snapshots, {str(snapshot.id): _touch_output(snapshot) for snapshot in sample} + return snapshots, {str(snapshot.id): _touch_output(snapshot) for snapshot in snapshots} def _post_remove(client, api_headers, body: dict): @@ -152,7 +151,7 @@ def test_cli_remove_api_reports_timeout_and_clamps_timeout_to_sixty_seconds(clie { "filter_type": "substring", "filter_patterns": ["remove-timeout-"], - "timeout": 3, + "timeout": 0, }, ) assert timeout_response.status_code == 200, timeout_response.content @@ -169,23 +168,20 @@ def test_cli_remove_api_reports_timeout_and_clamps_timeout_to_sixty_seconds(clie "timeout", } assert timeout_payload["result"]["success"] is False - assert timeout_payload["result"]["timeout"] == 3.0 + assert timeout_payload["result"]["timeout"] == 0.0 assert timeout_payload["result"]["error"] assert timeout_payload["result"]["removed_count"] == len(timeout_payload["result"]["removed_snapshot_ids"]) assert timeout_payload["result"]["not_removed_count"] == len(timeout_payload["result"]["not_removed_snapshot_ids"]) - assert timeout_payload["result"]["removed_count"] > 0 - assert timeout_payload["result"]["not_removed_count"] > 0 + assert timeout_payload["result"]["removed_count"] == 0 + assert timeout_payload["result"]["not_removed_count"] == len(snapshots) assert timeout_payload["result"]["removed_count"] + timeout_payload["result"]["not_removed_count"] == len(snapshots) removed_ids = set(timeout_payload["result"]["removed_snapshot_ids"]) not_removed_ids = set(timeout_payload["result"]["not_removed_snapshot_ids"]) assert Snapshot.objects.filter(url__icontains="remove-timeout-").count() == len(not_removed_ids) - assert removed_ids & set(output_dirs_by_id) - assert not_removed_ids & set(output_dirs_by_id) - for snapshot_id in removed_ids & set(output_dirs_by_id): - assert not Snapshot.objects.filter(pk=snapshot_id).exists() - assert not output_dirs_by_id[snapshot_id].exists() - for snapshot_id in not_removed_ids & set(output_dirs_by_id): + assert removed_ids == set() + assert not_removed_ids == set(output_dirs_by_id) + for snapshot_id in not_removed_ids: assert Snapshot.objects.filter(pk=snapshot_id).exists() assert output_dirs_by_id[snapshot_id].exists() diff --git a/archivebox/tests/test_api_v1_cli_schedule.py b/archivebox/tests/test_api_v1_cli_schedule.py index 97e91961..2a920e82 100644 --- a/archivebox/tests/test_api_v1_cli_schedule.py +++ b/archivebox/tests/test_api_v1_cli_schedule.py @@ -14,7 +14,7 @@ from .conftest import ( init_archive, start_archivebox_server, stop_server, - wait_for_http, + get_http_response, ) @@ -49,7 +49,7 @@ def test_api_v1_cli_schedule_creates_schedule_over_server(tmp_path, recursive_te try: start_archivebox_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") + get_http_response(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs") response = requests.post( f"http://127.0.0.1:{port}/api/v1/cli/schedule", diff --git a/archivebox/tests/test_api_v1_cli_update.py b/archivebox/tests/test_api_v1_cli_update.py index 23d75576..c10bb526 100644 --- a/archivebox/tests/test_api_v1_cli_update.py +++ b/archivebox/tests/test_api_v1_cli_update.py @@ -1,5 +1,3 @@ -import json - import pytest from .conftest import ( @@ -41,44 +39,30 @@ def test_cli_update_api_accepts_empty_json_without_traceback(client, tmp_path, a @pytest.mark.timeout(180) def test_cli_update_api_supports_all_snapshot_list_filters_with_real_rows(tmp_path): + from archivebox.core.models import Snapshot + from archivebox.tests.test_orm_helpers import use_archivebox_db + env = cli_env(disable_extractors=True) init_archive(tmp_path) - records = [ - { - "type": "Snapshot", - "url": "https://alpha.example.com/articles/needle", - "title": "Needle Alpha", - "tags": "api-keep", - "timestamp": "1700000000", - "bookmarked_at": "2023-11-14T22:13:20+00:00", - }, - { - "type": "Snapshot", - "url": "https://beta.example.org/posts/haystack", - "title": "Haystack Beta", - "tags": "api-other", - "timestamp": "1710000000", - "bookmarked_at": "2024-03-09T16:00:00+00:00", - }, - { - "type": "Snapshot", - "url": "https://docs.archivebox.io/manual", - "title": "Manual Gamma", - "tags": "api-docs", - "timestamp": "1720000000", - "bookmarked_at": "2024-07-03T09:46:40+00:00", - }, - ] - stdin = "\n".join(json.dumps(record) for record in records) + "\n" - run_archivebox_cmd(["snapshot", "create"], cwd=tmp_path, stdin=stdin, env=env, check=True) + rows = ( + ("https://alpha.example.com/articles/needle", "Needle Alpha", "api-keep", "1700000000", "2023-11-14T22:13:20+00:00"), + ("https://beta.example.org/posts/haystack", "Haystack Beta", "api-other", "1710000000", "2024-03-09T16:00:00+00:00"), + ("https://docs.archivebox.io/manual", "Manual Gamma", "api-docs", "1720000000", "2024-07-03T09:46:40+00:00"), + ) + for url, _title, tag, _timestamp, _bookmarked_at in rows: + run_archivebox_cmd(["snapshot", "create", f"--tag={tag}", url], cwd=tmp_path, env=env, check=True) + with use_archivebox_db(tmp_path): + for url, title, _tag, timestamp, bookmarked_at in rows: + Snapshot.objects.filter(url=url).update(title=title, timestamp=timestamp, bookmarked_at=bookmarked_at) list_result = run_archivebox_cmd(["snapshot", "list", "--sort", "timestamp"], cwd=tmp_path, env=env, check=True) initial_snapshots = {record["url"]: record for record in parse_jsonl_output(list_result.stdout) if record.get("type") == "Snapshot"} alpha = initial_snapshots["https://alpha.example.com/articles/needle"] + alpha_jsonl = next(line for line in list_result.stdout.splitlines() if alpha["id"] in line) + "\n" run_archivebox_cmd( ["snapshot", "update", "--status=paused"], cwd=tmp_path, - stdin=json.dumps(alpha), + stdin=alpha_jsonl, env=env, check=True, ) diff --git a/archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py b/archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py index bfbce1cc..c9a619d1 100644 --- a/archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py +++ b/archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py @@ -1,5 +1,3 @@ -import time - import pytest from archivebox.core.models import Snapshot @@ -64,19 +62,10 @@ def test_cli_api_add_search_update_remove_over_server(tmp_path): start_archivebox_server(tmp_path, env=env, port=port) wait_for_live_api(port) - deadline = time.time() + 180 - snapshot_id = None - snapshot_status = None - while time.time() < deadline: - with use_archivebox_db(tmp_path): - snapshot = Snapshot.objects.filter(crawl_id=crawl_id, url=target_url).first() - if snapshot is not None: - snapshot_id = str(snapshot.id) - snapshot_status = snapshot.status - break - time.sleep(1) - assert snapshot_id is not None - assert snapshot_status is not None + with use_archivebox_db(tmp_path): + snapshot = Snapshot.objects.get(crawl_id=crawl_id, url=target_url) + snapshot_id = str(snapshot.id) + snapshot_status = snapshot.status search_response = live_api_request( port, diff --git a/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py b/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py index 8a0862d1..ac197e3e 100644 --- a/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py +++ b/archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py @@ -1,6 +1,6 @@ import json -import time from pathlib import Path +from threading import Thread import pytest from django.utils import timezone @@ -8,6 +8,7 @@ from django.utils import timezone from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl from archivebox.tests.conftest import run_archivebox_cmd +from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook from archivebox.tests.test_orm_helpers import use_archivebox_db from archivebox.workers.models import RETRY_AT_MAX @@ -15,60 +16,19 @@ from .conftest import ( api_client_request, cli_env, create_admin_and_token, - get_crawl_runtime_state, + get_snapshot_file_text, get_free_port, init_archive, live_api_request, start_archivebox_server, stop_server, wait_for_live_api, - wait_for_snapshot_capture, ) pytestmark = pytest.mark.django_db(transaction=True) -def _seed_archiveresult( - snapshot: Snapshot, - *, - plugin: str, - hook_name: str, - status: str, - output_text: str = "", - output_path: str | None = None, -) -> ArchiveResult: - output_files = {} - output_size = 0 - output_mimetypes = "" - if output_path is not None: - output_bytes = output_text.encode() - absolute_path = Path(snapshot.output_dir) / output_path - absolute_path.parent.mkdir(parents=True, exist_ok=True) - absolute_path.write_bytes(output_bytes) - output_size = len(output_bytes) - output_mimetypes = "text/plain" - output_files[output_path] = { - "extension": Path(output_path).suffix.lstrip("."), - "mimetype": "text/plain", - "size": output_size, - } - - now = timezone.now() - return ArchiveResult.objects.create( - snapshot=snapshot, - plugin=plugin, - hook_name=hook_name, - status=status, - output_str=output_path or output_text, - output_files=output_files, - output_size=output_size, - output_mimetypes=output_mimetypes, - start_ts=now if status != ArchiveResult.StatusChoices.QUEUED else None, - end_ts=now if status in ArchiveResult.FINAL_STATES else None, - ) - - def _snapshot_hook_name(plugin_name: str) -> str: from abx_dl.models import discover_plugins @@ -117,28 +77,16 @@ def _paused_snapshot_state(cwd: Path, snapshot_id: str) -> dict[str, object]: } -def _wait_for_paused_scheduler_marker(cwd: Path, snapshot_id: str, timeout: int = 60) -> dict[str, object]: - deadline = time.time() + timeout - last_state: dict[str, object] = {} - while time.time() < deadline: - last_state = _paused_snapshot_state(cwd, snapshot_id) - if last_state["status"] == Snapshot.StatusChoices.PAUSED and last_state["retry_at"] == RETRY_AT_MAX: - return last_state - if last_state["status"] == Snapshot.StatusChoices.SEALED: - return last_state - time.sleep(1) - raise AssertionError(f"paused snapshot did not settle back to retry_at=MAX: {last_state}") - - -def _wait_for_crawl_snapshot_rows(cwd: Path, crawl_id: str, timeout: int = 45) -> dict[str, object]: - deadline = time.time() + timeout - latest_state: dict[str, object] | None = None - while time.time() < deadline: - latest_state = get_crawl_runtime_state(cwd, crawl_id) - if latest_state["snapshots"]: - return latest_state - time.sleep(0.2) - raise AssertionError(f"timed out waiting for snapshot rows for crawl {crawl_id}: {latest_state}") +def stop_runner_worker(cwd: Path) -> None: + script = """ +from archivebox.workers.supervisord_util import get_existing_supervisord_process, stop_worker +supervisor = get_existing_supervisord_process() +assert supervisor is not None +stop_worker(supervisor, "worker_runner") +print("stopped") +""" + result = run_archivebox_cmd(["manage", "shell", "-c", script], cwd=cwd, timeout=60) + assert result.returncode == 0, result.stderr or result.stdout def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers): @@ -151,9 +99,10 @@ def test_basic_success_case_request(client, tmp_path, api_admin_user, api_header def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_finished_rows( + request, tmp_path, client, - recursive_test_site, + blocking_http_server, ): init_archive(tmp_path) api_token = create_admin_and_token(tmp_path) @@ -165,7 +114,7 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_ "/api/v1/core/snapshots", api_token=api_token, payload={ - "url": recursive_test_site["root_url"], + "url": blocking_http_server.url, "depth": 0, "title": "Snapshot pause target", "tags": ["snapshot-pause-e2e"], @@ -175,33 +124,62 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_ assert create_response.status_code == 200, create_response.content.decode() snapshot_id = json.loads(create_response.content.decode())["id"] snapshot = Snapshot.objects.get(id=snapshot_id) + from archivebox.config.common import get_config + from archivebox.services.runner import run_due_snapshot - queued_result = _seed_archiveresult( + lib_dir = get_config().ABXPKG_LIB_DIR + snapshot.output_dir.mkdir(parents=True, exist_ok=True) + (snapshot.output_dir / "source.txt").write_text("finished result should stay finished", encoding="utf-8") + _succeeded_process, succeeded_result = _run_shipped_snapshot_hook( snapshot, - plugin="manualqueue", - hook_name="on_Snapshot__manual_queue", - status=ArchiveResult.StatusChoices.QUEUED, + plugin="hashes", + hook_name="on_Snapshot__93_hashes.py", + lib_dir=lib_dir, ) - started_result = _seed_archiveresult( + Snapshot.objects.filter(pk=snapshot.pk).update(url="http://127.0.0.1:1/") + snapshot.refresh_from_db() + _failed_process, failed_result = _run_shipped_snapshot_hook( snapshot, - plugin="manualstart", - hook_name="on_Snapshot__manual_start", - status=ArchiveResult.StatusChoices.STARTED, + plugin="title", + hook_name="on_Snapshot__54_title.js", + lib_dir=lib_dir, + expected_exit_codes=(1,), ) - succeeded_result = _seed_archiveresult( - snapshot, - plugin="manualdone", - hook_name="on_Snapshot__manual_done", - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_text="finished result should stay finished", - output_path="manualdone/final.txt", + now = timezone.now() + Snapshot.objects.filter(pk=snapshot.pk).update( + url=blocking_http_server.url, + status=Snapshot.StatusChoices.QUEUED, + retry_at=now, ) - failed_result = _seed_archiveresult( - snapshot, - plugin="manualfail", - hook_name="on_Snapshot__manual_fail", - status=ArchiveResult.StatusChoices.FAILED, - output_text="failed result should stay failed", + Crawl.objects.filter(pk=snapshot.crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now) + snapshot.refresh_from_db() + [started_result] = snapshot.create_pending_archiveresults(hooks=[("wget", "on_Snapshot__06_wget.finite.bg")]) + errors = [] + + def run_snapshot(): + try: + assert run_due_snapshot(snapshot, lock_seconds=60) is True + except BaseException as err: + errors.append(err) + finally: + blocking_http_server.request_started.set() + + runner = Thread(target=run_snapshot, name="archivebox-test-api-snapshot-wget-runner") + runner.start() + + def finish_runner(): + with use_archivebox_db(tmp_path): + blocking_http_server.release_response.set() + runner.join() + assert errors == [] + + request.addfinalizer(finish_runner) + blocking_http_server.request_started.wait() + assert errors == [] + started_result.refresh_from_db() + assert started_result.status == ArchiveResult.StatusChoices.STARTED + [queued_result] = snapshot.create_pending_archiveresults( + hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")], ) invalid_response = api_client_request( @@ -213,7 +191,7 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_ ) assert invalid_response.status_code == 400 snapshot = Snapshot.objects.get(id=snapshot_id) - assert snapshot.status == Snapshot.StatusChoices.QUEUED + assert snapshot.status == Snapshot.StatusChoices.STARTED pause_response = api_client_request( client, @@ -229,28 +207,29 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_ crawl = Crawl.objects.get(id=snapshot.crawl_id) assert snapshot.status == Snapshot.StatusChoices.PAUSED assert snapshot.retry_at == RETRY_AT_MAX - assert crawl.status == Crawl.StatusChoices.QUEUED + assert crawl.status == Crawl.StatusChoices.STARTED active_rows = { row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id]) } assert active_rows == { - "manualqueue": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), - "manualstart": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), + "parse_txt_urls": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), + "wget": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), } finished_rows = { row.plugin: (row.status, row.retry_at, row.output_size) for row in ArchiveResult.objects.filter(id__in=[succeeded_result.id, failed_result.id]) } - assert finished_rows["manualdone"][0] == ArchiveResult.StatusChoices.SUCCEEDED - assert finished_rows["manualdone"][1] is None - assert finished_rows["manualdone"][2] == len("finished result should stay finished") - assert finished_rows["manualfail"] == (ArchiveResult.StatusChoices.FAILED, None, 0) + assert finished_rows["hashes"][0] == ArchiveResult.StatusChoices.SUCCEEDED + assert finished_rows["hashes"][1] is None + assert finished_rows["hashes"][2] > 0 + assert finished_rows["title"][0] == ArchiveResult.StatusChoices.FAILED + assert finished_rows["title"][1] is None succeeded_row = ArchiveResult.objects.get(id=succeeded_result.id) - output_path = Path(snapshot.output_dir) / next(iter(succeeded_row.output_files)) - assert output_path.read_text() == "finished result should stay finished" + output_path = Path(snapshot.output_dir) / succeeded_row.plugin / next(iter(succeeded_row.output_files)) + assert output_path.is_file() resume_response = api_client_request( client, @@ -267,23 +246,23 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_ assert snapshot.status == Snapshot.StatusChoices.QUEUED assert snapshot.retry_at is not None assert snapshot.retry_at != RETRY_AT_MAX - assert crawl.status == Crawl.StatusChoices.QUEUED + assert crawl.status == Crawl.StatusChoices.STARTED assert crawl.retry_at is not None assert crawl.retry_at != RETRY_AT_MAX resumed_rows = { row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id]) } - assert resumed_rows["manualqueue"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["manualqueue"][1] is not None - assert resumed_rows["manualqueue"][1] != RETRY_AT_MAX - assert resumed_rows["manualstart"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["manualstart"][1] is not None - assert resumed_rows["manualstart"][1] != RETRY_AT_MAX + assert resumed_rows["parse_txt_urls"][0] == ArchiveResult.StatusChoices.QUEUED + assert resumed_rows["parse_txt_urls"][1] is not None + assert resumed_rows["parse_txt_urls"][1] != RETRY_AT_MAX + assert resumed_rows["wget"][0] == ArchiveResult.StatusChoices.QUEUED + assert resumed_rows["wget"][1] is not None + assert resumed_rows["wget"][1] != RETRY_AT_MAX assert ArchiveResult.objects.get(id=succeeded_result.id).status == ArchiveResult.StatusChoices.SUCCEEDED assert ArchiveResult.objects.get(id=failed_result.id).status == ArchiveResult.StatusChoices.FAILED - assert output_path.read_text() == "finished result should stay finished" + assert output_path.is_file() def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapshot_lifecycle( @@ -311,27 +290,34 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh assert snapshot_response.status_code == 200, snapshot_response.content.decode() snapshot_id = json.loads(snapshot_response.content.decode())["id"] snapshot = Snapshot.objects.get(id=snapshot_id) + from archivebox.config.common import get_config - wget_result = _seed_archiveresult( + lib_dir = get_config().ABXPKG_LIB_DIR + Snapshot.objects.filter(pk=snapshot.pk).update(url="http://127.0.0.1:1/") + snapshot.refresh_from_db() + _wget_process, wget_result = _run_shipped_snapshot_hook( snapshot, plugin="wget", - hook_name=_snapshot_hook_name("wget"), - status=ArchiveResult.StatusChoices.FAILED, - output_text="initial failure before targeted retry", + hook_name="on_Snapshot__06_wget.finite.bg.py", + event_hook_name=_snapshot_hook_name("wget"), + lib_dir=lib_dir, + env={"WGET_WARC_ENABLED": "False"}, + expected_exit_codes=(1,), ) - unrelated_result = _seed_archiveresult( - snapshot, - plugin="manualqueue", - hook_name="on_Snapshot__manual_queue", - status=ArchiveResult.StatusChoices.QUEUED, + assert wget_result.status == ArchiveResult.StatusChoices.FAILED + assert "wget failed (exit=4)" in wget_result.output_str + Snapshot.objects.filter(pk=snapshot.pk).update(url=recursive_test_site["root_url"]) + snapshot.refresh_from_db() + [unrelated_result] = snapshot.create_pending_archiveresults( + hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")], ) - finished_result = _seed_archiveresult( + snapshot.output_dir.mkdir(parents=True, exist_ok=True) + (snapshot.output_dir / "source.txt").write_text("finished row must survive targeted retry", encoding="utf-8") + _finished_process, finished_result = _run_shipped_snapshot_hook( snapshot, - plugin="manualdone", - hook_name="on_Snapshot__manual_done", - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_text="finished row must survive targeted retry", - output_path="manualdone/targeted.txt", + plugin="hashes", + hook_name="on_Snapshot__93_hashes.py", + lib_dir=lib_dir, ) pause_response = api_client_request( @@ -350,8 +336,8 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh assert ArchiveResult.objects.get(id=wget_result.id).status == ArchiveResult.StatusChoices.FAILED assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.PAUSED finished_row = ArchiveResult.objects.get(id=finished_result.id) - finished_output_path = Path(snapshot.output_dir) / next(iter(finished_row.output_files)) - assert finished_output_path.read_text() == "finished row must survive targeted retry" + finished_output_path = Path(snapshot.output_dir) / finished_row.plugin / next(iter(finished_row.output_files)) + assert finished_output_path.is_file() env = cli_env( port=get_free_port(), @@ -387,7 +373,7 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh finished = ArchiveResult.objects.get(id=finished_result.id) assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED assert finished.retry_at is None - assert finished_output_path.read_text() == "finished row must survive targeted retry" + assert finished_output_path.is_file() @pytest.mark.timeout(240) @@ -434,7 +420,7 @@ def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp start_archivebox_server(tmp_path, env=env, port=port) wait_for_live_api(port) - paused_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id) + paused_state = _paused_snapshot_state(tmp_path, snapshot_id) assert paused_state["status"] == Snapshot.StatusChoices.PAUSED assert paused_state["succeeded_results"] == 0 assert not list((paused_state["snapshot_dir"] / "wget").rglob("*.html")) @@ -443,10 +429,11 @@ def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp start_archivebox_server(tmp_path, env=env, port=port) wait_for_live_api(port) - restarted_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id) + restarted_state = _paused_snapshot_state(tmp_path, snapshot_id) assert restarted_state["status"] == Snapshot.StatusChoices.PAUSED assert restarted_state["succeeded_results"] == 0 + stop_runner_worker(tmp_path) resume_response = live_api_request( port, "patch", @@ -458,7 +445,10 @@ def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp assert resume_response.status_code == 200, resume_response.text assert resume_response.json()["status"] == Snapshot.StatusChoices.QUEUED - captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) + stop_server(tmp_path) + run_result = run_archivebox_cmd(["run", f"--crawl-id={crawl_id}"], cwd=tmp_path, timeout=180, env=env) + assert run_result.returncode == 0, run_result.stderr or run_result.stdout + captured_text = get_snapshot_file_text(tmp_path, recursive_test_site["root_url"]) assert "Root" in captured_text assert "About" in captured_text diff --git a/archivebox/tests/test_api_v1_core_snapshots.py b/archivebox/tests/test_api_v1_core_snapshots.py index 54edfa92..41e1518d 100644 --- a/archivebox/tests/test_api_v1_core_snapshots.py +++ b/archivebox/tests/test_api_v1_core_snapshots.py @@ -7,7 +7,7 @@ from archivebox.crawls.models import Crawl pytestmark = pytest.mark.django_db(transaction=True) -def test_snapshots_api_filters_status_column_and_rejects_legacy_status(client, api_admin_user, api_headers): +def test_snapshots_api_filters_status_column(client, api_admin_user, api_headers): crawl = Crawl.objects.create( urls="https://example.com", created_by=api_admin_user, @@ -36,11 +36,3 @@ def test_snapshots_api_filters_status_column_and_rejects_legacy_status(client, a items = payload["items"] if isinstance(payload, dict) and "items" in payload else payload assert [item["id"] for item in items] == [str(sealed_snapshot.id)] assert [item["status"] for item in items] == ["sealed"] - - legacy_response = client.get( - "/api/v1/core/snapshots", - {"status": "unarchived"}, - **api_headers, - ) - assert legacy_response.status_code == 400 - assert "Invalid snapshot status" in legacy_response.content.decode() diff --git a/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py index 5317bc54..f05639e9 100644 --- a/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py +++ b/archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py @@ -1,7 +1,7 @@ import json -import time from datetime import datetime, timedelta from pathlib import Path +from threading import Thread from typing import cast import pytest @@ -12,6 +12,7 @@ from django.utils import timezone from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl from archivebox.tests.test_orm_helpers import use_archivebox_db +from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook from archivebox.workers.models import RETRY_AT_MAX from .conftest import ( @@ -19,6 +20,7 @@ from .conftest import ( cli_env, create_admin_and_token, get_crawl_runtime_state, + get_snapshot_file_text, get_free_port, init_archive, live_api_request, @@ -26,7 +28,6 @@ from .conftest import ( start_archivebox_server, stop_server, wait_for_live_api, - wait_for_snapshot_capture, ) @@ -44,90 +45,16 @@ def other_user(db): ) -def _seed_archiveresult( - snapshot: Snapshot, - *, - plugin: str, - hook_name: str, - status: str, - output_text: str = "", - output_path: str | None = None, -) -> ArchiveResult: - output_files = {} - output_size = 0 - output_mimetypes = "" - if output_path is not None: - output_bytes = output_text.encode() - absolute_path = Path(snapshot.output_dir) / output_path - absolute_path.parent.mkdir(parents=True, exist_ok=True) - absolute_path.write_bytes(output_bytes) - output_size = len(output_bytes) - output_mimetypes = "text/plain" - output_files[output_path] = { - "extension": Path(output_path).suffix.lstrip("."), - "mimetype": "text/plain", - "size": output_size, - } - - now = timezone.now() - return ArchiveResult.objects.create( - snapshot=snapshot, - plugin=plugin, - hook_name=hook_name, - status=status, - output_str=output_path or output_text, - output_files=output_files, - output_size=output_size, - output_mimetypes=output_mimetypes, - start_ts=now if status != ArchiveResult.StatusChoices.QUEUED else None, - end_ts=now if status in ArchiveResult.FINAL_STATES else None, - ) - - -def wait_for_crawl_snapshot_rows(cwd, crawl_id, timeout=45): - deadline = time.time() + timeout - latest_state = None - while time.time() < deadline: - latest_state = get_crawl_runtime_state(cwd, crawl_id) - if latest_state["snapshots"]: - return latest_state - time.sleep(0.2) - raise AssertionError(f"timed out waiting for runner to create snapshots for crawl {crawl_id}: {latest_state}") - - -def wait_for_crawl_child_snapshots_paused_or_sealed(cwd, crawl_id, timeout=45): - deadline = time.time() + timeout - latest_state = None - while time.time() < deadline: - latest_state = get_crawl_runtime_state(cwd, crawl_id) - snapshots = latest_state["snapshots"] - if snapshots and all(snapshot["status"] in {"paused", "sealed"} for snapshot in snapshots): - return latest_state - time.sleep(0.2) - raise AssertionError(f"timed out waiting for runner to pause or seal snapshots for crawl {crawl_id}: {latest_state}") - - -def wait_for_crawl_wget_success_or_sealed(cwd, crawl_id, timeout=240): - deadline = time.time() + timeout - latest_state = None - while time.time() < deadline: - latest_state = get_crawl_runtime_state(cwd, crawl_id) - wget_results = [result for result in latest_state["results"] if result["plugin"] == "wget"] - if ( - latest_state["snapshots"] - and latest_state["snapshots"][0]["status"] == "sealed" - and any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results) - ): - return latest_state - if ( - latest_state["crawl_status"] == "sealed" - and latest_state["snapshots"] - and latest_state["snapshots"][0]["status"] == "sealed" - and all(result["status"] not in {"queued", "started", "paused"} for result in latest_state["results"]) - ): - return latest_state - time.sleep(2) - raise AssertionError(f"timed out waiting for crawl resume completion for crawl {crawl_id}: {latest_state}") +def stop_runner_worker(cwd: Path) -> None: + script = """ +from archivebox.workers.supervisord_util import get_existing_supervisord_process, stop_worker +supervisor = get_existing_supervisord_process() +assert supervisor is not None +stop_worker(supervisor, "worker_runner") +print("stopped") +""" + result = run_archivebox_cmd(["manage", "shell", "-c", script], cwd=cwd, timeout=60) + assert result.returncode == 0, result.stderr or result.stdout def seed_paused_crawl(client, cwd: Path, api_token: str, url: str, tag: str) -> tuple[str, str]: @@ -188,9 +115,10 @@ def test_basic_success_case_request(client, tmp_path, api_admin_user, api_header def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone( + request, tmp_path, client, - recursive_test_site, + blocking_http_server, ): init_archive(tmp_path) api_token = create_admin_and_token(tmp_path) @@ -202,10 +130,10 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap "/api/v1/crawls/crawls", api_token=api_token, payload={ - "urls": [recursive_test_site["root_url"]], + "urls": [blocking_http_server.url], "max_depth": 0, "tags": ["crawl-archiveresult-pause"], - "config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, + "config": {"PLUGINS": "wget,parse_txt_urls", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"}, }, ) assert crawl_response.status_code == 200, crawl_response.content.decode() @@ -218,7 +146,7 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap "/api/v1/core/snapshots", api_token=api_token, payload={ - "url": recursive_test_site["root_url"], + "url": blocking_http_server.url, "crawl_id": crawl_id, "depth": 0, "title": "Active child", @@ -234,7 +162,7 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap "/api/v1/core/snapshots", api_token=api_token, payload={ - "url": recursive_test_site["child_urls"][0], + "url": "https://example.com/already-sealed", "crawl_id": crawl_id, "depth": 0, "title": "Already sealed child", @@ -244,38 +172,61 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap assert sealed_response.status_code == 200, sealed_response.content.decode() sealed_snapshot_id = json.loads(sealed_response.content.decode())["id"] sealed_snapshot = Snapshot.objects.get(id=sealed_snapshot_id) - sealed_done = _seed_archiveresult( + from archivebox.config.common import get_config + + lib_dir = get_config().ABXPKG_LIB_DIR + sealed_snapshot.output_dir.mkdir(parents=True, exist_ok=True) + (sealed_snapshot.output_dir / "source.txt").write_text("sealed snapshot result remains finished", encoding="utf-8") + _sealed_process, sealed_done = _run_shipped_snapshot_hook( sealed_snapshot, - plugin="sealedone", - hook_name="on_Snapshot__sealed_done", - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_text="sealed snapshot result remains finished", - output_path="sealedone/final.txt", + plugin="hashes", + hook_name="on_Snapshot__93_hashes.py", + lib_dir=lib_dir, ) sealed_snapshot.sm.seal() sealed_snapshot.refresh_from_db() assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED assert sealed_snapshot.retry_at is None - active_queued = _seed_archiveresult( + active_snapshot.output_dir.mkdir(parents=True, exist_ok=True) + (active_snapshot.output_dir / "source.txt").write_text("parent cascade should not rewrite finished rows", encoding="utf-8") + _active_done_process, active_done = _run_shipped_snapshot_hook( active_snapshot, - plugin="manualqueue", - hook_name="on_Snapshot__manual_queue", - status=ArchiveResult.StatusChoices.QUEUED, + plugin="hashes", + hook_name="on_Snapshot__93_hashes.py", + lib_dir=lib_dir, ) - active_started = _seed_archiveresult( - active_snapshot, - plugin="manualstart", - hook_name="on_Snapshot__manual_start", - status=ArchiveResult.StatusChoices.STARTED, - ) - active_done = _seed_archiveresult( - active_snapshot, - plugin="manualdone", - hook_name="on_Snapshot__manual_done", - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_text="parent cascade should not rewrite finished rows", - output_path="manualdone/cascade.txt", + now = timezone.now() + Crawl.objects.filter(pk=crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now) + Snapshot.objects.filter(pk=active_snapshot.pk).update(status=Snapshot.StatusChoices.QUEUED, retry_at=now) + active_snapshot.refresh_from_db() + [active_started] = active_snapshot.create_pending_archiveresults(hooks=[("wget", "on_Snapshot__06_wget.finite.bg")]) + errors = [] + + def run_snapshot(): + try: + assert run_due_snapshot(active_snapshot, lock_seconds=60) is True + except BaseException as err: + errors.append(err) + finally: + blocking_http_server.request_started.set() + + runner = Thread(target=run_snapshot, name="archivebox-test-api-crawl-wget-runner") + runner.start() + + def finish_runner(): + with use_archivebox_db(tmp_path): + blocking_http_server.release_response.set() + runner.join() + assert errors == [] + + request.addfinalizer(finish_runner) + blocking_http_server.request_started.wait() + assert errors == [] + active_started.refresh_from_db() + assert active_started.status == ArchiveResult.StatusChoices.STARTED + [active_queued] = active_snapshot.create_pending_archiveresults( + hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")], ) pause_response = api_client_request( client, @@ -292,15 +243,6 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap crawl = Crawl.objects.get(id=crawl_id) assert crawl.status == Crawl.StatusChoices.PAUSED assert crawl.retry_at == RETRY_AT_MAX - assert active_snapshot.status == Snapshot.StatusChoices.QUEUED - assert active_snapshot.retry_at is not None - assert active_snapshot.retry_at <= timezone.now() - assert ArchiveResult.objects.get(id=active_queued.id).status == ArchiveResult.StatusChoices.QUEUED - assert ArchiveResult.objects.get(id=active_started.id).status == ArchiveResult.StatusChoices.STARTED - - assert run_due_snapshot(active_snapshot, lock_seconds=60) is True - active_snapshot.refresh_from_db() - sealed_snapshot.refresh_from_db() assert active_snapshot.status == Snapshot.StatusChoices.PAUSED assert active_snapshot.retry_at == RETRY_AT_MAX assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED @@ -309,21 +251,19 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap paused_rows = { row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id]) } - assert paused_rows == { - "manualqueue": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), - "manualstart": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX), - } + assert paused_rows["parse_txt_urls"] == (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX) + assert paused_rows["wget"] == (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX) active_done_row = ArchiveResult.objects.get(id=active_done.id) sealed_done_row = ArchiveResult.objects.get(id=sealed_done.id) - active_done_path = Path(active_snapshot.output_dir) / next(iter(active_done_row.output_files)) - sealed_done_path = Path(sealed_snapshot.output_dir) / next(iter(sealed_done_row.output_files)) + active_done_path = Path(active_snapshot.output_dir) / active_done_row.plugin / next(iter(active_done_row.output_files)) + sealed_done_path = Path(sealed_snapshot.output_dir) / sealed_done_row.plugin / next(iter(sealed_done_row.output_files)) assert active_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED assert active_done_row.retry_at is None - assert active_done_path.read_text() == "parent cascade should not rewrite finished rows" + assert active_done_path.is_file() assert sealed_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED assert sealed_done_row.retry_at is None - assert sealed_done_path.read_text() == "sealed snapshot result remains finished" + assert sealed_done_path.is_file() resume_response = api_client_request( client, @@ -350,16 +290,24 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap resumed_rows = { row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id]) } - assert resumed_rows["manualqueue"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["manualqueue"][1] is not None - assert resumed_rows["manualqueue"][1] != RETRY_AT_MAX - assert resumed_rows["manualstart"][0] == ArchiveResult.StatusChoices.QUEUED - assert resumed_rows["manualstart"][1] is not None - assert resumed_rows["manualstart"][1] != RETRY_AT_MAX + assert resumed_rows["parse_txt_urls"][0] == ArchiveResult.StatusChoices.QUEUED + assert resumed_rows["parse_txt_urls"][1] is not None + assert resumed_rows["parse_txt_urls"][1] != RETRY_AT_MAX + assert resumed_rows["wget"][0] == ArchiveResult.StatusChoices.QUEUED + assert resumed_rows["wget"][1] is not None + assert resumed_rows["wget"][1] != RETRY_AT_MAX + assert ArchiveResult.objects.get(id=active_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED + assert ArchiveResult.objects.get(id=sealed_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED + assert active_done_path.is_file() + assert sealed_done_path.is_file() + + blocking_http_server.release_response.set() + runner.join() + assert errors == [] + active_started.refresh_from_db() + assert active_started.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS) assert ArchiveResult.objects.get(id=active_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED assert ArchiveResult.objects.get(id=sealed_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED - assert active_done_path.read_text() == "parent cascade should not rewrite finished rows" - assert sealed_done_path.read_text() == "sealed snapshot result remains finished" @pytest.mark.timeout(240) @@ -375,7 +323,7 @@ def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resu start_archivebox_server(tmp_path, env=env, port=port) wait_for_live_api(port) - paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id) + paused_state = get_crawl_runtime_state(tmp_path, crawl_id) assert paused_state["crawl_status"] == "paused" assert paused_state["crawl_retry_at"] == paused_state["retry_at_max"] assert len(paused_state["snapshots"]) == 1 @@ -393,6 +341,7 @@ def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resu assert restarted_state["snapshots"][0]["retry_at"] == restarted_state["retry_at_max"] assert not any(result["status"] == "succeeded" for result in restarted_state["results"]) + stop_runner_worker(tmp_path) resume_response = live_api_request( port, "patch", @@ -404,7 +353,10 @@ def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resu assert resume_response.status_code == 200, resume_response.text assert resume_response.json()["status"] == "queued" - captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) + stop_server(tmp_path) + run_result = run_archivebox_cmd(["run", f"--crawl-id={crawl_id}"], cwd=tmp_path, timeout=180, env=env) + assert run_result.returncode == 0, run_result.stderr or run_result.stdout + captured_text = get_snapshot_file_text(tmp_path, recursive_test_site["root_url"]) assert "Root" in captured_text assert "About" in captured_text @@ -465,6 +417,7 @@ def test_update_index_only_leaves_paused_snapshot_on_normal_lifecycle_path(clien assert still_paused_state["snapshots"][0]["status"] == "paused" assert not any(result["plugin"] == "wget" and result["status"] == "succeeded" for result in still_paused_state["results"]) + stop_runner_worker(tmp_path) resume_response = live_api_request( port, "patch", @@ -476,12 +429,15 @@ def test_update_index_only_leaves_paused_snapshot_on_normal_lifecycle_path(clien assert resume_response.status_code == 200, resume_response.text assert resume_response.json()["status"] == "queued" - resumed_state = wait_for_crawl_wget_success_or_sealed(tmp_path, crawl_id, timeout=240) + stop_server(tmp_path) + run_result = run_archivebox_cmd(["run", f"--crawl-id={crawl_id}"], cwd=tmp_path, timeout=240, env=env) + assert run_result.returncode == 0, run_result.stderr or run_result.stdout + resumed_state = get_crawl_runtime_state(tmp_path, crawl_id) assert resumed_state["snapshots"][0]["status"] == "sealed" wget_results = [result for result in resumed_state["results"] if result["plugin"] == "wget"] assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results) - captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=60) + captured_text = get_snapshot_file_text(tmp_path, recursive_test_site["root_url"]) assert "Root" in captured_text assert "About" in captured_text finally: diff --git a/archivebox/tests/test_api_v1_machine_binaries.py b/archivebox/tests/test_api_v1_machine_binaries.py index 9ae80e67..7b975c9f 100644 --- a/archivebox/tests/test_api_v1_machine_binaries.py +++ b/archivebox/tests/test_api_v1_machine_binaries.py @@ -1,6 +1,7 @@ import pytest -from archivebox.machine.models import Binary, Machine +from archivebox.machine.models import Machine +from archivebox.tests.conftest import install_real_binary pytestmark = pytest.mark.django_db(transaction=True) @@ -8,15 +9,13 @@ pytestmark = pytest.mark.django_db(transaction=True) def test_basic_success_case_request(client, tmp_path, api_headers): machine = Machine.current(refresh=True) - Binary.objects.create( - machine=machine, - name="api-basic-bin", - binprovider="env", - abspath="/usr/bin/env", - version="1.0", - status=Binary.StatusChoices.INSTALLED, - ) + binary = install_real_binary("python3", machine=machine) response = client.get("/api/v1/machine/binaries", **api_headers) assert response.status_code == 200, response.content + payload = response.json() + assert payload["count"] == 1 + assert payload["items"][0]["id"] == str(binary.id) + assert payload["items"][0]["abspath"] == binary.abspath + assert payload["items"][0]["version"] == binary.version diff --git a/archivebox/tests/test_api_v1_machine_binary_binary_id.py b/archivebox/tests/test_api_v1_machine_binary_binary_id.py index a8c1d541..a06e9072 100644 --- a/archivebox/tests/test_api_v1_machine_binary_binary_id.py +++ b/archivebox/tests/test_api_v1_machine_binary_binary_id.py @@ -1,6 +1,7 @@ import pytest -from archivebox.machine.models import Binary, Machine +from archivebox.machine.models import Machine +from archivebox.tests.conftest import install_real_binary pytestmark = pytest.mark.django_db(transaction=True) @@ -8,15 +9,11 @@ pytestmark = pytest.mark.django_db(transaction=True) def test_basic_success_case_request(client, tmp_path, api_headers): machine = Machine.current(refresh=True) - binary = Binary.objects.create( - machine=machine, - name="api-basic-bin", - binprovider="env", - abspath="/usr/bin/env", - version="1.0", - status=Binary.StatusChoices.INSTALLED, - ) + binary = install_real_binary("python3", machine=machine) response = client.get(f"/api/v1/machine/binary/{binary.id}", **api_headers) assert response.status_code == 200, response.content + assert response.json()["id"] == str(binary.id) + assert response.json()["abspath"] == binary.abspath + assert response.json()["version"] == binary.version diff --git a/archivebox/tests/test_api_v1_machine_binary_by_name_name.py b/archivebox/tests/test_api_v1_machine_binary_by_name_name.py index 45744fa2..ce64c998 100644 --- a/archivebox/tests/test_api_v1_machine_binary_by_name_name.py +++ b/archivebox/tests/test_api_v1_machine_binary_by_name_name.py @@ -1,6 +1,7 @@ import pytest -from archivebox.machine.models import Binary, Machine +from archivebox.machine.models import Machine +from archivebox.tests.conftest import install_real_binary pytestmark = pytest.mark.django_db(transaction=True) @@ -8,15 +9,13 @@ pytestmark = pytest.mark.django_db(transaction=True) def test_basic_success_case_request(client, tmp_path, api_headers): machine = Machine.current(refresh=True) - Binary.objects.create( - machine=machine, - name="api-basic-bin", - binprovider="env", - abspath="/usr/bin/env", - version="1.0", - status=Binary.StatusChoices.INSTALLED, - ) + binary = install_real_binary("python3", machine=machine) - response = client.get("/api/v1/machine/binary/by-name/api-basic-bin", **api_headers) + response = client.get("/api/v1/machine/binary/by-name/python3", **api_headers) assert response.status_code == 200, response.content + payload = response.json() + assert len(payload) == 1 + assert payload[0]["id"] == str(binary.id) + assert payload[0]["abspath"] == binary.abspath + assert payload[0]["version"] == binary.version diff --git a/archivebox/tests/test_archive_result_service.py b/archivebox/tests/test_archive_result_service.py index 6e7583a1..8ded5435 100644 --- a/archivebox/tests/test_archive_result_service.py +++ b/archivebox/tests/test_archive_result_service.py @@ -1,11 +1,17 @@ from pathlib import Path +from importlib.resources import files +import json +import os + import pytest from abxpkg.binary_service import BinaryRequestEvent -from abx_dl.events import ArchiveResultEvent, ProcessCompletedEvent, ProcessEvent, ProcessStartedEvent, SnapshotEvent +from abx_dl.events import ProcessCompletedEvent, ProcessEvent, ProcessStartedEvent, SnapshotEvent from abx_dl.orchestrator import create_bus from abx_dl.output_files import OutputFile +from archivebox.tests.conftest import resolve_abxpkg_binary_env +from archivebox.tests.conftest import install_real_binary pytestmark = pytest.mark.django_db(transaction=True) @@ -17,6 +23,107 @@ def _cleanup_machine_process_rows() -> None: Process.objects.all().delete() +def _run_shipped_snapshot_hook( + snapshot, + *, + plugin: str, + hook_name: str, + event_hook_name: str | None = None, + lib_dir: Path, + env: dict | None = None, + expected_exit_codes: tuple[int, ...] = (0,), +): + """Run one shipped hook through the production process/result bus services.""" + import asyncio + + from abx_dl.services.process_service import ProcessService as HookProcessService + from archivebox.core.models import ArchiveResult + from archivebox.machine.models import Process + from archivebox.services.archive_result_service import ArchiveResultService + from archivebox.services.process_service import ProcessService as PersistedProcessService + + hook_path = Path(str(files(f"abx_plugins.plugins.{plugin}").joinpath(hook_name))) + projected_hook_name = event_hook_name or hook_name + hook_config = hook_path.parent / "config.json" + binary_env = resolve_abxpkg_binary_env(lib_dir, deps_from=hook_config) + output_dir = Path(snapshot.output_dir) / plugin + output_dir.mkdir(parents=True, exist_ok=True) + bus = create_bus(name=f"test_real_{plugin}_{snapshot.id}") + HookProcessService(bus, emit_jsonl=False, interactive_tty=False) + PersistedProcessService(bus) + ArchiveResultService(bus) + + async def run() -> None: + try: + snapshot_event = SnapshotEvent( + url=snapshot.url, + snapshot_id=str(snapshot.id), + output_dir=str(snapshot.output_dir), + ) + await bus.emit(snapshot_event).now() + process_event = bus.emit( + ProcessEvent( + plugin_name=plugin, + hook_name=projected_hook_name, + hook_path=str(hook_path), + hook_args=[f"--url={snapshot.url}"], + env={ + **binary_env, + "ABXPKG_LIB_DIR": str(lib_dir), + "SNAP_DIR": str(snapshot.output_dir), + "PATH": f"{Path(os.sys.executable).parent}{os.pathsep}{os.environ['PATH']}", + **(env or {}), + }, + output_dir=str(output_dir), + timeout=60, + is_background=".bg." in hook_name, + url=snapshot.url, + process_type="hook", + worker_type="hook", + event_parent_id=snapshot_event.event_id, + ), + ) + await process_event.now() + if ".bg." in hook_name: + completed_event = await bus.find( + ProcessCompletedEvent, + child_of=process_event, + past=True, + future=90, + ) + assert completed_event is not None + await completed_event.wait(timeout=90) + await completed_event.event_results_list() + await bus.wait_until_idle() + finally: + await bus.destroy(clear=False) + + asyncio.run(run()) + process = Process.objects.filter(pwd=str(output_dir)).order_by("-created_at").first() + assert process is not None + process.refresh_from_db() + assert process.exit_code in expected_exit_codes, (process.stdout, process.stderr) + result = ArchiveResult.objects.get(snapshot=snapshot, plugin=plugin, hook_name=projected_hook_name) + return process, result + + +def _run_real_title_crawl(url: str, lib_dir: Path): + import asyncio + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services.runner import CrawlRunner + + crawl = Crawl.objects.create( + urls=url, + config={"ABXPKG_LIB_DIR": str(lib_dir), "PLUGINS": "title"}, + created_by_id=get_or_create_system_user_pk(), + ) + asyncio.run(CrawlRunner(crawl, selected_plugins=["title"], show_progress=False).run()) + return Snapshot.objects.get(crawl=crawl, url=url) + + def _create_snapshot(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl @@ -37,95 +144,62 @@ def _create_snapshot(): return snapshot -def test_process_completed_projects_inline_archiveresult(): +def test_process_completed_projects_inline_archiveresult(tmp_path, hermetic_lib_dir): from archivebox.core.models import ArchiveResult - from archivebox.services.archive_result_service import ArchiveResultService - import asyncio snapshot = _create_snapshot() - plugin_dir = Path(snapshot.output_dir) / "wget" - plugin_dir.mkdir(parents=True, exist_ok=True) - (plugin_dir / "index.html").write_text("ok") - - bus = create_bus(name="test_inline_archiveresult") - service = ArchiveResultService(bus) - - event = ArchiveResultEvent( - snapshot_id=str(snapshot.id), - plugin="wget", - hook_name="on_Snapshot__06_wget.finite.bg", - status="succeeded", - output_str="wget/index.html", - output_files=[OutputFile(path="index.html", extension="html", mimetype="text/html", size=15)], - start_ts="2026-03-22T12:00:00+00:00", - end_ts="2026-03-22T12:00:01+00:00", + snapshot_dir = Path(snapshot.output_dir) + snapshot_dir.mkdir(parents=True, exist_ok=True) + (snapshot_dir / "source.txt").write_text("real hook input", encoding="utf-8") + process, result = _run_shipped_snapshot_hook( + snapshot, + plugin="hashes", + hook_name="on_Snapshot__93_hashes.py", + lib_dir=hermetic_lib_dir, ) - async def emit_event() -> None: - await service.on_ArchiveResultEvent__save_to_db(event) - - asyncio.run(emit_event()) - - result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget.finite.bg") assert result.status == ArchiveResult.StatusChoices.SUCCEEDED - assert result.output_str == "wget/index.html" - assert "index.html" in result.output_files - assert result.output_files["index.html"] == {"extension": "html", "mimetype": "text/html", "size": 15} - assert result.output_size == 15 + assert result.process_id == process.id + assert result.output_str.endswith(json.loads((snapshot_dir / "hashes" / "hashes.json").read_text())["root_hash"][:12]) + assert result.output_files == { + "hashes.json": { + "extension": "json", + "mimetype": "application/json", + "size": (snapshot_dir / "hashes" / "hashes.json").stat().st_size, + }, + } + assert result.output_size == (snapshot_dir / "hashes" / "hashes.json").stat().st_size _cleanup_machine_process_rows() -def test_archiveresult_event_retry_updates_existing_hook_row(): +def test_archiveresult_event_retry_updates_existing_hook_row(tmp_path, hermetic_lib_dir): from archivebox.core.models import ArchiveResult - from archivebox.services.archive_result_service import ArchiveResultService - import asyncio snapshot = _create_snapshot() - plugin_dir = Path(snapshot.output_dir) / "wget" - plugin_dir.mkdir(parents=True, exist_ok=True) - (plugin_dir / "index.html").write_text("ok") - - service = ArchiveResultService(create_bus(name="test_archiveresult_retry_updates_existing_hook_row")) - first_event = ArchiveResultEvent( - snapshot_id=str(snapshot.id), - plugin="wget", - hook_name="on_Snapshot__06_wget.finite.bg", - status="failed", - output_str="timed out", - start_ts="2026-03-22T12:00:00+00:00", - end_ts="2026-03-22T12:00:01+00:00", + snapshot_dir = Path(snapshot.output_dir) + snapshot_dir.mkdir(parents=True, exist_ok=True) + (snapshot_dir / "source.txt").write_text("first input", encoding="utf-8") + _, first_result = _run_shipped_snapshot_hook( + snapshot, + plugin="hashes", + hook_name="on_Snapshot__93_hashes.py", + lib_dir=hermetic_lib_dir, + env={"HASHES_ENABLED": "False"}, ) - retry_event = ArchiveResultEvent( - snapshot_id=str(snapshot.id), - plugin="wget", - hook_name="on_Snapshot__06_wget.finite.bg", - status="succeeded", - output_str="wget/index.html", - output_files=[OutputFile(path="index.html", extension="html", mimetype="text/html", size=15)], - start_ts="2026-03-22T12:01:00+00:00", - end_ts="2026-03-22T12:01:01+00:00", + first_result_id = first_result.id + assert first_result.status == ArchiveResult.StatusChoices.SKIPPED + + (snapshot_dir / "source.txt").write_text("retry input", encoding="utf-8") + _, retry_result = _run_shipped_snapshot_hook( + snapshot, + plugin="hashes", + hook_name="on_Snapshot__93_hashes.py", + lib_dir=hermetic_lib_dir, + env={"HASHES_ENABLED": "True"}, ) - - async def emit_events() -> None: - await service.on_ArchiveResultEvent__save_to_db(first_event) - first_result_id = await ArchiveResult.objects.values_list("id", flat=True).aget( - snapshot=snapshot, - plugin="wget", - hook_name="on_Snapshot__06_wget.finite.bg", - ) - await service.on_ArchiveResultEvent__save_to_db(retry_event) - retry_result = await ArchiveResult.objects.aget( - snapshot=snapshot, - plugin="wget", - hook_name="on_Snapshot__06_wget.finite.bg", - ) - assert retry_result.id == first_result_id - assert retry_result.status == ArchiveResult.StatusChoices.SUCCEEDED - assert retry_result.output_str == "wget/index.html" - - asyncio.run(emit_events()) - - assert ArchiveResult.objects.filter(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget.finite.bg").count() == 1 + assert retry_result.id == first_result_id + assert retry_result.status == ArchiveResult.StatusChoices.SUCCEEDED + assert ArchiveResult.objects.filter(snapshot=snapshot, plugin="hashes", hook_name="on_Snapshot__93_hashes.py").count() == 1 _cleanup_machine_process_rows() @@ -150,72 +224,37 @@ def test_archiveresult_duplicate_hook_rows_are_rejected(): ) -def test_process_completed_projects_synthetic_failed_archiveresult(): +def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_path, hermetic_lib_dir): from archivebox.core.models import ArchiveResult - from archivebox.services.archive_result_service import ArchiveResultService - import asyncio snapshot = _create_snapshot() - plugin_dir = Path(snapshot.output_dir) / "chrome" - plugin_dir.mkdir(parents=True, exist_ok=True) - - bus = create_bus(name="test_synthetic_archiveresult") - service = ArchiveResultService(bus) - - event = ArchiveResultEvent( - snapshot_id=str(snapshot.id), - plugin="chrome", - hook_name="on_Snapshot__11_chrome_wait", - status="failed", - output_str="Hook timed out after 60 seconds", - error="Hook timed out after 60 seconds", - start_ts="2026-03-22T12:00:00+00:00", - end_ts="2026-03-22T12:01:00+00:00", + process, result = _run_shipped_snapshot_hook( + snapshot, + plugin="title", + hook_name="on_Snapshot__54_title.js", + lib_dir=hermetic_lib_dir, + expected_exit_codes=(1,), ) - - async def emit_event() -> None: - await service.on_ArchiveResultEvent__save_to_db(event) - - asyncio.run(emit_event()) - - result = ArchiveResult.objects.get(snapshot=snapshot, plugin="chrome", hook_name="on_Snapshot__11_chrome_wait") assert result.status == ArchiveResult.StatusChoices.FAILED - assert result.output_str == "Hook timed out after 60 seconds" - assert "Hook timed out" in result.notes + assert result.process_id == process.id + assert "Chrome session" in result.output_str + assert result.output_str in result.notes _cleanup_machine_process_rows() -def test_failed_title_archiveresult_does_not_overwrite_snapshot_title(): +def test_failed_title_archiveresult_does_not_overwrite_snapshot_title(tmp_path, hermetic_lib_dir): from archivebox.core.models import ArchiveResult - from archivebox.services.archive_result_service import ArchiveResultService - import asyncio snapshot = _create_snapshot() - plugin_dir = Path(snapshot.output_dir) / "title" - plugin_dir.mkdir(parents=True, exist_ok=True) - - bus = create_bus(name="test_failed_title_does_not_update_snapshot") - service = ArchiveResultService(bus) - - event = ArchiveResultEvent( - snapshot_id=str(snapshot.id), + _, result = _run_shipped_snapshot_hook( + snapshot, plugin="title", hook_name="on_Snapshot__54_title.js", - status="failed", - output_str="No Chrome session found (chrome plugin must run first)", - error="No Chrome session found (chrome plugin must run first)", - start_ts="2026-03-22T12:00:00+00:00", - end_ts="2026-03-22T12:00:01+00:00", + lib_dir=hermetic_lib_dir, + expected_exit_codes=(1,), ) - - async def emit_event() -> None: - await service.on_ArchiveResultEvent__save_to_db(event) - - asyncio.run(emit_event()) - - result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", hook_name="on_Snapshot__54_title.js") assert result.status == ArchiveResult.StatusChoices.FAILED - assert result.output_str == "No Chrome session found (chrome plugin must run first)" + assert "Chrome session" in result.output_str snapshot.refresh_from_db() assert snapshot.title in (None, "") assert snapshot.resolved_title == "" @@ -240,36 +279,21 @@ def test_snapshot_resolved_title_ignores_failed_title_output_str(): _cleanup_machine_process_rows() -def test_snapshot_title_ignores_noresults_title_output_str(): +def test_snapshot_title_ignores_noresults_hook_output_str(tmp_path, hermetic_lib_dir): from archivebox.core.models import ArchiveResult - from archivebox.services.archive_result_service import ArchiveResultService - import asyncio snapshot = _create_snapshot() - plugin_dir = Path(snapshot.output_dir) / "title" - plugin_dir.mkdir(parents=True, exist_ok=True) - - bus = create_bus(name="test_noresults_title_does_not_update_snapshot") - service = ArchiveResultService(bus) - - event = ArchiveResultEvent( - snapshot_id=str(snapshot.id), - plugin="title", - hook_name="on_Snapshot__54_title.js", - status="noresults", - output_str="TimeoutError: Navigation timeout of 54172 ms exceeded", - start_ts="2026-03-22T12:00:00+00:00", - end_ts="2026-03-22T12:00:01+00:00", + staticfile_dir = Path(snapshot.output_dir) / "staticfile" + staticfile_dir.mkdir(parents=True, exist_ok=True) + (staticfile_dir / "input.txt").write_text("plain text without links", encoding="utf-8") + _, result = _run_shipped_snapshot_hook( + snapshot, + plugin="parse_txt_urls", + hook_name="on_Snapshot__71_parse_txt_urls.py", + lib_dir=hermetic_lib_dir, ) - - async def emit_event() -> None: - await service.on_ArchiveResultEvent__save_to_db(event) - - asyncio.run(emit_event()) - - result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", hook_name="on_Snapshot__54_title.js") assert result.status == ArchiveResult.StatusChoices.NORESULTS - assert result.output_str == "TimeoutError: Navigation timeout of 54172 ms exceeded" + assert result.output_str == "0 URLs parsed" snapshot.refresh_from_db() assert snapshot.title in (None, "") assert snapshot.resolved_title == "" @@ -299,84 +323,41 @@ def test_snapshot_save_normalizes_url_title_to_none(): _cleanup_machine_process_rows() -def test_process_completed_projects_noresults_archiveresult(): +def test_process_completed_projects_noresults_archiveresult(tmp_path, hermetic_lib_dir): from archivebox.core.models import ArchiveResult - from archivebox.services.archive_result_service import ArchiveResultService - import asyncio snapshot = _create_snapshot() - plugin_dir = Path(snapshot.output_dir) / "title" - plugin_dir.mkdir(parents=True, exist_ok=True) - - bus = create_bus(name="test_noresults_archiveresult") - service = ArchiveResultService(bus) - - event = ArchiveResultEvent( - snapshot_id=str(snapshot.id), - plugin="title", - hook_name="on_Snapshot__54_title.js", - status="noresults", - output_str="No title found", - start_ts="2026-03-22T12:00:00+00:00", - end_ts="2026-03-22T12:00:01+00:00", + staticfile_dir = Path(snapshot.output_dir) / "staticfile" + staticfile_dir.mkdir(parents=True, exist_ok=True) + (staticfile_dir / "input.txt").write_text("plain text without links", encoding="utf-8") + process, result = _run_shipped_snapshot_hook( + snapshot, + plugin="parse_txt_urls", + hook_name="on_Snapshot__71_parse_txt_urls.py", + lib_dir=hermetic_lib_dir, ) - - async def emit_event() -> None: - await service.on_ArchiveResultEvent__save_to_db(event) - - asyncio.run(emit_event()) - - result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", hook_name="on_Snapshot__54_title.js") assert result.status == ArchiveResult.StatusChoices.NORESULTS - assert result.output_str == "No title found" + assert result.output_str == "0 URLs parsed" + assert result.process_id == process.id -def test_process_completed_without_archive_result_does_not_infer_success_from_output_files(snapshot): +def test_skipped_shipped_hook_does_not_infer_success_from_snapshot_files(snapshot, hermetic_lib_dir): from archivebox.core.models import ArchiveResult - from archivebox.services.archive_result_service import ArchiveResultService - import asyncio - plugin_dir = Path(snapshot.output_dir) / "wget" - plugin_dir.mkdir(parents=True, exist_ok=True) - (plugin_dir / "index.html").write_text("downloaded but not reported") - - bus = create_bus(name="test_process_completed_without_archive_result_output_files") - ArchiveResultService(bus) - - snapshot_event = SnapshotEvent( - url=snapshot.url, - snapshot_id=str(snapshot.id), - output_dir=str(snapshot.output_dir), + snapshot_dir = Path(snapshot.output_dir) + snapshot_dir.mkdir(parents=True, exist_ok=True) + (snapshot_dir / "source.txt").write_text("real input remains present", encoding="utf-8") + _, result = _run_shipped_snapshot_hook( + snapshot, + plugin="hashes", + hook_name="on_Snapshot__93_hashes.py", + lib_dir=hermetic_lib_dir, + env={"HASHES_ENABLED": "False"}, ) - completed_event = ProcessCompletedEvent( - plugin_name="wget", - hook_name="on_Snapshot__06_wget.finite.bg", - hook_path="/usr/bin/env", - hook_args=[], - env={}, - timeout=60, - stdout="", - stderr="", - exit_code=0, - status="succeeded", - output_dir=str(plugin_dir), - output_files=[OutputFile(path="index.html", extension="html", mimetype="text/html", size=36)], - start_ts="2026-03-22T12:00:00+00:00", - end_ts="2026-03-22T12:00:01+00:00", - event_parent_id=snapshot_event.event_id, - ) - - async def emit_events() -> None: - await bus.emit(snapshot_event).now() - await bus.emit(completed_event).now() - await bus.wait_until_idle() - - asyncio.run(emit_events()) - - result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget.finite.bg") - assert result.status == ArchiveResult.StatusChoices.NORESULTS - assert result.output_str == "" - assert result.output_files == {"index.html": {"extension": "html", "mimetype": "text/html", "size": 36}} + assert result.status == ArchiveResult.StatusChoices.SKIPPED + assert result.output_str == "HASHES_ENABLED=False" + assert "hashes.json" not in result.output_files + assert not (snapshot_dir / "hashes" / "hashes.json").exists() _cleanup_machine_process_rows() @@ -416,115 +397,24 @@ def test_retry_failed_archiveresults_requeues_snapshot_in_queued_state(): _cleanup_machine_process_rows() -def test_retry_failed_archiveresults_preserves_legacy_plugin_rows_without_hook_name(): - from archivebox.core.models import ArchiveResult, Snapshot +def test_process_completed_projects_snapshot_title_from_output_str(recursive_test_site, hermetic_lib_dir): + snapshot = _run_real_title_crawl(recursive_test_site["root_url"], hermetic_lib_dir) + result = snapshot.archiveresult_set.get(plugin="title") - snapshot = _create_snapshot() - legacy_result = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="wget", - hook_name="", - status=ArchiveResult.StatusChoices.FAILED, - output_str="legacy failure", - output_files={"index.html": {"size": 123}}, - output_size=123, - output_mimetypes="text/html", - ) - hook_result = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="wget", - hook_name="on_Snapshot__06_wget.finite.bg", - status=ArchiveResult.StatusChoices.FAILED, - output_str="hook failure", - output_files={"stderr.log": {}}, - output_size=10, - output_mimetypes="text/plain", - ) - - reset_count = snapshot.retry_failed_archiveresults() - - snapshot.refresh_from_db() - snapshot.crawl.refresh_from_db() - legacy_result.refresh_from_db() - hook_result.refresh_from_db() - - assert reset_count == 2 - assert snapshot.status == Snapshot.StatusChoices.QUEUED - assert snapshot.retry_at is not None - assert snapshot.crawl.status == snapshot.crawl.StatusChoices.QUEUED - assert snapshot.crawl.retry_at is not None - assert legacy_result.status == ArchiveResult.StatusChoices.FAILED - assert legacy_result.output_str == "legacy failure" - assert legacy_result.output_files == {"index.html": {"size": 123}} - assert legacy_result.output_size == 123 - assert hook_result.status == ArchiveResult.StatusChoices.QUEUED - assert hook_result.output_str == "" - assert hook_result.output_files == {} - assert hook_result.output_size == 0 + assert result.status == result.StatusChoices.SUCCEEDED + assert result.output_str == "Root" + assert snapshot.title == "Root" _cleanup_machine_process_rows() -def test_process_completed_projects_snapshot_title_from_output_str(): - from archivebox.services.archive_result_service import ArchiveResultService - import asyncio +def test_process_completed_projects_snapshot_title_from_title_file(recursive_test_site, hermetic_lib_dir): + snapshot = _run_real_title_crawl(recursive_test_site["root_url"], hermetic_lib_dir) + title_file = Path(snapshot.output_dir) / "title" / "title.txt" + result = snapshot.archiveresult_set.get(plugin="title") - snapshot = _create_snapshot() - plugin_dir = Path(snapshot.output_dir) / "title" - plugin_dir.mkdir(parents=True, exist_ok=True) - - bus = create_bus(name="test_snapshot_title_output_str") - service = ArchiveResultService(bus) - - event = ArchiveResultEvent( - snapshot_id=str(snapshot.id), - plugin="title", - hook_name="on_Snapshot__54_title.js", - status="succeeded", - output_str="Example Domain", - start_ts="2026-03-22T12:00:00+00:00", - end_ts="2026-03-22T12:00:01+00:00", - ) - - async def emit_event() -> None: - await service.on_ArchiveResultEvent__save_to_db(event) - - asyncio.run(emit_event()) - - snapshot.refresh_from_db() - assert snapshot.title == "Example Domain" - _cleanup_machine_process_rows() - - -def test_process_completed_projects_snapshot_title_from_title_file(): - from archivebox.services.archive_result_service import ArchiveResultService - import asyncio - - snapshot = _create_snapshot() - plugin_dir = Path(snapshot.output_dir) / "title" - plugin_dir.mkdir(parents=True, exist_ok=True) - (plugin_dir / "title.txt").write_text("Example Domain") - - bus = create_bus(name="test_snapshot_title_file") - service = ArchiveResultService(bus) - - event = ArchiveResultEvent( - snapshot_id=str(snapshot.id), - plugin="title", - hook_name="on_Snapshot__54_title.js", - status="noresults", - output_str="No title found", - output_files=[OutputFile(path="title.txt", extension="txt", mimetype="text/plain", size=14)], - start_ts="2026-03-22T12:00:00+00:00", - end_ts="2026-03-22T12:00:01+00:00", - ) - - async def emit_event() -> None: - await service.on_ArchiveResultEvent__save_to_db(event) - - asyncio.run(emit_event()) - - snapshot.refresh_from_db() - assert snapshot.title == "Example Domain" + assert title_file.read_text() == "Root" + assert result.output_files["title.txt"]["size"] == title_file.stat().st_size + assert snapshot.resolved_title == title_file.read_text() _cleanup_machine_process_rows() @@ -589,8 +479,13 @@ def test_collect_output_metadata_detects_warc_gz_mimetype(tmp_path): @pytest.mark.django_db(transaction=True) -def test_process_started_hydrates_binary_and_iface_from_existing_binary_records(tmp_path): - from archivebox.machine.models import Binary, NetworkInterface +def test_process_started_hydrates_binary_and_iface_from_existing_binary_records( + tmp_path, + hermetic_lib_dir, + recursive_test_site, +): + from abx_plugins.plugins.base.utils import get_hydrated_required_binary + from archivebox.machine.models import NetworkInterface from archivebox.machine.models import Process as MachineProcess from archivebox.services.process_service import ProcessService as ArchiveBoxProcessService from abx_dl.services.process_service import ProcessService as DlProcessService @@ -598,19 +493,28 @@ def test_process_started_hydrates_binary_and_iface_from_existing_binary_records( iface = NetworkInterface.current() machine = iface.machine - binary = Binary.objects.create( - machine=machine, - name="postlight-parser", - abspath="/tmp/postlight-parser", - version="2.2.3", - binprovider="npm", - binproviders="npm", - status=Binary.StatusChoices.INSTALLED, + lib_dir = hermetic_lib_dir + mercury_config = Path(str(files("abx_plugins.plugins.mercury").joinpath("config.json"))) + required_binary = get_hydrated_required_binary( + "postlight-parser", + mercury_config, + environ=os.environ, ) + binary = install_real_binary( + "postlight-parser", + machine=machine, + binproviders=required_binary["binproviders"], + overrides=required_binary["overrides"], + ) + mercury_env = resolve_abxpkg_binary_env( + lib_dir, + deps_from=mercury_config, + install=False, + ) + mercury_path = Path(mercury_env["MERCURY_BINARY"]) + assert Path(binary.abspath).resolve() == mercury_path.resolve() - hook_path = tmp_path / "on_Snapshot__57_mercury.py" - hook_path.write_text("#!/bin/bash\nexit 0\n", encoding="utf-8") - hook_path.chmod(0o755) + hook_path = Path(str(files("abx_plugins.plugins.mercury").joinpath("on_Snapshot__57_mercury.py"))) output_dir = tmp_path / "mercury" output_dir.mkdir() @@ -624,15 +528,17 @@ def test_process_started_hydrates_binary_and_iface_from_existing_binary_records( plugin_name="mercury", hook_name="on_Snapshot__57_mercury.py", hook_path=str(hook_path), - hook_args=["--url=https://example.com"], + hook_args=[f"--url={recursive_test_site['root_url']}"], is_background=False, output_dir=str(output_dir), env={ + **mercury_env, + "ABXPKG_LIB_DIR": str(lib_dir), "MERCURY_BINARY": binary.abspath, - "NODE_BINARY": "/tmp/node", + "SNAP_DIR": str(tmp_path), }, timeout=60, - url="https://example.com", + url=recursive_test_site["root_url"], ), ).now() started = await bus.find( @@ -652,15 +558,24 @@ def test_process_started_hydrates_binary_and_iface_from_existing_binary_records( process = MachineProcess.objects.get( pwd=str(output_dir), - cmd=[str(hook_path), "--url=https://example.com"], + cmd=[str(hook_path), f"--url={recursive_test_site['root_url']}"], ) assert process.binary_id == binary.id assert process.iface_id == iface.id + assert process.exit_code == 0, process.stderr + assert (output_dir / "content.html").read_text() == ( + ' About Blog Contact ' + ) + assert (output_dir / "content.txt").read_text() == "About Blog Contact" + article = json.loads((output_dir / "article.json").read_text()) + assert article["title"] == "Root" + assert article["url"] == recursive_test_site["root_url"] + assert article["word_count"] == 3 @pytest.mark.django_db(transaction=True) -def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp_path): - from archivebox.machine.models import Binary, NetworkInterface +def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp_path, hermetic_lib_dir): + from archivebox.machine.models import NetworkInterface from archivebox.machine.models import Process as MachineProcess from archivebox.services.process_service import ProcessService as ArchiveBoxProcessService from abx_dl.services.process_service import ProcessService as DlProcessService @@ -668,21 +583,17 @@ def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp iface = NetworkInterface.current() machine = iface.machine - node = Binary.objects.create( - machine=machine, - name="node", - abspath="/tmp/node", - version="22.0.0", - binprovider="env", - binproviders="env", - status=Binary.StatusChoices.INSTALLED, - ) + lib_dir = hermetic_lib_dir + chrome_config = Path(str(files("abx_plugins.plugins.chrome").joinpath("config.json"))) + node_env = resolve_abxpkg_binary_env(lib_dir, deps_from=chrome_config) + node_path = Path(node_env["NODE_BINARY"]) + node = install_real_binary("node", machine=machine) + assert Path(node.abspath).resolve() == node_path.resolve() - hook_path = tmp_path / "on_Snapshot__75_parse_dom_outlinks.js" - hook_path.write_text("#!/bin/bash\nexit 0\n", encoding="utf-8") - hook_path.chmod(0o755) - output_dir = tmp_path / "parse-dom-outlinks" - output_dir.mkdir() + hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js"))) + crawl_dir = tmp_path / "crawl" + output_dir = crawl_dir / "chrome" + output_dir.mkdir(parents=True) bus = create_bus(name="test_process_started_node_fallback") DlProcessService(bus, emit_jsonl=False, interactive_tty=False) @@ -691,13 +602,20 @@ def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp async def run_test() -> None: await bus.emit( ProcessEvent( - plugin_name="parse_dom_outlinks", - hook_name="on_Snapshot__75_parse_dom_outlinks.js", + plugin_name="chrome", + hook_name="on_CrawlSetup__89_chrome_kill_zombies.js", hook_path=str(hook_path), - hook_args=["--url=https://example.com"], + hook_args=[], is_background=False, output_dir=str(output_dir), - env={"NODE_BINARY": node.abspath}, + env={ + **node_env, + "ABXPKG_LIB_DIR": str(lib_dir), + "NODE_BINARY": node.abspath, + "CRAWL_DIR": str(crawl_dir), + "SNAP_DIR": str(crawl_dir / "snapshot"), + "CHROME_USER_DATA_DIR": str(output_dir / "profile"), + }, timeout=60, url="https://example.com", ), @@ -706,7 +624,7 @@ def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp ProcessStartedEvent, past=True, future=False, - hook_name="on_Snapshot__75_parse_dom_outlinks.js", + hook_name="on_CrawlSetup__89_chrome_kill_zombies.js", output_dir=str(output_dir), ) assert started is not None @@ -719,32 +637,25 @@ def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp process = MachineProcess.objects.get( pwd=str(output_dir), - cmd=[str(hook_path), "--url=https://example.com"], + cmd=[str(hook_path)], ) assert process.binary_id == node.id assert process.iface_id == iface.id + assert process.exit_code == 0, process.stderr + assert "chrome zombies. cpu usage:" in process.stdout def test_binary_event_reuses_existing_installed_binary_row(): from archivebox.machine.models import Binary, Machine from archivebox.services.binary_service import ArchiveBoxDBBinaryCacheBackend - from abxpkg import PROVIDER_CLASS_BY_NAME from abxpkg.binary_service import BinaryCacheService, BinaryService import asyncio machine = Machine.current() - wget_path = PROVIDER_CLASS_BY_NAME["env"]().get_abspath("wget", quiet=True, no_cache=True) - assert wget_path - - binary = Binary.objects.create( - machine=machine, - name="wget", - abspath=str(wget_path), - version="9.9.9", - binprovider="env", - binproviders="env,apt,brew", - status=Binary.StatusChoices.INSTALLED, - ) + binary = install_real_binary("wget", machine=machine, binproviders="env,apt,brew") + installed_abspath = binary.abspath + installed_version = binary.version + installed_provider = binary.binprovider bus = create_bus(name="test_binary_event_reuses_existing_installed_binary_row") BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend()) @@ -754,7 +665,7 @@ def test_binary_event_reuses_existing_installed_binary_row(): binproviders=binary.binproviders, extra_context={ "plugin_name": "wget", - "output_dir": "/tmp/wget", + "output_dir": str(binary.output_dir), }, ) @@ -767,7 +678,7 @@ def test_binary_event_reuses_existing_installed_binary_row(): binary.refresh_from_db() assert Binary.objects.filter(machine=machine, name="wget").count() == 1 assert binary.status == Binary.StatusChoices.INSTALLED - assert binary.abspath == str(wget_path) - assert binary.version == "9.9.9" - assert binary.binprovider == "env" + assert binary.abspath == installed_abspath + assert binary.version == installed_version + assert binary.binprovider == installed_provider assert binary.binproviders == "env,apt,brew" diff --git a/archivebox/tests/test_binary_service.py b/archivebox/tests/test_binary_service.py index cdf3f2e2..ac49a46d 100644 --- a/archivebox/tests/test_binary_service.py +++ b/archivebox/tests/test_binary_service.py @@ -1,4 +1,3 @@ -import json import sys import uuid import asyncio @@ -8,7 +7,7 @@ import pytest from django.utils import timezone from archivebox.machine.models import Binary, Machine, Process -from archivebox.tests.conftest import parse_jsonl_output, run_archivebox_cmd +from archivebox.tests.conftest import run_archivebox_cmd from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) @@ -22,7 +21,25 @@ def _runtime_env(data_dir: Path, *, lib_dir: Path | None = None, **extra: str) - } -def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event(monkeypatch): +def _run_real_binary_state_machine(data_dir: Path, *, name: str, binproviders: str, env: dict[str, str]): + """Run a real Binary model through its abxpkg-backed state machine.""" + script = ( + "from archivebox.machine.models import Binary, Machine; " + f"binary = Binary.objects.create(machine=Machine.current(), name={name!r}, binproviders={binproviders!r}, status=Binary.StatusChoices.QUEUED); " + "assert binary.tick_claimed(lock_seconds=600); " + "print('BINARY_STATE_MACHINE_E2E_DONE')" + ) + return run_archivebox_cmd( + ["shell", "-c", script], + cwd=data_dir, + timeout=120, + env=env, + default_cli_env=True, + disable_extractors=True, + ) + + +def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event(): from abxpkg.binary_service import BinaryCacheService, BinaryEvent, BinaryRequestEvent, BinaryService from abx_dl.orchestrator import create_bus from archivebox.services.binary_service import ArchiveBoxDBBinaryCacheBackend @@ -34,7 +51,6 @@ def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event(m "module_name": "imagesize", }, } - monkeypatch.setenv("PYTHON3_BINARY", sys.executable) binary = Binary.objects.create( machine=machine, name="python3", @@ -85,26 +101,11 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized name = "python" provider_bin_dir = initialized_archive / "lib" / "env" / "bin" runtime_env = _runtime_env(initialized_archive) - request_record = { - "type": "BinaryRequest", - "name": name, - "binproviders": "env", - } - - _cmd_result = run_archivebox_cmd( - ["run"], - cwd=initialized_archive, - stdin=json.dumps(request_record) + "\n", - timeout=120, - env=runtime_env, - default_cli_env=True, - disable_extractors=True, - ) + _cmd_result = _run_real_binary_state_machine(initialized_archive, name=name, binproviders="env", env=runtime_env) stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, stderr - output_records = parse_jsonl_output(stdout) - assert any(record["type"] == "BinaryRequest" and record["name"] == name for record in output_records) + assert "BINARY_STATE_MACHINE_E2E_DONE" in stdout with use_archivebox_db(initialized_archive): binary = Binary.objects.get(name=name) @@ -173,10 +174,15 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized lib_dir=changed_lib_dir, ) + with use_archivebox_db(initialized_archive): + Binary.objects.get(pk=first_binary_id).update_and_requeue( + status=Binary.StatusChoices.QUEUED, + retry_at=None, + ) + _cmd_result = run_archivebox_cmd( - ["run"], + ["run", f"--binary-id={first_binary_id}"], cwd=initialized_archive, - stdin=json.dumps(request_record) + "\n", timeout=120, env=changed_runtime_env, default_cli_env=True, @@ -200,19 +206,11 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res provider_bin_dir = initialized_archive / "lib" / "pip" / "venv" / "bin" runtime_env = _runtime_env(initialized_archive) - _cmd_result = run_archivebox_cmd( - ["run"], - cwd=initialized_archive, - stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n", - timeout=120, - env=runtime_env, - default_cli_env=True, - disable_extractors=True, - ) + _cmd_result = _run_real_binary_state_machine(initialized_archive, name=name, binproviders="env", env=runtime_env) stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode - assert returncode == 0, stderr - assert any(record["type"] == "BinaryRequest" and record["name"] == name for record in parse_jsonl_output(stdout)), stdout + stderr + assert returncode != 0, stdout + stderr + assert "Binary http installation failed" in stderr with use_archivebox_db(initialized_archive): queued = Binary.objects.get(name=name) diff --git a/archivebox/tests/test_cli_add.py b/archivebox/tests/test_cli_add.py index dbddd471..63c50aaf 100644 --- a/archivebox/tests/test_cli_add.py +++ b/archivebox/tests/test_cli_add.py @@ -6,7 +6,6 @@ Verify add creates snapshots in DB, crawls, source files, and archive directorie import os import json -import time from pathlib import Path import pytest @@ -18,11 +17,8 @@ from archivebox.machine.models import Process from archivebox.tests.conftest import ( cli_env, find_snapshot_dir, - get_free_port, run_archivebox_cmd, run_queued_crawls, - start_archivebox_server, - stop_server, resolve_abxpkg_chrome_env, ) @@ -158,43 +154,18 @@ IMPORT_FORMAT_ENV = { } -def wait_for_import_processing(cwd: Path, expected_urls: set[str], *, timeout: float = 120.0) -> None: - import time - - deadline = time.time() + timeout - counts = {url: 0 for url in expected_urls} - while time.time() < deadline: - with use_archivebox_db(cwd): - rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", flat=True)) - counts = {url: 0 for url in expected_urls} - for url in rows: - counts[url] += 1 - if all(count >= 1 for count in counts.values()): - return - time.sleep(1) - raise AssertionError(f"timed out waiting for import crawl processing to start, got counts={counts}") - - -def wait_for_expected_import_snapshots(cwd: Path, expected_urls: set[str], *, timeout: float = 180.0) -> None: - import time - +def assert_expected_import_snapshots(cwd: Path, expected_urls: set[str]) -> None: allowed_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED} - deadline = time.time() + timeout - while time.time() < deadline: - with use_archivebox_db(cwd): - rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", "status")) - counts = {url: 0 for url in expected_urls} - bad_statuses = [] - for url, status in rows: - counts[url] += 1 - if status not in allowed_statuses: - bad_statuses.append((url, status)) - if all(count == 1 for count in counts.values()) and not bad_statuses: - return - time.sleep(1) - raise AssertionError( - f"timed out waiting for one queued/started/sealed snapshot per URL, got counts={counts}, bad_statuses={bad_statuses}", - ) + with use_archivebox_db(cwd): + rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", "status")) + counts = {url: 0 for url in expected_urls} + bad_statuses = [] + for url, status in rows: + counts[url] += 1 + if status not in allowed_statuses: + bad_statuses.append((url, status)) + assert all(count == 1 for count in counts.values()), counts + assert not bad_statuses, bad_statuses def malicious_add_inputs(tmp_path: Path, *, safe_url: str) -> tuple[list[str], Path]: @@ -263,8 +234,7 @@ def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initial """`archivebox add < import-file` should normalize rich import formats before crawling URLs.""" import_files = write_import_format_files(initialized_archive) expected_urls = {case["url"] for case in IMPORT_FORMAT_EXPECTATIONS.values()} - port = get_free_port() - env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV) + env = cli_env(**IMPORT_FORMAT_ENV) for import_path in import_files.values(): source_text = import_path.read_text(encoding="utf-8") @@ -282,40 +252,26 @@ def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initial assert crawl.snapshot_set.count() == 0 assert crawl.urls == source_text - try: - start_archivebox_server(initialized_archive, env=env, port=port) - deadline = time.time() + 120 - root_counts = {} - while time.time() < deadline: - with use_archivebox_db(initialized_archive): - root_counts = { - str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all() - } - if root_counts and all(count == 1 for count in root_counts.values()): - break - time.sleep(1) - assert root_counts and all(count == 1 for count in root_counts.values()), root_counts - with use_archivebox_db(initialized_archive): - for crawl in Crawl.objects.all(): - root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL) - root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") - assert root_input == crawl.urls - wait_for_import_processing(initialized_archive, expected_urls) - stop_server(initialized_archive) - start_archivebox_server(initialized_archive, env=env, port=port) - wait_for_expected_import_snapshots(initialized_archive, expected_urls) + run_queued_crawls(initialized_archive, env=env, timeout=240) + with use_archivebox_db(initialized_archive): + root_counts = {str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()} + assert root_counts and all(count == 1 for count in root_counts.values()), root_counts + with use_archivebox_db(initialized_archive): + for crawl in Crawl.objects.all(): + root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL) + root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") + assert root_input == crawl.urls + assert_expected_import_snapshots(initialized_archive, expected_urls) - list_result = run_archivebox_cmd( - ["list", "--json"], - cwd=initialized_archive, - env=env, - timeout=60, - ) - assert list_result.returncode == 0, list_result.stderr or list_result.stdout - for expected_url in expected_urls: - assert expected_url in list_result.stdout - finally: - stop_server(initialized_archive) + list_result = run_archivebox_cmd( + ["list", "--json"], + cwd=initialized_archive, + env=env, + timeout=60, + ) + assert list_result.returncode == 0, list_result.stderr or list_result.stdout + for expected_url in expected_urls: + assert expected_url in list_result.stdout with use_archivebox_db(initialized_archive): crawls = list(Crawl.objects.order_by("created_at")) @@ -345,8 +301,7 @@ def test_add_rejects_file_path_and_shell_injection_payloads(initialized_archive) """CLI add must not turn user-supplied local paths or shell payloads into snapshots.""" safe_url = "https://example.com/?archivebox-cli-security=1" inputs, canary = malicious_add_inputs(initialized_archive, safe_url=safe_url) - port = get_free_port() - env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV) + env = cli_env(**IMPORT_FORMAT_ENV) result = run_archivebox_cmd( ["add", "--bg", "--depth=0", "--tag=cli-security"], @@ -357,11 +312,8 @@ def test_add_rejects_file_path_and_shell_injection_payloads(initialized_archive) ) assert result.returncode == 0, result.stderr or result.stdout - try: - start_archivebox_server(initialized_archive, env=env, port=port) - wait_for_expected_import_snapshots(initialized_archive, {safe_url}, timeout=120) - finally: - stop_server(initialized_archive) + run_queued_crawls(initialized_archive, env=env, timeout=120) + assert_expected_import_snapshots(initialized_archive, {safe_url}) assert_no_file_or_shell_payload_snapshots(initialized_archive, canary=canary) with use_archivebox_db(initialized_archive): diff --git a/archivebox/tests/test_cli_archiveresult.py b/archivebox/tests/test_cli_archiveresult.py index 681a8a1f..3ef1d83b 100644 --- a/archivebox/tests/test_cli_archiveresult.py +++ b/archivebox/tests/test_cli_archiveresult.py @@ -126,21 +126,31 @@ class TestArchiveResultCreate: assert "Snapshot" in types assert "ArchiveResult" in types - def test_create_pass_through_only_when_no_snapshots(self, initialized_archive): - """Only pass-through records but no new snapshots returns success.""" - crawl_record = {"type": "Crawl", "id": "fake-id", "urls": "https://example.com"} - - _cmd_result = run_archivebox_cmd( - ["archiveresult", "create"], - stdin=json.dumps(crawl_record), + def test_create_passes_through_cli_crawl_when_no_snapshots(self, initialized_archive): + """A real Crawl with no Snapshot input passes through successfully.""" + crawl_result = run_archivebox_cmd( + ["crawl", "create", create_test_url()], cwd=initialized_archive, default_cli_env=True, disable_extractors=True, ) - _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode + assert crawl_result.returncode == 0, crawl_result.stderr + crawl_record = parse_jsonl_output(crawl_result.stdout)[0] + + _cmd_result = run_archivebox_cmd( + ["archiveresult", "create"], + stdin=crawl_result.stdout, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert code == 0 assert "Passed through" in stderr + records = parse_jsonl_output(stdout) + assert len(records) == 1 + assert records[0]["id"] == crawl_record["id"] class TestArchiveResultList: diff --git a/archivebox/tests/test_cli_binary.py b/archivebox/tests/test_cli_binary.py index 480728ea..ea9a4a59 100644 --- a/archivebox/tests/test_cli_binary.py +++ b/archivebox/tests/test_cli_binary.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """ Tests for archivebox binary command. - -TODO: expand beyond command discovery into create/list/update/delete behavior. """ from archivebox.tests.conftest import run_archivebox_cmd diff --git a/archivebox/tests/test_cli_config.py b/archivebox/tests/test_cli_config.py index 09ca1342..298c3174 100644 --- a/archivebox/tests/test_cli_config.py +++ b/archivebox/tests/test_cli_config.py @@ -158,6 +158,28 @@ def test_config_search_finds_keys(initialized_archive): assert "TIMEOUT" in result.stdout +def test_config_search_finds_plugin_options(initialized_archive): + """Test that config --search finds plugin keys and descriptions.""" + + result = run_archivebox_cmd( + ["config", "--search", "wget"], + ) + + assert result.returncode == 0 + assert "WGET_BINARY" in result.stdout + + +def test_config_search_finds_core_aliases(initialized_archive): + """Test that config --search finds core options by partial alias.""" + + result = run_archivebox_cmd( + ["config", "--search", "URL_BLACK"], + ) + + assert result.returncode == 0 + assert "URL_DENYLIST" in result.stdout + + def test_config_preserves_existing_values(initialized_archive): """Test that setting new values preserves existing ones.""" diff --git a/archivebox/tests/test_cli_crawl.py b/archivebox/tests/test_cli_crawl.py index 850f40ef..268059cc 100644 --- a/archivebox/tests/test_cli_crawl.py +++ b/archivebox/tests/test_cli_crawl.py @@ -106,11 +106,18 @@ class TestCrawlCreate: records = parse_jsonl_output(stdout) assert "test-tag" in records[0].get("tags_str", "") - def test_create_pass_through_other_types(self, initialized_archive): - """Pass-through records of other types unchanged.""" - tag_record = {"type": "Tag", "id": "fake-tag-id", "name": "test"} + def test_create_passes_through_tag_emitted_by_cli(self, initialized_archive): + """A real Tag emitted by the CLI remains available to the next stage.""" + tag_result = run_archivebox_cmd( + ["tag", "create", "crawl-input-tag"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + assert tag_result.returncode == 0, tag_result.stderr + tag_record = parse_jsonl_output(tag_result.stdout)[0] url = create_test_url() - stdin = json.dumps(tag_record) + "\n" + json.dumps({"url": url}) + stdin = tag_result.stdout + json.dumps({"url": url}) + "\n" _cmd_result = run_archivebox_cmd( ["crawl", "create"], @@ -124,10 +131,8 @@ class TestCrawlCreate: assert code == 0 records = parse_jsonl_output(stdout) - # Should have both the passed-through Tag and the new Crawl - types = [r.get("type") for r in records] - assert "Tag" in types - assert "Crawl" in types + assert any(record.get("type") == "Tag" and record["id"] == tag_record["id"] for record in records) + assert any(record.get("type") == "Crawl" and url in record["urls"] for record in records) def test_create_pass_through_existing_crawl(self, initialized_archive): """Existing Crawl records (with id) are passed through.""" diff --git a/archivebox/tests/test_cli_extract_input.py b/archivebox/tests/test_cli_extract_input.py index 1ceb4df1..e1424543 100644 --- a/archivebox/tests/test_cli_extract_input.py +++ b/archivebox/tests/test_cli_extract_input.py @@ -1,7 +1,6 @@ """Tests for archivebox extract input handling and pipelines.""" import subprocess -import json import pytest @@ -219,15 +218,19 @@ def test_extract_stdin_jsonl_input(initialized_archive): env = cli_env(PLUGINS="wget,title") create_extract_snapshot(initialized_archive, env) - with use_archivebox_db(initialized_archive): - snapshot_id = Snapshot.objects.values_list("id", flat=True).first() - - jsonl_input = json.dumps({"type": "Snapshot", "id": str(snapshot_id)}) + "\n" + list_result = run_archivebox_cmd( + ["snapshot", "list", "--url__icontains=example.com"], + cwd=initialized_archive, + env=env, + check=True, + ) + snapshot_record = next(record for record in parse_jsonl_output(list_result.stdout) if record.get("type") == "Snapshot") + snapshot_id = snapshot_record["id"] result = run_archivebox_cmd( ["extract", "--plugins=wget,title"], cwd=initialized_archive, - input=jsonl_input, + input=list_result.stdout, env=env, timeout=90, ) diff --git a/archivebox/tests/test_cli_list.py b/archivebox/tests/test_cli_list.py index bb791cef..eff87c18 100644 --- a/archivebox/tests/test_cli_list.py +++ b/archivebox/tests/test_cli_list.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python3 """ Tests for archivebox list command. Verify list emits snapshot JSONL and applies the documented filters. """ import json -import sys import pytest +from django.contrib.auth import get_user_model from django.db import connection from django.utils import timezone from archivebox.core.models import Snapshot +from archivebox.cli.archivebox_snapshot import iter_snapshot_json from archivebox.tests.conftest import create_test_url, parse_jsonl_output, run_archivebox_cmd, run_queued_crawls, cli_env from archivebox.tests.test_orm_helpers import use_archivebox_db @@ -19,102 +19,128 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) -class CountingStdout: - encoding = "utf-8" - - def __init__(self): - self.rows = 0 - self._pending = "" - - def isatty(self): - return False - - def write(self, text): - self._pending += text - lines = self._pending.split("\n") - self._pending = lines.pop() - self.rows += sum(1 for line in lines if line.startswith("{")) - return len(text) - - def flush(self): - return None - - -def test_list_limit_zero_streams_one_million_snapshots_without_materializing(admin_user, monkeypatch): - """Regression: archivebox list --limit=0 must stream unbounded result sets.""" - from archivebox.cli.archivebox_snapshot import list_snapshots +def test_streaming_json_matches_snapshot_serializer(initialized_archive): from archivebox.crawls.models import Crawl - crawl = Crawl.objects.create( - urls="https://example.com", - created_by=admin_user, - status=Crawl.StatusChoices.SEALED, - retry_at=None, - ) - now = timezone.now().isoformat() - with connection.cursor() as cursor: - cursor.execute( - """ - WITH RECURSIVE seq(n) AS ( - SELECT 1 - UNION ALL - SELECT n + 1 FROM seq WHERE n < 1000000 + with use_archivebox_db(initialized_archive): + user = get_user_model().objects.create_user(username="streaming-json-parity") + crawl = Crawl.objects.create( + urls="https://example.com/a\nhttps://example.com/b", + created_by=user, + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + populated = Snapshot.objects.create( + crawl=crawl, + url="https://example.com/a", + timestamp="20260721220000000000000000000001", + title="Populated title", + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + output_size=42, + ) + populated.save_tags(["Êclair", "Zulu", "alpha"]) + empty = Snapshot.objects.create( + crawl=crawl, + url="https://example.com/b", + timestamp="20260721220000000000000000000002", + title=None, + status=Snapshot.StatusChoices.QUEUED, + output_size=0, + ) + queryset = Snapshot.objects.filter(id__in=(populated.id, empty.id)).order_by("url") + + expected = [snapshot.to_json() for snapshot in queryset.prefetch_related("tags")] + actual = list(iter_snapshot_json(queryset)) + + assert actual == expected + assert [record["url"] for record in actual] == ["https://example.com/a", "https://example.com/b"] + + +def test_list_limit_zero_streams_one_million_snapshots_without_materializing(initialized_archive, tmp_path): + """Regression: archivebox list --limit=0 must stream unbounded result sets.""" + from archivebox.crawls.models import Crawl + + with use_archivebox_db(initialized_archive): + user = get_user_model().objects.create_user(username="million-snapshot-list") + crawl = Crawl.objects.create( + urls="https://example.com", + created_by=user, + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + now = timezone.now().isoformat() + with connection.cursor() as cursor: + cursor.execute( + """ + WITH RECURSIVE seq(n) AS ( + SELECT 1 + UNION ALL + SELECT n + 1 FROM seq WHERE n < 1000000 + ) + INSERT INTO core_snapshot ( + id, + url, + timestamp, + title, + bookmarked_at, + created_at, + modified_at, + downloaded_at, + fs_version, + crawl_id, + config, + current_step, + depth, + notes, + num_uses_failed, + num_uses_succeeded, + retry_at, + status, + delete_at, + output_size, + parent_snapshot_id + ) + SELECT + lower(hex(randomblob(16))), + 'https://example.com/page-' || n, + printf('9%031d', n), + '', + %s, + %s, + %s, + NULL, + '0.9.0', + %s, + '{}', + 0, + 0, + '', + 0, + 0, + NULL, + 'sealed', + NULL, + 0, + NULL + FROM seq + """, + [now, now, now, str(crawl.id).replace("-", "")], ) - INSERT INTO core_snapshot ( - id, - url, - timestamp, - title, - bookmarked_at, - created_at, - modified_at, - downloaded_at, - fs_version, - crawl_id, - config, - current_step, - depth, - notes, - num_uses_failed, - num_uses_succeeded, - retry_at, - status, - delete_at, - output_size, - parent_snapshot_id - ) - SELECT - lower(hex(randomblob(16))), - 'https://example.com/page-' || n, - printf('9%031d', n), - '', - %s, - %s, - %s, - NULL, - '0.9.0', - %s, - '{}', - 0, - 0, - '', - 0, - 0, - NULL, - 'sealed', - NULL, - 0, - NULL - FROM seq - """, - [now, now, now, str(crawl.id).replace("-", "")], + + output_path = tmp_path / "million-snapshots.jsonl" + with output_path.open("w") as stdout: + result = run_archivebox_cmd( + ["list", "--limit=0"], + cwd=initialized_archive, + stdout=stdout, + default_cli_env=True, + disable_extractors=True, ) - stdout = CountingStdout() - monkeypatch.setattr(sys, "stdout", stdout) - - assert list_snapshots(limit=0) == 0 - assert stdout.rows == 1000000 + assert result.returncode == 0, result.stderr + with output_path.open() as stdout: + assert sum(1 for line in stdout if line.startswith("{")) == 1000000 def test_list_outputs_existing_snapshots_as_jsonl(initialized_archive): @@ -127,6 +153,8 @@ def test_list_outputs_existing_snapshots_as_jsonl(initialized_archive): check=True, ) run_queued_crawls(initialized_archive, env) + with use_archivebox_db(initialized_archive): + Snapshot.objects.get(url="https://example.com").save_tags(["z-tag", "a-tag"]) result = run_archivebox_cmd( ["list"], @@ -135,10 +163,12 @@ def test_list_outputs_existing_snapshots_as_jsonl(initialized_archive): rows = parse_jsonl_output(result.stdout) urls = {row["url"] for row in rows} + rows_by_url = {row["url"]: row for row in rows} assert result.returncode == 0, result.stderr assert "https://example.com" in urls assert "https://iana.org" in urls + assert rows_by_url["https://example.com"]["tags"] == "a-tag,z-tag" def test_list_filters_by_url_icontains(initialized_archive): diff --git a/archivebox/tests/test_cli_machine.py b/archivebox/tests/test_cli_machine.py index 602ea12d..5e445998 100644 --- a/archivebox/tests/test_cli_machine.py +++ b/archivebox/tests/test_cli_machine.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """ Tests for archivebox machine command. - -TODO: expand beyond command discovery into list/filter behavior. """ from archivebox.tests.conftest import run_archivebox_cmd diff --git a/archivebox/tests/test_cli_mcp.py b/archivebox/tests/test_cli_mcp.py index 82b2c97b..4090ae6f 100644 --- a/archivebox/tests/test_cli_mcp.py +++ b/archivebox/tests/test_cli_mcp.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """ Tests for archivebox mcp command. - -TODO: expand beyond command discovery into JSON-RPC stdio behavior. """ from archivebox.tests.conftest import run_archivebox_cmd diff --git a/archivebox/tests/test_cli_oneshot.py b/archivebox/tests/test_cli_oneshot.py new file mode 100644 index 00000000..84c1376f --- /dev/null +++ b/archivebox/tests/test_cli_oneshot.py @@ -0,0 +1,19 @@ +from importlib.metadata import version + +from archivebox.tests.conftest import run_archivebox_cmd + + +def test_oneshot_runs_abx_dl_through_abxpkg_env_projection(tmp_path): + lib_dir = tmp_path / "lib" + + result = run_archivebox_cmd( + ["oneshot", "--version"], + cwd=tmp_path, + env={"ABXPKG_LIB_DIR": str(lib_dir)}, + ) + + abx_dl_projection = lib_dir / "env" / "bin" / "abx-dl" + assert result.returncode == 0, result.stderr + assert version("abx-dl") in result.stdout + assert abx_dl_projection.is_symlink() + assert abx_dl_projection.resolve().is_file() diff --git a/archivebox/tests/test_cli_persona.py b/archivebox/tests/test_cli_persona.py index 6f45cc54..6379facc 100644 --- a/archivebox/tests/test_cli_persona.py +++ b/archivebox/tests/test_cli_persona.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """ Tests for archivebox persona command. - -TODO: expand beyond command discovery into create/list/update/delete behavior. """ from archivebox.tests.conftest import run_archivebox_cmd diff --git a/archivebox/tests/test_cli_piping.py b/archivebox/tests/test_cli_piping.py index 4c857651..a274db3a 100644 --- a/archivebox/tests/test_cli_piping.py +++ b/archivebox/tests/test_cli_piping.py @@ -6,9 +6,12 @@ This file covers both: - subprocess integration for the supported records `archivebox run` consumes """ +import os +import pty import sys import uuid -from io import StringIO +from importlib.resources import files +from pathlib import Path import pytest @@ -33,18 +36,33 @@ PIPE_TEST_ENV = { } -class MockTTYStringIO(StringIO): - def __init__(self, initial_value: str = "", *, is_tty: bool): - super().__init__(initial_value) - self._is_tty = is_tty +def run_real_txt_parser(tmp_path, text): + """Run the shipped text parser and return its real snapshot output directory.""" + from archivebox.plugins.hooks import run_hook - def isatty(self) -> bool: - return self._is_tty + snap_dir = tmp_path / "parser-snapshot" + staticfile_dir = snap_dir / "staticfile" + output_dir = snap_dir / "parse_txt_urls" + staticfile_dir.mkdir(parents=True) + output_dir.mkdir(parents=True) + (staticfile_dir / "input.txt").write_text(text, encoding="utf-8") + hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py"))) + process = run_hook( + hook_path, + output_dir, + config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)}, + timeout=30, + url="file:///input.txt", + depth=0, + ) + process.refresh_from_db() + assert process.exit_code == 0, process.stderr + return snap_dir def test_parse_line_accepts_supported_piping_inputs(): """The JSONL parser should normalize the input forms CLI pipes accept.""" - from archivebox.misc.jsonl import TYPE_CRAWL, TYPE_SNAPSHOT, parse_line + from archivebox.misc.jsonl import TYPE_SNAPSHOT, parse_line assert parse_line("") is None assert parse_line(" ") is None @@ -57,27 +75,8 @@ def test_parse_line_accepts_supported_piping_inputs(): assert parse_line("file:///tmp/example.txt") is None - snapshot_json = parse_line('{"type":"Snapshot","url":"https://example.com","tags":"tag1,tag2"}') - assert snapshot_json is not None - assert snapshot_json["type"] == TYPE_SNAPSHOT - assert snapshot_json["tags"] == "tag1,tag2" - crawl_json = parse_line('{"type":"Crawl","id":"abc123","urls":"https://example.com","max_depth":1}') - assert crawl_json is not None - assert crawl_json["type"] == TYPE_CRAWL - assert crawl_json["id"] == "abc123" - assert crawl_json["max_depth"] == 1 - - snapshot_id = "01234567-89ab-cdef-0123-456789abcdef" - parsed_id = parse_line(snapshot_id) - assert parsed_id == {"type": TYPE_SNAPSHOT, "id": snapshot_id} - - compact_snapshot_id = "0123456789abcdef0123456789abcdef" - compact_parsed_id = parse_line(compact_snapshot_id) - assert compact_parsed_id == {"type": TYPE_SNAPSHOT, "id": compact_snapshot_id} - - -def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl(tmp_path): +def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl(tmp_path, initialized_archive): """Piping helpers should consume args, structured JSONL, and pass-through records.""" from archivebox.misc.jsonl import TYPE_CRAWL, read_args_or_stdin @@ -88,80 +87,67 @@ def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl(tmp_path): local_file.write_text("https://from-file-arg.example\n") assert list(read_args_or_stdin((str(local_file),))) == [] - stdin_records = list( - read_args_or_stdin( - (), - stream=MockTTYStringIO( - "https://plain-url.com\n" - '{"type":"Snapshot","url":"https://jsonl-url.com","tags":"test"}\n' - '{"type":"Tag","id":"tag-1","name":"example"}\n' - "01234567-89ab-cdef-0123-456789abcdef\n" - "not valid json\n", - is_tty=False, - ), - ), + snapshot_result = run_archivebox_cmd( + ["snapshot", "create", "--tag=test", "https://jsonl-url.com"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + check=True, ) - assert len(stdin_records) == 4 - assert stdin_records[0]["url"] == "https://plain-url.com" - assert stdin_records[1]["url"] == "https://jsonl-url.com" - assert stdin_records[1]["tags"] == "test" - assert stdin_records[2]["type"] == "Tag" - assert stdin_records[2]["name"] == "example" - assert stdin_records[3]["id"] == "01234567-89ab-cdef-0123-456789abcdef" - - crawl_records = list( - read_args_or_stdin( - (), - stream=MockTTYStringIO( - '{"type":"Crawl","id":"crawl-1","urls":"https://example.com\\nhttps://foo.com"}\n', - is_tty=False, - ), - ), + crawl_result = run_archivebox_cmd( + ["crawl", "create", "https://crawl-url.com"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + check=True, ) - assert len(crawl_records) == 1 - assert crawl_records[0]["type"] == TYPE_CRAWL - assert crawl_records[0]["id"] == "crawl-1" + snapshot_record = next(record for record in parse_jsonl_output(snapshot_result.stdout) if record.get("type") == "Snapshot") - tty_records = list(read_args_or_stdin((), stream=MockTTYStringIO("https://example.com", is_tty=True))) - assert tty_records == [] + read_fd, write_fd = os.pipe() + os.write( + write_fd, + f"https://plain-url.com\n{snapshot_result.stdout}{crawl_result.stdout}{snapshot_record['id']}\nnot valid json\n".encode(), + ) + os.close(write_fd) + with os.fdopen(read_fd, encoding="utf-8") as pipe_stream: + assert pipe_stream.isatty() is False + stdin_records = list(read_args_or_stdin((), stream=pipe_stream)) + assert any(record.get("url") == "https://plain-url.com" for record in stdin_records) + assert any(record.get("type") == "Snapshot" and record.get("id") == snapshot_record["id"] for record in stdin_records) + assert any(record.get("type") == TYPE_CRAWL and record.get("urls") == "https://crawl-url.com" for record in stdin_records) + assert any( + record.get("type") == "Snapshot" and record.get("id") == snapshot_record["id"] and len(record) == 2 for record in stdin_records + ) + + master_fd, slave_fd = pty.openpty() + try: + with os.fdopen(slave_fd, encoding="utf-8") as tty_stream: + assert tty_stream.isatty() is True + assert list(read_args_or_stdin((), stream=tty_stream)) == [] + finally: + os.close(master_fd) def test_collect_urls_from_plugins_reads_only_parser_outputs(tmp_path): """Parser extractor `urls.jsonl` outputs should be discoverable for recursive piping.""" from archivebox.plugins.hooks import collect_urls_from_plugins - (tmp_path / "wget").mkdir() - (tmp_path / "wget" / "urls.jsonl").write_text( - '{"url":"https://wget-link-1.com"}\n{"url":"https://wget-link-2.com"}\n', - encoding="utf-8", - ) - (tmp_path / "parse_html_urls").mkdir() - (tmp_path / "parse_html_urls" / "urls.jsonl").write_text( - '{"url":"https://html-link-1.com"}\n{"url":"https://html-link-2.com","title":"HTML Link 2"}\n', - encoding="utf-8", - ) - (tmp_path / "screenshot").mkdir() + snap_dir = run_real_txt_parser(tmp_path, "https://html-link-1.com https://html-link-2.com") + (snap_dir / "screenshot").mkdir() - urls = collect_urls_from_plugins(tmp_path) - assert len(urls) == 4 - assert {url["plugin"] for url in urls} == {"wget", "parse_html_urls"} - titled = [url for url in urls if url.get("title") == "HTML Link 2"] - assert len(titled) == 1 - assert titled[0]["url"] == "https://html-link-2.com" + urls = collect_urls_from_plugins(snap_dir) + assert {url["url"] for url in urls} == {"https://html-link-1.com", "https://html-link-2.com"} + assert {url["plugin"] for url in urls} == {"parse_txt_urls"} - assert collect_urls_from_plugins(tmp_path / "nonexistent") == [] + assert collect_urls_from_plugins(snap_dir / "nonexistent") == [] def test_collect_urls_from_plugins_trims_markdown_suffixes(tmp_path): from archivebox.plugins.hooks import collect_urls_from_plugins - (tmp_path / "parse_html_urls").mkdir() - (tmp_path / "parse_html_urls" / "urls.jsonl").write_text( - '{"url":"https://docs.sweeting.me/s/youtube-favorites)**"}\n', - encoding="utf-8", - ) + snap_dir = run_real_txt_parser(tmp_path, "[favorites](https://docs.sweeting.me/s/youtube-favorites)**") - urls = collect_urls_from_plugins(tmp_path) + urls = collect_urls_from_plugins(snap_dir) assert len(urls) == 1 assert urls[0]["url"] == "https://docs.sweeting.me/s/youtube-favorites" @@ -169,13 +155,9 @@ def test_collect_urls_from_plugins_trims_markdown_suffixes(tmp_path): def test_collect_urls_from_plugins_trims_trailing_punctuation(tmp_path): from archivebox.plugins.hooks import collect_urls_from_plugins - (tmp_path / "parse_html_urls").mkdir() - (tmp_path / "parse_html_urls" / "urls.jsonl").write_text( - ('{"url":"https://github.com/ArchiveBox/ArchiveBox."}\n{"url":"https://github.com/abc?abc#234234?."}\n'), - encoding="utf-8", - ) + snap_dir = run_real_txt_parser(tmp_path, "https://github.com/ArchiveBox/ArchiveBox. https://github.com/abc?abc#234234?.") - urls = collect_urls_from_plugins(tmp_path) + urls = collect_urls_from_plugins(snap_dir) assert [url["url"] for url in urls] == [ "https://github.com/ArchiveBox/ArchiveBox", "https://github.com/abc?abc#234234", diff --git a/archivebox/tests/test_cli_pluginmap.py b/archivebox/tests/test_cli_pluginmap.py index d7266ec3..37ddc097 100644 --- a/archivebox/tests/test_cli_pluginmap.py +++ b/archivebox/tests/test_cli_pluginmap.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """ Tests for archivebox pluginmap command. - -TODO: expand beyond command discovery into quiet/event output behavior. """ from archivebox.tests.conftest import run_archivebox_cmd diff --git a/archivebox/tests/test_cli_process.py b/archivebox/tests/test_cli_process.py index 1968fc50..3a5b2706 100644 --- a/archivebox/tests/test_cli_process.py +++ b/archivebox/tests/test_cli_process.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """ Tests for archivebox process command. - -TODO: expand beyond command discovery into list/filter behavior. """ from archivebox.tests.conftest import run_archivebox_cmd diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index d9d5a370..08258d6b 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -7,14 +7,12 @@ Tests cover: - pass-through output (for chaining) """ -import json import os import signal import subprocess -import sys -import time import pytest +import psutil from archivebox.tests.conftest import ( cleanup_process_group, @@ -22,9 +20,8 @@ from archivebox.tests.conftest import ( run_archivebox_cmd, parse_jsonl_output, create_test_url, - create_test_crawl_json, - create_test_snapshot_json, pid_is_alive, + wait_for_log, wait_for_pid_to_disappear, ) @@ -48,133 +45,72 @@ def _install_real_chrome_for_test(data_dir, env, *, isolation): @pytest.mark.django_db(transaction=True) -@pytest.mark.timeout(90) -def test_cli_run_signal_cleans_background_hook_process_group(initialized_archive): +@pytest.mark.timeout(660) +def test_cli_run_signal_cleans_real_chrome_hook_process_group(initialized_archive, recursive_test_site): + from archivebox.core.models import Snapshot + from archivebox.tests.test_orm_helpers import use_archivebox_db - plugins_root = initialized_archive / "runtime_plugins" - plugin_dir = plugins_root / "cancel_group" - plugin_dir.mkdir(parents=True) - daemon_hook = plugin_dir / "on_CrawlSetup__10_daemon.daemon.bg.sh" - foreground_hook = plugin_dir / "on_CrawlSetup__20_foreground.sh" - daemon_hook.write_text( - "\n".join( - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - 'test_dir="${LEAK_TEST_DIR:?}"', - "sleep 600 &", - 'echo $$ > "$test_dir/daemon.pid"', - 'echo $! > "$test_dir/daemon-child.pid"', - 'echo ready > "$test_dir/daemon.ready"', - "trap 'echo cleaned > \"$test_dir/daemon.cleaned\"; exit 0' TERM INT", - "wait", - "", - ], - ), - ) - foreground_hook.write_text( - "\n".join( - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - 'test_dir="${LEAK_TEST_DIR:?}"', - 'echo $$ > "$test_dir/foreground.pid"', - 'echo ready > "$test_dir/foreground.ready"', - "trap 'echo cleaned > \"$test_dir/foreground.cleaned\"; exit 0' TERM INT", - "while true; do sleep 1; done", - "", - ], - ), - ) - daemon_hook.chmod(0o755) - foreground_hook.chmod(0o755) - - leak_test_dir = initialized_archive / "leak-check" - leak_test_dir.mkdir() - env = os.environ.copy() - env.update( - { - "ABX_PLUGINS_DIR": str(plugins_root), - "LEAK_TEST_DIR": str(leak_test_dir), - "PLUGINS": "cancel_group", - "TIMEOUT": "30", - "USE_COLOR": "false", - "SHOW_PROGRESS": "false", - }, - ) + env = cli_env(live=True, PLUGINS="chrome", CHROME_ISOLATION="crawl", CHROME_HEADLESS="true", CHROME_SANDBOX="false") + _install_real_chrome_for_test(initialized_archive, env, isolation="crawl") _cmd_result = run_archivebox_cmd( - ["crawl", "create", "https://example.com"], + ["snapshot", "create", recursive_test_site["root_url"]], cwd=initialized_archive, env=env, timeout=60, ) stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, stderr or stdout - crawl_records = [json.loads(line) for line in stdout.splitlines() if line.strip().startswith("{")] - crawl_id = next(record["id"] for record in crawl_records if record.get("type") == "Crawl") + records = parse_jsonl_output(stdout) + snapshot_id = next(record["id"] for record in records if record.get("type") == "Snapshot") + with use_archivebox_db(initialized_archive): + browser_state = Snapshot.objects.get(id=snapshot_id).output_dir / "chrome" / "browser.json" - daemon_pid: int | None = None - daemon_child_pid: int | None = None - foreground_pid: int | None = None + run_log = initialized_archive / "run-signal-chrome.log" + run_log_handle = run_log.open("w", encoding="utf-8") run_process = run_archivebox_cmd( - ["run", f"--crawl-id={crawl_id}"], + ["run", f"--snapshot-id={snapshot_id}"], cwd=initialized_archive, env=env, - stdout=subprocess.PIPE, + stdout=run_log_handle, stderr=subprocess.STDOUT, start_new_session=True, wait=False, ) + run_log_handle.close() try: - deadline = time.time() + 20 - while time.time() < deadline: - if (leak_test_dir / "daemon.ready").exists() and (leak_test_dir / "foreground.ready").exists(): - break - if run_process.poll() is not None: - output = run_process.communicate(timeout=1)[0] - raise AssertionError(f"archivebox run exited before hooks were ready:\n{output}") - time.sleep(0.05) - assert (leak_test_dir / "daemon.ready").exists() - assert (leak_test_dir / "foreground.ready").exists() - - daemon_pid = int((leak_test_dir / "daemon.pid").read_text().strip()) - daemon_child_pid = int((leak_test_dir / "daemon-child.pid").read_text().strip()) - foreground_pid = int((leak_test_dir / "foreground.pid").read_text().strip()) - assert pid_is_alive(daemon_pid) - assert pid_is_alive(daemon_child_pid) - assert pid_is_alive(foreground_pid) + wait_for_log(browser_state, '"ready": true', timeout=120) + child_pids = [child.pid for child in psutil.Process(run_process.pid).children(recursive=True)] + assert child_pids + assert all(pid_is_alive(pid) for pid in child_pids) run_process.send_signal(signal.SIGTERM) - output = run_process.communicate(timeout=20)[0] + run_process.wait(timeout=30) + output = run_log.read_text(encoding="utf-8", errors="replace") assert "Runner error" not in output - - wait_for_pid_to_disappear(daemon_pid, timeout=5) - wait_for_pid_to_disappear(daemon_child_pid, timeout=5) - wait_for_pid_to_disappear(foreground_pid, timeout=5) - assert (leak_test_dir / "daemon.cleaned").read_text().strip() == "cleaned" - assert (leak_test_dir / "foreground.cleaned").read_text().strip() == "cleaned" + for pid in child_pids: + wait_for_pid_to_disappear(pid, timeout=15) finally: - if run_process.poll() is None: - try: - os.killpg(run_process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - run_process.communicate(timeout=5) - cleanup_process_group(daemon_pid, daemon_child_pid) - cleanup_process_group(foreground_pid) + cleanup_process_group(run_process.pid) class TestRunWithCrawl: """Tests for `archivebox run` with Crawl input.""" def test_run_with_new_crawl(self, initialized_archive): - """Run creates and processes a new Crawl (no id).""" - crawl_record = create_test_crawl_json() + """Run processes a Crawl emitted by the public create command.""" + create_result = run_archivebox_cmd( + ["crawl", "create", create_test_url()], + cwd=initialized_archive, + env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, + ) + assert create_result.returncode == 0, create_result.stderr _cmd_result = run_archivebox_cmd( ["run"], - stdin=json.dumps(crawl_record), + stdin=create_result.stdout, cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, @@ -204,12 +140,10 @@ class TestRunWithCrawl: disable_extractors=True, ) stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode - crawl = parse_jsonl_output(stdout1)[0] - # Run with the existing crawl _cmd_result = run_archivebox_cmd( ["run"], - stdin=json.dumps(crawl), + stdin=stdout1, cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, @@ -227,12 +161,19 @@ class TestRunWithSnapshot: """Tests for `archivebox run` with Snapshot input.""" def test_run_with_new_snapshot(self, initialized_archive): - """Run creates and processes a new Snapshot (no id, just url).""" - snapshot_record = create_test_snapshot_json() + """Run processes a Snapshot emitted by the public create command.""" + create_result = run_archivebox_cmd( + ["snapshot", "create", create_test_url()], + cwd=initialized_archive, + env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, + ) + assert create_result.returncode == 0, create_result.stderr _cmd_result = run_archivebox_cmd( ["run"], - stdin=json.dumps(snapshot_record), + stdin=create_result.stdout, cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, @@ -261,12 +202,10 @@ class TestRunWithSnapshot: disable_extractors=True, ) stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode - snapshot = parse_jsonl_output(stdout1)[0] - # Run with the existing snapshot _cmd_result = run_archivebox_cmd( ["run"], - stdin=json.dumps(snapshot), + stdin=stdout1, cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, @@ -282,11 +221,9 @@ class TestRunWithSnapshot: def test_run_with_plain_url(self, initialized_archive): """Run accepts plain URL records (no type field).""" url = create_test_url() - url_record = {"url": url} - _cmd_result = run_archivebox_cmd( ["run"], - stdin=json.dumps(url_record), + stdin=url + "\n", cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, @@ -316,34 +253,64 @@ class TestRunWithArchiveResult: disable_extractors=True, ) stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode - snapshot = parse_jsonl_output(stdout1)[0] - _cmd_result = run_archivebox_cmd( ["archiveresult", "create", "--plugin=favicon"], - stdin=json.dumps(snapshot), + stdin=stdout1, cwd=initialized_archive, env=RUN_TEST_ENV, default_cli_env=True, disable_extractors=True, ) stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode - ar = next(r for r in parse_jsonl_output(stdout2) if r.get("type") == "ArchiveResult") + assert any(record.get("type") == "ArchiveResult" for record in parse_jsonl_output(stdout2)) - # Update to failed - ar["status"] = "failed" - run_archivebox_cmd( - ["archiveresult", "update", "--status=failed"], - stdin=json.dumps(ar), + initial_run = run_archivebox_cmd( + ["run"], + stdin=stdout2, + cwd=initialized_archive, + timeout=120, + env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, + ) + assert initial_run.returncode == 0, initial_run.stderr + persisted_result = run_archivebox_cmd( + ["archiveresult", "list", "--plugin=favicon"], cwd=initialized_archive, env=RUN_TEST_ENV, default_cli_env=True, disable_extractors=True, ) + assert persisted_result.returncode == 0, persisted_result.stderr + assert any(record.get("type") == "ArchiveResult" for record in parse_jsonl_output(persisted_result.stdout)) + + # Update to failed + update_result = run_archivebox_cmd( + ["archiveresult", "update", "--status=failed"], + stdin=persisted_result.stdout, + cwd=initialized_archive, + env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, + ) + assert update_result.returncode == 0, update_result.stderr + failed_result = run_archivebox_cmd( + ["archiveresult", "list", "--plugin=favicon"], + cwd=initialized_archive, + env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, + ) + assert failed_result.returncode == 0, failed_result.stderr + failed_records = [record for record in parse_jsonl_output(failed_result.stdout) if record.get("type") == "ArchiveResult"] + assert len(failed_records) == 1 + assert failed_records[0]["status"] == "failed" + failed_jsonl = next(line for line in failed_result.stdout.splitlines() if failed_records[0]["id"] in line) + "\n" # Now run should re-queue it _cmd_result = run_archivebox_cmd( ["run"], - stdin=json.dumps(ar), + stdin=failed_jsonl, cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, @@ -417,13 +384,20 @@ class TestRunRecovery: class TestRunPassThrough: """Tests for pass-through behavior in `archivebox run`.""" - def test_run_passes_through_unknown_types(self, initialized_archive): - """Run passes through records with unknown types.""" - unknown_record = {"type": "Unknown", "id": "fake-id", "data": "test"} + def test_run_passes_through_tag_emitted_by_cli(self, initialized_archive): + """Run passes through a real non-runnable Tag record.""" + tag_result = run_archivebox_cmd( + ["tag", "create", "run-input-tag"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + assert tag_result.returncode == 0, tag_result.stderr + tag_record = parse_jsonl_output(tag_result.stdout)[0] _cmd_result = run_archivebox_cmd( ["run"], - stdin=json.dumps(unknown_record), + stdin=tag_result.stdout, cwd=initialized_archive, default_cli_env=True, disable_extractors=True, @@ -432,18 +406,25 @@ class TestRunPassThrough: assert code == 0 records = parse_jsonl_output(stdout) - unknown_records = [r for r in records if r.get("type") == "Unknown"] - assert len(unknown_records) == 1 - assert unknown_records[0]["data"] == "test" + tag_records = [record for record in records if record.get("type") == "Tag"] + assert len(tag_records) == 1 + assert tag_records[0]["id"] == tag_record["id"] def test_run_outputs_all_processed_records(self, initialized_archive): """Run outputs all processed records for chaining.""" url = create_test_url() - crawl_record = create_test_crawl_json(urls=[url]) + create_result = run_archivebox_cmd( + ["crawl", "create", url], + cwd=initialized_archive, + env=RUN_TEST_ENV, + default_cli_env=True, + disable_extractors=True, + ) + assert create_result.returncode == 0, create_result.stderr _cmd_result = run_archivebox_cmd( ["run"], - stdin=json.dumps(crawl_record), + stdin=create_result.stdout, cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, @@ -461,23 +442,34 @@ class TestRunPassThrough: class TestRunMixedInput: """Tests for `archivebox run` with mixed record types.""" - def test_run_handles_mixed_types(self, initialized_archive): - """Run handles mixed Crawl/Snapshot/ArchiveResult input.""" - crawl = create_test_crawl_json() - snapshot = create_test_snapshot_json() - unknown = {"type": "Tag", "id": "fake", "name": "test"} - - stdin = "\n".join( - [ - json.dumps(crawl), - json.dumps(snapshot), - json.dumps(unknown), - ], + def test_run_handles_mixed_records_emitted_by_cli(self, initialized_archive): + """Run handles real Crawl, Snapshot, and Tag records from CLI stages.""" + tag_result = run_archivebox_cmd( + ["tag", "create", "mixed-run-tag"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + assert tag_result.returncode == 0, tag_result.stderr + crawl_result = run_archivebox_cmd( + ["crawl", "create", create_test_url()], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + assert crawl_result.returncode == 0, crawl_result.stderr + snapshot_result = run_archivebox_cmd( + ["snapshot", "create"], + stdin=crawl_result.stdout, + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + assert snapshot_result.returncode == 0, snapshot_result.stderr _cmd_result = run_archivebox_cmd( ["run"], - stdin=stdin, + stdin=tag_result.stdout + snapshot_result.stdout, cwd=initialized_archive, timeout=120, env=RUN_TEST_ENV, @@ -489,9 +481,8 @@ class TestRunMixedInput: assert code == 0 records = parse_jsonl_output(stdout) - types = {r.get("type") for r in records} - # Should have processed Crawl and Snapshot, passed through Tag - assert "Crawl" in types or "Snapshot" in types or "Tag" in types + types = {record.get("type") for record in records} + assert {"Crawl", "Snapshot", "Tag"}.issubset(types) class TestRunEmpty: @@ -510,13 +501,19 @@ class TestRunEmpty: assert code == 0 - def test_run_no_records_to_process(self, initialized_archive): - """Run with only pass-through records shows message.""" - unknown = {"type": "Unknown", "id": "fake"} + def test_run_no_runnable_records_to_process(self, initialized_archive): + """Run with only a real non-runnable Tag reports no work.""" + tag_result = run_archivebox_cmd( + ["tag", "create", "non-runnable-tag"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + assert tag_result.returncode == 0, tag_result.stderr _cmd_result = run_archivebox_cmd( ["run"], - stdin=json.dumps(unknown), + stdin=tag_result.stdout, cwd=initialized_archive, default_cli_env=True, disable_extractors=True, @@ -537,72 +534,69 @@ class TestRunDaemonMode: snapshot_url = None if stdin_kind == "valid-snapshot": snapshot_url = create_test_url() - piped_stdin = json.dumps(create_test_snapshot_json(url=snapshot_url)) + "\n" + snapshot_result = run_archivebox_cmd( + ["snapshot", "create", snapshot_url], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + assert snapshot_result.returncode == 0, snapshot_result.stderr + piped_stdin = snapshot_result.stdout else: piped_stdin = "{this is not jsonl}\n" - env = cli_env() + env = cli_env(PLUGINS="__archivebox_test_no_plugins__") + queued = run_archivebox_cmd( + ["crawl", "create", create_test_url()], + cwd=initialized_archive, + env=env, + timeout=60, + ) + assert queued.returncode == 0, queued.stderr or queued.stdout + daemon_log = initialized_archive / f"run-daemon-{stdin_kind}.log" + daemon_log_handle = daemon_log.open("w", encoding="utf-8") proc = run_archivebox_cmd( ["run", "--daemon"], cwd=initialized_archive, env=env, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=daemon_log_handle, + stderr=subprocess.STDOUT, start_new_session=True, wait=False, ) + daemon_log_handle.close() assert proc.stdin is not None - assert proc.stdout is not None - assert proc.stderr is not None try: proc.stdin.write(piped_stdin) proc.stdin.close() - deadline = time.monotonic() + 20 - started = False - while time.monotonic() < deadline: - if proc.poll() is not None: - stdout = proc.stdout.read() - stderr = proc.stderr.read() - raise AssertionError( - f"daemon exited before starting runner: code={proc.returncode}\nstdout={stdout}\nstderr={stderr}", - ) - with use_archivebox_db(initialized_archive): - started = Process.objects.filter( - process_type=Process.TypeChoices.ORCHESTRATOR, - status=Process.StatusChoices.RUNNING, - pid=proc.pid, - ).exists() - if started: - break - time.sleep(0.25) - - assert started is True + wait_for_log(daemon_log, "[Crawl#", timeout=30) + with use_archivebox_db(initialized_archive): + started = Process.objects.filter( + process_type=Process.TypeChoices.ORCHESTRATOR, + status=Process.StatusChoices.RUNNING, + pid=proc.pid, + ).exists() + assert started if snapshot_url is not None: with use_archivebox_db(initialized_archive): assert not Snapshot.objects.filter(url=snapshot_url).exists() finally: - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGTERM) - try: - proc.wait(timeout=15) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait(timeout=5) + os.kill(proc.pid, signal.SIGTERM) + proc.wait(timeout=15) - stdout = proc.stdout.read() - stderr = proc.stderr.read() - assert proc.returncode == 143, stdout + stderr - assert "No records to process" not in stderr + output = daemon_log.read_text(encoding="utf-8", errors="replace") + assert proc.returncode == 143, output + assert "No records to process" not in output def test_run_daemon_takeover_has_single_active_runner_gate(self, initialized_archive, db): from archivebox.machine.models import Process from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE from archivebox.tests.test_orm_helpers import use_archivebox_db - env = cli_env() + env = cli_env(PLUGINS="__archivebox_test_no_plugins__") def active_runners(): with use_archivebox_db(initialized_archive): @@ -617,67 +611,65 @@ class TestRunDaemonMode: if proc.is_running ] - def wait_for_stable_single_active(*, timeout: float, stable_seconds: float = 1.0, exclude_pid: int | None = None): - deadline = time.monotonic() + timeout - stable_pid = None - stable_since = None - while time.monotonic() < deadline: - active = active_runners() - assert len(active) <= 1 - if len(active) == 1 and active[0].pid != exclude_pid: - pid = active[0].pid - if pid != stable_pid: - stable_pid = pid - stable_since = time.monotonic() - elif stable_since is not None and time.monotonic() - stable_since >= stable_seconds: - return pid - else: - stable_pid = None - stable_since = None - time.sleep(0.25) - return None - - procs = [ - run_archivebox_cmd( + queued = run_archivebox_cmd(["crawl", "create", create_test_url()], cwd=initialized_archive, env=env, timeout=60) + assert queued.returncode == 0, queued.stderr or queued.stdout + procs = [] + logs = [] + for index in range(2): + log_path = initialized_archive / f"run-daemon-takeover-{index}.log" + log_handle = log_path.open("w", encoding="utf-8") + proc = run_archivebox_cmd( ["run", "--daemon"], cwd=initialized_archive, env=env, stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=log_handle, + stderr=subprocess.STDOUT, start_new_session=True, wait=False, ) - for _ in range(2) - ] + log_handle.close() + procs.append(proc) + logs.append(log_path) + if index == 0: + wait_for_log(log_path, "[Crawl#", timeout=30) try: - active_pid = wait_for_stable_single_active(timeout=30) - assert active_pid is not None + wait_for_log(logs[1], "Stopping older ArchiveBox runner process", timeout=30) + queued = run_archivebox_cmd(["crawl", "create", create_test_url()], cwd=initialized_archive, env=env, timeout=60) + assert queued.returncode == 0, queued.stderr or queued.stdout + wait_for_log(logs[1], "[Crawl#", timeout=30) + active = active_runners() + assert len(active) == 1 + active_pid = active[0].pid + assert active_pid == procs[1].pid - os.killpg(active_pid, signal.SIGKILL) + os.kill(active_pid, signal.SIGTERM) + wait_for_pid_to_disappear(active_pid, timeout=15) + replacement_log = initialized_archive / "run-daemon-takeover-replacement.log" + replacement_log_handle = replacement_log.open("w", encoding="utf-8") replacement = run_archivebox_cmd( ["run", "--daemon"], cwd=initialized_archive, env=env, stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=replacement_log_handle, + stderr=subprocess.STDOUT, start_new_session=True, wait=False, ) + replacement_log_handle.close() procs.append(replacement) - recovered_pid = wait_for_stable_single_active(timeout=30, exclude_pid=active_pid) - assert recovered_pid is not None + queued = run_archivebox_cmd(["crawl", "create", create_test_url()], cwd=initialized_archive, env=env, timeout=60) + assert queued.returncode == 0, queued.stderr or queued.stdout + wait_for_log(replacement_log, "[Crawl#", timeout=30) + recovered = active_runners() + assert len(recovered) == 1 + assert recovered[0].pid == replacement.pid + assert recovered[0].pid != active_pid finally: for proc in procs: - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGTERM) - for proc in procs: - try: - proc.wait(timeout=15) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait(timeout=5) + cleanup_process_group(proc.pid) + proc.wait(timeout=15) @pytest.mark.django_db @@ -1114,11 +1106,12 @@ class TestRecoverOrchestratorState: sealed_crawl.cancel() paused_child.refresh_from_db() + paused_result.refresh_from_db() sealed_child.refresh_from_db() sealed_started_child.refresh_from_db() - assert paused_child.status == Snapshot.StatusChoices.STARTED - assert paused_child.retry_at is not None - assert paused_child.retry_at <= timezone.now() + assert paused_child.status == Snapshot.StatusChoices.PAUSED + assert paused_child.retry_at == RETRY_AT_MAX + assert paused_result.status == ArchiveResult.StatusChoices.PAUSED assert sealed_child.status == Snapshot.StatusChoices.PAUSED assert sealed_child.retry_at is not None assert sealed_child.retry_at <= timezone.now() @@ -1126,13 +1119,6 @@ class TestRecoverOrchestratorState: assert sealed_started_child.retry_at is not None assert sealed_started_child.retry_at <= timezone.now() - assert run_due_snapshot(paused_child, lock_seconds=60) is True - paused_child.refresh_from_db() - paused_result.refresh_from_db() - assert paused_child.status == Snapshot.StatusChoices.PAUSED - assert paused_child.retry_at == RETRY_AT_MAX - assert paused_result.status == ArchiveResult.StatusChoices.PAUSED - assert run_due_snapshot(sealed_child, lock_seconds=60) is True sealed_child.refresh_from_db() assert sealed_child.status == Snapshot.StatusChoices.SEALED @@ -1243,7 +1229,7 @@ class TestRecoverOrchestratorState: assert crawl.retry_at < future assert snapshot.retry_at < future - def test_recover_orchestrator_state_preserves_future_started_snapshot_with_live_result_process(self): + def test_recover_orchestrator_state_preserves_future_started_snapshot_with_live_result_process(self, initialized_archive): from datetime import timedelta from django.utils import timezone @@ -1254,13 +1240,19 @@ class TestRecoverOrchestratorState: from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.core.recovery_util import recover_orchestrator_state - worker = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(60)"], + worker = run_archivebox_cmd( + ["manage", "shell"], + cwd=initialized_archive, + env=cli_env(live=True), + stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - text=True, + capture_output=False, start_new_session=True, + wait=False, ) + assert worker.stdin is not None + assert pid_is_alive(worker.pid) try: future = timezone.now() + timedelta(seconds=45) crawl = Crawl.objects.create( @@ -1303,13 +1295,9 @@ class TestRecoverOrchestratorState: assert snapshot.status == Snapshot.StatusChoices.STARTED assert snapshot.retry_at == future finally: - if worker.poll() is None: - os.killpg(worker.pid, signal.SIGTERM) - try: - worker.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(worker.pid, signal.SIGKILL) - worker.wait(timeout=5) + worker.stdin.close() + worker.wait(timeout=20) + assert not pid_is_alive(worker.pid) def test_recover_orchestrator_state_does_not_resume_paused_rows_with_max_retry_at(self): from archivebox.base_models.models import get_or_create_system_user_pk @@ -1628,7 +1616,7 @@ class TestRecoverOrchestratorState: update_process = run_archivebox_cmd( ["archiveresult", "update", "--status=queued"], - stdin=json.dumps(wait_record) + "\n", + stdin=next(line for line in list_process.stdout.splitlines() if wait_record["id"] in line) + "\n", cwd=initialized_archive, env=env, timeout=60, @@ -1649,47 +1637,27 @@ class TestRecoverOrchestratorState: run_process.stdin.write(update_process.stdout) run_process.stdin.close() - resumed_full_plugin = False try: - deadline = time.time() + 90 - last_wait_status = None - last_tab_process_id = None - while time.time() < deadline: - with use_archivebox_db(initialized_archive): - wait_result = ArchiveResult.objects.get( - snapshot_id=snapshot_id, - plugin="chrome", - hook_name="on_Snapshot__11_chrome_wait", - ) - tab_result = ArchiveResult.objects.get( - snapshot_id=snapshot_id, - plugin="chrome", - hook_name="on_Snapshot__10_chrome_tab.daemon.bg", - ) - last_wait_status = wait_result.status - last_tab_process_id = tab_result.process_id - if wait_result.status == ArchiveResult.StatusChoices.SUCCEEDED and tab_result.process_id != first_tab_process_id: - resumed_full_plugin = True - break - if run_process.poll() is not None: - break - time.sleep(0.5) - - if resumed_full_plugin: - try: - run_process.wait(timeout=30) - except subprocess.TimeoutExpired: - cleanup_process_group(run_process.pid) - run_process.wait(timeout=10) + run_process.wait(timeout=120) finally: - if run_process.poll() is None: - cleanup_process_group(run_process.pid) - run_process.wait(timeout=10) + cleanup_process_group(run_process.pid) + + with use_archivebox_db(initialized_archive): + wait_result = ArchiveResult.objects.get( + snapshot_id=snapshot_id, + plugin="chrome", + hook_name="on_Snapshot__11_chrome_wait", + ) + tab_result = ArchiveResult.objects.get( + snapshot_id=snapshot_id, + plugin="chrome", + hook_name="on_Snapshot__10_chrome_tab.daemon.bg", + ) assert run_process.returncode == 0 - assert last_wait_status == ArchiveResult.StatusChoices.SUCCEEDED - assert last_tab_process_id is not None - assert last_tab_process_id != first_tab_process_id + assert wait_result.status == ArchiveResult.StatusChoices.SUCCEEDED + assert tab_result.process_id is not None + assert tab_result.process_id != first_tab_process_id def test_recover_orchestrator_state_ignores_sealed_downloaded_snapshot_without_results(self): from django.utils import timezone @@ -1912,13 +1880,15 @@ class TestRunDueCrawlState: assert finished.output_files == {"favicon.ico": {"size": 1}} def test_finished_parser_result_projects_children_before_resume_seals_snapshot(self): - import json + from importlib.resources import files + from pathlib import Path from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.plugins.hooks import extract_records_from_process, run_hook from archivebox.services.runner import run_due_snapshot crawl = Crawl.objects.create( @@ -1935,17 +1905,34 @@ class TestRunDueCrawlState: status=Snapshot.StatusChoices.STARTED, retry_at=timezone.now(), ) + staticfile_dir = root.output_dir / "staticfile" parser_dir = root.output_dir / "parse_txt_urls" + staticfile_dir.mkdir(parents=True, exist_ok=True) parser_dir.mkdir(parents=True, exist_ok=True) - (parser_dir / "urls.jsonl").write_text( - json.dumps({"type": "Snapshot", "url": "https://example.org/"}) + "\n", + (staticfile_dir / "input.txt").write_text( + "Plain text import containing https://example.org/\n", encoding="utf-8", ) + hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py"))) + process = run_hook( + hook_path, + parser_dir, + config={"ABXPKG_LIB_DIR": str(root.output_dir.parent.parent / "lib"), "SNAP_DIR": str(root.output_dir)}, + timeout=30, + url=root.url, + depth=root.depth, + snapshot_id=str(root.id), + ) + process.refresh_from_db() + assert process.exit_code == 0, process.stderr + result_record = next(record for record in extract_records_from_process(process) if record.get("type") == "ArchiveResult") ArchiveResult.objects.create( snapshot=root, - plugin="parse_txt_urls", - hook_name="on_Snapshot__71_parse_txt_urls", - status=ArchiveResult.StatusChoices.SUCCEEDED, + process=process, + plugin=result_record["plugin"], + hook_name=result_record["hook_name"], + status=result_record["status"], + output_str=result_record.get("output_str", ""), output_files={"urls.jsonl": {"size": (parser_dir / "urls.jsonl").stat().st_size}}, ) diff --git a/archivebox/tests/test_cli_schedule.py b/archivebox/tests/test_cli_schedule.py index 81bae29e..ff970f35 100644 --- a/archivebox/tests/test_cli_schedule.py +++ b/archivebox/tests/test_cli_schedule.py @@ -15,7 +15,7 @@ from .conftest import ( make_latest_schedule_due, start_archivebox_server, stop_server, - wait_for_http, + get_http_response, wait_for_snapshot_capture, ) @@ -162,7 +162,7 @@ def test_schedule_due_crawl_runs_over_server_and_saves_real_content(tmp_path, re try: start_archivebox_server(tmp_path, env=env, port=port) - wait_for_http(port, host=f"web.archivebox.localhost:{port}") + get_http_response(port, host=f"web.archivebox.localhost:{port}") captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) assert "Root" in captured_text assert "About" in captured_text diff --git a/archivebox/tests/test_cli_search.py b/archivebox/tests/test_cli_search.py index f3780796..9dc1a127 100644 --- a/archivebox/tests/test_cli_search.py +++ b/archivebox/tests/test_cli_search.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """ Tests for archivebox search command. - -TODO: keep search-specific tests here instead of folding all coverage into test_cli_list.py. """ import json @@ -62,12 +60,3 @@ def test_cli_search_status_filters_snapshot_status_column(tmp_path, initialized_ rows = [json.loads(line) for line in result.stdout.splitlines() if line.strip().startswith("{")] assert [row["status"] for row in rows] == ["sealed"] assert [row["url"] for row in rows] == ["https://example.com/search-status-sealed"] - - legacy_result = run_archivebox_cmd( - ["search", "--status", "unarchived", "search-status"], - env=env, - timeout=30, - ) - - assert legacy_result.returncode != 0 - assert "Invalid snapshot status" in legacy_result.stderr diff --git a/archivebox/tests/test_cli_server.py b/archivebox/tests/test_cli_server.py index 56c4e136..c6be16f0 100644 --- a/archivebox/tests/test_cli_server.py +++ b/archivebox/tests/test_cli_server.py @@ -8,10 +8,10 @@ import os import asyncio import json import signal +import shlex import socket import subprocess import sys -import time from datetime import datetime from pathlib import Path from types import SimpleNamespace @@ -19,15 +19,18 @@ from types import SimpleNamespace import pytest from archivebox.tests.conftest import ( + _wait_for_archivebox_workers, assert_no_processes_for_data_dir, + assert_port_open, + find_process, get_free_port, kill_processes_for_data_dir, cli_env, + pid_is_alive, start_archivebox_server, stop_archivebox_process, + wait_for_log_count, wait_for_pid_to_disappear, - wait_for_port_open, - wait_for_process, run_archivebox_cmd, resolve_abxpkg_binary_env, ) @@ -42,12 +45,13 @@ def _resolve_sonic_env(data_dir: Path) -> dict[str, str]: return resolved -def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path, monkeypatch): +def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path): """Admin sessions must survive `archivebox server` restarts for a collection.""" from archivebox.config.collection import write_config_file (tmp_path / ".archivebox_id").write_text("testcoll") - monkeypatch.setenv("BASE_URL", "http://archivebox.localhost:9292") + first_env = os.environ.copy() + first_env["BASE_URL"] = "http://archivebox.localhost:9292" first = subprocess.run( [ @@ -70,6 +74,8 @@ def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path, mon capture_output=True, text=True, check=True, + cwd=tmp_path, + env=first_env, ) first_lines = first.stdout.strip().splitlines() assert first_lines[0], first.stderr @@ -77,8 +83,9 @@ def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path, mon # Simulate the next `archivebox server` process, reading only persisted # collection config. If SECRET_KEY falls back to the random default_factory # here, Django will reject existing signed session cookies after restart. - monkeypatch.delenv("BASE_URL", raising=False) write_config_file({"BASE_URL": "http://archivebox.localhost:9292"}) + second_env = os.environ.copy() + second_env.pop("BASE_URL", None) second = subprocess.run( [ sys.executable, @@ -100,6 +107,8 @@ def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path, mon capture_output=True, text=True, check=True, + cwd=tmp_path, + env=second_env, ) assert second.stdout.strip().splitlines() == first_lines @@ -181,13 +190,13 @@ def test_server_help_lists_runtime_options(initialized_archive): assert "--reload" in result.stdout -def test_runner_worker_uses_current_interpreter(): - """The supervised runner should use the active Python environment, not PATH.""" - from archivebox.workers.supervisord_util import RUNNER_WORKER +def test_runner_worker_uses_abxpkg_projected_archivebox(): + from archivebox.workers.supervisord_util import RUNNER_WORKER, resolve_env_binary - assert RUNNER_WORKER["command"] == f"{sys.executable} -m archivebox run --daemon" - assert RUNNER_WORKER["autorestart"] == "true" - assert 'ARCHIVEBOX_RUNNER_DAEMON="1"' in RUNNER_WORKER["environment"] + worker = RUNNER_WORKER() + assert shlex.split(worker["command"]) == [str(resolve_env_binary("archivebox")), "run", "--daemon"] + assert worker["autorestart"] == "true" + assert 'ARCHIVEBOX_RUNNER_DAEMON="1"' in worker["environment"] def test_daphne_worker_uses_default_application_close_timeout(): @@ -199,20 +208,26 @@ def test_daphne_worker_uses_default_application_close_timeout(): assert "--application-close-timeout=0" not in command -def test_reload_workers_use_current_interpreter_and_supervisord_managed_runner(): - from archivebox.workers.supervisord_util import RUNNER_WATCH_WORKER, RUNSERVER_WORKER +def test_reload_workers_use_abxpkg_projected_archivebox(): + from archivebox.workers.supervisord_util import RUNNER_WATCH_WORKER, RUNSERVER_WORKER, resolve_env_binary runserver = RUNSERVER_WORKER("127.0.0.1", "8000", reload=True) watcher = RUNNER_WATCH_WORKER("http://127.0.0.1:8000") + archivebox_binary = resolve_env_binary("archivebox") assert runserver["name"] == "worker_runserver" - assert runserver["command"] == f"{sys.executable} -m archivebox manage runserver 127.0.0.1:8000" + assert shlex.split(runserver["command"]) == [str(archivebox_binary), "manage", "runserver", "127.0.0.1:8000"] assert 'ARCHIVEBOX_RUNSERVER="1"' in runserver["environment"] assert 'ARCHIVEBOX_AUTORELOAD="1"' in runserver["environment"] assert 'ARCHIVEBOX_RUNSERVER_BIND_URL="http://127.0.0.1:8000"' in runserver["environment"] assert watcher["name"] == "worker_runner_watch" - assert watcher["command"] == f"{sys.executable} -m archivebox manage runner_watch --bind-url=http://127.0.0.1:8000" + assert shlex.split(watcher["command"]) == [ + str(archivebox_binary), + "manage", + "runner_watch", + "--bind-url=http://127.0.0.1:8000", + ] def test_server_daemon_starts_real_plugin_owned_sonic_worker(initialized_archive, archivebox_daemon_server): @@ -235,18 +250,17 @@ def test_server_daemon_restarts_runner_killed_by_signal(archivebox_daemon_server ) state = server.wait_for_workers(("worker_daphne", "worker_runner")) old_runner_pid = state["worker_runner"]["pid"] + supervisord_log = server.data_dir / "logs" / "supervisord.log" + spawn_text = "spawned: 'worker_runner' with pid" + spawn_count = supervisord_log.read_text(encoding="utf-8", errors="replace").count(spawn_text) os.kill(old_runner_pid, signal.SIGTERM) - deadline = time.time() + 30 - while time.time() < deadline: - state = server.worker_state() - runner = state.get("worker_runner", {}) - if runner.get("statename") == "RUNNING" and runner.get("pid") and runner.get("pid") != old_runner_pid: - break - time.sleep(0.5) - else: - raise AssertionError(f"worker_runner did not restart after SIGTERM: {state}") + wait_for_log_count(supervisord_log, spawn_text, spawn_count + 1, timeout=30) + state = server.worker_state() + runner = state["worker_runner"] + assert runner["statename"] == "RUNNING", state + assert runner["pid"] != old_runner_pid, state assert state["worker_daphne"]["statename"] == "RUNNING", state @@ -472,8 +486,7 @@ def test_supervisord_takeover_stops_all_live_process_rows(initialized_archive, d ).exists() finally: for proc in procs: - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGKILL) + stop_archivebox_process(proc, signal.SIGTERM) @pytest.mark.timeout(300) @@ -497,11 +510,7 @@ def test_live_server_signal_exit_and_resume_uses_existing_supervisor_state(initi server_log = server.log_path os.kill(server.pid, stop_signal) - try: - server.wait(timeout=20 if stop_signal != signal.SIGKILL else 5) - except subprocess.TimeoutExpired: - os.kill(server.pid, signal.SIGKILL) - server.wait(timeout=5) + server.wait(timeout=20) if expected_notice: log_text = server_log.read_text(encoding="utf-8", errors="replace") @@ -523,8 +532,8 @@ def test_live_server_signal_exit_and_resume_uses_existing_supervisor_state(initi assert_no_processes_for_data_dir(initialized_archive, timeout=12) finally: for proc in (server, resumed): - if proc is not None and proc.poll() is None: - stop_archivebox_process(proc, signal.SIGKILL) + if proc is not None: + stop_archivebox_process(proc, signal.SIGTERM) kill_processes_for_data_dir(initialized_archive) @@ -543,19 +552,20 @@ def test_live_daemonized_server_keeps_supervisord_owned_by_archivebox_parent(ini ) stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, stderr or stdout - wait_for_port_open("127.0.0.1", port, timeout=30) + _wait_for_archivebox_workers(initialized_archive, env, ("worker_daphne", "worker_runner"), timeout=30) + assert_port_open("127.0.0.1", port, timeout=30) - server_process = wait_for_process( + server_process = find_process( lambda _proc, command: "archivebox" in command and " server " in f" {command} " and bind_url.replace("http://", "") in command, ) - supervisord = wait_for_process( + supervisord = find_process( lambda proc, command: proc.ppid() == server_process.pid and "supervisord" in command, ) - wait_for_process( + find_process( lambda proc, command: proc.ppid() == supervisord.pid and "supervisord_watchdog" in command, ) - os.kill(server_process.pid, signal.SIGKILL) + os.kill(server_process.pid, signal.SIGTERM) wait_for_pid_to_disappear(server_process.pid, timeout=10) wait_for_pid_to_disappear(supervisord.pid, timeout=20) assert_no_processes_for_data_dir(initialized_archive, timeout=12) @@ -608,7 +618,7 @@ def test_live_servers_in_different_data_dirs_do_not_interfere(initialized_archiv stop_archivebox_process(first, signal.SIGTERM) first = None - assert second.poll() is None, "stopping one DATA_DIR server must not stop another DATA_DIR server" + assert pid_is_alive(second.pid), "stopping one DATA_DIR server must not stop another DATA_DIR server" first_resumed = start_archivebox_server( first_data_dir, @@ -616,10 +626,10 @@ def test_live_servers_in_different_data_dirs_do_not_interfere(initialized_archiv log_name="server-first-data-dir-resumed.log", env=cli_env(live=True), ) - assert second.poll() is None, "restarting one DATA_DIR server must not take over another DATA_DIR supervisor" + assert pid_is_alive(second.pid), "restarting one DATA_DIR server must not take over another DATA_DIR supervisor" finally: for proc in (first, first_resumed, second): - if proc is not None and proc.poll() is None: + if proc is not None: stop_archivebox_process(proc, signal.SIGTERM) kill_processes_for_data_dir(first_data_dir) kill_processes_for_data_dir(second_data_dir) diff --git a/archivebox/tests/test_cli_snapshot.py b/archivebox/tests/test_cli_snapshot.py index 21a12078..b3e1ed90 100644 --- a/archivebox/tests/test_cli_snapshot.py +++ b/archivebox/tests/test_cli_snapshot.py @@ -95,11 +95,18 @@ class TestSnapshotCreate: records = parse_jsonl_output(stdout) assert "test-tag" in records[0].get("tags", "") - def test_create_pass_through_other_types(self, initialized_archive): - """Pass-through records of other types unchanged.""" - tag_record = {"type": "Tag", "id": "fake-tag-id", "name": "test"} + def test_create_passes_through_tag_emitted_by_cli(self, initialized_archive): + """A real Tag emitted by the CLI remains available to the next stage.""" + tag_result = run_archivebox_cmd( + ["tag", "create", "snapshot-input-tag"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, + ) + assert tag_result.returncode == 0, tag_result.stderr + tag_record = parse_jsonl_output(tag_result.stdout)[0] url = create_test_url() - stdin = json.dumps(tag_record) + "\n" + json.dumps({"url": url}) + stdin = tag_result.stdout + url + "\n" _cmd_result = run_archivebox_cmd( ["snapshot", "create"], @@ -113,9 +120,8 @@ class TestSnapshotCreate: assert code == 0 records = parse_jsonl_output(stdout) - types = [r.get("type") for r in records] - assert "Tag" in types - assert "Snapshot" in types + assert any(record.get("type") == "Tag" and record["id"] == tag_record["id"] for record in records) + assert any(record.get("type") == "Snapshot" and record["url"] == url for record in records) def test_create_multiple_urls(self, initialized_archive): """Create snapshots from multiple URLs.""" diff --git a/archivebox/tests/test_cli_tag.py b/archivebox/tests/test_cli_tag.py index 2af1f78d..1c0cc009 100644 --- a/archivebox/tests/test_cli_tag.py +++ b/archivebox/tests/test_cli_tag.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """ Tests for archivebox tag command. - -TODO: expand beyond command discovery into create/list/update/delete behavior. """ from archivebox.tests.conftest import run_archivebox_cmd diff --git a/archivebox/tests/test_config_DELETE_AFTER.py b/archivebox/tests/test_config_DELETE_AFTER.py index 9886c47c..cdca03a0 100644 --- a/archivebox/tests/test_config_DELETE_AFTER.py +++ b/archivebox/tests/test_config_DELETE_AFTER.py @@ -14,7 +14,7 @@ ADMIN_HOST = "admin.archivebox.localhost:8000" API_HOST = "api.archivebox.localhost:8000" -def test_delete_after_real_cli_and_orchestrator_paths_cover_all_retained_models(tmp_path): +def test_delete_after_real_cli_and_orchestrator_paths_cover_all_retained_models(tmp_path, recursive_test_site): env = cli_env(disable_extractors=True) _cmd_result = run_archivebox_cmd(["init", "--quick"], cwd=tmp_path, timeout=90) stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode @@ -22,13 +22,14 @@ def test_delete_after_real_cli_and_orchestrator_paths_cover_all_retained_models( run_env = { **env, + "ABXPKG_LIB_DIR": str(tmp_path / "lib"), "DELETE_AFTER": "1hr", "USE_COLOR": "False", "SHOW_PROGRESS": "False", } - url = "https://example.com/delete-after-cli" + url = recursive_test_site["root_url"] _cmd_result = run_archivebox_cmd( - ["add", "--index-only", "--depth=0", url], + ["add", "--plugins=wget,hashes", "--depth=0", url], cwd=tmp_path, timeout=120, env=run_env, @@ -62,69 +63,39 @@ print(json.dumps({{ setup_script = f""" import json +import os from pathlib import Path from datetime import timedelta from django.utils import timezone from archivebox.core.models import ArchiveResult, Snapshot -from archivebox.machine.models import Machine, NetworkInterface, Process +from archivebox.machine.models import Process snapshot = Snapshot.objects.select_related("crawl").get(id="{created["snapshot_id"]}") crawl = snapshot.crawl -if snapshot.status == snapshot.StatusChoices.QUEUED: - snapshot.sm.tick() - snapshot.refresh_from_db() -if snapshot.status == snapshot.StatusChoices.STARTED: - snapshot.sm.seal() - snapshot.refresh_from_db() -crawl.refresh_from_db() -if crawl.status == crawl.StatusChoices.QUEUED: - crawl.sm.tick() - crawl.refresh_from_db() -if crawl.status == crawl.StatusChoices.STARTED: - crawl.sm.seal() - crawl.refresh_from_db() if snapshot.status != snapshot.StatusChoices.SEALED or crawl.status != crawl.StatusChoices.SEALED: raise RuntimeError(f"expected sealed snapshot/crawl, got {{snapshot.status}}/{{crawl.status}}") -Path(crawl.output_dir).mkdir(parents=True, exist_ok=True) -Path(snapshot.output_dir).mkdir(parents=True, exist_ok=True) -(Path(crawl.output_dir) / "crawl-retention.txt").write_text("crawl") -(Path(snapshot.output_dir) / "snapshot-retention.txt").write_text("snapshot") - -result = ArchiveResult.objects.create( +result = ArchiveResult.objects.select_related("process").get( snapshot=snapshot, - plugin="title", - hook_name="on_Snapshot__54_title.py", - status=ArchiveResult.StatusChoices.SUCCEEDED, + plugin="hashes", + hook_name="on_Snapshot__93_hashes", ) -Path(result.output_dir).mkdir(parents=True, exist_ok=True) -(Path(result.output_dir) / "title.txt").write_text("Example") - -machine = Machine.current() -iface = NetworkInterface.objects.filter(machine=machine).first() -if iface is None: - iface = NetworkInterface.objects.create( - machine=machine, - mac_address="00:00:00:00:00:00", - ip_public="203.0.113.10", - ip_local="127.0.0.1", - dns_server="1.1.1.1", - hostname=machine.hostname, - iface="lo", - isp="Test ISP", - city="Test City", - region="Test Region", - country="Test Country", - ) -process = Process.objects.create( - machine=machine, - iface=iface, - process_type=Process.TypeChoices.HOOK, - pwd=str(result.output_dir), - cmd=["echo", "ok"], - env={{"DELETE_AFTER": "1hr"}}, - status=Process.StatusChoices.EXITED, +if result.status != ArchiveResult.StatusChoices.SUCCEEDED or result.process_id is None: + raise RuntimeError(f"expected successful projected hashes result, got {{result.status}}/{{result.process_id}}") +process = result.process +if process.status != Process.StatusChoices.EXITED or process.exit_code != 0: + raise RuntimeError(f"expected successful real hook process, got {{process.status}}/{{process.exit_code}}") +if not Path(result.output_dir).is_dir(): + raise RuntimeError(f"expected real hook output directory at {{result.output_dir}}") +runner = next( + (row for row in Process.objects.filter(worker_type="worker_runner").order_by("created_at") if "--no-stdin" in row.cmd), + None, ) +if runner is None: + raise RuntimeError("expected persisted worker_runner process") +archivebox_projection = Path(os.environ["ABXPKG_LIB_DIR"]) / "env" / "bin" / "archivebox" +if not archivebox_projection.is_symlink() or str(archivebox_projection) not in runner.cmd or "-m" in runner.cmd: + raise RuntimeError(f"worker did not execute projected archivebox: {{runner.cmd}}") due_at = timezone.now() - timedelta(hours=1) ArchiveResult.objects.filter(pk=result.pk).update(delete_at=due_at) @@ -137,7 +108,8 @@ print(json.dumps({{ "crawl_id": str(crawl.id), "snapshot_id": str(snapshot.id), "archiveresult_id": str(result.id), - "process_id": str(process.id), + "process_id": str(process.id), + "runner_cmd": runner.cmd, "crawl_dir": str(crawl.output_dir), "snapshot_dir": str(snapshot.output_dir), "archiveresult_dir": str(result.output_dir), diff --git a/archivebox/tests/test_config_MAX_limits.py b/archivebox/tests/test_config_MAX_limits.py index 415be2ed..3fba65fc 100644 --- a/archivebox/tests/test_config_MAX_limits.py +++ b/archivebox/tests/test_config_MAX_limits.py @@ -2,6 +2,7 @@ import asyncio import json +from importlib.resources import files from pathlib import Path import pytest @@ -66,6 +67,7 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.plugins.hooks import run_hook from archivebox.services.runner import CrawlRunner crawl = Crawl.objects.create( @@ -80,17 +82,26 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path): status=Snapshot.StatusChoices.SEALED, depth=0, ) - parser_dir = Path(snapshot.output_dir) / "parse_html_urls" + snap_dir = Path(snapshot.output_dir) + staticfile_dir = snap_dir / "staticfile" + parser_dir = snap_dir / "parse_txt_urls" + staticfile_dir.mkdir(parents=True, exist_ok=True) parser_dir.mkdir(parents=True, exist_ok=True) - (parser_dir / "urls.jsonl").write_text( - "\n".join( - [ - json.dumps({"type": "Snapshot", "url": "https://example.com/child-a", "depth": 1}), - json.dumps({"type": "Snapshot", "url": "https://example.com/child-b", "depth": 1}), - "", - ], - ), + (staticfile_dir / "input.txt").write_text( + "https://example.com/child-a\nhttps://example.com/child-b\n", + encoding="utf-8", ) + hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py"))) + process = run_hook( + hook_path, + parser_dir, + config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)}, + timeout=30, + url=snapshot.url, + depth=snapshot.depth, + ) + process.refresh_from_db() + assert process.exit_code == 0, process.stderr runner = CrawlRunner(crawl) Crawl.objects.filter(id=crawl.id).update(max_depth=1) @@ -220,7 +231,7 @@ def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size(): ) bus = create_bus(name=f"test_snapshot_limit_cancel_{str(crawl.id).replace('-', '_')}") - service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: None) + service = SnapshotService(bus, crawl_id=str(crawl.id)) try: async def emit_event() -> None: diff --git a/archivebox/tests/test_config_SAVE_TITLE.py b/archivebox/tests/test_config_SAVE_TITLE.py index 40f527c5..4092e689 100644 --- a/archivebox/tests/test_config_SAVE_TITLE.py +++ b/archivebox/tests/test_config_SAVE_TITLE.py @@ -1,7 +1,6 @@ -import time from pathlib import Path -from archivebox.tests.conftest import run_archivebox_cmd, cli_env +from archivebox.tests.conftest import cli_env, run_archivebox_cmd, run_queued_crawls import pytest @@ -25,18 +24,6 @@ def _install_chrome(tmp_path, env): env.update(resolve_abxpkg_chrome_env(Path(env["ABXPKG_LIB_DIR"]), env)) -def _wait_for_snapshot_title(data_dir, *, timeout=60): - deadline = time.time() + timeout - title = None - while time.time() < deadline: - with use_archivebox_db(data_dir): - title = Snapshot.objects.get().resolved_title - if title: - return title - time.sleep(0.5) - return title - - def test_title_is_extracted(tmp_path, initialized_archive, recursive_test_site): """Test that title is extracted from the page.""" env = cli_env(disable_extractors=True) @@ -48,8 +35,10 @@ def test_title_is_extracted(tmp_path, initialized_archive, recursive_test_site): env=env, ) assert add_process.returncode == 0, add_process.stderr or add_process.stdout + run_queued_crawls(tmp_path, env=env) - title = _wait_for_snapshot_title(tmp_path) + with use_archivebox_db(tmp_path): + title = Snapshot.objects.get().resolved_title assert title is not None assert "Root" in title diff --git a/archivebox/tests/test_crawl_runner.py b/archivebox/tests/test_crawl_runner.py index 17ed484a..0376766a 100644 --- a/archivebox/tests/test_crawl_runner.py +++ b/archivebox/tests/test_crawl_runner.py @@ -1,6 +1,7 @@ import asyncio -import sys +from importlib.resources import files from pathlib import Path +import sys import pytest from asgiref.sync import sync_to_async @@ -33,15 +34,13 @@ def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus(): abort_event_holder: dict[str, CrawlAbortEvent | None] = {"event": None} async def on_CrawlEvent(event: CrawlEvent) -> None: - watcher = asyncio.create_task(runner.watch_for_cancelled_crawl(event, poll_interval=0.01)) - await asyncio.sleep(0.02) await sync_to_async(Crawl.objects.filter(id=crawl.id).update, thread_sensitive=True)( status=Crawl.StatusChoices.SEALED, retry_at=None, ) + await runner.watch_for_cancelled_crawl(event, poll_interval=0) abort_event = await runner.bus.find(CrawlAbortEvent, child_of=event, past=True, future=1.0) abort_event_holder["event"] = abort_event if isinstance(abort_event, CrawlAbortEvent) else None - await watcher runner.bus.on(CrawlEvent, on_CrawlEvent) await runner.bus.emit( @@ -59,7 +58,6 @@ def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus(): assert abort_event is not None -@pytest.mark.django_db(transaction=True) @pytest.mark.django_db(transaction=True) def test_snapshot_payload_uses_crawl_chrome_dirs_by_default(): from archivebox.base_models.models import get_or_create_system_user_pk @@ -145,22 +143,19 @@ def test_snapshot_payload_uses_snapshot_chrome_dirs_when_snapshot_isolated(): assert Path(config["SNAP_DIR"]) == snapshot.output_dir -def test_ensure_background_runner_skips_under_pytest_guard(): - from archivebox.services.runner import ensure_background_runner - - assert ensure_background_runner() is False - - @pytest.mark.django_db(transaction=True) -def test_ensure_background_runner_skips_with_real_running_orchestrator_record(): +def test_ensure_background_runner_does_not_start_duplicate_orchestrator(): import os from datetime import datetime import psutil from archivebox.machine.models import Machine, Process from archivebox.services.runner import ensure_background_runner + from archivebox.workers.supervisord_util import get_existing_supervisord_process, stop_existing_supervisord_process from django.utils import timezone + stop_existing_supervisord_process() + assert get_existing_supervisord_process(quiet=True) is None os_proc = psutil.Process(os.getpid()) process = Process.objects.create( machine=Machine.current(), @@ -171,7 +166,7 @@ def test_ensure_background_runner_skips_with_real_running_orchestrator_record(): timeout=1, ) - assert ensure_background_runner(allow_under_pytest=True) is False + assert ensure_background_runner() is False process.refresh_from_db() assert process.status == Process.StatusChoices.RUNNING @@ -184,7 +179,7 @@ def test_ensure_background_runner_does_not_spawn_runner_without_supervisord(): stop_existing_supervisord_process() assert get_existing_supervisord_process(quiet=True) is None - assert ensure_background_runner(allow_under_pytest=True) is False + assert ensure_background_runner() is False assert get_existing_supervisord_process(quiet=True) is None @@ -783,11 +778,9 @@ def test_wait_for_snapshot_tasks_returns_after_completed_tasks_are_pruned(): ) crawl_runner = runner_module.CrawlRunner(crawl) - async def finish_snapshot() -> None: - await asyncio.sleep(0) - async def run_test(): - task = asyncio.create_task(finish_snapshot()) + task = asyncio.get_running_loop().create_future() + task.set_result(None) crawl_runner.snapshot_tasks["snap-1"] = task await asyncio.wait_for(crawl_runner.wait_for_snapshot_tasks(), timeout=0.5) assert crawl_runner.snapshot_tasks == {} @@ -795,10 +788,11 @@ def test_wait_for_snapshot_tasks_returns_after_completed_tasks_are_pruned(): asyncio.run(run_test()) -def test_abx_process_service_background_process_finishes_after_process_exit(tmp_path): +def test_abx_process_service_background_process_finishes_after_process_exit(tmp_path, recursive_test_site, hermetic_lib_dir): from abx_dl.events import ProcessCompletedEvent, ProcessEvent from abx_dl.orchestrator import create_bus from abx_dl.services.process_service import ProcessService + from archivebox.machine.models import Process bus = create_bus(name="test_abx_process_service_background_process_finishes_after_process_exit") ProcessService(bus, emit_jsonl=False, interactive_tty=False) @@ -809,21 +803,29 @@ def test_abx_process_service_background_process_finishes_after_process_exit(tmp_ bus.on(ProcessCompletedEvent, collect_completed) - plugin_output_dir = tmp_path / "chrome" - plugin_output_dir.mkdir() + snap_dir = tmp_path / "snapshot" + plugin_output_dir = snap_dir / "wget" + plugin_output_dir.mkdir(parents=True) + hook_path = Path(str(files("abx_plugins.plugins.wget").joinpath("on_Snapshot__06_wget.finite.bg.py"))) + wget_config = Path(str(files("abx_plugins.plugins.wget").joinpath("config.json"))) + hook_env = resolve_abxpkg_binary_env(hermetic_lib_dir, deps_from=wget_config) async def run_test(): try: event = ProcessEvent( - plugin_name="chrome", - hook_name="on_CrawlSetup__90_chrome_launch.daemon.bg", - hook_path=sys.executable, - hook_args=["-c", "print('daemon output')"], - env={}, + plugin_name="wget", + hook_name=hook_path.name, + hook_path=str(hook_path), + hook_args=[f"--url={recursive_test_site['root_url']}"], + env={ + **hook_env, + "ABXPKG_LIB_DIR": str(hermetic_lib_dir), + "SNAP_DIR": str(snap_dir), + }, output_dir=str(plugin_output_dir), timeout=60, is_background=True, - url="https://example.org/", + url=recursive_test_site["root_url"], process_type="hook", worker_type="hook", ) @@ -832,7 +834,14 @@ def test_abx_process_service_background_process_finishes_after_process_exit(tmp_ assert isinstance(completed, ProcessCompletedEvent) await completed.event_results_list() assert completed.status == "succeeded" - assert completed.stdout.strip() == "daemon output" + records = [record for record in Process.parse_records_from_text(completed.stdout) if record.get("type") == "ArchiveResult"] + assert records == [ + { + "type": "ArchiveResult", + "status": "succeeded", + "output_str": f"wget/127.0.0.1+{recursive_test_site['base_url'].rsplit(':', 1)[-1]}/index.html", + }, + ] assert completed.output_dir == str(plugin_output_dir) assert bus.event_is_child_of(completed, event) finally: @@ -840,7 +849,8 @@ def test_abx_process_service_background_process_finishes_after_process_exit(tmp_ asyncio.run(run_test()) - assert not list(plugin_output_dir.glob("on_CrawlSetup__90_chrome_launch.daemon.bg.*.pid")) + assert list(plugin_output_dir.rglob("index.html")) + assert not list(plugin_output_dir.glob(f"{hook_path.name}.*.pid")) assert any(isinstance(event, ProcessCompletedEvent) for event in emitted_events) @@ -1076,7 +1086,7 @@ def test_snapshot_completed_event_defers_finished_crawl_seal(): ) bus = create_bus(name=f"test_snapshot_completed_finished_crawl_{str(crawl.id).replace('-', '_')}") - service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: asyncio.sleep(0)) + service = SnapshotService(bus, crawl_id=str(crawl.id)) try: async def emit_completed() -> None: @@ -1123,7 +1133,7 @@ def test_snapshot_completed_event_bus_defers_finished_crawl_seal(): ) bus = create_bus(name=f"test_snapshot_completed_bus_finished_crawl_{str(crawl.id).replace('-', '_')}") - service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: asyncio.sleep(0)) + service = SnapshotService(bus, crawl_id=str(crawl.id)) assert service is not None try: diff --git a/archivebox/tests/test_frozen_crawl_config.py b/archivebox/tests/test_frozen_crawl_config.py index 9ce9f734..fd92ebb4 100644 --- a/archivebox/tests/test_frozen_crawl_config.py +++ b/archivebox/tests/test_frozen_crawl_config.py @@ -172,12 +172,12 @@ def test_search_backend_engine_derives_default_backend_enabled_without_entering_ assert "DATABASE_NAME" not in runtime_config -def test_plugin_selection_enabled_keys_are_derived_from_plugins_not_frozen_or_env_overridden(archivebox_db, monkeypatch): - from archivebox.config.common import get_config +def test_plugin_selection_enabled_keys_are_derived_from_plugins_not_frozen_or_base_config_overridden(archivebox_db): + from archivebox.config.common import ArchiveBoxConfig, get_config from archivebox.crawls.models import Crawl from archivebox.plugins.discovery import get_plugin_special_config - monkeypatch.setenv("ARCHIVEDOTORG_ENABLED", "False") + process_defaults = ArchiveBoxConfig(ARCHIVEDOTORG_ENABLED=False) user = _user("frozen-config-enabled-admin") persona = _persona(user, name="Enabled Persona") @@ -190,8 +190,8 @@ def test_plugin_selection_enabled_keys_are_derived_from_plugins_not_frozen_or_en retry_at=timezone.now(), ) env_default_crawl.save() - env_default_config = get_config(crawl=env_default_crawl, include_machine=False) - assert env_default_config.ARCHIVEDOTORG_ENABLED is False + base_default_config = get_config(crawl=env_default_crawl, include_machine=False, base_config=process_defaults) + assert base_default_config.ARCHIVEDOTORG_ENABLED is False crawl = Crawl( urls="https://example.com/enabled", @@ -206,14 +206,12 @@ def test_plugin_selection_enabled_keys_are_derived_from_plugins_not_frozen_or_en assert crawl.config["PLUGINS"] == "archivedotorg" assert "ARCHIVEDOTORG_ENABLED" not in crawl.config assert "DEFAULT_PERSONA" not in crawl.config - runtime_config = get_config(crawl=crawl, include_machine=False) + runtime_config = get_config(crawl=crawl, include_machine=False, base_config=process_defaults) assert runtime_config.ARCHIVEDOTORG_ENABLED is True assert runtime_config.WGET_ENABLED is False assert get_plugin_special_config("archivedotorg", runtime_config)["enabled"] is True assert get_plugin_special_config("wget", runtime_config)["enabled"] is False - monkeypatch.delenv("ARCHIVEDOTORG_ENABLED") - Crawl.objects.filter(id=crawl.id).update( config={ **crawl.config, @@ -223,7 +221,7 @@ def test_plugin_selection_enabled_keys_are_derived_from_plugins_not_frozen_or_en }, ) crawl.refresh_from_db() - stale_runtime_config = get_config(crawl=crawl, include_machine=False) + stale_runtime_config = get_config(crawl=crawl, include_machine=False, base_config=process_defaults) assert stale_runtime_config.ARCHIVEDOTORG_ENABLED is True assert stale_runtime_config.YTDLP_ENABLED is False assert stale_runtime_config.WGET_ENABLED is False diff --git a/archivebox/tests/test_hooks.py b/archivebox/tests/test_hooks.py index d48b5c92..6f96e848 100755 --- a/archivebox/tests/test_hooks.py +++ b/archivebox/tests/test_hooks.py @@ -10,15 +10,13 @@ Run with: """ import json +import hashlib import os -import shutil import subprocess -import sys -import textwrap +from importlib.resources import files from pathlib import Path import pytest -import rich_click as click from archivebox.tests.conftest import resolve_abxpkg_binary_env @@ -27,70 +25,8 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.settings") REPO_ROOT = Path(__file__).resolve().parents[2] WORKSPACE_ROOT = REPO_ROOT.parent -RESULT_PREFIX = "__ARCHIVEBOX_TEST_RESULT__=" - - -def create_test_plugin_structure(plugins_dir: Path) -> None: - """Create a minimal plugin tree for hook discovery tests.""" - plugins_dir.mkdir() - - wget_dir = plugins_dir / "wget" - wget_dir.mkdir() - (wget_dir / "on_Snapshot__50_wget.py").write_text("# test hook") - - chrome_dir = plugins_dir / "chrome" - chrome_dir.mkdir(exist_ok=True) - (chrome_dir / "on_Snapshot__20_chrome_tab.daemon.bg.js").write_text("// background hook") - - consolelog_dir = plugins_dir / "consolelog" - consolelog_dir.mkdir() - (consolelog_dir / "on_Snapshot__21_consolelog.daemon.bg.js").write_text("// background hook") - - -def run_plugin_discovery_subprocess(tmp_path: Path, plugins_dir: Path, script: str): - env = os.environ.copy() - data_dir = tmp_path / "data" - data_dir.mkdir() - cwd_plugins_dir = data_dir / "custom_plugins" - if plugins_dir != cwd_plugins_dir: - shutil.copytree(plugins_dir, cwd_plugins_dir) - existing_pythonpath = [ - str(Path(entry).expanduser().resolve(strict=False)) - for entry in env.get("PYTHONPATH", "").split(os.pathsep) - if entry and Path(entry).expanduser().is_absolute() - ] - env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys([str(REPO_ROOT), *existing_pythonpath])) - subprocess_script = "\n".join( - [ - "import json", - f"RESULT_PREFIX = {RESULT_PREFIX!r}", - "", - "def emit(value):", - " print(RESULT_PREFIX + json.dumps(value))", - "", - textwrap.dedent(script), - ], - ) - - result = subprocess.run( - [ - sys.executable, - "-c", - subprocess_script, - ], - cwd=data_dir, - env=env, - capture_output=True, - text=True, - timeout=30, - ) - assert result.returncode == 0, result.stderr - - for line in reversed(result.stdout.splitlines()): - if line.startswith(RESULT_PREFIX): - return json.loads(line.removeprefix(RESULT_PREFIX)) - - raise AssertionError(f"Subprocess did not emit a result line.\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}") +WGET_CONFIG = Path(str(files("abx_plugins.plugins.wget").joinpath("config.json"))) +CHROME_CONFIG = Path(str(files("abx_plugins.plugins.chrome").joinpath("config.json"))) def test_cli_env_does_not_emit_relative_pythonpath_entries(): @@ -116,104 +52,129 @@ def test_cli_env_does_not_emit_relative_pythonpath_entries(): class TestBackgroundHookDetection: - """Test that background hooks are detected by .bg. suffix.""" + """Test background classification against the shipped hook suite.""" - def test_bg_js_suffix_detected(self): - """Hooks with .bg.js suffix should be detected as background.""" - from archivebox.plugins.hooks import is_background_hook + def test_shipped_hooks_are_classified_by_bg_marker(self): + from archivebox.plugins.hooks import discover_hooks, is_background_hook - assert is_background_hook("on_Snapshot__21_consolelog.daemon.bg.js") + hooks = discover_hooks("Snapshot", filter_disabled=False) + background_hooks = [hook for hook in hooks if is_background_hook(hook.name)] + foreground_hooks = [hook for hook in hooks if not is_background_hook(hook.name)] - def test_bg_py_suffix_detected(self): - """Hooks with .bg.py suffix should be detected as background.""" - from archivebox.plugins.hooks import is_background_hook - - assert is_background_hook("on_Snapshot__24_responses.finite.bg.py") - - def test_bg_sh_suffix_detected(self): - """Hooks with .bg.sh suffix should be detected as background.""" - from archivebox.plugins.hooks import is_background_hook - - assert is_background_hook("on_Snapshot__23_ssl.daemon.bg.sh") - - def test_legacy_background_suffix_detected(self): - """Hooks with __background in stem should be detected (backwards compat).""" - from archivebox.plugins.hooks import is_background_hook - - assert is_background_hook("on_Snapshot__21_consolelog__background.js") - - def test_foreground_hook_not_detected(self): - """Hooks without .bg. or __background should NOT be detected as background.""" - from archivebox.plugins.hooks import is_background_hook - - assert not is_background_hook("on_Snapshot__11_favicon.js") - - def test_foreground_py_hook_not_detected(self): - """Python hooks without .bg. should NOT be detected as background.""" - from archivebox.plugins.hooks import is_background_hook - - assert not is_background_hook("on_Snapshot__50_wget.py") + assert hooks + assert background_hooks + assert foreground_hooks + assert all(".bg." in hook.name for hook in background_hooks) + assert all(".bg." not in hook.name for hook in foreground_hooks) + assert any(hook.name == "on_Snapshot__10_chrome_tab.daemon.bg.js" for hook in background_hooks) + assert any(hook.name == "on_Snapshot__06_wget.finite.bg.py" for hook in background_hooks) + assert any(hook.name == "on_Snapshot__93_hashes.py" for hook in foreground_hooks) +@pytest.mark.django_db(transaction=True) class TestJSONLParsing: - """Test JSONL parsing in run_hook() output processing.""" + """Test JSONL parsing against output from shipped hooks.""" - def test_parse_clean_jsonl(self): - """Clean JSONL format should be parsed correctly.""" - stdout = '{"type": "ArchiveResult", "status": "succeeded", "output_str": "Done"}' + @staticmethod + def run_hashes_hook(tmp_path): + from archivebox.plugins.hooks import run_hook + + snap_dir = tmp_path / "hash-snapshot" + output_dir = snap_dir / "hashes" + output_dir.mkdir(parents=True) + (snap_dir / "source.txt").write_text("real parser input", encoding="utf-8") + hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py"))) + process = run_hook( + hook_path, + output_dir, + config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)}, + timeout=30, + url="https://example.com/hash-parser", + ) + process.refresh_from_db() + assert process.exit_code == 0, process.stderr + return process + + @staticmethod + def run_parser_hook(tmp_path): + from archivebox.plugins.hooks import run_hook + + snap_dir = tmp_path / "parser-snapshot" + staticfile_dir = snap_dir / "staticfile" + output_dir = snap_dir / "parse_txt_urls" + staticfile_dir.mkdir(parents=True) + output_dir.mkdir(parents=True) + (staticfile_dir / "input.txt").write_text( + "links: https://one.example/path and https://two.example/path", + encoding="utf-8", + ) + hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py"))) + process = run_hook( + hook_path, + output_dir, + config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)}, + timeout=30, + url="file:///input.txt", + ) + process.refresh_from_db() + assert process.exit_code == 0, process.stderr + return process + + def test_parse_clean_jsonl(self, tmp_path): + """Clean JSONL emitted by a shipped hook should be parsed correctly.""" from archivebox.machine.models import Process - records = Process.parse_records_from_text(stdout) + process = self.run_hashes_hook(tmp_path) + records = Process.parse_records_from_text(process.stdout) assert len(records) == 1 assert records[0]["type"] == "ArchiveResult" assert records[0]["status"] == "succeeded" - assert records[0]["output_str"] == "Done" + assert records[0]["output_str"].endswith(hashlib.sha256(b"real parser input").hexdigest()[:12]) - def test_parse_multiple_jsonl_records(self): - """Multiple JSONL records should all be parsed.""" - stdout = """{"type": "ArchiveResult", "status": "succeeded", "output_str": "Done"} -{"type": "Binary", "name": "wget", "abspath": "/usr/bin/wget"}""" + def test_parse_multiple_jsonl_records(self, tmp_path): + """Every record emitted by a shipped parser hook should be parsed.""" from archivebox.machine.models import Process - records = Process.parse_records_from_text(stdout) + process = self.run_parser_hook(tmp_path) + records = Process.parse_records_from_text(process.stdout) - assert len(records) == 2 - assert records[0]["type"] == "ArchiveResult" - assert records[1]["type"] == "Binary" + assert [record["type"] for record in records] == ["Snapshot", "Snapshot", "ArchiveResult"] + assert {record["url"] for record in records[:-1]} == { + "https://one.example/path", + "https://two.example/path", + } + assert records[-1]["status"] == "succeeded" - def test_parse_jsonl_with_log_output(self): - """JSONL should be extracted from mixed stdout with log lines.""" - stdout = """Starting hook execution... -Processing URL: https://example.com -{"type": "ArchiveResult", "status": "succeeded", "output_str": "Downloaded"} -Hook completed successfully""" + def test_parse_jsonl_with_log_output(self, tmp_path): + """JSONL should be extracted from a shipped hook's mixed stdout.""" from archivebox.machine.models import Process + process = self.run_parser_hook(tmp_path) + assert "parsing 1 files for urls..." in process.stdout + assert "2 URLs parsed" in process.stdout + records = Process.parse_records_from_text(process.stdout) + + assert len(records) == 3 + assert records[-1]["status"] == "succeeded" + + def test_ignore_invalid_json(self, tmp_path): + """Malformed non-record lines must not hide real hook records.""" + from archivebox.machine.models import Process + + process = self.run_hashes_hook(tmp_path) + stdout = f"{process.stdout}\n{{invalid json here}}\nnot json at all\n" records = Process.parse_records_from_text(stdout) assert len(records) == 1 - assert records[0]["status"] == "succeeded" + assert records[0]["type"] == "ArchiveResult" - def test_ignore_invalid_json(self): - """Invalid JSON should be silently ignored.""" - stdout = """{"type": "ArchiveResult", "status": "succeeded"} -{invalid json here} -not json at all -{"type": "BinaryRequest", "name": "wget"}""" + def test_json_without_type_ignored(self, tmp_path): + """A non-record object must not hide the shipped hook's real record.""" from archivebox.machine.models import Process - records = Process.parse_records_from_text(stdout) - - assert len(records) == 2 - - def test_json_without_type_ignored(self): - """JSON objects without 'type' field should be ignored.""" - stdout = """{"status": "succeeded", "output_str": "Done"} -{"type": "ArchiveResult", "status": "succeeded"}""" - from archivebox.machine.models import Process - - records = Process.parse_records_from_text(stdout) + process = self.run_hashes_hook(tmp_path) + records = Process.parse_records_from_text(f'{process.stdout}\n{{"status":"succeeded"}}\n') assert len(records) == 1 assert records[0]["type"] == "ArchiveResult" @@ -222,68 +183,53 @@ not json at all class TestRequiredBinaryConfigHandling: """Test that required_binaries keep configured XYZ_BINARY values intact.""" - def test_binary_env_var_absolute_path_handling(self): - """Absolute binary paths should pass through unchanged.""" - configured_binary = "/custom/path/to/wget2" - binary_name = configured_binary + def test_binary_env_var_absolute_path_handling(self, tmp_path): + """abxpkg should expose the resolved binary as an absolute path.""" + resolved = resolve_abxpkg_binary_env(tmp_path / "lib", deps_from=WGET_CONFIG) - assert binary_name == "/custom/path/to/wget2" + assert Path(resolved["WGET_BINARY"]).is_absolute() + assert Path(resolved["WGET_BINARY"]).is_file() - def test_binary_env_var_name_only_handling(self): - """Binary command names should pass through unchanged.""" - configured_binary = "wget2" - binary_name = configured_binary + def test_binary_env_var_name_only_handling(self, tmp_path): + """The projected command name should execute the resolved host binary.""" + lib_dir = tmp_path / "lib" + resolve_abxpkg_binary_env(lib_dir, deps_from=WGET_CONFIG) + projection = lib_dir / "env" / "bin" / "wget" + result = subprocess.run([projection, "--version"], capture_output=True, text=True) - assert binary_name == "wget2" + assert projection.is_symlink() + assert result.returncode == 0, result.stderr + assert "Wget" in result.stdout def test_binary_env_var_empty_default(self): - """Empty configured binary values should keep the schema default.""" - configured_binary = "" - if configured_binary: - binary_name = configured_binary - else: - binary_name = "wget" + """The shipped wget schema should retain wget as its required binary.""" + config = json.loads(files("abx_plugins.plugins.wget").joinpath("config.json").read_text()) - assert binary_name == "wget" + assert config["required_binaries"][0]["name"] == "{WGET_BINARY}" + assert config["properties"]["WGET_BINARY"]["default"] == "wget" class TestHookDiscovery: """Test hook discovery functions.""" - def test_discover_hooks_by_event(self, tmp_path): + def test_discover_hooks_by_event(self): """discover_hooks() should find all hooks for an event.""" - plugins_dir = tmp_path / "plugins" - create_test_plugin_structure(plugins_dir) + from archivebox.plugins.hooks import discover_hooks - hooks = [] - for ext in ("sh", "py", "js"): - pattern = f"*/on_Snapshot__*.{ext}" - hooks.extend(plugins_dir.glob(pattern)) + hooks = discover_hooks("Snapshot", filter_disabled=False) - hooks = sorted(set(hooks), key=lambda p: p.name) - - assert len(hooks) == 3 hook_names = [h.name for h in hooks] - assert "on_Snapshot__20_chrome_tab.daemon.bg.js" in hook_names + assert "on_Snapshot__10_chrome_tab.daemon.bg.js" in hook_names assert "on_Snapshot__21_consolelog.daemon.bg.js" in hook_names - assert "on_Snapshot__50_wget.py" in hook_names + assert "on_Snapshot__06_wget.finite.bg.py" in hook_names + assert all(hook.is_file() for hook in hooks) - def test_discover_hooks_sorted_by_name(self, tmp_path): + def test_discover_hooks_sorted_by_name(self): """Hooks should be sorted by filename (numeric prefix ordering).""" - plugins_dir = tmp_path / "plugins" - create_test_plugin_structure(plugins_dir) + from archivebox.plugins.hooks import discover_hooks - hooks = [] - for ext in ("sh", "py", "js"): - pattern = f"*/on_Snapshot__*.{ext}" - hooks.extend(plugins_dir.glob(pattern)) - - hooks = sorted(set(hooks), key=lambda p: p.name) - - # Check numeric ordering - assert hooks[0].name == "on_Snapshot__20_chrome_tab.daemon.bg.js" - assert hooks[1].name == "on_Snapshot__21_consolelog.daemon.bg.js" - assert hooks[2].name == "on_Snapshot__50_wget.py" + hook_names = [hook.name for hook in discover_hooks("Snapshot", filter_disabled=False)] + assert hook_names == sorted(hook_names) def test_normalize_hook_event_name_accepts_event_classes(self): """Hook discovery should normalize bus event class names to hook families.""" @@ -303,167 +249,51 @@ class TestHookDiscovery: assert hooks_module.normalize_hook_event_name("SnapshotCleanupEvent") == "SnapshotCleanup" assert hooks_module.normalize_hook_event_name("CrawlCleanupEvent") == "CrawlCleanup" - def test_discover_hooks_skips_plugins_with_disabled_required_dependencies(self, tmp_path): + def test_discover_hooks_skips_plugins_with_disabled_required_dependencies(self): """Plugins whose required_plugins are disabled should not run.""" - plugins_dir = tmp_path / "plugins" - create_test_plugin_structure(plugins_dir) + from archivebox.plugins.hooks import discover_hooks - chrome_dir = plugins_dir / "chrome" - chrome_dir.mkdir(exist_ok=True) - (chrome_dir / "config.json").write_text( - json.dumps( - { - "type": "object", - "required_plugins": [], - "properties": { - "CHROME_ENABLED": { - "type": "boolean", - "default": True, - "x-aliases": ["USE_CHROME"], - }, - }, - }, - ), - ) - (chrome_dir / "on_Snapshot__20_chrome.js").write_text("// chrome hook") - - accessibility_dir = plugins_dir / "accessibility" - accessibility_dir.mkdir(exist_ok=True) - (accessibility_dir / "config.json").write_text( - json.dumps( - { - "type": "object", - "required_plugins": ["chrome"], - "properties": { - "ACCESSIBILITY_ENABLED": { - "type": "boolean", - "default": True, - }, - }, - }, - ), - ) - (accessibility_dir / "on_Snapshot__10_accessibility.js").write_text("// accessibility hook") - - wget_dir = plugins_dir / "wget" - (wget_dir / "config.json").write_text( - json.dumps( - { - "type": "object", - "required_plugins": [], - "properties": { - "WGET_ENABLED": { - "type": "boolean", - "default": True, - "x-aliases": ["SAVE_WGET"], - }, - }, - }, - ), - ) - - hook_names = run_plugin_discovery_subprocess( - tmp_path, - plugins_dir, - """ - from archivebox.plugins import hooks as hooks_module - - hooks = hooks_module.discover_hooks("Snapshot", config={"CHROME_ENABLED": False, "WGET_ENABLED": True}) - emit([hook.parent.name for hook in hooks]) - """, - ) + hook_names = [hook.parent.name for hook in discover_hooks("Snapshot", config={"CHROME_ENABLED": False, "WGET_ENABLED": True})] assert "wget" in hook_names assert "chrome" not in hook_names assert "accessibility" not in hook_names - def test_get_plugins_includes_config_only_plugin_dirs(self, tmp_path): - """get_plugins() should include config-only plugins with standardized metadata.""" - plugins_dir = tmp_path / "plugins" - create_test_plugin_structure(plugins_dir) + def test_get_plugins_includes_config_only_plugin_dirs(self): + """get_plugins() should include shipped plugin directories without hooks.""" + from archivebox.plugins.discovery import BUILTIN_PLUGINS_DIR, get_plugins - helper_dir = plugins_dir / "helper" - helper_dir.mkdir() - (helper_dir / "config.json").write_text('{"type": "object", "properties": {}}') + plugins = get_plugins() + assert "base" in plugins + base_dir = BUILTIN_PLUGINS_DIR / "base" + assert (base_dir / "config.json").is_file() + assert list(base_dir.glob("on_*__*.*")) == [] - plugins = run_plugin_discovery_subprocess( - tmp_path, - plugins_dir, - """ - from archivebox.plugins import hooks as hooks_module - - from archivebox.plugins.discovery import get_plugins - get_plugins.cache_clear() - emit(get_plugins()) - """, - ) - assert "helper" in plugins - - def test_discover_binary_hooks_returns_empty(self, tmp_path): + def test_discover_binary_hooks_returns_empty(self): """Binary provider hooks are owned by abxpkg, not ArchiveBox plugin discovery.""" - plugins_dir = tmp_path / "plugins" - create_test_plugin_structure(plugins_dir) + from archivebox.plugins.hooks import discover_hooks - hook_names = run_plugin_discovery_subprocess( - tmp_path, - plugins_dir, - """ - from archivebox.plugins import hooks as hooks_module - - from archivebox.plugins.discovery import get_plugins - get_plugins.cache_clear() - hooks = hooks_module.discover_hooks("BinaryRequest", filter_disabled=False) - emit([hook.name for hook in hooks]) - """, - ) + hook_names = [hook.name for hook in discover_hooks("BinaryRequest", filter_disabled=False)] assert hook_names == [] - def test_discover_hooks_accepts_event_class_names(self, tmp_path): + def test_discover_hooks_accepts_event_class_names(self): """discover_hooks should accept CrawlSetupEvent / SnapshotEvent class names.""" - plugins_dir = tmp_path / "plugins" - create_test_plugin_structure(plugins_dir) - chrome_dir = plugins_dir / "chrome" - (chrome_dir / "on_CrawlSetup__90_chrome_launch.daemon.bg.js").write_text("// crawl hook") + from archivebox.plugins.hooks import discover_hooks - hook_names = run_plugin_discovery_subprocess( - tmp_path, - plugins_dir, - """ - from archivebox.plugins import hooks as hooks_module - - from archivebox.plugins.discovery import get_plugins - get_plugins.cache_clear() - crawl_setup_hooks = hooks_module.discover_hooks("CrawlSetupEvent", filter_disabled=False) - snapshot_hooks = hooks_module.discover_hooks("SnapshotEvent", filter_disabled=False) - emit({ - "crawl_setup": [hook.name for hook in crawl_setup_hooks], - "snapshot": [hook.name for hook in snapshot_hooks], - }) - """, - ) + hook_names = { + "crawl_setup": [hook.name for hook in discover_hooks("CrawlSetupEvent", filter_disabled=False)], + "snapshot": [hook.name for hook in discover_hooks("SnapshotEvent", filter_disabled=False)], + } assert "on_CrawlSetup__90_chrome_launch.daemon.bg.js" in hook_names["crawl_setup"] - assert "on_Snapshot__50_wget.py" in hook_names["snapshot"] + assert "on_Snapshot__06_wget.finite.bg.py" in hook_names["snapshot"] - def test_discover_hooks_returns_empty_for_non_hook_lifecycle_events(self, tmp_path): + def test_discover_hooks_returns_empty_for_non_hook_lifecycle_events(self): """Lifecycle events without a hook family should return no hooks.""" - plugins_dir = tmp_path / "plugins" - create_test_plugin_structure(plugins_dir) + from archivebox.plugins.hooks import discover_hooks - hooks = run_plugin_discovery_subprocess( - tmp_path, - plugins_dir, - """ - from archivebox.plugins import hooks as hooks_module - - from archivebox.plugins.discovery import get_plugins - get_plugins.cache_clear() - emit({ - "binary": [hook.name for hook in hooks_module.discover_hooks("BinaryEvent", filter_disabled=False)], - "crawl_cleanup": [ - hook.name for hook in hooks_module.discover_hooks("CrawlCleanupEvent", filter_disabled=False) - ], - }) - """, - ) + hooks = { + "binary": [hook.name for hook in discover_hooks("BinaryEvent", filter_disabled=False)], + "crawl_cleanup": [hook.name for hook in discover_hooks("CrawlCleanupEvent", filter_disabled=False)], + } assert hooks["binary"] == [] assert hooks["crawl_cleanup"] == [] @@ -503,20 +333,28 @@ class TestHookExecution: def test_python_hook_execution(self, tmp_path): """Python hook should execute and output JSONL.""" - hook_path = tmp_path / "test_hook.py" - hook_path.write_text("""#!/usr/bin/env python3 -import json -print(json.dumps({"type": "ArchiveResult", "status": "succeeded", "output_str": "Test passed"})) -""") - - result = subprocess.run( - [sys.executable, str(hook_path)], - cwd=tmp_path, - capture_output=True, - text=True, + snap_dir = tmp_path / "snapshot" + output_dir = snap_dir / "hashes" + output_dir.mkdir(parents=True) + (snap_dir / "source.txt").write_text("real hook input", encoding="utf-8") + hook_path = Path( + str( + files("abx_plugins.plugins.hashes").joinpath( + "on_Snapshot__93_hashes.py", + ), + ), ) - assert result.returncode == 0 + result = subprocess.run( + [str(hook_path), "--url=https://example.com"], + cwd=output_dir, + capture_output=True, + text=True, + env={**os.environ, "SNAP_DIR": str(snap_dir)}, + timeout=30, + ) + + assert result.returncode == 0, result.stderr from archivebox.machine.models import Process records = Process.parse_records_from_text(result.stdout) @@ -525,212 +363,213 @@ print(json.dumps({"type": "ArchiveResult", "status": "succeeded", "output_str": assert records[0]["status"] == "succeeded" def test_js_hook_execution(self, tmp_path): - """JavaScript hook should execute and output JSONL.""" + """A shipped JavaScript hook should execute through projected Node.""" lib_dir = tmp_path / "lib" - node_env = resolve_abxpkg_binary_env(lib_dir, "node") + chrome_config = Path( + str(files("abx_plugins.plugins.chrome").joinpath("config.json")), + ) + node_env = resolve_abxpkg_binary_env( + lib_dir, + deps_from=chrome_config, + ) node_binary = lib_dir / "env" / "bin" / "node" assert node_binary.is_symlink() - hook_path = tmp_path / "test_hook.js" - hook_path.write_text("""#!/usr/bin/env node -console.log(JSON.stringify({type: 'ArchiveResult', status: 'succeeded', output_str: 'JS test'})); -""") + crawl_dir = tmp_path / "crawl" + output_dir = crawl_dir / "chrome" + output_dir.mkdir(parents=True) + hook_path = Path( + str( + files("abx_plugins.plugins.chrome").joinpath( + "on_CrawlSetup__89_chrome_kill_zombies.js", + ), + ), + ) result = subprocess.run( [str(node_binary), str(hook_path)], - cwd=tmp_path, + cwd=output_dir, capture_output=True, text=True, - env={**os.environ, **node_env}, + env={ + **os.environ, + **node_env, + "CRAWL_DIR": str(crawl_dir), + "SNAP_DIR": str(crawl_dir / "snapshot"), + "CHROME_USER_DATA_DIR": str(output_dir / "profile"), + }, + timeout=30, ) - assert result.returncode == 0 - from archivebox.machine.models import Process + assert result.returncode == 0, result.stderr + assert "chrome zombies" in result.stdout - records = Process.parse_records_from_text(result.stdout) - assert records - assert records[0]["type"] == "ArchiveResult" - assert records[0]["status"] == "succeeded" + @pytest.mark.django_db(transaction=True) + def test_real_js_hook_runs_through_abxpkg_node_projection(self, tmp_path): + from archivebox.plugins.hooks import run_hook + + lib_dir = tmp_path / "lib" + node_env = resolve_abxpkg_binary_env(lib_dir, deps_from=CHROME_CONFIG) + node_projection = lib_dir / "env" / "bin" / "node" + crawl_dir = tmp_path / "crawl" + snap_dir = crawl_dir / "snapshot" + hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js"))) + + process = run_hook( + hook_path, + crawl_dir / "chrome", + config={ + **node_env, + "ABXPKG_LIB_DIR": str(lib_dir), + "CRAWL_DIR": str(crawl_dir), + "SNAP_DIR": str(snap_dir), + "CHROME_USER_DATA_DIR": str(crawl_dir / "chrome" / "profile"), + }, + timeout=30, + ) + process.refresh_from_db() + + assert process.cmd[0] == str(node_projection) + assert process.exit_code == 0, process.stderr + assert "chrome zombies" in process.stdout def test_hook_receives_cli_args(self, tmp_path): """Hook should receive CLI arguments.""" - hook_path = tmp_path / "test_hook.py" - hook_path.write_text("""#!/usr/bin/env python3 -import sys -import json -# Simple arg parsing -args = {} -for arg in sys.argv[1:]: - if arg.startswith('--') and '=' in arg: - key, val = arg[2:].split('=', 1) - args[key.replace('-', '_')] = val -print(json.dumps({"type": "ArchiveResult", "status": "succeeded", "url": args.get("url", "")})) -""") - - result = subprocess.run( - [sys.executable, str(hook_path), "--url=https://example.com"], - cwd=tmp_path, - capture_output=True, - text=True, + snap_dir = tmp_path / "snapshot" + output_dir = snap_dir / "hashes" + output_dir.mkdir(parents=True) + (snap_dir / "source.txt").write_text("real CLI argument input", encoding="utf-8") + hook_path = Path( + str( + files("abx_plugins.plugins.hashes").joinpath( + "on_Snapshot__93_hashes.py", + ), + ), ) - assert result.returncode == 0 + result = subprocess.run( + [str(hook_path), "--url=https://example.com/real-hook-argument"], + cwd=output_dir, + capture_output=True, + text=True, + env={**os.environ, "SNAP_DIR": str(snap_dir)}, + timeout=30, + ) + + assert result.returncode == 0, result.stderr from archivebox.machine.models import Process records = Process.parse_records_from_text(result.stdout) - assert records - assert records[0]["url"] == "https://example.com" + source_hash = hashlib.sha256(b"real CLI argument input").hexdigest() + assert records == [{"type": "ArchiveResult", "status": "succeeded", "output_str": f"0.0MB {source_hash[:12]}"}] + hashes = json.loads((output_dir / "hashes.json").read_text()) + assert hashes["files"][0]["hash"] == source_hash class TestDependencyRecordOutput: - """Test dependency record output format compliance.""" + """Test Binary JSONL emitted by the real CLI and persisted model.""" - def test_dependency_record_outputs_binary(self): - """Dependency resolution should output Binary JSONL when binary is found.""" - hook_output = json.dumps( - { - "type": "Binary", - "name": "wget", - "abspath": "/usr/bin/wget", - "version": "1.21.3", - "sha256": None, - "binprovider": "apt", - }, + @pytest.mark.django_db(transaction=True) + def test_binary_cli_emits_resolved_dependency_record(self, initialized_archive, tmp_path): + wget_path = resolve_abxpkg_binary_env(tmp_path / "lib", deps_from=WGET_CONFIG)["WGET_BINARY"] + version = subprocess.run([wget_path, "--version"], capture_output=True, text=True, check=True).stdout.split()[2] + from archivebox.tests.conftest import parse_jsonl_output, run_archivebox_cmd + + result = run_archivebox_cmd( + ["binary", "create", "--name=wget", f"--abspath={wget_path}", f"--version={version}"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) + assert result.returncode == 0, result.stderr - from archivebox.machine.models import Process - - data = Process.parse_records_from_text(hook_output)[0] + data = parse_jsonl_output(result.stdout)[0] assert data["type"] == "Binary" assert data["name"] == "wget" - assert data["abspath"].startswith("/") + assert data["abspath"] == wget_path + assert data["version"] == version - def test_dependency_record_outputs_binary_jsonl(self): - """Dependency resolution should output Binary JSONL.""" - hook_output = json.dumps( - { - "type": "Binary", - "name": "wget", - "abspath": "/usr/bin/wget", - "version": "1.21.3", - "binprovider": "env", - }, + list_result = run_archivebox_cmd( + ["binary", "list", "--name=wget"], + cwd=initialized_archive, + default_cli_env=True, + disable_extractors=True, ) - - from archivebox.machine.models import Process - - data = Process.parse_records_from_text(hook_output)[0] - assert data["type"] == "Binary" - assert data["name"] == "wget" - assert data["abspath"] == "/usr/bin/wget" + assert list_result.returncode == 0, list_result.stderr + listed = parse_jsonl_output(list_result.stdout) + assert any(record["id"] == data["id"] and record["abspath"] == wget_path for record in listed) class TestSnapshotHookOutput: - """Test snapshot hook output format compliance.""" + """Test ArchiveResult records emitted by a shipped snapshot hook.""" - def test_snapshot_hook_basic_output(self): - """Snapshot hook should output clean ArchiveResult JSONL.""" - hook_output = json.dumps( - { - "type": "ArchiveResult", - "status": "succeeded", - "output_str": "Downloaded 5 files", + @pytest.mark.django_db(transaction=True) + @pytest.mark.parametrize( + ("enabled", "expected_status"), + [(True, "succeeded"), (False, "skipped")], + ) + def test_hashes_hook_emits_real_archive_result(self, tmp_path, enabled, expected_status): + from archivebox.plugins.hooks import extract_records_from_process, run_hook + + snap_dir = tmp_path / f"snapshot-{expected_status}" + output_dir = snap_dir / "hashes" + output_dir.mkdir(parents=True) + (snap_dir / "source.txt").write_text("real hook protocol input", encoding="utf-8") + hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py"))) + + process = run_hook( + hook_path, + output_dir, + config={ + "ABXPKG_LIB_DIR": str(tmp_path / "lib"), + "SNAP_DIR": str(snap_dir), + "HASHES_ENABLED": enabled, }, + timeout=30, + url="https://example.com/real-hook-record", ) + process.refresh_from_db() - from archivebox.machine.models import Process - - data = Process.parse_records_from_text(hook_output)[0] - assert data["type"] == "ArchiveResult" - assert data["status"] == "succeeded" - assert "output_str" in data - - def test_snapshot_hook_with_cmd(self): - """Snapshot hook should include cmd for binary FK lookup.""" - hook_output = json.dumps( - { - "type": "ArchiveResult", - "status": "succeeded", - "output_str": "Archived with wget", - "cmd": ["/usr/bin/wget", "-p", "-k", "https://example.com"], - }, - ) - - from archivebox.machine.models import Process - - data = Process.parse_records_from_text(hook_output)[0] - assert data["type"] == "ArchiveResult" - assert isinstance(data["cmd"], list) - assert data["cmd"][0] == "/usr/bin/wget" - - def test_snapshot_hook_with_output_json(self): - """Snapshot hook can include structured metadata in output_json.""" - hook_output = json.dumps( - { - "type": "ArchiveResult", - "status": "succeeded", - "output_str": "Got headers", - "output_json": { - "content-type": "text/html", - "server": "nginx", - "status-code": 200, - }, - }, - ) - - from archivebox.machine.models import Process - - data = Process.parse_records_from_text(hook_output)[0] - assert data["type"] == "ArchiveResult" - assert isinstance(data["output_json"], dict) - assert data["output_json"]["status-code"] == 200 - - def test_snapshot_hook_skipped_status(self): - """Snapshot hook should support skipped status.""" - hook_output = json.dumps( - { - "type": "ArchiveResult", - "status": "skipped", - "output_str": "SAVE_WGET=False", - }, - ) - - from archivebox.machine.models import Process - - data = Process.parse_records_from_text(hook_output)[0] - assert data["status"] == "skipped" - - def test_snapshot_hook_failed_status(self): - """Snapshot hook should support failed status.""" - hook_output = json.dumps( - { - "type": "ArchiveResult", - "status": "failed", - "output_str": "404 Not Found", - }, - ) - - from archivebox.machine.models import Process - - data = Process.parse_records_from_text(hook_output)[0] - assert data["status"] == "failed" + assert process.exit_code == 0, process.stderr + records = extract_records_from_process(process) + assert len(records) == 1 + assert records[0]["type"] == "ArchiveResult" + assert records[0]["status"] == expected_status + assert records[0]["plugin"] == "hashes" + assert records[0]["hook_name"] == hook_path.name + assert records[0]["plugin_hook"] == str(hook_path) + if enabled: + assert (output_dir / "hashes.json").is_file() + else: + assert records[0]["output_str"] == "HASHES_ENABLED=False" class TestPluginMetadata: """Test that plugin metadata is added to JSONL records.""" - def test_plugin_name_added(self): - """run_hook() should add plugin name to records.""" - # Simulate what run_hook() does - script = Path("/abx_plugins/plugins/wget/on_Snapshot__50_wget.py") - plugin_name = script.parent.name + @pytest.mark.django_db(transaction=True) + def test_python_hook_metadata_comes_from_executed_shipped_hook(self, tmp_path): + from archivebox.plugins.hooks import extract_records_from_process, run_hook - record = {"type": "ArchiveResult", "status": "succeeded"} - record["plugin"] = plugin_name - record["plugin_hook"] = str(script) + snap_dir = tmp_path / "snapshot-metadata" + output_dir = snap_dir / "hashes" + output_dir.mkdir(parents=True) + (snap_dir / "source.txt").write_text("metadata", encoding="utf-8") + script = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py"))) + process = run_hook( + script, + output_dir, + config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)}, + timeout=30, + url="https://example.com/metadata", + ) + process.refresh_from_db() - assert record["plugin"] == "wget" - assert "on_Snapshot__50_wget.py" in record["plugin_hook"] + assert process.exit_code == 0, process.stderr + records = extract_records_from_process(process) + assert records[0]["plugin"] == "hashes" + assert records[0]["hook_name"] == script.name + assert records[0]["plugin_hook"] == str(script) @pytest.mark.django_db(transaction=True) @@ -739,93 +578,67 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat from archivebox.plugins.hooks import run_hook lib_dir = tmp_path / "lib" - node_modules_dir = lib_dir / "pnpm" / "packages" / "chrome" / "node_modules" - configured_node_path = os.pathsep.join( - [ - "/home/archivebox/.pnpm/packages/chrome/node_modules", - "/usr/lib/node_modules", - str(node_modules_dir), - "/usr/share/archivebox/lib/pnpm/packages/chrome/node_modules", - ], + chrome_config = Path(str(files("abx_plugins.plugins.chrome").joinpath("config.json"))) + node_env = resolve_abxpkg_binary_env( + lib_dir, + deps_from=chrome_config, ) - - plugin_dir = tmp_path / "plugins" / "envprobe" - plugin_dir.mkdir(parents=True) - hook_path = plugin_dir / "on_Snapshot__99_envprobe.py" - hook_path.write_text( - """#!/usr/bin/env python3 -import json -import os - -print(json.dumps({ - "NODE_PATH": os.environ.get("NODE_PATH"), - "NODE_MODULES_DIR": os.environ.get("NODE_MODULES_DIR"), - "NODE_MODULE_DIR": os.environ.get("NODE_MODULE_DIR"), -})) -""", - encoding="utf-8", - ) - hook_path.chmod(0o755) - - output_dir = tmp_path / "archive" / "users" / "system" / "snapshots" / "20260513" / "example.com" / "test" / "envprobe" + configured_node_path = node_env["NODE_PATH"] + node_modules_dir = Path(node_env["NODE_MODULES_DIR"]) + crawl_dir = tmp_path / "crawl" + output_dir = crawl_dir / "chrome" + hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js"))) process = run_hook( hook_path, output_dir, config={ + **node_env, "ABXPKG_LIB_DIR": str(lib_dir), - "NODE_PATH": configured_node_path, + "CRAWL_DIR": str(crawl_dir), + "SNAP_DIR": str(crawl_dir / "snapshot"), + "CHROME_USER_DATA_DIR": str(output_dir / "profile"), }, - timeout=10, + timeout=30, ) process.refresh_from_db() assert process.exit_code == 0, process.stderr - payload = json.loads(process.stdout.strip()) - assert payload["NODE_MODULES_DIR"] == str(node_modules_dir) - assert payload["NODE_MODULE_DIR"] == str(node_modules_dir) - assert payload["NODE_PATH"].split(os.pathsep) == configured_node_path.split(os.pathsep) assert process.env["NODE_MODULES_DIR"] == str(node_modules_dir) + assert process.env["NODE_MODULE_DIR"] == str(node_modules_dir) + assert process.env["NODE_PATH"].split(os.pathsep) == configured_node_path.split(os.pathsep) + assert "chrome zombies" in process.stdout @pytest.mark.django_db(transaction=True) def test_run_hook_executes_python_hooks_through_script_shebang(tmp_path): - """Python hooks must use their abxpkg script header instead of sys.executable.""" + """Python hooks must execute through their abxpkg script header.""" from archivebox.plugins.hooks import run_hook - plugin_dir = tmp_path / "plugins" / "shebangprobe" - plugin_dir.mkdir(parents=True) - hook_path = plugin_dir / "on_Snapshot__99_shebangprobe.py" - hook_path.write_text( - """#!/usr/bin/env -S abxpkg run --script python3 -# /// script -# requires-python = ">=3.12" -# /// -import json -import os -import rich_click - -print(json.dumps({ - "ABXPKG_FAST_SCRIPT": os.environ.get("ABXPKG_FAST_SCRIPT"), - "RICH_CLICK_FILE": rich_click.__file__, -})) -""", - encoding="utf-8", - ) - hook_path.chmod(0o755) - - output_dir = tmp_path / "archive" / "users" / "system" / "snapshots" / "20260603" / "example.com" / "test" / "shebangprobe" + snap_dir = tmp_path / "snapshot" + output_dir = snap_dir / "hashes" + output_dir.mkdir(parents=True) + (snap_dir / "source.txt").write_text("real shebang hook input", encoding="utf-8") + hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py"))) process = run_hook( hook_path, output_dir, config={ "ABXPKG_LIB_DIR": str(tmp_path / "lib"), + "SNAP_DIR": str(snap_dir), }, - timeout=10, + timeout=30, + url="https://example.com/shebang", ) process.refresh_from_db() assert process.cmd[0] == str(hook_path) assert process.exit_code == 0, process.stderr - payload = json.loads(process.stdout.strip()) - assert payload["ABXPKG_FAST_SCRIPT"] == "1" - assert Path(payload["RICH_CLICK_FILE"]).resolve() == Path(click.__file__).resolve() + assert process.env["ABXPKG_FAST_SCRIPT"] == "1" + records = process.parse_records_from_text(process.stdout) + source_hash = hashlib.sha256(b"real shebang hook input").hexdigest() + assert records == [{"type": "ArchiveResult", "status": "succeeded", "output_str": f"0.0MB {source_hash[:12]}"}] + hashes = json.loads((output_dir / "hashes.json").read_text()) + source = hashes["files"][0] + assert source["path"] == "source.txt" + assert source["size"] == len("real shebang hook input") + assert source["hash"] == source_hash diff --git a/archivebox/tests/test_machine_models.py b/archivebox/tests/test_machine_models.py index 8c3c29cf..2c8b8901 100644 --- a/archivebox/tests/test_machine_models.py +++ b/archivebox/tests/test_machine_models.py @@ -14,7 +14,7 @@ Tests cover: import os import subprocess import sys -from datetime import timedelta +from datetime import datetime, timedelta from pathlib import Path from typing import cast @@ -38,7 +38,44 @@ from archivebox.machine.models import ( from archivebox.machine.detect import unknown_if_blank from archivebox.tests.conftest import resolve_abxpkg_binary_env -pytestmark = pytest.mark.django_db +pytestmark = pytest.mark.django_db(transaction=True) + + +def _current_process_started_at(): + import psutil + + return datetime.fromtimestamp( + psutil.Process(os.getpid()).create_time(), + tz=timezone.get_current_timezone(), + ) + + +def _spawn_blocked_process(binary_abspath: str): + import psutil + + process = subprocess.Popen( + [binary_abspath, "-c", "print('READY', flush=True); input()"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert process.stdout is not None + assert process.stdout.readline() == "READY\n" + started_at = datetime.fromtimestamp( + psutil.Process(process.pid).create_time(), + tz=timezone.get_current_timezone(), + ) + return process, started_at + + +def _reaped_process_identity(binary_abspath: str) -> tuple[int, datetime]: + process, started_at = _spawn_blocked_process(binary_abspath) + assert process.stdin is not None + process.stdin.write("\n") + process.stdin.flush() + assert process.wait(timeout=5) == 0 + return process.pid, started_at def _reset_machine_model_caches(): @@ -64,20 +101,37 @@ def machine(): @pytest.fixture def binary(machine): - return Binary.objects.create( + from archivebox.tests.conftest import install_real_binary + + return install_real_binary("python", machine=machine) + + +@pytest.fixture +def process(machine, binary, tmp_path): + return Process.objects.create( machine=machine, - name="test-binary", - binproviders="env", + binary=binary, + cmd=[binary.abspath, "--version"], + pwd=str(tmp_path), ) @pytest.fixture -def process(machine): - return Process.objects.create( - machine=machine, - cmd=["echo", "test"], - pwd="/tmp", - ) +def live_process_identity_factory(binary): + processes = [] + + def spawn() -> tuple[int, datetime]: + process, started_at = _spawn_blocked_process(binary.abspath) + processes.append(process) + return process.pid, started_at + + yield spawn + for process in processes: + if process.poll() is None: + assert process.stdin is not None + process.stdin.write("\n") + process.stdin.flush() + assert process.wait(timeout=5) == 0 @pytest.fixture @@ -343,6 +397,7 @@ class TestNetworkInterfaceModel: ) +@pytest.mark.django_db(transaction=True) class TestBinaryModel: """Test the Binary model.""" @@ -351,52 +406,41 @@ class TestBinaryModel: self.machine = machine def test_binary_creation(self): - """Binary should be created with default values.""" - binary = Binary.objects.create( - machine=self.machine, - name="wget", - binproviders="apt,brew,env", - ) + """A resolved Binary should persist its detected installation.""" + from archivebox.tests.conftest import install_real_binary + + binary = install_real_binary("python", machine=self.machine) assert binary.id is not None - assert binary.name == "wget" - assert binary.status == Binary.StatusChoices.QUEUED - assert not binary.is_valid + assert binary.name == "python" + assert binary.status == Binary.StatusChoices.INSTALLED + assert binary.is_valid def test_binary_is_valid(self): """Binary.is_valid should be True for installed binaries with a resolved path.""" - binary = Binary.objects.create( - machine=self.machine, - name="python", - abspath=sys.executable, - version=f"{sys.version_info.major}.{sys.version_info.minor}", - status=Binary.StatusChoices.INSTALLED, - ) + from archivebox.tests.conftest import install_real_binary + + binary = install_real_binary("python", machine=self.machine) assert binary.is_valid def test_binary_manager_get_valid_binary(self): """BinaryManager.get_valid_binary() should find valid binaries.""" - # Create invalid binary (no abspath) - Binary.objects.create(machine=self.machine, name="python") + from archivebox.tests.conftest import install_real_binary - # Create valid binary - Binary.objects.create( - machine=self.machine, - name="python", - abspath=sys.executable, - version=f"{sys.version_info.major}.{sys.version_info.minor}", - status=Binary.StatusChoices.INSTALLED, - ) + binary = install_real_binary("python", machine=self.machine) result = cast(BinaryManager, Binary.objects).get_valid_binary("python") assert result is not None - assert result.abspath == sys.executable + assert result.id == binary.id + assert Path(result.abspath).resolve() == Path(sys.executable).resolve() def test_binary_update_and_requeue(self): """Binary.update_and_requeue() should update fields and save.""" - binary = Binary.objects.create(machine=self.machine, name="test") + from archivebox.tests.conftest import install_real_binary + + binary = install_real_binary("python", machine=self.machine) old_modified = binary.modified_at binary.update_and_requeue( @@ -416,86 +460,41 @@ class TestBinaryModel: "custom": {"install": "bash -lc 'echo ok'"}, } - binary = Binary.from_json( - { - "name": "chrome", - "binproviders": "apt,pnpm,custom", - "overrides": overrides, - }, - ) + from archivebox.tests.conftest import install_real_binary + + binary = install_real_binary("python", machine=self.machine, overrides=overrides) assert binary is not None assert binary.overrides == overrides - def test_binary_from_json_canonicalizes_path_like_names(self): - """Binary.from_json() should store command names, not path cache values.""" - binary = Binary.from_json( - { - "name": "/tmp/old-lib/pip/venv/bin/trafilatura", - "binproviders": "env,pip", - "overrides": {"pip": {"install_args": ["trafilatura"]}}, - }, - ) + def test_binary_from_json_preserves_provider_package_metadata(self): + """A real Binary install should preserve provider-specific package metadata.""" + from archivebox.tests.conftest import install_real_binary - assert binary is not None - assert binary.name == "trafilatura" - - def test_binary_from_json_does_not_coerce_legacy_override_shapes(self): - """Binary.from_json() should no longer translate legacy non-dict provider overrides.""" - overrides = { - "apt": ["chromium"], - "pnpm": "puppeteer", - } - - binary = Binary.from_json( - { - "name": "chrome", - "binproviders": "apt,pnpm", - "overrides": overrides, - }, - ) - - assert binary is not None - assert binary.overrides == overrides - - def test_binary_from_json_preserves_readability_package_metadata(self): - """Binary.from_json() should preserve readability's pnpm package metadata.""" - binary = Binary.from_json( - { - "name": "readability-extractor", - "binproviders": "env,pnpm", - "overrides": { - "pnpm": { - "install_args": ["readability-extractor"], - }, - }, - }, + binary = install_real_binary( + "python", + machine=self.machine, + overrides={"pip": {"install_args": ["python"]}}, ) assert binary is not None assert binary.overrides == { - "pnpm": { - "install_args": ["readability-extractor"], + "pip": { + "install_args": ["python"], }, } @pytest.mark.django_db(transaction=True) def test_binary_lib_bin_symlink_waits_for_outer_transaction_commit(self, tmp_path): """Binary DB projection writes can be direct, but convenience symlinks must run after commit.""" - provider_lib = tmp_path / "provider" - resolve_abxpkg_binary_env(provider_lib, "node") - source = provider_lib / "env" / "bin" / "node" + from archivebox.tests.conftest import install_real_binary + + binary = install_real_binary("python", machine=self.machine) + source = Path(binary.abspath) lib_bin_dir = tmp_path / "lib" / "bin" - symlink = lib_bin_dir / "abx-test-binary" + symlink = lib_bin_dir / "python" with transaction.atomic(): - binary = Binary.objects.create( - machine=self.machine, - name="abx-test-binary", - abspath=str(source), - version="1.0.0", - status=Binary.StatusChoices.INSTALLED, - ) binary.symlink_to_lib_bin_after_commit(lib_bin_dir) assert not symlink.exists() @@ -530,19 +529,22 @@ class TestProcessModel: """Test the Process model.""" @pytest.fixture(autouse=True) - def setup_machine(self, machine): + def setup_machine(self, machine, binary, tmp_path): self.machine = machine + self.binary = binary + self.pwd = str(tmp_path) def test_process_creation(self): """Process should be created with default values.""" process = Process.objects.create( machine=self.machine, - cmd=["echo", "hello"], - pwd="/tmp", + binary=self.binary, + cmd=[self.binary.abspath, "--version"], + pwd=self.pwd, ) assert process.id is not None - assert process.cmd == ["echo", "hello"] + assert process.cmd == [self.binary.abspath, "--version"] assert process.status == Process.StatusChoices.QUEUED assert process.pid is None assert process.exit_code is None @@ -551,30 +553,36 @@ class TestProcessModel: """Process.to_json() should serialize correctly.""" process = Process.objects.create( machine=self.machine, - cmd=["echo", "hello"], - pwd="/tmp", + binary=self.binary, + cmd=[self.binary.abspath, "--version"], + pwd=self.pwd, timeout=60, ) json_data = process.to_json() assert json_data["type"] == "Process" - assert json_data["cmd"] == ["echo", "hello"] - assert json_data["pwd"] == "/tmp" + assert json_data["cmd"] == [self.binary.abspath, "--version"] + assert json_data["pwd"] == self.pwd assert json_data["timeout"] == 60 def test_process_update_and_requeue(self): """Process.update_and_requeue() should update fields and save.""" - process = Process.objects.create(machine=self.machine, cmd=["test"]) + process = Process.objects.create( + machine=self.machine, + binary=self.binary, + cmd=[self.binary.abspath, "--version"], + pwd=self.pwd, + ) process.update_and_requeue( status=Process.StatusChoices.RUNNING, - pid=12345, - started_at=timezone.now(), + pid=os.getpid(), + started_at=_current_process_started_at(), ) process.refresh_from_db() assert process.status == Process.StatusChoices.RUNNING - assert process.pid == 12345 + assert process.pid == os.getpid() assert process.started_at is not None @@ -640,33 +648,23 @@ class TestProcessCurrent: finally: sys.argv = old_argv - def test_process_proc_allows_interpreter_wrapped_script(self, tmp_path): + def test_process_proc_allows_interpreter_wrapped_script(self, binary): """Process.proc should accept a script recorded in DB when wrapped by an interpreter in psutil.""" import psutil - script = tmp_path / "on_CrawlSetup__90_chrome_launch.daemon.bg.py" - script.write_text("import time\ntime.sleep(30)\n", encoding="utf-8") + script = Path(__file__).parents[1] / "cli" / "archivebox_manage.py" process = subprocess.Popen( - [sys.executable, str(script), "--url=https://example.com/"], - stdin=subprocess.DEVNULL, + [binary.abspath, str(script), "shell"], + stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) - def cleanup_process(): - if process.poll() is None: - process.terminate() - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=5) - try: os_proc = psutil.Process(process.pid) proc = Process.objects.create( machine=Machine.current(), - cmd=[str(script), "--url=https://example.com/"], + cmd=[str(script), "shell"], pid=process.pid, status=Process.StatusChoices.RUNNING, started_at=timezone.datetime.fromtimestamp(os_proc.create_time(), tz=timezone.get_current_timezone()), @@ -676,7 +674,9 @@ class TestProcessCurrent: assert resolved_proc is not None assert resolved_proc.pid == process.pid finally: - cleanup_process() + assert process.stdin is not None + process.stdin.close() + process.wait(timeout=30) class TestProcessHierarchy: @@ -686,14 +686,16 @@ class TestProcessHierarchy: def setup_machine(self, machine): self.machine = machine - def test_process_parent_child(self): + def test_process_parent_child(self, live_process_identity_factory): """Process should track parent/child relationships.""" + parent_pid, parent_started_at = live_process_identity_factory() + child_pid, child_started_at = live_process_identity_factory() parent = Process.objects.create( machine=self.machine, process_type=Process.TypeChoices.CLI, status=Process.StatusChoices.RUNNING, - pid=1, - started_at=timezone.now(), + pid=parent_pid, + started_at=parent_started_at, ) child = Process.objects.create( @@ -701,50 +703,60 @@ class TestProcessHierarchy: parent=parent, process_type=Process.TypeChoices.WORKER, status=Process.StatusChoices.RUNNING, - pid=2, - started_at=timezone.now(), + pid=child_pid, + started_at=child_started_at, ) assert child.parent == parent assert child in parent.children.all() - def test_process_root(self): + def test_process_root(self, live_process_identity_factory): """Process.root should return the root of the hierarchy.""" + root_pid, root_started_at = live_process_identity_factory() + child_pid, child_started_at = live_process_identity_factory() + grandchild_pid, grandchild_started_at = live_process_identity_factory() root = Process.objects.create( machine=self.machine, process_type=Process.TypeChoices.CLI, status=Process.StatusChoices.RUNNING, - started_at=timezone.now(), + pid=root_pid, + started_at=root_started_at, ) child = Process.objects.create( machine=self.machine, parent=root, status=Process.StatusChoices.RUNNING, - started_at=timezone.now(), + pid=child_pid, + started_at=child_started_at, ) grandchild = Process.objects.create( machine=self.machine, parent=child, status=Process.StatusChoices.RUNNING, - started_at=timezone.now(), + pid=grandchild_pid, + started_at=grandchild_started_at, ) assert grandchild.root == root assert child.root == root assert root.root == root - def test_process_depth(self): + def test_process_depth(self, live_process_identity_factory): """Process.depth should return depth in tree.""" + root_pid, root_started_at = live_process_identity_factory() + child_pid, child_started_at = live_process_identity_factory() root = Process.objects.create( machine=self.machine, status=Process.StatusChoices.RUNNING, - started_at=timezone.now(), + pid=root_pid, + started_at=root_started_at, ) child = Process.objects.create( machine=self.machine, parent=root, status=Process.StatusChoices.RUNNING, - started_at=timezone.now(), + pid=child_pid, + started_at=child_started_at, ) assert root.depth == 0 @@ -755,42 +767,41 @@ class TestProcessLifecycle: """Test Process lifecycle methods.""" @pytest.fixture(autouse=True) - def setup_machine(self, machine): + def setup_machine(self, machine, binary): self.machine = machine + self.binary = binary def test_process_is_running_current_pid(self): """is_running should be True for current PID.""" - import psutil - from datetime import datetime - - proc_start = datetime.fromtimestamp(psutil.Process(os.getpid()).create_time(), tz=timezone.get_current_timezone()) proc = Process.objects.create( machine=self.machine, status=Process.StatusChoices.RUNNING, pid=os.getpid(), - started_at=proc_start, + started_at=_current_process_started_at(), ) assert proc.is_running - def test_process_is_running_fake_pid(self): - """is_running should be False for non-existent PID.""" + def test_process_is_running_reaped_process(self): + """is_running should be False after the recorded OS process exits.""" + pid, started_at = _reaped_process_identity(self.binary.abspath) proc = Process.objects.create( machine=self.machine, status=Process.StatusChoices.RUNNING, - pid=999999, - started_at=timezone.now(), + pid=pid, + started_at=started_at, ) assert not proc.is_running def test_process_poll_detects_exit(self): """poll() should detect exited process.""" + pid, started_at = _reaped_process_identity(self.binary.abspath) proc = Process.objects.create( machine=self.machine, status=Process.StatusChoices.RUNNING, - pid=999999, - started_at=timezone.now(), + pid=pid, + started_at=started_at, ) exit_code = proc.poll() @@ -801,12 +812,13 @@ class TestProcessLifecycle: def test_process_poll_normalizes_negative_exit_code(self): """poll() should normalize -1 exit codes to 137.""" + pid, started_at = _reaped_process_identity(self.binary.abspath) proc = Process.objects.create( machine=self.machine, status=Process.StatusChoices.EXITED, - pid=999999, + pid=pid, exit_code=-1, - started_at=timezone.now(), + started_at=started_at, ) exit_code = proc.poll() @@ -817,11 +829,12 @@ class TestProcessLifecycle: def test_process_terminate_dead_process(self): """terminate() should handle already-dead process.""" + pid, started_at = _reaped_process_identity(self.binary.abspath) proc = Process.objects.create( machine=self.machine, status=Process.StatusChoices.RUNNING, - pid=999999, - started_at=timezone.now(), + pid=pid, + started_at=started_at, ) result = proc.terminate() @@ -835,8 +848,9 @@ class TestProcessClassMethods: """Test Process class methods for querying.""" @pytest.fixture(autouse=True) - def setup_machine(self, machine): + def setup_machine(self, machine, binary): self.machine = machine + self.binary = binary def test_get_running(self): """get_running should return running processes.""" @@ -844,8 +858,8 @@ class TestProcessClassMethods: machine=self.machine, process_type=Process.TypeChoices.HOOK, status=Process.StatusChoices.RUNNING, - pid=99999, - started_at=timezone.now(), + pid=os.getpid(), + started_at=_current_process_started_at(), ) running = Process.get_running(process_type=Process.TypeChoices.HOOK) @@ -859,8 +873,8 @@ class TestProcessClassMethods: machine=self.machine, process_type=Process.TypeChoices.HOOK, status=Process.StatusChoices.RUNNING, - pid=99900 + i, - started_at=timezone.now(), + pid=os.getpid(), + started_at=_current_process_started_at(), ) count = Process.get_running_count(process_type=Process.TypeChoices.HOOK) @@ -868,10 +882,11 @@ class TestProcessClassMethods: def test_cleanup_stale_running(self): """cleanup_stale_running should mark stale processes as exited.""" + pid, _started_at = _reaped_process_identity(self.binary.abspath) stale = Process.objects.create( machine=self.machine, status=Process.StatusChoices.RUNNING, - pid=999999, + pid=pid, started_at=timezone.now() - PID_REUSE_WINDOW - timedelta(hours=1), ) @@ -883,11 +898,12 @@ class TestProcessClassMethods: def test_cleanup_stale_running_marks_timed_out_rows_exited(self): """cleanup_stale_running should retire RUNNING rows that exceed timeout + grace.""" + pid, _started_at = _reaped_process_identity(self.binary.abspath) stale = Process.objects.create( machine=self.machine, process_type=Process.TypeChoices.HOOK, status=Process.StatusChoices.RUNNING, - pid=999998, + pid=pid, timeout=5, started_at=timezone.now() - PROCESS_TIMEOUT_GRACE - timedelta(seconds=10), ) @@ -917,16 +933,13 @@ class TestProcessClassMethods: def test_cleanup_orphaned_workers_marks_dead_root_children_exited(self): """cleanup_orphaned_workers should retire rows whose CLI/orchestrator root is gone.""" - import psutil - from datetime import datetime - - started_at = datetime.fromtimestamp(psutil.Process(os.getpid()).create_time(), tz=timezone.get_current_timezone()) + parent_pid, parent_started_at = _reaped_process_identity(self.binary.abspath) parent = Process.objects.create( machine=self.machine, process_type=Process.TypeChoices.CLI, status=Process.StatusChoices.RUNNING, - pid=999997, - started_at=timezone.now() - timedelta(minutes=5), + pid=parent_pid, + started_at=parent_started_at, ) child = Process.objects.create( machine=self.machine, @@ -934,7 +947,7 @@ class TestProcessClassMethods: process_type=Process.TypeChoices.HOOK, status=Process.StatusChoices.RUNNING, pid=os.getpid(), - started_at=started_at, + started_at=_current_process_started_at(), ) cleaned = Process.cleanup_orphaned_workers() @@ -945,12 +958,13 @@ class TestProcessClassMethods: def test_cleanup_orphaned_workers_marks_non_running_children_exited(self): """cleanup_orphaned_workers should retire child rows whose OS process is already gone.""" + pid, started_at = _reaped_process_identity(self.binary.abspath) child = Process.objects.create( machine=self.machine, process_type=Process.TypeChoices.HOOK, status=Process.StatusChoices.RUNNING, - pid=999997, - started_at=timezone.now() - timedelta(minutes=5), + pid=pid, + started_at=started_at, ) cleaned = Process.cleanup_orphaned_workers() diff --git a/archivebox/tests/test_migrations_08_to_09.py b/archivebox/tests/test_migrations_08_to_09.py index 9d156bda..f5e0753a 100644 --- a/archivebox/tests/test_migrations_08_to_09.py +++ b/archivebox/tests/test_migrations_08_to_09.py @@ -316,36 +316,6 @@ def test_hyphenated_crawl_ids_are_normalized_before_snapshot_saves(migration_08_ assert "FOREIGN KEY constraint failed" not in output -def test_migration_removes_seed_id_column(migration_08_data): - """Migration should remove seed_id column from archivebox.crawls.crawl.""" - work_dir, db_path, original_data = migration_08_data - result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) - assert result.returncode == 0, f"Init failed: {result.stderr}" - - conn = sqlite3.connect(str(db_path)) - cursor = conn.cursor() - cursor.execute("PRAGMA table_info(crawls_crawl)") - columns = [row[1] for row in cursor.fetchall()] - conn.close() - - assert "seed_id" not in columns, f"seed_id column should have been removed by migration. Columns: {columns}" - - -def test_migration_removes_seed_table(migration_08_data): - """Migration should remove crawls_seed table.""" - work_dir, db_path, original_data = migration_08_data - result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) - assert result.returncode == 0, f"Init failed: {result.stderr}" - - conn = sqlite3.connect(str(db_path)) - cursor = conn.cursor() - cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='crawls_seed'") - table_exists = cursor.fetchone() is not None - conn.close() - - assert not table_exists, "crawls_seed table should have been removed by migration" - - def test_add_works_after_migration(migration_08_data): """Adding new URLs should work after migration from 0.8.x.""" work_dir, db_path, original_data = migration_08_data diff --git a/archivebox/tests/test_migrations_fresh.py b/archivebox/tests/test_migrations_fresh.py index d3cacdda..cfdddfb7 100644 --- a/archivebox/tests/test_migrations_fresh.py +++ b/archivebox/tests/test_migrations_fresh.py @@ -144,9 +144,6 @@ def test_crawl_table_has_required_columns(tmp_path): for col in required: assert col in columns, f"Missing column: {col}" - # seed_id should NOT exist (removed in 0.9.x) - assert "seed_id" not in columns, "seed_id column should not exist in 0.9.x" - def test_add_urls_separately(tmp_path): """Should be able to add multiple URLs one at a time.""" diff --git a/archivebox/tests/test_misc_checks.py b/archivebox/tests/test_misc_checks.py index ecf7e47c..f44aad00 100644 --- a/archivebox/tests/test_misc_checks.py +++ b/archivebox/tests/test_misc_checks.py @@ -1,12 +1,14 @@ import os import signal +import subprocess +import sys +import textwrap import pytest from archivebox.core.shutdown_util import foreground_shutdown_signals from archivebox.core.shutdown_util import raise_if_shutdown_requested from archivebox.misc.checks import _migration_interrupt_message -from archivebox.misc.checks import _exit_on_migration_interrupt def test_migration_interrupt_message_prints_resume_command_and_atomic_safety(): @@ -25,23 +27,37 @@ def test_migration_interrupt_message_before_apply_says_no_changes_applied(): assert "archivebox init" in message -def test_migration_interrupt_handler_exits_for_sigint_and_sigterm(monkeypatch): - def fake_exit(code): - raise SystemExit(code) +@pytest.mark.parametrize("sig", [signal.SIGINT, signal.SIGTERM]) +def test_migration_interrupt_handler_exits_for_sigint_and_sigterm(sig): + process = subprocess.Popen( + [ + sys.executable, + "-c", + textwrap.dedent( + """ + import signal - monkeypatch.setattr("archivebox.misc.checks.os._exit", fake_exit) + from archivebox.misc.checks import _exit_on_migration_interrupt - for sig in (signal.SIGINT, signal.SIGTERM): - previous_handler = signal.getsignal(sig) - try: - with _exit_on_migration_interrupt(): - assert signal.getsignal(sig) != previous_handler - os.kill(os.getpid(), sig) - except SystemExit as err: - assert err.code == 130 - else: - raise AssertionError(f"{sig.name} should exit during migration auto-apply") - assert signal.getsignal(sig) == previous_handler + with _exit_on_migration_interrupt(): + print("READY", flush=True) + signal.pause() + """, + ), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=os.environ.copy(), + ) + assert process.stdout is not None + assert process.stdout.readline() == "READY\n" + + process.send_signal(sig) + stdout, stderr = process.communicate(timeout=30) + + assert process.returncode == 130, (stdout, stderr) def test_nested_foreground_signal_state_propagates_to_outer_context(): diff --git a/archivebox/tests/test_opencode_agent.py b/archivebox/tests/test_opencode_agent.py index 8fcdcc14..2876841f 100644 --- a/archivebox/tests/test_opencode_agent.py +++ b/archivebox/tests/test_opencode_agent.py @@ -52,8 +52,6 @@ def opencode_archive_config(initialized_archive): env = os.environ.copy() env.update( { - "ABXPKG_INSTALL_TIMEOUT": "900", - "ABXPKG_MIN_RELEASE_AGE": "0", "ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS": "true", "OPENCODE_ENABLED": "True", "OPENCODE_HOST": "127.0.0.1", @@ -113,10 +111,7 @@ def live_opencode(opencode_archive_config): finally: if views._PROCESS and views._PROCESS.poll() is None: views._PROCESS.terminate() - try: - views._PROCESS.wait(timeout=10) - except Exception: - views._PROCESS.kill() + views._PROCESS.wait(timeout=10) views._PROCESS = None diff --git a/archivebox/tests/test_process_runtime_paths.py b/archivebox/tests/test_process_runtime_paths.py index c5b51a85..d3f3e22f 100644 --- a/archivebox/tests/test_process_runtime_paths.py +++ b/archivebox/tests/test_process_runtime_paths.py @@ -1,34 +1,56 @@ -import os +from importlib.resources import files from pathlib import Path +import pytest -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.settings") +from archivebox.tests.conftest import run_archivebox_cmd +from archivebox.tests.test_orm_helpers import use_archivebox_db - -from archivebox.machine.models import Process +pytestmark = pytest.mark.django_db(transaction=True) class TestProcessRuntimePaths: - def test_hook_processes_use_isolated_runtime_dir(self): - process = Process( - process_type=Process.TypeChoices.HOOK, - pwd="/tmp/archive/example/chrome", - cmd=["node", "/plugins/chrome/on_Snapshot__11_chrome_wait.js", "--url=https://example.com"], - ) + def test_hook_processes_use_isolated_runtime_dir(self, tmp_path): + from archivebox.plugins.hooks import run_hook - expected_dir = Path("/tmp/archive/example/chrome/.hooks/on_Snapshot__11_chrome_wait.js") + snap_dir = tmp_path / "snapshot" + output_dir = snap_dir / "hashes" + output_dir.mkdir(parents=True) + (snap_dir / "source.txt").write_text("real runtime path input", encoding="utf-8") + hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py"))) + process = run_hook( + hook_path, + output_dir, + config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)}, + timeout=30, + url="https://example.com/runtime-path", + ) + process.refresh_from_db() + assert process.exit_code == 0, process.stderr + + expected_dir = output_dir / ".hooks" / hook_path.name assert process.runtime_dir == expected_dir assert process.stdout_file == expected_dir / "stdout.log" assert process.stderr_file == expected_dir / "stderr.log" - def test_non_hook_processes_keep_runtime_files_in_pwd(self): - process = Process( - process_type=Process.TypeChoices.WORKER, - pwd="/tmp/archive/example", - cmd=["archivebox", "run", "--snapshot-id", "123"], + def test_non_hook_processes_keep_runtime_files_in_pwd(self, tmp_path): + init_result = run_archivebox_cmd(["init", "--quick"], cwd=tmp_path, timeout=90) + assert init_result.returncode == 0, init_result.stderr or init_result.stdout + add_result = run_archivebox_cmd( + ["add", "--index-only", "--depth=0", "https://example.com/runtime-path"], + cwd=tmp_path, + timeout=90, ) + assert add_result.returncode == 0, add_result.stderr or add_result.stdout - expected_dir = Path("/tmp/archive/example") + with use_archivebox_db(tmp_path): + from archivebox.machine.models import Process + + process = next( + row for row in Process.objects.order_by("-created_at") if "add" in row.cmd and row.process_type != Process.TypeChoices.HOOK + ) + + expected_dir = Path(process.pwd) assert process.runtime_dir == expected_dir assert process.stdout_file == expected_dir / "stdout.log" assert process.stderr_file == expected_dir / "stderr.log" diff --git a/archivebox/tests/test_process_service.py b/archivebox/tests/test_process_service.py index 2c1484aa..f399e71d 100644 --- a/archivebox/tests/test_process_service.py +++ b/archivebox/tests/test_process_service.py @@ -5,60 +5,36 @@ pytestmark = pytest.mark.django_db @pytest.mark.django_db(transaction=True) -def test_process_completed_persists_with_uncached_network_interface(tmp_path): +def test_process_completed_persists_with_uncached_network_interface(tmp_path, recursive_test_site): import asyncio - from abx_dl.events import CrawlCleanupEvent, ProcessCompletedEvent - from abx_dl.orchestrator import create_bus + import archivebox.machine.models as machine_models + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl from archivebox.machine.models import Machine, NetworkInterface, Process - from archivebox.services.process_service import ProcessService + from archivebox.services.runner import CrawlRunner machine = Machine.current() iface = NetworkInterface.current() + crawl = Crawl.objects.create( + urls=recursive_test_site["root_url"], + config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "PLUGINS": "headers"}, + created_by_id=get_or_create_system_user_pk(), + ) - output_dir = tmp_path / "headers" - output_dir.mkdir() - bus = create_bus(name="test_process_completed_uncached_iface") - ProcessService(bus) + machine_models._CURRENT_INTERFACE = None + runner = CrawlRunner(crawl, selected_plugins=["headers"], show_progress=False) + asyncio.run(runner.run()) - async def run_event() -> None: - event = bus.emit( - ProcessCompletedEvent( - plugin_name="headers", - hook_name="on_Snapshot__27_headers.daemon.bg", - hook_path="/bin/echo", - hook_args=["--url=https://example.com"], - is_background=True, - output_dir=str(output_dir), - env={}, - timeout=60, - pid=123, - stdout="", - stderr="", - exit_code=0, - status="succeeded", - output_files=[], - start_ts="2026-05-13T07:22:00+00:00", - end_ts="2026-05-13T07:22:01+00:00", - ), - ) - await event.now() - await event.wait() - await event.event_results_list() - cleanup = bus.emit( - CrawlCleanupEvent( - url="https://example.com", - snapshot_id="test-snapshot", - output_dir=str(output_dir), - ), - ) - await cleanup.now() - await cleanup.wait() - - asyncio.run(run_event()) - - process = Process.objects.get(pwd=str(output_dir), cmd=["/bin/echo", "--url=https://example.com"]) + process = next( + process + for process in Process.objects.filter(process_type=Process.TypeChoices.HOOK) + if process.cmd and "on_Snapshot__27_headers.daemon.bg.js" in str(process.cmd[0]) + ) assert process.machine_id == machine.id assert process.iface_id == iface.id assert process.process_type == Process.TypeChoices.HOOK assert process.status == Process.StatusChoices.EXITED + assert process.started_at is not None + assert process.ended_at is not None + assert process.pid is not None diff --git a/archivebox/tests/test_recursive_crawl.py b/archivebox/tests/test_recursive_crawl.py index 23bc6a17..6dc00df2 100644 --- a/archivebox/tests/test_recursive_crawl.py +++ b/archivebox/tests/test_recursive_crawl.py @@ -3,8 +3,6 @@ import json import os -import subprocess -import time from pathlib import Path import pytest @@ -18,40 +16,18 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) -def wait_for_db_condition(timeout, condition, interval=0.5): - deadline = time.time() + timeout - while time.time() < deadline: - if os.path.exists("index.sqlite3"): - with use_archivebox_db("."): - if condition(): - return True - time.sleep(interval) - return False - - -def stop_process(proc): - if proc.poll() is None: - proc.terminate() - try: - return proc.communicate(timeout=5) - except subprocess.TimeoutExpired: - proc.kill() - return proc.communicate() - - def run_add_until(args, env, condition, timeout=120): assert args[0] == "archivebox" - proc = run_archivebox_cmd( + result = run_archivebox_cmd( args[1:], cwd=Path.cwd(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, env=env, - wait=False, + timeout=timeout, ) - - assert wait_for_db_condition(timeout=timeout, condition=condition), f"Timed out waiting for condition while running: {' '.join(args)}" - return stop_process(proc) + assert result.returncode == 0, result.stderr or result.stdout + with use_archivebox_db("."): + assert condition(), f"Condition was false after command completed: {' '.join(args)}" + return result.stdout, result.stderr def test_background_hooks_dont_block_parser_extractors(tmp_path, initialized_archive, recursive_test_site): @@ -83,24 +59,16 @@ def test_background_hooks_dont_block_parser_extractors(tmp_path, initialized_arc }, ) - proc = run_archivebox_cmd( - ["add", "--depth=1", "--plugins=favicon,parse_html_urls", recursive_test_site["root_url"]], - cwd=tmp_path, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout, stderr = run_add_until( + ["archivebox", "add", "--depth=1", "--plugins=favicon,parse_html_urls", recursive_test_site["root_url"]], env=env, - wait=False, - ) - - assert wait_for_db_condition( timeout=120, condition=lambda: ArchiveResult.objects.filter( plugin__startswith="parse_", plugin__endswith="_urls", status__in=("started", "succeeded", "failed"), ).exists(), - ), "Parser extractors never progressed beyond queued status" - stdout, stderr = stop_process(proc) + ) if stderr: print(f"\n=== STDERR ===\n{stderr}\n=== END STDERR ===\n") @@ -426,7 +394,6 @@ def test_add_archivewebpage_installs_required_chrome_dependency(initialized_arch "USE_COLOR": "false", "SHOW_PROGRESS": "false", "TIMEOUT": "120", - "ABXPKG_INSTALL_TIMEOUT": "900", "ABXPKG_LIB_DIR": str(initialized_archive / "lib"), "CHROME_HEADLESS": "true", "CHROME_SANDBOX": "false", @@ -550,7 +517,6 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel( "ABXPKG_LIB_DIR": str(initialized_archive / "lib"), "CHROMEWEBSTORE_EXTENSIONS_DIR": str(initialized_archive / "lib/chromewebstore/extensions"), "TIMEOUT": "90", - "ABXPKG_INSTALL_TIMEOUT": "900", "CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3", "SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1", "SEARCH_BACKEND_SONIC_PORT": str(free_tcp_port_factory()), diff --git a/archivebox/tests/test_search.py b/archivebox/tests/test_search.py index 661f1f1c..5091e041 100644 --- a/archivebox/tests/test_search.py +++ b/archivebox/tests/test_search.py @@ -1,4 +1,3 @@ -import json import os import re import time @@ -13,10 +12,10 @@ import requests from asgiref.sync import async_to_sync from django.contrib.auth import get_user_model from django.core.cache import cache -from django.test import override_settings from django.urls import reverse from archivebox.misc.logging import AttrDict +from archivebox.machine.models import Machine from archivebox.tests.conftest import ( cli_env, create_admin_and_token, @@ -25,7 +24,7 @@ from archivebox.tests.conftest import ( resolve_abxpkg_binary_env, start_archivebox_server, stop_archivebox_process, - wait_for_http, + get_http_response, ) @@ -131,10 +130,10 @@ def test_search_backend_env_exposes_resolved_runtime_config(tmp_path): os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] = old_env -def test_search_mode_options_use_canonical_backend_names(monkeypatch): +def test_search_mode_options_use_canonical_backend_names(): from archivebox.search.config import get_search_mode_options - monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep") + Machine.from_json({"config": {"SEARCH_BACKEND_ENGINE": "ripgrep"}}) options = get_search_mode_options() @@ -178,8 +177,8 @@ def test_snapshot_metadata_search_includes_notes_crawl_fields_username_and_confi class TestAdminSnapshotSearch: - def test_admin_search_mode_selector_defaults_to_configured_deep_backend_for_ripgrep(self, client, admin_user, monkeypatch): - monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep") + def test_admin_search_mode_selector_defaults_to_configured_deep_backend_for_ripgrep(self, client, admin_user): + Machine.from_json({"config": {"SEARCH_BACKEND_ENGINE": "ripgrep"}}) client.login(username="testadmin", password="testpassword") response = client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_HOST) @@ -192,8 +191,8 @@ class TestAdminSnapshotSearch: assert b'value="deep:ripgrep"' in response.content assert b">deep:ripgrep<" in response.content - def test_admin_search_mode_selector_defaults_to_configured_deep_backend_for_sqlite(self, client, admin_user, monkeypatch): - monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "sqlite") + def test_admin_search_mode_selector_defaults_to_configured_deep_backend_for_sqlite(self, client, admin_user): + Machine.from_json({"config": {"SEARCH_BACKEND_ENGINE": "sqlite"}}) client.login(username="testadmin", password="testpassword") response = client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_HOST) @@ -222,10 +221,10 @@ class TestAdminSnapshotSearch: assert b'id="changelist"' in response.content assert b"search-mode-contents" in response.content - def test_admin_search_stream_uses_real_ripgrep_backend_for_deep_results(self, client, admin_user, crawl, monkeypatch): + def test_admin_search_stream_uses_real_ripgrep_backend_for_deep_results(self, client, admin_user, crawl): from archivebox.core.models import Snapshot - monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep") + Machine.from_json({"config": {"SEARCH_BACKEND_ENGINE": "ripgrep"}}) fulltext_snapshot = Snapshot.objects.create( url="https://example.com/fulltext-only", title="Unrelated Title", @@ -279,10 +278,10 @@ class TestAdminSnapshotSearch: assert result_ids[:3] == [prefix_snapshot.pk, title_snapshot.pk, contains_snapshot.pk] assert {title_snapshot.pk, contains_snapshot.pk, prefix_snapshot.pk}.issubset(result_ids) - def test_admin_contents_search_stream_uses_real_backend_results(self, client, admin_user, crawl, monkeypatch): + def test_admin_contents_search_stream_uses_real_backend_results(self, client, admin_user, crawl): from archivebox.core.models import Snapshot - monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep") + Machine.from_json({"config": {"SEARCH_BACKEND_ENGINE": "ripgrep"}}) metadata_snapshot = Snapshot.objects.create( url="https://example.com/google-meta", title="Google Metadata Match", @@ -382,7 +381,10 @@ class TestAdminSnapshotSearch: class TestPublicIndexSearch: - @override_settings(PUBLIC_INDEX=True) + @pytest.fixture(autouse=True) + def public_index_enabled(self): + Machine.from_json({"config": {"PUBLIC_INDEX": True}}) + def test_public_search_by_url(self, client, public_snapshot): cache.clear() response = client.get("/public/", {"q": "public-example.com"}, HTTP_HOST=WEB_HOST) @@ -391,9 +393,8 @@ class TestPublicIndexSearch: assert b"matching snapshots..." in response.content assert b"No snapshots found." not in response.content - @override_settings(PUBLIC_INDEX=True) - def test_public_search_mode_selector_defaults_to_configured_deep_backend_for_ripgrep(self, client, monkeypatch): - monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep") + def test_public_search_mode_selector_defaults_to_configured_deep_backend_for_ripgrep(self, client): + Machine.from_json({"config": {"SEARCH_BACKEND_ENGINE": "ripgrep"}}) response = client.get("/public/", HTTP_HOST=WEB_HOST) @@ -405,11 +406,10 @@ class TestPublicIndexSearch: assert b'value="deep:ripgrep"' in response.content assert b">deep:ripgrep<" in response.content - @override_settings(PUBLIC_INDEX=True) - def test_public_search_uses_streamed_metadata_order(self, client, crawl, monkeypatch): + def test_public_search_uses_streamed_metadata_order(self, client, crawl): from archivebox.core.models import Snapshot - monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "ripgrep") + Machine.from_json({"config": {"SEARCH_BACKEND_ENGINE": "ripgrep"}}) metadata_snapshot = Snapshot.objects.create( url="https://public-example.com/google-meta", title="Google Metadata Match", @@ -441,7 +441,6 @@ class TestPublicIndexSearch: content = response.content.decode() assert content.index(str(metadata_snapshot.url)) < content.index(str(fulltext_snapshot.url)) - @override_settings(PUBLIC_INDEX=True) def test_public_metadata_search_prioritizes_common_url_prefixes(self, client, crawl): from archivebox.core.models import Snapshot @@ -473,14 +472,12 @@ class TestPublicIndexSearch: assert content.index(str(prefix_match.url)) < content.index(str(broad_match.url)) - @override_settings(PUBLIC_INDEX=True) def test_public_search_by_title(self, client, public_snapshot): response = client.get("/public/", {"q": "Public Example"}, HTTP_HOST=WEB_HOST) assert response.status_code == 200 assert b"archivebox-search-stream-status" in response.content - @override_settings(PUBLIC_INDEX=True) def test_public_search_stream_preserves_search_form_dom(self, client, public_snapshot): response = client.get("/public/", {"q": "Public Example"}, HTTP_HOST=WEB_HOST) @@ -489,7 +486,6 @@ class TestPublicIndexSearch: assert b"replaceRegionFromDocument(doc, '#table-bookmarks tbody')" in response.content assert b"currentList.replaceWith(nextList)" not in response.content - @override_settings(PUBLIC_INDEX=True) def test_public_search_stream_populates_public_results_cache(self, client, public_snapshot): search_params = {"q": "Public Example", "search_mode": "meta"} search_url = f"/public/?{urlencode(search_params)}" @@ -510,7 +506,6 @@ class TestPublicIndexSearch: assert b"Public Example Website" in response.content assert b"No snapshots found." not in response.content - @override_settings(PUBLIC_INDEX=True) def test_public_index_shows_exact_total_count_and_page_count_for_100_plus_snapshots(self, client, crawl, public_snapshot): from archivebox.core.models import Snapshot @@ -572,7 +567,6 @@ class TestPublicIndexSearch: assert "last »" not in last_content assert "private-page-test" not in last_content - @override_settings(PUBLIC_INDEX=True) def test_public_index_preview_respects_root_relative_screenshot_output(self, client, public_snapshot): from archivebox.core.models import ArchiveResult @@ -593,7 +587,6 @@ class TestPublicIndexSearch: assert "/screenshot.png" in content assert "/screenshot/screenshot.png" not in content - @override_settings(PUBLIC_INDEX=True) def test_public_index_preview_respects_plugin_relative_screenshot_output(self, client, public_snapshot): from archivebox.core.models import ArchiveResult @@ -613,7 +606,6 @@ class TestPublicIndexSearch: content = response.content.decode() assert "screenshot/screenshot.png" in content - @override_settings(PUBLIC_INDEX=True) def test_public_index_preview_falls_back_to_extension_screenshots(self, client, public_snapshot): from archivebox.core.models import ArchiveResult @@ -638,7 +630,6 @@ class TestPublicIndexSearch: assert "/screenshot/screenshot.png" not in content assert "chrome_extension_screenshot/screenshot-2.png" not in content - @override_settings(PUBLIC_INDEX=True) def test_public_index_snapshot_without_preview_renders_placeholder(self, client, public_snapshot): response = client.get("/public/", HTTP_HOST=WEB_HOST) @@ -647,7 +638,6 @@ class TestPublicIndexSearch: assert "snapshot-preview-empty" in content assert "screenshot/screenshot.png" not in content - @override_settings(PUBLIC_INDEX=True) def test_public_index_pending_snapshot_uses_small_preview_spinner(self, client, crawl): from archivebox.core.models import Snapshot @@ -665,7 +655,6 @@ class TestPublicIndexSearch: assert "snapshot-preview-spinner" in content assert "spinner.gif" in content - @override_settings(PUBLIC_INDEX=True) def test_public_index_finished_snapshot_without_title_falls_back_to_url(self, client, public_snapshot): public_snapshot.title = "" public_snapshot.save(update_fields=["title"]) @@ -677,19 +666,16 @@ class TestPublicIndexSearch: assert "https://public-example.com" in content assert "Loading..." not in content - @override_settings(PUBLIC_INDEX=True) def test_public_search_query_type_meta(self, client, public_snapshot): response = client.get("/public/", {"q": "example", "query_type": "meta"}, HTTP_HOST=WEB_HOST) assert response.status_code == 200 - @override_settings(PUBLIC_INDEX=True) def test_public_search_query_type_url(self, client, public_snapshot): response = client.get("/public/", {"q": "public-example.com", "query_type": "url"}, HTTP_HOST=WEB_HOST) assert response.status_code == 200 - @override_settings(PUBLIC_INDEX=True) def test_public_search_query_type_title(self, client, public_snapshot): response = client.get("/public/", {"q": "Website", "query_type": "title"}, HTTP_HOST=WEB_HOST) @@ -784,16 +770,15 @@ class TestSearchBackendsE2E: if body is None: self.send_response(404) self.end_headers() - return - - self.send_response(200) - self.send_header("Content-Type", "text/html; charset=utf-8") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) + else: + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) def log_message(self, _format, *args): - return + pass fixture_server = ThreadingHTTPServer(("127.0.0.1", 0), SearchMatrixHandler) fixture_thread = Thread(target=fixture_server.serve_forever, daemon=True) @@ -885,70 +870,36 @@ class TestSearchBackendsE2E: ) assert second_add_result.returncode == 0, second_add_result.stderr or second_add_result.stdout - metadata_snapshot_records = [ - { - "type": "Snapshot", - "url": url_only_url, - "title": "URL Only Precision Page", - "tags": "search-matrix", - "depth": 0, - }, - { - "type": "Snapshot", - "url": title_only_url, - "title": title_only_needle, - "tags": "search-matrix", - "depth": 0, - }, - { - "type": "Snapshot", - "url": tag_only_url, - "title": "Tag Only Precision Page", - "tags": f"search-matrix,{tag_only_needle}", - "depth": 0, - }, - { - "type": "Snapshot", - "url": title_prefix_order_url, - "title": title_prefix_order_title, - "tags": "search-matrix", - "depth": 0, - }, - { - "type": "Snapshot", - "url": url_contains_order_url, - "title": "URL Contains Ordering Page", - "tags": "search-matrix", - "depth": 0, - }, - { - "type": "Snapshot", - "url": title_contains_order_url, - "title": title_contains_order_title, - "tags": "search-matrix", - "depth": 0, - }, - { - "type": "Snapshot", - "url": tag_order_url, - "title": "Tag Ordering Page", - "tags": f"search-matrix,{order_needle}", - "depth": 0, - }, + metadata_snapshots = [ + (url_only_url, "URL Only Precision Page", "search-matrix"), + (title_only_url, title_only_needle, "search-matrix"), + (tag_only_url, "Tag Only Precision Page", f"search-matrix,{tag_only_needle}"), + (title_prefix_order_url, title_prefix_order_title, "search-matrix"), + (url_contains_order_url, "URL Contains Ordering Page", "search-matrix"), + (title_contains_order_url, title_contains_order_title, "search-matrix"), + (tag_order_url, "Tag Ordering Page", f"search-matrix,{order_needle}"), ] - metadata_create_result = run_archivebox_cmd( - ["snapshot", "create"], - cwd=initialized_archive, - env=env, - input="\n".join(json.dumps(record) for record in metadata_snapshot_records) + "\n", - timeout=60, - ) - assert metadata_create_result.returncode == 0, metadata_create_result.stderr or metadata_create_result.stdout + metadata_create_outputs = [] + for url, _title, tags in metadata_snapshots: + create_result = run_archivebox_cmd( + ["snapshot", "create", f"--tag={tags}", url], + cwd=initialized_archive, + env=env, + timeout=60, + ) + assert create_result.returncode == 0, create_result.stderr or create_result.stdout + metadata_create_outputs.append(create_result.stdout) + from archivebox.core.models import Snapshot + from archivebox.tests.test_orm_helpers import use_archivebox_db + + with use_archivebox_db(initialized_archive): + for url, title, _tags in metadata_snapshots: + Snapshot.objects.filter(url=url).update(title=title) metadata_seal_result = run_archivebox_cmd( ["snapshot", "update", "--status=sealed"], cwd=initialized_archive, env=env, - input=metadata_create_result.stdout, + input="".join(metadata_create_outputs), timeout=60, ) assert metadata_seal_result.returncode == 0, metadata_seal_result.stderr or metadata_seal_result.stdout @@ -978,13 +929,13 @@ class TestSearchBackendsE2E: log_name="search-matrix-server.log", env=env, ) - wait_for_http( + get_http_response( archivebox_port, host=f"web.archivebox.localhost:{archivebox_port}", path="/public/", process=archivebox_server, ) - wait_for_http( + get_http_response( archivebox_port, host=f"admin.archivebox.localhost:{archivebox_port}", path="/admin/login/", diff --git a/archivebox/tests/test_server_security_browser.py b/archivebox/tests/test_server_security_browser.py index 4ce86eda..3b907fb8 100644 --- a/archivebox/tests/test_server_security_browser.py +++ b/archivebox/tests/test_server_security_browser.py @@ -7,7 +7,6 @@ import json import os import subprocess import textwrap -import time from pathlib import Path from urllib.parse import urlencode @@ -21,7 +20,8 @@ from .conftest import ( start_archivebox_server as start_daemon_server, stop_archivebox_process, stop_server as stop_daemon_server, - wait_for_http, + get_http_response, + wait_for_log, ) @@ -169,39 +169,34 @@ async function frameText(frame) { } } -async function findPreviewText(page, expectedText, timeoutMs) { - const deadline = Date.now() + timeoutMs; - let lastState = []; - while (Date.now() < deadline) { - const frames = page.frames(); - const previewFrame = frames.find((frame) => { - const url = frame.url(); - return url.includes("/archivewebpage/archivewebpage.wacz") && url.includes("preview=1"); - }); +async function readPreviewText(page, expectedText) { + const frames = page.frames(); + const previewFrame = frames.find((frame) => { + const url = frame.url(); + return url.includes("/archivewebpage/archivewebpage.wacz") && url.includes("preview=1"); + }); - lastState = []; - for (const frame of frames) { - const text = await frameText(frame); - lastState.push({ - name: frame.name(), - url: frame.url(), - isPreview: frame === previewFrame, - underPreview: previewFrame ? isDescendantOf(frame, previewFrame) : false, - textSample: text.slice(0, 240), - }); - if (previewFrame && (frame === previewFrame || isDescendantOf(frame, previewFrame)) && text.includes(expectedText)) { - return { - matched: true, - previewUrl: previewFrame.url(), - matchedFrameUrl: frame.url(), - matchedFrameName: frame.name(), - textSample: text.slice(0, 400), - }; - } + const frameState = []; + for (const frame of frames) { + const text = await frameText(frame); + frameState.push({ + name: frame.name(), + url: frame.url(), + isPreview: frame === previewFrame, + underPreview: previewFrame ? isDescendantOf(frame, previewFrame) : false, + textSample: text.slice(0, 240), + }); + if (previewFrame && (frame === previewFrame || isDescendantOf(frame, previewFrame)) && text.includes(expectedText)) { + return { + matched: true, + previewUrl: previewFrame.url(), + matchedFrameUrl: frame.url(), + matchedFrameName: frame.name(), + textSample: text.slice(0, 400), + }; } - await new Promise((resolve) => setTimeout(resolve, 500)); } - return {matched: false, frames: lastState}; + return {matched: false, frames: frameState}; } async function main() { @@ -236,7 +231,8 @@ async function main() { waitUntil: "domcontentloaded", timeout: 30000, }); - const previewResult = await findPreviewText(page, config.expectedText, 60000); + await page.waitForNetworkIdle({idleTime: 500, timeout: 60000}); + const previewResult = await readPreviewText(page, config.expectedText); console.log(JSON.stringify({ detailUrl: config.detailUrl, @@ -260,8 +256,6 @@ main().catch((error) => { def browser_runtime(initialized_archive: Path): shared_lib = initialized_archive / "lib" env = cli_env( - ABXPKG_INSTALL_TIMEOUT="900", - ABXPKG_MIN_RELEASE_AGE="0", ABXPKG_LIB_DIR=str(shared_lib), CHROME_HEADLESS="True", CHROME_SANDBOX="False", @@ -484,19 +478,23 @@ def _run_browser_probe( "USE_CHROME": "False", }, ) - process = run_archivebox_cmd( - ["server", "--debug", "--nothreading", f"127.0.0.1:{port}"], - cwd=data_dir, - env=server_env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - start_new_session=True, - wait=False, - ) + server_log_path = tmp_path / f"{mode}_server.log" + with server_log_path.open("w", encoding="utf-8") as server_log_file: + process = run_archivebox_cmd( + ["server", "--debug", "--nothreading", f"127.0.0.1:{port}"], + cwd=data_dir, + env=server_env, + stdout=server_log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + wait=False, + ) try: - wait_for_http(port, f"archivebox.localhost:{port}", process=process) + wait_for_log(server_log_path, "Listening on TCP", timeout=30) + get_http_response(port, f"archivebox.localhost:{port}", process=process) except AssertionError as exc: - server_log = stop_archivebox_process(process) + stop_archivebox_process(process) + server_log = server_log_path.read_text(encoding="utf-8", errors="replace") raise AssertionError(f"{exc}\n\nSERVER LOG:\n{server_log}") from exc probe_path = tmp_path / "server_security_probe.js" @@ -520,50 +518,32 @@ def _run_browser_probe( timeout=120, ) finally: - server_log = stop_archivebox_process(process) + stop_archivebox_process(process) + server_log = server_log_path.read_text(encoding="utf-8", errors="replace") assert result.returncode == 0, f"{result.stderr}\n\nSERVER LOG:\n{server_log}" return json.loads(result.stdout.strip()) -def _wait_for_archivewebpage_capture(data_dir: Path, url: str, timeout: float = 300.0) -> dict[str, str]: +def _get_archivewebpage_capture(data_dir: Path, url: str) -> dict[str, str]: from archivebox.core.models import ArchiveResult, Snapshot from archivebox.tests.test_orm_helpers import use_archivebox_db - deadline = time.time() + timeout - last_state = {} - while time.time() < deadline: - with use_archivebox_db(data_dir): - snapshot = Snapshot.objects.filter(url=url).order_by("-created_at").first() - if snapshot is None: - last_state = {"snapshot": "missing"} - else: - result = ( - ArchiveResult.objects.filter(snapshot=snapshot, plugin="archivewebpage") - .order_by("-created_at") - .values("status", "output_files", "output_str") - .first() - ) - wacz_path = Path(snapshot.output_dir) / "archivewebpage" / "archivewebpage.wacz" - last_state = { - "snapshot_id": str(snapshot.id), - "snapshot_status": str(snapshot.status), - "result": str(result), - "wacz_path": str(wacz_path), - "wacz_exists": str(wacz_path.is_file()), - } - if ( - snapshot.status == Snapshot.StatusChoices.SEALED - and result is not None - and result["status"] == ArchiveResult.StatusChoices.SUCCEEDED - and wacz_path.is_file() - ): - return { - "snapshot_id": str(snapshot.id), - "wacz_path": str(wacz_path), - } - time.sleep(2) - raise AssertionError(f"timed out waiting for archivewebpage capture: {last_state}") + with use_archivebox_db(data_dir): + snapshot = Snapshot.objects.get(url=url) + result = ArchiveResult.objects.get( + snapshot=snapshot, + plugin="archivewebpage", + hook_name="on_Snapshot__65_archivewebpage_stop", + ) + wacz_path = Path(snapshot.output_dir) / "archivewebpage" / "archivewebpage.wacz" + assert snapshot.status == Snapshot.StatusChoices.SEALED + assert result.status == ArchiveResult.StatusChoices.SUCCEEDED + assert wacz_path.is_file() + return { + "snapshot_id": str(snapshot.id), + "wacz_path": str(wacz_path), + } def _run_wacz_preview_probe( @@ -700,31 +680,28 @@ def test_server_security_modes_in_chrome( assert probe_results["admin"]["ok"] is expected["admin_ok"] assert probe_results["api"]["ok"] is expected["api_ok"] assert any("CORS policy" in text for text in console_texts) - return - - if mode == "safe-onedomain-nojsreplay": + elif mode == "safe-onedomain-nojsreplay": assert probe_results == {} assert "Dangerous Replay Fixture" in page_state["bodyText"] assert any("Blocked script execution" in text for text in console_texts) - return + else: + assert probe_results["own"]["ok"] is True + assert probe_results["own"]["status"] == 200 + assert "ATTACKER_SECRET" in probe_results["own"]["sample"] + assert probe_results["victim"]["ok"] is expected["victim_ok"] + assert probe_results["victim"]["status"] == expected["victim_status"] + assert "VICTIM_SECRET" in probe_results["victim"]["sample"] + assert probe_results["admin"]["ok"] is expected["admin_ok"] + assert probe_results["admin"]["status"] == expected["admin_status"] + assert probe_results["api"]["ok"] is expected["api_ok"] + assert probe_results["api"]["status"] == expected["api_status"] - assert probe_results["own"]["ok"] is True - assert probe_results["own"]["status"] == 200 - assert "ATTACKER_SECRET" in probe_results["own"]["sample"] - assert probe_results["victim"]["ok"] is expected["victim_ok"] - assert probe_results["victim"]["status"] == expected["victim_status"] - assert "VICTIM_SECRET" in probe_results["victim"]["sample"] - assert probe_results["admin"]["ok"] is expected["admin_ok"] - assert probe_results["admin"]["status"] == expected["admin_status"] - assert probe_results["api"]["ok"] is expected["api_ok"] - assert probe_results["api"]["status"] == expected["api_status"] - - if mode == "unsafe-onedomain-noadmin": - assert "control plane disabled" in probe_results["admin"]["sample"].lower() - assert "control plane disabled" in probe_results["api"]["sample"].lower() - elif mode == "danger-onedomain-fullreplay": - assert "ArchiveBox" in probe_results["admin"]["sample"] - assert "swagger" in probe_results["api"]["sample"].lower() + if mode == "unsafe-onedomain-noadmin": + assert "control plane disabled" in probe_results["admin"]["sample"].lower() + assert "control plane disabled" in probe_results["api"]["sample"].lower() + elif mode == "danger-onedomain-fullreplay": + assert "ArchiveBox" in probe_results["admin"]["sample"] + assert "swagger" in probe_results["api"]["sample"].lower() @pytest.mark.django_db(transaction=True) @@ -770,17 +747,17 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi ) start_daemon_server(initialized_archive, env=env, port=port) - wait_for_http(port, host=f"archivebox.localhost:{port}", path="/") + get_http_response(port, host=f"archivebox.localhost:{port}", path="/") _cmd_result = run_archivebox_cmd( - ["add", "--bg", "--depth=0", "--max-urls=1", "--plugins=archivewebpage", url], + ["add", "--depth=0", "--max-urls=1", "--plugins=archivewebpage", url], cwd=initialized_archive, env=env, timeout=120, ) stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode - assert returncode == 0, f"archivebox add --bg failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + assert returncode == 0, f"archivebox add failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" - capture = _wait_for_archivewebpage_capture(initialized_archive, url, timeout=360) + capture = _get_archivewebpage_capture(initialized_archive, url) snapshot_host = f"{get_snapshot_subdomain(capture['snapshot_id'])}.archivebox.localhost:{port}" detail_url = f"http://{snapshot_host}/#archivewebpage/archivewebpage.wacz" result = _run_wacz_preview_probe(initialized_archive, browser_runtime, detail_url, tmp_path) diff --git a/archivebox/tests/test_shutdown_util.py b/archivebox/tests/test_shutdown_util.py index f881f750..c4e50143 100644 --- a/archivebox/tests/test_shutdown_util.py +++ b/archivebox/tests/test_shutdown_util.py @@ -1,76 +1,136 @@ +import os import signal - -import pytest - -from archivebox.cli import archivebox_run -from archivebox.core import shutdown_util +import subprocess +import sys +import textwrap -def test_foreground_shutdown_second_signal_exits_immediately(monkeypatch): - def fake_exit(code): - raise SystemExit(code) - - monkeypatch.setattr(shutdown_util.os, "_exit", fake_exit) - - with shutdown_util.foreground_shutdown_signals() as state: - handler = signal.getsignal(signal.SIGTERM) - - with pytest.raises(KeyboardInterrupt): - handler(signal.SIGTERM, None) - assert state.signal_name == "SIGTERM" - - with pytest.raises(SystemExit) as err: - handler(signal.SIGTERM, None) - assert err.value.code == 130 +def _start_signal_process(source: str) -> subprocess.Popen[str]: + process = subprocess.Popen( + [sys.executable, "-c", textwrap.dedent(source)], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=os.environ.copy(), + ) + assert process.stdout is not None + ready_line = process.stdout.readline() + if ready_line != "READY\n": + assert process.stderr is not None + stderr = process.stderr.read() + process.wait(timeout=30) + raise AssertionError(f"signal subprocess exited before readiness: {ready_line!r}\n{stderr}") + return process -def test_foreground_shutdown_can_request_cooperative_shutdown_without_raising(monkeypatch): - def fake_exit(code): - raise SystemExit(code) - - seen = [] - monkeypatch.setattr(shutdown_util.os, "_exit", fake_exit) - - with shutdown_util.foreground_shutdown_signals( - first_signal_message=None, - on_signal=seen.append, - raise_on_first_signal=False, - ) as state: - handler = signal.getsignal(signal.SIGTERM) - - handler(signal.SIGTERM, None) - assert state.signal_name == "SIGTERM" - assert seen == [signal.SIGTERM] - - with pytest.raises(SystemExit) as err: - handler(signal.SIGTERM, None) - assert err.value.code == 130 +def _signal_and_collect(process: subprocess.Popen[str], sig: signal.Signals) -> tuple[str, str]: + process.send_signal(sig) + return process.communicate(timeout=30) -def test_daemon_runner_signal_exit_is_unexpected_for_supervisor(monkeypatch): - def fake_exit(code): - raise SystemExit(code) +def test_foreground_shutdown_second_signal_exits_immediately(): + process = _start_signal_process( + """ + import signal - monkeypatch.setattr(archivebox_run.os, "_exit", fake_exit) + from archivebox.core.shutdown_util import foreground_shutdown_signals - with pytest.raises(SystemExit) as err: - archivebox_run._exit_daemon_runner_on_signal(signal.SIGTERM) + with foreground_shutdown_signals(first_signal_message=None) as state: + print("READY", flush=True) + try: + signal.pause() + except KeyboardInterrupt: + print(f"FIRST:{state.signal_name}", flush=True) + signal.pause() + """, + ) - assert err.value.code == 143 + process.send_signal(signal.SIGTERM) + assert process.stdout is not None + assert process.stdout.readline() == "FIRST:SIGTERM\n" + stdout, stderr = _signal_and_collect(process, signal.SIGTERM) + + assert process.returncode == 130, (stdout, stderr) -def test_crawl_runner_daemon_signal_exits_before_async_cleanup(monkeypatch): - from archivebox.services import runner as runner_module - from archivebox.services.runner import CrawlRunner +def test_foreground_shutdown_can_request_cooperative_shutdown_without_raising(): + process = _start_signal_process( + """ + import signal - def fake_exit(code): - raise SystemExit(code) + from archivebox.core.shutdown_util import foreground_shutdown_signals - runner = object.__new__(CrawlRunner) - monkeypatch.setenv("ARCHIVEBOX_RUNNER_DAEMON", "1") - monkeypatch.setattr(runner_module.os, "_exit", fake_exit) + def on_signal(sig): + print(f"SIGNAL:{sig.name}", flush=True) - with pytest.raises(SystemExit) as err: - runner._request_abort_from_signal(signal.SIGTERM) + with foreground_shutdown_signals( + first_signal_message=None, + on_signal=on_signal, + raise_on_first_signal=False, + ): + print("READY", flush=True) + signal.pause() + signal.pause() + """, + ) - assert err.value.code == 143 + process.send_signal(signal.SIGTERM) + assert process.stdout is not None + assert process.stdout.readline() == "SIGNAL:SIGTERM\n" + stdout, stderr = _signal_and_collect(process, signal.SIGTERM) + + assert process.returncode == 130, (stdout, stderr) + + +def test_daemon_runner_signal_exit_is_unexpected_for_supervisor(): + process = _start_signal_process( + """ + import signal + + from archivebox.cli.archivebox_run import _exit_daemon_runner_on_signal + + signal.signal(signal.SIGTERM, lambda signum, _frame: _exit_daemon_runner_on_signal(signal.Signals(signum))) + print("READY", flush=True) + signal.pause() + """, + ) + + stdout, stderr = _signal_and_collect(process, signal.SIGTERM) + + assert process.returncode == 143, (stdout, stderr) + + +def test_crawl_runner_daemon_signal_exits_before_async_cleanup(): + process = _start_signal_process( + """ + import os + import signal + import uuid + + import django + + django.setup() + + from archivebox.crawls.models import Crawl + from archivebox.core.shutdown_util import foreground_shutdown_signals + from archivebox.services.runner import CrawlRunner + + runner = CrawlRunner( + Crawl(urls="https://example.com", created_by_id=uuid.uuid4()), + show_progress=False, + ) + os.environ["ARCHIVEBOX_RUNNER_DAEMON"] = "1" + with foreground_shutdown_signals( + first_signal_message=None, + on_signal=runner._request_abort_from_signal, + raise_on_first_signal=False, + ): + print("READY", flush=True) + signal.pause() + """, + ) + + stdout, stderr = _signal_and_collect(process, signal.SIGTERM) + + assert process.returncode == 143, (stdout, stderr) diff --git a/archivebox/tests/test_takeover_util.py b/archivebox/tests/test_takeover_util.py index cda17486..52104591 100644 --- a/archivebox/tests/test_takeover_util.py +++ b/archivebox/tests/test_takeover_util.py @@ -3,10 +3,10 @@ import os import json +import re import signal import subprocess import sys -import time from pathlib import Path import pytest @@ -25,7 +25,7 @@ from archivebox.tests.conftest import ( start_archivebox_server, stop_archivebox_process, supervisor_pid_from_log, - wait_for_http, + get_http_response, wait_for_log, wait_for_log_count, wait_for_log_pattern, @@ -39,13 +39,16 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) -def test_pid_is_alive_treats_unreaped_zombie_as_exited(): - proc = subprocess.Popen([sys.executable, "-c", "pass"]) +def test_pid_is_alive_treats_unreaped_archivebox_cli_as_exited(tmp_path, initialized_archive): + proc = run_archivebox_cmd( + ["version"], + cwd=tmp_path, + default_cli_env=True, + disable_extractors=True, + wait=False, + ) try: - deadline = time.time() + 5 - while time.time() < deadline and psutil.Process(proc.pid).status() != psutil.STATUS_ZOMBIE: - time.sleep(0.01) - + os.waitid(os.P_PID, proc.pid, os.WEXITED | os.WNOWAIT) assert psutil.Process(proc.pid).status() == psutil.STATUS_ZOMBIE assert not pid_is_alive(proc.pid) finally: @@ -115,7 +118,7 @@ def test_behavior_update_index_only_keeps_server_http_and_search_visible(tmp_pat server = None try: server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-update.log", env=env) - assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500 update = run_archivebox_cmd( ["update", "--index-only", "--batch-size=1"], @@ -125,7 +128,7 @@ def test_behavior_update_index_only_keeps_server_http_and_search_visible(tmp_pat ) assert update.returncode == 0, update.stderr or update.stdout - assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500 search = run_archivebox_cmd( ["list", "--search=contents", "--csv=url", "Root"], @@ -137,7 +140,7 @@ def test_behavior_update_index_only_keeps_server_http_and_search_visible(tmp_pat assert search.returncode == 0, search.stderr or search.stdout assert root_url in search.stdout finally: - if server is not None and server.poll() is None: + if server is not None: stop_archivebox_process(server, signal.SIGTERM) kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) @@ -174,7 +177,7 @@ def test_behavior_update_yields_to_server_then_finishes_visible_indexing(tmp_pat wait_for_log(update_log, "[*] Reindexing", timeout=90) server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-takes-update.log", env=env) - assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500 wait_for_log(update_log, "A newer archivebox process took over the orchestrator, sonic", timeout=90) stop_archivebox_process(server, signal.SIGTERM) @@ -193,9 +196,9 @@ def test_behavior_update_yields_to_server_then_finishes_visible_indexing(tmp_pat assert search.returncode == 0, search.stderr or search.stdout assert root_url in search.stdout finally: - if update_proc is not None and update_proc.poll() is None: + if update_proc is not None: stop_archivebox_process(update_proc, signal.SIGTERM) - if server is not None and server.poll() is None: + if server is not None: stop_archivebox_process(server, signal.SIGTERM) kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) @@ -209,7 +212,7 @@ def test_behavior_foreground_add_keeps_existing_server_http_visible(tmp_path, in server = None try: server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-add.log", env=env) - assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500 add = run_archivebox_cmd( [ @@ -224,11 +227,11 @@ def test_behavior_foreground_add_keeps_existing_server_http_visible(tmp_path, in ) assert add.returncode == 0, add.stderr or add.stdout - assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500 captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=120) assert "Root" in captured_text finally: - if server is not None and server.poll() is None: + if server is not None: stop_archivebox_process(server, signal.SIGTERM) kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) @@ -242,7 +245,7 @@ def test_behavior_background_add_returns_and_server_archives_visible_url(tmp_pat server = None try: server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-bg-add.log", env=env) - assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500 add = run_archivebox_cmd( [ @@ -259,11 +262,11 @@ def test_behavior_background_add_returns_and_server_archives_visible_url(tmp_pat assert add.returncode == 0, add.stderr or add.stdout assert "background runner will process" in add.stdout - assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500 captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) assert "Root" in captured_text finally: - if server is not None and server.poll() is None: + if server is not None: stop_archivebox_process(server, signal.SIGTERM) kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) @@ -277,14 +280,14 @@ def test_behavior_daemonized_server_restarts_cleanly_after_forced_stop(tmp_path, try: first = start_archivebox_server(tmp_path, port=port, env=env, daemonize=True) assert first.returncode == 0, first.stderr or first.stdout - assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500 kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) second = start_archivebox_server(tmp_path, port=port, env=env, daemonize=True) assert second.returncode == 0, second.stderr or second.stdout - assert wait_for_http(port, host=f"archivebox.localhost:{port}").status_code < 500 + assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500 finally: kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) @@ -303,7 +306,7 @@ def test_live_second_server_takes_over_existing_server_process(tmp_path, initial second = start_archivebox_server(tmp_path, port=port, log_name="server-second.log", env=env) second_log = second.log_path - assert first.poll() is None + assert pid_is_alive(first.pid) first_text = first_log.read_text(encoding="utf-8", errors="replace") second_text = second_log.read_text(encoding="utf-8", errors="replace") assert "A newer archivebox process took over the orchestrator, server" in first_text @@ -322,12 +325,12 @@ def test_live_second_server_takes_over_existing_server_process(tmp_path, initial stop_archivebox_process(second, signal.SIGTERM) second = None wait_for_log_count(first_log, "Other newer archivebox process", first_resumes + 1, timeout=35) - assert first.poll() is None + assert pid_is_alive(first.pid) finally: - if second is not None and second.poll() is None: + if second is not None: stop_archivebox_process(second, signal.SIGTERM) - if first is not None and first.poll() is None: - stop_archivebox_process(first, signal.SIGKILL) + if first is not None: + stop_archivebox_process(first, signal.SIGTERM) kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) @@ -353,7 +356,7 @@ def test_live_update_index_only_does_not_take_over_server_runtime(tmp_path, init update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert update_returncode == 0, update_stderr or update_stdout - assert server.poll() is None + assert pid_is_alive(server.pid) assert pid_is_alive(supervisor_pid_before) assert pid_is_alive(daphne_pid_before) assert supervisor_pid_from_log(server_log) == supervisor_pid_before @@ -363,7 +366,7 @@ def test_live_update_index_only_does_not_take_over_server_runtime(tmp_path, init errors="replace", ) finally: - if server is not None and server.poll() is None: + if server is not None: stop_archivebox_process(server, signal.SIGTERM) kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) @@ -389,6 +392,9 @@ def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tm daphne_pid_before = worker_pid_from_log(server_log, "worker_daphne") runner_pid_before = worker_pid_from_log(server_log, "worker_runner") sonic_pid_before = worker_pid_from_log(server_log, "worker_sonic") + supervisord_log = tmp_path / "logs" / "supervisord.log" + runner_spawn_text = "spawned: 'worker_runner' with pid" + runner_spawn_count = supervisord_log.read_text(encoding="utf-8", errors="replace").count(runner_spawn_text) _cmd_result = run_archivebox_cmd( ["update", "--index-only", "--batch-size=1"], @@ -399,7 +405,7 @@ def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tm update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert update_returncode == 0, update_stderr or update_stdout - assert server.poll() is None + assert pid_is_alive(server.pid) assert supervisor_pid_from_log(server_log) == supervisor_pid_before assert pid_is_alive(daphne_pid_before) assert pid_is_alive(sonic_pid_before) @@ -411,37 +417,15 @@ def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tm ) assert "Stopping older ArchiveBox runner process" in update_stdout - deadline = time.time() + 180 - runner_pid_after = runner_pid_before - while time.time() < deadline: - with use_archivebox_db(tmp_path): - rows = list( - Process.objects.filter( - process_type=Process.TypeChoices.ORCHESTRATOR, - worker_type="worker_runner", - status=Process.StatusChoices.RUNNING, - ).values("pid"), - ) - for row in rows: - pid = int(row["pid"]) - if pid != runner_pid_before and pid_is_alive(pid): - runner_pid_after = pid - break - if runner_pid_after != runner_pid_before: - break - time.sleep(0.25) + supervisord_text = wait_for_log_count(supervisord_log, runner_spawn_text, runner_spawn_count + 1, timeout=30) + runner_pid_after = int(re.findall(r"spawned: 'worker_runner' with pid (\d+)", supervisord_text)[-1]) assert runner_pid_after != runner_pid_before + assert pid_is_alive(runner_pid_after) - deadline = time.time() + 180 - indexed_results: list[str] = [] - while time.time() < deadline: - with use_archivebox_db(tmp_path): - indexed_results = list( - ArchiveResult.objects.filter(plugin="search_backend_sqlite").values_list("status", flat=True), - ) - if indexed_results and all(status in ArchiveResult.FINAL_STATES for status in indexed_results): - break - time.sleep(0.25) + with use_archivebox_db(tmp_path): + indexed_results = list( + ArchiveResult.objects.filter(plugin="search_backend_sqlite").values_list("status", flat=True), + ) assert indexed_results assert all(status in ArchiveResult.FINAL_STATES for status in indexed_results) @@ -452,7 +436,7 @@ def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tm wait_for_pid_to_disappear(runner_pid_after, timeout=20) assert_no_processes_for_data_dir(tmp_path, timeout=12) finally: - if server is not None and server.poll() is None: + if server is not None: stop_archivebox_process(server, signal.SIGTERM) kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) @@ -498,8 +482,8 @@ def test_live_update_yields_to_server_then_reclaims_real_sqlite_indexing(tmp_pat server = start_archivebox_server(tmp_path, port=port, log_name="server-takes-real-sqlite-update.log", env=env) server_log = server.log_path wait_for_log(update_log, "A newer archivebox process took over the orchestrator, sonic", timeout=90) - assert update_proc.poll() is None - assert server.poll() is None + assert pid_is_alive(update_proc.pid) + assert pid_is_alive(server.pid) server_text = server_log.read_text(encoding="utf-8", errors="replace") # The older update process can yield orchestrator ownership just before # the server logs its takeover, but sonic must always move to the server. @@ -528,23 +512,17 @@ def test_live_update_yields_to_server_then_reclaims_real_sqlite_indexing(tmp_pat wait_for_pid_to_disappear(server_runner_pid, timeout=20) wait_for_pid_to_disappear(server_sonic_pid, timeout=20) - deadline = time.time() + 30 - indexed_results: list[str] = [] - while time.time() < deadline: - with use_archivebox_db(tmp_path): - indexed_results = list( - ArchiveResult.objects.filter(plugin="search_backend_sqlite").values_list("status", flat=True), - ) - if indexed_results and all(status in ArchiveResult.FINAL_STATES for status in indexed_results): - break - time.sleep(0.25) + with use_archivebox_db(tmp_path): + indexed_results = list( + ArchiveResult.objects.filter(plugin="search_backend_sqlite").values_list("status", flat=True), + ) assert indexed_results assert all(status in ArchiveResult.FINAL_STATES for status in indexed_results) assert_no_processes_for_data_dir(tmp_path, timeout=12) finally: - if update_proc is not None and update_proc.poll() is None: + if update_proc is not None: stop_archivebox_process(update_proc, signal.SIGTERM) - if server is not None and server.poll() is None: + if server is not None: stop_archivebox_process(server, signal.SIGTERM) kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) @@ -572,7 +550,7 @@ def test_live_repeated_server_startups_take_over_cleanly(tmp_path, initialized_a previous_server = servers[index - 1] previous_log = (tmp_path / f"server-chaos-{index - 1}.log").read_text(encoding="utf-8", errors="replace") current_log = log_path.read_text(encoding="utf-8", errors="replace") - assert previous_server.poll() is None + assert pid_is_alive(previous_server.pid) assert pid_is_alive(server_pids[index - 1]) assert "A newer archivebox process took over the orchestrator, server" in previous_log assert "Starting orchestrator, server" in current_log @@ -587,10 +565,9 @@ def test_live_repeated_server_startups_take_over_cleanly(tmp_path, initialized_a ) stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert returncode == 0, stderr or stdout - time.sleep(5) - assert servers[-1].poll() is None - assert all(server.poll() is None for server in servers) + assert pid_is_alive(servers[-1].pid) + assert all(pid_is_alive(server.pid) for server in servers) listener = subprocess.run( ["lsof", "-nP", f"-iTCP:{port}", "-sTCP:LISTEN"], capture_output=True, @@ -606,70 +583,20 @@ def test_live_repeated_server_startups_take_over_cleanly(tmp_path, initialized_a ) stop_archivebox_process(servers[-1], signal.SIGTERM) wait_for_log_count(previous_log_path, "Other newer archivebox process", previous_takeovers + 1, timeout=35) - assert servers[3].poll() is None + assert pid_is_alive(servers[3].pid) finally: for server in reversed(servers): - if server.poll() is None: - stop_archivebox_process(server, signal.SIGTERM) + stop_archivebox_process(server, signal.SIGTERM) kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) @pytest.mark.timeout(420) -def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initialized_archive, recursive_test_site): - plugins_root = tmp_path / "runtime_plugins" - marker_dir = tmp_path / "slow-plugin-markers" - plugin_dir = plugins_root / "slow_exit" - plugin_dir.mkdir(parents=True, exist_ok=True) - counter_hook = plugin_dir / "on_Snapshot__08_counter.sh" - counter_hook.write_text( - "\n".join( - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - f"marker_dir={str(marker_dir)!r}", - 'mkdir -p "$marker_dir/counter-seen"', - 'snapshot_key="${SNAPSHOT_ID:-$(basename "${SNAP_DIR:-unknown}")}"', - 'seen_file="$marker_dir/counter-seen/$snapshot_key"', - 'if [[ -e "$seen_file" ]]; then', - ' echo "$snapshot_key" >> "$marker_dir/counter-duplicates.txt"', - " exit 42", - "fi", - 'touch "$seen_file"', - 'echo "$snapshot_key" >> "$marker_dir/counter-runs.txt"', - "", - ], - ), - encoding="utf-8", - ) - counter_hook.chmod(0o755) - hook = plugin_dir / "on_Snapshot__09_slow_exit.sh" - hook.write_text( - "\n".join( - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - f"marker_dir={str(marker_dir)!r}", - 'mkdir -p "$marker_dir"', - 'echo $$ >> "$marker_dir/hook-pids.txt"', - 'touch "$marker_dir/hook-started"', - "trap 'touch \"$marker_dir/hook-stopped\"; exit 143' TERM INT HUP", - 'while [[ ! -f "$marker_dir/allow-finish" ]]; do sleep 0.1; done', - 'touch "$marker_dir/hook-finished"', - "", - ], - ), - encoding="utf-8", - ) - hook.chmod(0o755) - - env = cli_env(live=True, plugins_root=plugins_root) +def test_live_background_add_survives_server_exit_and_foreground_run_reclaims(tmp_path, initialized_archive, recursive_test_site): + env = cli_env(live=True, SEARCH_BACKEND_ENGINE="ripgrep") port = get_free_port() server = None server2 = None - server3 = None - add_proc = None - add_proc2 = None try: server = start_archivebox_server(tmp_path, port=port, log_name="server-add-owner-1.log", env=env) server_log = server.log_path @@ -683,95 +610,48 @@ def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initi ) update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode assert update_returncode == 0, update_stderr or update_stdout - assert server.poll() is None + assert pid_is_alive(server.pid) assert pid_is_alive(supervisor_pid_before) assert supervisor_pid_from_log(server_log) == supervisor_pid_before - add_log = tmp_path / "archivebox-add-1.log" - add_log_handle = add_log.open("w", encoding="utf-8") - add_proc = run_archivebox_cmd( + add = run_archivebox_cmd( [ "add", + "--bg", "--depth=1", - "--max-urls=2", + "--max-urls=20", "--crawl-max-size=50mb", - "--plugins=wget,parse_html_urls,slow_exit", + "--plugins=wget,parse_html_urls", recursive_test_site["root_url"], - recursive_test_site["child_urls"][0], ], cwd=tmp_path, env=env, - stdout=add_log_handle, - stderr=subprocess.STDOUT, - start_new_session=True, - wait=False, + timeout=60, ) - add_log_handle.close() + assert add.returncode == 0, add.stderr or add.stdout + assert "background runner will process" in add.stdout - pid_file = marker_dir / "hook-pids.txt" - deadline = time.time() + 45 - hook_pids: list[int] = [] - while time.time() < deadline: - if pid_file.exists(): - hook_pids = [int(line.strip()) for line in pid_file.read_text().splitlines() if line.strip()] - if len(hook_pids) >= 1: - break - time.sleep(0.25) - assert len(hook_pids) >= 1 - - deadline = time.time() + 30 - snapshot_started = False - while time.time() < deadline: - with use_archivebox_db(tmp_path): - snapshot_started = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).exists() - if snapshot_started: - break - time.sleep(0.25) - assert snapshot_started - - os.kill(server.pid, signal.SIGTERM) - server.wait(timeout=20) - assert add_proc.poll() is None, "foreground add should keep owning its crawl after the server exits" + stop_archivebox_process(server, signal.SIGTERM) + server = None assert "Got SIGTERM" in server_log.read_text(encoding="utf-8", errors="replace") - - stop_archivebox_process(add_proc, signal.SIGKILL, timeout=30) - add_output = add_log.read_text(encoding="utf-8", errors="replace") - assert "Runner error" not in add_output - kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) + with use_archivebox_db(tmp_path): + crawl_id = str(Crawl.objects.get().id) + run = run_archivebox_cmd( + ["run", f"--crawl-id={crawl_id}"], + cwd=tmp_path, + env=env, + timeout=180, + ) + assert run.returncode == 0, run.stderr or run.stdout + server2 = start_archivebox_server(tmp_path, port=port, log_name="server-add-owner-2.log", env=env) - _server2_log = server2.log_path - (marker_dir / "allow-finish").touch() - - deadline = time.time() + 90 - crawls = [] - snapshots = [] - bad_results = [] - while time.time() < deadline: - with use_archivebox_db(tmp_path): - crawls = list(Crawl.objects.order_by("created_at").values_list("status", "retry_at")) - snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "status", "retry_at")) - bad_results = list( - ArchiveResult.objects.filter( - status__in=[ - ArchiveResult.StatusChoices.FAILED, - ArchiveResult.StatusChoices.SKIPPED, - ], - ).values_list("plugin", "status", "output_str"), - ) - if ( - crawls - and snapshots - and all(status == Crawl.StatusChoices.SEALED for status, _retry_at in crawls) - and all(status == Snapshot.StatusChoices.SEALED for _url, status, _retry_at in snapshots) - and not bad_results - ): - break - time.sleep(0.25) - - os.kill(server2.pid, signal.SIGTERM) - server2.wait(timeout=20) + assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500 + captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180) + assert "Root" in captured_text + stop_archivebox_process(server2, signal.SIGTERM) + server2 = None with use_archivebox_db(tmp_path): crawls = list(Crawl.objects.order_by("created_at").values_list("status", "retry_at")) snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "status", "retry_at")) @@ -787,20 +667,10 @@ def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initi assert snapshots assert all(status == Crawl.StatusChoices.SEALED for status, _retry_at in crawls) assert all(status == Snapshot.StatusChoices.SEALED for _url, status, _retry_at in snapshots) - counter_runs = (marker_dir / "counter-runs.txt").read_text(encoding="utf-8").splitlines() - assert counter_runs - assert len(counter_runs) == len(set(counter_runs)) - assert not (marker_dir / "counter-duplicates.txt").exists() - - # The interrupted hook should be retried directly without rerunning the - # previous hook in the same plugin. That keeps plugin-level shell hooks - # idempotent across runner takeover instead of depending on each hook to - # detect partial prior work itself. assert not bad_results - assert (marker_dir / "hook-finished").exists() finally: - for proc in (add_proc, add_proc2, server, server2, server3): - if proc is not None and proc.poll() is None: + for proc in (server, server2): + if proc is not None: stop_archivebox_process(proc, signal.SIGTERM, timeout=10) kill_processes_for_data_dir(tmp_path) assert_no_processes_for_data_dir(tmp_path, timeout=12) @@ -809,20 +679,41 @@ def test_live_add_update_jobs_survive_server_and_cli_owner_exits(tmp_path, initi # Utility-level takeover selection tests. -def test_runtime_stack_owner_prefers_newer_server_over_older_update(tmp_path): - from archivebox.machine.models import Machine, Process +def _start_archivebox_shell(tmp_path: Path): + process = run_archivebox_cmd( + ["manage", "shell"], + cwd=tmp_path, + env=cli_env(live=True), + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + capture_output=False, + start_new_session=True, + wait=False, + ) + assert process.pid is not None + assert process.stdin is not None + assert pid_is_alive(process.pid) + return process + + +def _stop_archivebox_shells(processes) -> None: + for process in processes: + if process.stdin is not None and not process.stdin.closed: + process.stdin.close() + for process in processes: + process.wait(timeout=20) + assert not pid_is_alive(process.pid) + + +def test_runtime_stack_owner_prefers_newer_server_over_older_update(tmp_path, initialized_archive): + from archivebox.machine.models import Machine from archivebox.core.takeover_util import runtime_stack_owner procs: list[subprocess.Popen[str]] = [] try: for process_type in (Process.TypeChoices.UPDATE, Process.TypeChoices.SERVER): - proc = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(60)"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - text=True, - start_new_session=True, - ) + proc = _start_archivebox_shell(tmp_path) procs.append(proc) Process.objects.create( machine=Machine.current(), @@ -833,7 +724,6 @@ def test_runtime_stack_owner_prefers_newer_server_over_older_update(tmp_path): pid=proc.pid, status=Process.StatusChoices.RUNNING, ) - time.sleep(0.05) owner = runtime_stack_owner(data_dir=tmp_path) @@ -841,31 +731,17 @@ def test_runtime_stack_owner_prefers_newer_server_over_older_update(tmp_path): assert owner.process_type == Process.TypeChoices.SERVER assert owner.pid == procs[-1].pid finally: - for proc in procs: - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGTERM) - for proc in procs: - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait(timeout=5) + _stop_archivebox_shells(procs) -def test_runtime_stack_owner_keeps_server_over_newer_update(tmp_path): - from archivebox.machine.models import Machine, Process +def test_runtime_stack_owner_keeps_server_over_newer_update(tmp_path, initialized_archive): + from archivebox.machine.models import Machine from archivebox.core.takeover_util import runtime_stack_owner procs: list[subprocess.Popen[str]] = [] try: for process_type in (Process.TypeChoices.SERVER, Process.TypeChoices.UPDATE): - proc = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(60)"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - text=True, - start_new_session=True, - ) + proc = _start_archivebox_shell(tmp_path) procs.append(proc) Process.objects.create( machine=Machine.current(), @@ -876,7 +752,6 @@ def test_runtime_stack_owner_keeps_server_over_newer_update(tmp_path): pid=proc.pid, status=Process.StatusChoices.RUNNING, ) - time.sleep(0.05) owner = runtime_stack_owner(data_dir=tmp_path) @@ -884,31 +759,17 @@ def test_runtime_stack_owner_keeps_server_over_newer_update(tmp_path): assert owner.process_type == Process.TypeChoices.SERVER assert owner.pid == procs[0].pid finally: - for proc in procs: - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGTERM) - for proc in procs: - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait(timeout=5) + _stop_archivebox_shells(procs) -def test_foreground_runner_owner_prefers_newer_update_over_server(tmp_path): - from archivebox.machine.models import Machine, Process +def test_foreground_runner_owner_prefers_newer_update_over_server(tmp_path, initialized_archive): + from archivebox.machine.models import Machine from archivebox.core.takeover_util import foreground_runner_owner, runtime_stack_owner procs: list[subprocess.Popen[str]] = [] try: for process_type in (Process.TypeChoices.SERVER, Process.TypeChoices.UPDATE): - proc = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(60)"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - text=True, - start_new_session=True, - ) + proc = _start_archivebox_shell(tmp_path) procs.append(proc) Process.objects.create( machine=Machine.current(), @@ -919,7 +780,6 @@ def test_foreground_runner_owner_prefers_newer_update_over_server(tmp_path): pid=proc.pid, status=Process.StatusChoices.RUNNING, ) - time.sleep(0.05) runtime_owner = runtime_stack_owner(data_dir=tmp_path) runner_owner = foreground_runner_owner(data_dir=tmp_path) @@ -930,31 +790,17 @@ def test_foreground_runner_owner_prefers_newer_update_over_server(tmp_path): assert runner_owner.process_type == Process.TypeChoices.UPDATE assert runner_owner.pid == procs[-1].pid finally: - for proc in procs: - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGTERM) - for proc in procs: - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait(timeout=5) + _stop_archivebox_shells(procs) -def test_foreground_runner_owner_prefers_newer_server_over_update(tmp_path): - from archivebox.machine.models import Machine, Process +def test_foreground_runner_owner_prefers_newer_server_over_update(tmp_path, initialized_archive): + from archivebox.machine.models import Machine from archivebox.core.takeover_util import foreground_runner_owner, runtime_stack_owner procs: list[subprocess.Popen[str]] = [] try: for process_type in (Process.TypeChoices.UPDATE, Process.TypeChoices.SERVER): - proc = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(60)"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - text=True, - start_new_session=True, - ) + proc = _start_archivebox_shell(tmp_path) procs.append(proc) Process.objects.create( machine=Machine.current(), @@ -965,7 +811,6 @@ def test_foreground_runner_owner_prefers_newer_server_over_update(tmp_path): pid=proc.pid, status=Process.StatusChoices.RUNNING, ) - time.sleep(0.05) runtime_owner = runtime_stack_owner(data_dir=tmp_path) runner_owner = foreground_runner_owner(data_dir=tmp_path) @@ -976,19 +821,11 @@ def test_foreground_runner_owner_prefers_newer_server_over_update(tmp_path): assert runner_owner.process_type == Process.TypeChoices.SERVER assert runner_owner.pid == procs[-1].pid finally: - for proc in procs: - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGTERM) - for proc in procs: - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait(timeout=5) + _stop_archivebox_shells(procs) -def test_runtime_stack_owner_keeps_server_over_newer_supervised_runner(tmp_path): - from archivebox.machine.models import Machine, Process +def test_runtime_stack_owner_keeps_server_over_newer_supervised_runner(tmp_path, initialized_archive): + from archivebox.machine.models import Machine from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, runtime_stack_owner procs: list[subprocess.Popen[str]] = [] @@ -997,13 +834,7 @@ def test_runtime_stack_owner_keeps_server_over_newer_supervised_runner(tmp_path) (Process.TypeChoices.SERVER, ""), (Process.TypeChoices.ORCHESTRATOR, RUNNER_ACTIVE_WORKER_TYPE), ): - proc = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(60)"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - text=True, - start_new_session=True, - ) + proc = _start_archivebox_shell(tmp_path) procs.append(proc) Process.objects.create( machine=Machine.current(), @@ -1014,7 +845,6 @@ def test_runtime_stack_owner_keeps_server_over_newer_supervised_runner(tmp_path) pid=proc.pid, status=Process.StatusChoices.RUNNING, ) - time.sleep(0.05) owner = runtime_stack_owner(data_dir=tmp_path) @@ -1022,19 +852,11 @@ def test_runtime_stack_owner_keeps_server_over_newer_supervised_runner(tmp_path) assert owner.process_type == Process.TypeChoices.SERVER assert owner.pid == procs[0].pid finally: - for proc in procs: - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGTERM) - for proc in procs: - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait(timeout=5) + _stop_archivebox_shells(procs) -def test_runtime_stack_owner_reaps_dead_server_without_promoting_update(tmp_path): - from archivebox.machine.models import Machine, Process +def test_runtime_stack_owner_reaps_dead_server_without_promoting_update(tmp_path, initialized_archive): + from archivebox.machine.models import Machine from archivebox.core.takeover_util import runtime_stack_owner procs: list[subprocess.Popen[str]] = [] @@ -1042,13 +864,7 @@ def test_runtime_stack_owner_reaps_dead_server_without_promoting_update(tmp_path newer_row = None try: for process_type in (Process.TypeChoices.UPDATE, Process.TypeChoices.SERVER): - proc = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(60)"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - text=True, - start_new_session=True, - ) + proc = _start_archivebox_shell(tmp_path) procs.append(proc) row = Process.objects.create( machine=Machine.current(), @@ -1063,13 +879,12 @@ def test_runtime_stack_owner_reaps_dead_server_without_promoting_update(tmp_path older_row = row else: newer_row = row - time.sleep(0.05) assert older_row is not None assert newer_row is not None - os.killpg(procs[-1].pid, signal.SIGTERM) - procs[-1].wait(timeout=5) + os.kill(procs[-1].pid, signal.SIGTERM) + procs[-1].wait(timeout=20) owner = runtime_stack_owner(data_dir=tmp_path) @@ -1079,31 +894,17 @@ def test_runtime_stack_owner_reaps_dead_server_without_promoting_update(tmp_path older_row.refresh_from_db() assert older_row.status == Process.StatusChoices.RUNNING finally: - for proc in procs: - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGTERM) - for proc in procs: - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait(timeout=5) + _stop_archivebox_shells(procs) -def test_runtime_stack_owner_ignores_supervised_orphan_runner(tmp_path): - from archivebox.machine.models import Machine, Process +def test_runtime_stack_owner_ignores_supervised_orphan_runner(tmp_path, initialized_archive): + from archivebox.machine.models import Machine from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, runtime_stack_owner procs: list[subprocess.Popen[str]] = [] try: for _ in range(2): - proc = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(60)"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - text=True, - start_new_session=True, - ) + proc = _start_archivebox_shell(tmp_path) procs.append(proc) supervisor_row = Process.objects.create( @@ -1128,28 +929,14 @@ def test_runtime_stack_owner_ignores_supervised_orphan_runner(tmp_path): assert runtime_stack_owner(data_dir=tmp_path) is None finally: - for proc in procs: - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGTERM) - for proc in procs: - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait(timeout=5) + _stop_archivebox_shells(procs) -def test_runtime_stack_owner_allows_top_level_runner_when_no_parent_command_exists(tmp_path): - from archivebox.machine.models import Machine, Process +def test_runtime_stack_owner_allows_top_level_runner_when_no_parent_command_exists(tmp_path, initialized_archive): + from archivebox.machine.models import Machine from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, runtime_stack_owner - proc = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(60)"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - text=True, - start_new_session=True, - ) + proc = _start_archivebox_shell(tmp_path) try: runner_row = Process.objects.create( machine=Machine.current(), @@ -1166,10 +953,4 @@ def test_runtime_stack_owner_allows_top_level_runner_when_no_parent_command_exis assert owner is not None assert owner.id == runner_row.id finally: - if proc.poll() is None: - os.killpg(proc.pid, signal.SIGTERM) - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(proc.pid, signal.SIGKILL) - proc.wait(timeout=5) + _stop_archivebox_shells([proc]) diff --git a/archivebox/tests/test_ui_add_view.py b/archivebox/tests/test_ui_add_view.py index 3baabfbd..869717e2 100644 --- a/archivebox/tests/test_ui_add_view.py +++ b/archivebox/tests/test_ui_add_view.py @@ -6,6 +6,7 @@ from django.urls import reverse from archivebox.config.common import ArchiveBoxConfig from archivebox.core.models import Snapshot, Tag from archivebox.crawls.models import Crawl +from archivebox.machine.models import Machine from archivebox.personas.models import Persona from archivebox.services.runner import CrawlRunner from archivebox.workers.models import RETRY_AT_MAX @@ -27,9 +28,12 @@ def admin_user(db): ) -def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +@pytest.fixture +def public_add_enabled(): + return Machine.from_json({"config": {"PUBLIC_ADD_VIEW": True}}) + +def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, public_add_enabled): response = client.get(reverse("add"), HTTP_HOST=WEB_HOST) form = response.context["form"] @@ -58,8 +62,7 @@ def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, m assert b"skip URLs you've previously saved" in response.content or b"skip URLs you've previously saved" in response.content -def test_add_view_admin_renders_plugin_config_grid(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_admin_renders_plugin_config_grid(client, admin_user): client.force_login(admin_user) response = client.get(reverse("add"), HTTP_HOST=ADMIN_HOST) @@ -167,8 +170,7 @@ def test_add_view_staff_user_cannot_override_raw_or_plugin_config(client): assert crawl.config.get("YTDLP_ARGS_EXTRA") != ["--exec", "touch /tmp/owned"] -def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_user): client.force_login(admin_user) default_persona = Persona.get_or_create_default() default_persona.config = { @@ -189,8 +191,7 @@ def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_ assert persona_config_map["Private"]["effective_config"]["YTDLP_COOKIES_FILE"] == "/tmp/archivebox-private-cookies.txt" -def test_add_view_public_only_lists_public_personas(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_public_only_lists_public_personas(client, admin_user, public_add_enabled): secret_value = "SHOULD_NOT_LEAK_PUBLIC_PERSONA_SECRET" default_persona = Persona.get_or_create_default() default_persona.config = {"PERMISSIONS": "public", "NODE_BINARY": "/secret/node", "TWOCAPTCHA_API_KEY": secret_value} @@ -223,9 +224,9 @@ def test_persona_config_grid_allows_binary_fields(client, admin_user): assert b"plugin_config__wget__WGET_ENABLED" in response.content -def test_add_view_hides_search_backend_plugins(client, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") - monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "sqlite") +def test_add_view_hides_search_backend_plugins(client, public_add_enabled): + public_add_enabled.config["SEARCH_BACKEND_ENGINE"] = "sqlite" + public_add_enabled.save(update_fields=["config"]) response = client.get(reverse("add"), HTTP_HOST=WEB_HOST) form = response.context["form"] @@ -234,8 +235,7 @@ def test_add_view_hides_search_backend_plugins(client, monkeypatch): assert form.plugin_groups == [] -def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_user): client.force_login(admin_user) response = client.post( @@ -320,8 +320,7 @@ def test_add_view_sanitizes_crawl_notes_before_safe_update(client, admin_user): assert "" not in crawl.notes -def test_add_view_unchecked_only_new_sets_crawl_override(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_unchecked_only_new_sets_crawl_override(client, admin_user): client.force_login(admin_user) response = client.post( @@ -351,8 +350,7 @@ def test_add_view_unchecked_only_new_sets_crawl_override(client, admin_user, mon assert crawl.config["ONLY_NEW"] is False -def test_add_view_selected_persona_wins_over_stale_config_override(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_selected_persona_wins_over_stale_config_override(client, admin_user): client.force_login(admin_user) private_persona = Persona.objects.create(name="Private", created_by=admin_user) private_persona.ensure_dirs() @@ -396,8 +394,7 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin assert runtime_config["COOKIES_FILE"] == str(private_cookies_file) -def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_applies_plugin_config_overrides(client, admin_user): client.force_login(admin_user) response = client.post( @@ -439,8 +436,7 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatc assert "NODE_BINARY" not in crawl.config -def test_add_view_public_submission_ignores_plugin_and_custom_config(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_public_submission_ignores_plugin_and_custom_config(client, admin_user, public_add_enabled): response = client.post( reverse("add"), @@ -492,8 +488,7 @@ def test_add_view_public_submission_ignores_plugin_and_custom_config(client, adm assert crawl.schedule is None -def test_add_view_queues_crawl_for_background_runner(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_queues_crawl_for_background_runner(client, admin_user): client.force_login(admin_user) response = client.post( @@ -527,8 +522,7 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user, monkeyp assert crawl.snapshot_set.count() == 0 -def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, admin_user): client.force_login(admin_user) response = client.post( @@ -563,8 +557,7 @@ def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, ad assert crawl.config.get("INDEX_ONLY") is not True -def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user): client.force_login(admin_user) response = client.post( @@ -614,8 +607,7 @@ def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkey assert crawl.snapshot_set.count() == 0 -def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_user): client.force_login(admin_user) response = client.post( @@ -659,8 +651,7 @@ def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_us assert crawl.snapshot_set.count() == 0 -def test_add_view_exposes_api_token_for_tag_widget_autocomplete(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") +def test_add_view_exposes_api_token_for_tag_widget_autocomplete(client, admin_user): client.force_login(admin_user) response = client.get(reverse("add"), HTTP_HOST=ADMIN_HOST) @@ -676,8 +667,8 @@ def _create_tagged_snapshot(user, *, permissions="public"): return snapshot -def test_tags_autocomplete_requires_auth_when_public_index_disabled(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_INDEX", "false") +def test_tags_autocomplete_requires_auth_when_public_index_disabled(client, admin_user): + Machine.from_json({"config": {"PUBLIC_INDEX": False}}) _create_tagged_snapshot(admin_user) response = client.get( @@ -689,8 +680,8 @@ def test_tags_autocomplete_requires_auth_when_public_index_disabled(client, admi assert response.status_code == 401 -def test_tags_autocomplete_lists_only_public_snapshot_tags(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_INDEX", "true") +def test_tags_autocomplete_lists_only_public_snapshot_tags(client, admin_user): + Machine.from_json({"config": {"PUBLIC_INDEX": True}}) _create_tagged_snapshot(admin_user) _create_tagged_snapshot(admin_user, permissions="unlisted") Tag.objects.create(name="private-empty") @@ -705,8 +696,8 @@ def test_tags_autocomplete_lists_only_public_snapshot_tags(client, admin_user, m assert response.json()["tags"][0]["name"] == "archive" -def test_tags_autocomplete_allows_authenticated_user_when_public_index_disabled(client, admin_user, monkeypatch): - monkeypatch.setenv("PUBLIC_INDEX", "false") +def test_tags_autocomplete_allows_authenticated_user_when_public_index_disabled(client, admin_user): + Machine.from_json({"config": {"PUBLIC_INDEX": False}}) Tag.objects.create(name="archive") client.force_login(admin_user) diff --git a/archivebox/tests/test_ui_add_view_runtime.py b/archivebox/tests/test_ui_add_view_runtime.py index 4ffeda61..c255b730 100644 --- a/archivebox/tests/test_ui_add_view_runtime.py +++ b/archivebox/tests/test_ui_add_view_runtime.py @@ -1,5 +1,4 @@ import re -import time import json from pathlib import Path @@ -18,7 +17,7 @@ from .conftest import ( run_archivebox_cmd, start_archivebox_server, stop_server, - wait_for_http, + get_http_response, ) pytestmark = pytest.mark.django_db(transaction=True) @@ -140,37 +139,6 @@ def write_import_format_files(base_dir: Path) -> dict[str, Path]: return files -def wait_for_import_processing(cwd: Path, expected_urls: set[str], *, timeout: float = 120.0) -> None: - deadline = time.time() + timeout - while time.time() < deadline: - with use_archivebox_db(cwd): - snapshot_started = Snapshot.objects.filter(url__in=expected_urls).exists() - if snapshot_started: - return - time.sleep(1) - raise AssertionError("timed out waiting for import crawl processing to start") - - -def wait_for_expected_import_snapshots(cwd: Path, expected_urls: set[str], *, timeout: float = 180.0) -> None: - allowed_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED} - deadline = time.time() + timeout - while time.time() < deadline: - with use_archivebox_db(cwd): - rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", "status")) - counts = {url: 0 for url in expected_urls} - bad_statuses = [] - for url, status in rows: - counts[url] += 1 - if status not in allowed_statuses: - bad_statuses.append((url, status)) - if all(count == 1 for count in counts.values()) and not bad_statuses: - return - time.sleep(1) - raise AssertionError( - f"timed out waiting for one queued/started/sealed snapshot per URL, got counts={counts}, bad_statuses={bad_statuses}", - ) - - def malicious_add_inputs(tmp_path: Path, *, safe_url: str) -> tuple[list[str], Path]: other_crawl_source = tmp_path / "sources" / "other_crawl_source.txt" other_crawl_source.parent.mkdir(parents=True, exist_ok=True) @@ -221,13 +189,12 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s port=port, PLUGINS="wget", PUBLIC_ADD_VIEW="True", - PYTEST_CURRENT_TEST="", ) create_admin_and_token(tmp_path) try: start_archivebox_server(tmp_path, env=env, port=port) - _wait_for_worker_state(tmp_path, "worker_runner", "RUNNING") + assert _worker_state(tmp_path, "worker_runner") == "RUNNING" _stop_worker(tmp_path, "worker_runner") assert _worker_state(tmp_path, "worker_runner") != "RUNNING" @@ -258,7 +225,7 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s ) assert response.status_code in (302, 303), response.text - _wait_for_worker_state(tmp_path, "worker_runner", "RUNNING") + assert _worker_state(tmp_path, "worker_runner") == "RUNNING" with use_archivebox_db(tmp_path): crawl = Crawl.objects.order_by("-created_at").first() assert crawl is not None @@ -274,7 +241,7 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s def _login_to_add_view(port: int) -> tuple[requests.Session, str]: session = requests.Session() - wait_for_http(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/") + get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/") login_page = session.get( f"http://127.0.0.1:{port}/admin/login/", headers={"Host": f"admin.archivebox.localhost:{port}"}, @@ -296,7 +263,7 @@ def _login_to_add_view(port: int) -> tuple[requests.Session, str]: allow_redirects=False, ) assert login_response.status_code in (302, 303), login_response.text - add_page = wait_for_http(port, host=f"admin.archivebox.localhost:{port}", path="/add/") + add_page = get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/add/") assert add_page.status_code == 200 add_csrf_match = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', add_page.text) assert add_csrf_match, add_page.text[:500] @@ -333,17 +300,6 @@ print("stopped") assert returncode == 0, stderr or stdout -def _wait_for_worker_state(cwd, worker_name: str, statename: str, timeout: int = 45) -> None: - deadline = time.time() + timeout - state = None - while time.time() < deadline: - state = _worker_state(cwd, worker_name) - if state == statename: - return - time.sleep(1) - raise AssertionError(f"Timed out waiting for {worker_name}={statename}, last state={state}") - - @pytest.mark.timeout(240) def test_public_add_view_depth_one_crawl_skips_unreadable_persona_profile_entries(tmp_path, recursive_test_site): init_archive(tmp_path) @@ -369,9 +325,10 @@ def test_public_add_view_depth_one_crawl_skips_unreadable_persona_profile_entrie try: start_archivebox_server(tmp_path, env=env, port=port) - add_page = wait_for_http(port, host=f"web.archivebox.localhost:{port}", path="/add/") + add_page = get_http_response(port, host=f"web.archivebox.localhost:{port}", path="/add/") assert add_page.status_code == 200 assert 'name="depth"' in add_page.text + _stop_worker(tmp_path, "worker_runner") response = requests.post( f"http://127.0.0.1:{port}/add/", @@ -396,27 +353,11 @@ def test_public_add_view_depth_one_crawl_skips_unreadable_persona_profile_entrie allow_redirects=False, ) assert response.status_code in (302, 303), response.text - - deadline = time.time() + 180 - while time.time() < deadline: - with use_archivebox_db(tmp_path): - crawl = Crawl.objects.order_by("-created_at").first() - depth_counts = get_depth_counts(tmp_path) - child_urls = set(Snapshot.objects.filter(depth=1).values_list("url", flat=True)) - root_status = Snapshot.objects.filter(url=recursive_test_site["root_url"], depth=0).values_list("status", flat=True).first() - failed_results = list( - ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.FAILED).values_list("plugin", "output_str"), - ) - if ( - depth_counts.get(0, 0) >= 1 - and set(recursive_test_site["child_urls"]).issubset(child_urls) - and root_status == Snapshot.StatusChoices.SEALED - ): - break - assert not failed_results - time.sleep(2) - else: - raise AssertionError(f"timed out waiting for depth=1 crawl, got depth counts {get_depth_counts(tmp_path)}") + with use_archivebox_db(tmp_path): + submitted_crawl = Crawl.objects.order_by("-created_at").get() + stop_server(tmp_path) + run_result = run_archivebox_cmd(["run", f"--crawl-id={submitted_crawl.id}"], cwd=tmp_path, timeout=180, env=env) + assert run_result.returncode == 0, run_result.stderr or run_result.stdout with use_archivebox_db(tmp_path): crawl = Crawl.objects.order_by("-created_at").first() @@ -462,10 +403,12 @@ def test_public_add_view_import_text_formats_preserve_metadata_and_resume_withou try: start_archivebox_server(tmp_path, env=env, port=port) - add_page = wait_for_http(port, host=f"web.archivebox.localhost:{port}", path="/add/") + add_page = get_http_response(port, host=f"web.archivebox.localhost:{port}", path="/add/") assert add_page.status_code == 200 assert 'name="url"' in add_page.text + _stop_worker(tmp_path, "worker_runner") + crawl_ids = [] for import_path in import_files.values(): source_text = import_path.read_text(encoding="utf-8") response = requests.post( @@ -491,24 +434,24 @@ def test_public_add_view_import_text_formats_preserve_metadata_and_resume_withou allow_redirects=False, ) assert response.status_code in (302, 303), response.text - deadline = time.time() + 60 - root_input = None - while time.time() < deadline: - with use_archivebox_db(tmp_path): - crawl = Crawl.objects.order_by("-created_at").first() - assert crawl is not None - assert crawl.urls == source_text - root_snapshot = crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).first() - if root_snapshot: - root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") - break - time.sleep(1) - assert root_input == source_text + with use_archivebox_db(tmp_path): + crawl = Crawl.objects.order_by("-created_at").first() + assert crawl is not None + assert crawl.urls == source_text + crawl_ids.append(crawl.id) - wait_for_import_processing(tmp_path, expected_urls) stop_server(tmp_path) + for crawl_id in crawl_ids: + run_result = run_archivebox_cmd(["run", f"--crawl-id={crawl_id}"], cwd=tmp_path, timeout=180, env=env) + assert run_result.returncode == 0, run_result.stderr or run_result.stdout + + with use_archivebox_db(tmp_path): + for crawl in Crawl.objects.order_by("created_at"): + root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL) + root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") + assert root_input == crawl.urls + start_archivebox_server(tmp_path, env=env, port=port) - wait_for_expected_import_snapshots(tmp_path, expected_urls) public_index = requests.get( f"http://127.0.0.1:{port}/", @@ -566,8 +509,9 @@ def test_public_add_view_rejects_file_path_and_shell_injection_payloads(tmp_path try: start_archivebox_server(tmp_path, env=env, port=port) - add_page = wait_for_http(port, host=f"web.archivebox.localhost:{port}", path="/add/") + add_page = get_http_response(port, host=f"web.archivebox.localhost:{port}", path="/add/") assert add_page.status_code == 200 + _stop_worker(tmp_path, "worker_runner") response = requests.post( f"http://127.0.0.1:{port}/add/", @@ -593,10 +537,12 @@ def test_public_add_view_rejects_file_path_and_shell_injection_payloads(tmp_path ) assert response.status_code in (302, 303), response.text - wait_for_import_processing(tmp_path, {safe_url}) + with use_archivebox_db(tmp_path): + submitted_crawl = Crawl.objects.order_by("-created_at").get() stop_server(tmp_path) + run_result = run_archivebox_cmd(["run", f"--crawl-id={submitted_crawl.id}"], cwd=tmp_path, timeout=180, env=env) + assert run_result.returncode == 0, run_result.stderr or run_result.stdout start_archivebox_server(tmp_path, env=env, port=port) - wait_for_expected_import_snapshots(tmp_path, {safe_url}, timeout=120) public_index = requests.get( f"http://127.0.0.1:{port}/", @@ -629,7 +575,7 @@ def test_add_view_post_creates_schedule_over_server(tmp_path, recursive_test_sit try: start_archivebox_server(tmp_path, env=env, port=port) session = requests.Session() - wait_for_http(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/") + get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/") login_page = session.get( f"http://admin.archivebox.localhost:{port}/admin/login/", timeout=10, @@ -650,7 +596,7 @@ def test_add_view_post_creates_schedule_over_server(tmp_path, recursive_test_sit allow_redirects=False, ) assert login_response.status_code in (302, 303), login_response.text - wait_for_http(port, host=f"admin.archivebox.localhost:{port}", path="/add/") + get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/add/") response = session.post( f"http://admin.archivebox.localhost:{port}/add/", @@ -703,7 +649,7 @@ def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursiv try: start_archivebox_server(tmp_path, env=env, port=port) session = requests.Session() - wait_for_http(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/") + get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/") login_page = session.get( f"http://127.0.0.1:{port}/admin/login/", headers={"Host": f"admin.archivebox.localhost:{port}"}, @@ -725,10 +671,11 @@ def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursiv allow_redirects=False, ) assert login_response.status_code in (302, 303), login_response.text - add_page = wait_for_http(port, host=f"admin.archivebox.localhost:{port}", path="/add/") + add_page = get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/add/") assert add_page.status_code == 200 assert 'name="depth"' in add_page.text assert 'name="url"' in add_page.text + _stop_worker(tmp_path, "worker_runner") response = session.post( f"http://127.0.0.1:{port}/add/", @@ -756,19 +703,12 @@ def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursiv allow_redirects=False, ) assert response.status_code in (302, 303), response.text - - deadline = time.time() + 180 - while time.time() < deadline: - depth_counts = get_depth_counts(tmp_path) - if ( - depth_counts.get(0, 0) >= 1 - and depth_counts.get(1, 0) >= len(recursive_test_site["child_urls"]) - and depth_counts.get(2, 0) >= len(recursive_test_site["deep_urls"]) - ): - break - time.sleep(2) - else: - raise AssertionError(f"timed out waiting for depth=2 crawl, got depth counts {get_depth_counts(tmp_path)}") + with use_archivebox_db(tmp_path): + submitted_crawl = Crawl.objects.order_by("-created_at").get() + stop_server(tmp_path) + run_result = run_archivebox_cmd(["run", f"--crawl-id={submitted_crawl.id}"], cwd=tmp_path, timeout=180, env=env) + assert run_result.returncode == 0, run_result.stderr or run_result.stdout + start_archivebox_server(tmp_path, env=env, port=port) with use_archivebox_db(tmp_path): depth_counts = get_depth_counts(tmp_path) diff --git a/archivebox/tests/test_ui_admin_archiveresult.py b/archivebox/tests/test_ui_admin_archiveresult.py index f878f70e..8fbf3302 100644 --- a/archivebox/tests/test_ui_admin_archiveresult.py +++ b/archivebox/tests/test_ui_admin_archiveresult.py @@ -1,25 +1,86 @@ """ArchiveResult admin UI tests.""" +import asyncio +import os +from importlib.resources import files +from pathlib import Path + import pytest from django.urls import reverse -from archivebox.tests.conftest import ADMIN_TEST_HOST +from archivebox.tests.conftest import ADMIN_TEST_HOST, resolve_abxpkg_binary_env -pytestmark = pytest.mark.django_db +pytestmark = pytest.mark.django_db(transaction=True) + + +@pytest.fixture +def projected_noresults(snapshot, cached_abxpkg_lib_dir): + from abx_dl.events import ProcessEvent, SnapshotEvent + from abx_dl.orchestrator import create_bus + from abx_dl.services.process_service import ProcessService as HookProcessService + from archivebox.core.models import ArchiveResult + from archivebox.services.archive_result_service import ArchiveResultService + from archivebox.services.process_service import ProcessService as PersistedProcessService + + plugin = "parse_txt_urls" + hook_name = "on_Snapshot__71_parse_txt_urls.py" + hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath(hook_name))) + binary_env = resolve_abxpkg_binary_env(cached_abxpkg_lib_dir, deps_from=hook_path.parent / "config.json") + staticfile_dir = snapshot.output_dir / "staticfile" + output_dir = snapshot.output_dir / plugin + staticfile_dir.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + (staticfile_dir / "input.txt").write_text("plain text without links", encoding="utf-8") + bus = create_bus(name=f"test_admin_noresults_{snapshot.id}") + HookProcessService(bus, emit_jsonl=False, interactive_tty=False) + PersistedProcessService(bus) + ArchiveResultService(bus) + + async def run() -> None: + try: + snapshot_event = SnapshotEvent( + url=snapshot.url, + snapshot_id=str(snapshot.id), + output_dir=str(snapshot.output_dir), + ) + await bus.emit(snapshot_event).now() + await bus.emit( + ProcessEvent( + plugin_name=plugin, + hook_name=hook_name, + hook_path=str(hook_path), + hook_args=[f"--url={snapshot.url}"], + env={ + **binary_env, + "ABXPKG_LIB_DIR": str(cached_abxpkg_lib_dir), + "SNAP_DIR": str(snapshot.output_dir), + "PATH": f"{Path(os.sys.executable).parent}{os.pathsep}{os.environ['PATH']}", + }, + output_dir=str(output_dir), + timeout=60, + is_background=False, + url=snapshot.url, + process_type="hook", + worker_type="hook", + event_parent_id=snapshot_event.event_id, + ), + ).now() + await bus.wait_until_idle() + finally: + await bus.destroy(clear=False) + + asyncio.run(run()) + return ArchiveResult.objects.get(snapshot=snapshot, plugin=plugin, hook_name=hook_name) class TestArchiveResultAdminListView: - def test_list_view_renders_readonly_tags_and_noresults_status(self, client, admin_user, snapshot): + def test_list_view_renders_readonly_tags_and_noresults_status(self, client, admin_user, snapshot, projected_noresults): from archivebox.core.models import ArchiveResult, Tag tag = Tag.objects.create(name="Alpha Research") snapshot.tags.add(tag) - ArchiveResult.objects.create( - snapshot=snapshot, - plugin="title", - status=ArchiveResult.StatusChoices.NORESULTS, - output_str="No title found", - ) + assert projected_noresults.status == ArchiveResult.StatusChoices.NORESULTS + assert projected_noresults.process_id is not None client.force_login(admin_user) response = client.get(reverse("admin:core_archiveresult_changelist"), HTTP_HOST=ADMIN_TEST_HOST) diff --git a/archivebox/tests/test_ui_admin_config_widget.py b/archivebox/tests/test_ui_admin_config_widget.py index 829b3d4c..536703b2 100644 --- a/archivebox/tests/test_ui_admin_config_widget.py +++ b/archivebox/tests/test_ui_admin_config_widget.py @@ -1,21 +1,7 @@ from archivebox.base_models.admin import KeyValueWidget -def test_key_value_widget_renders_enum_autocomplete_metadata(monkeypatch): - monkeypatch.setattr( - KeyValueWidget, - "_get_config_options", - lambda self: { - "CHROME_WAIT_FOR": { - "plugin": "chrome", - "type": "string", - "default": "networkidle2", - "description": "Page load completion condition", - "enum": ["domcontentloaded", "load", "networkidle0", "networkidle2"], - }, - }, - ) - +def test_key_value_widget_renders_enum_autocomplete_metadata(): html = str( KeyValueWidget().render( "config", @@ -32,33 +18,10 @@ def test_key_value_widget_renders_enum_autocomplete_metadata(monkeypatch): assert "validateValueAgainstMeta_id_config" in html -def test_key_value_widget_renders_numeric_and_pattern_constraints(monkeypatch): - monkeypatch.setattr( - KeyValueWidget, - "_get_config_options", - lambda self: { - "TIMEOUT": { - "plugin": "base", - "type": "integer", - "default": 60, - "description": "Timeout in seconds", - "minimum": 5, - "maximum": 120, - }, - "CHROME_RESOLUTION": { - "plugin": "chrome", - "type": "string", - "default": "1440,2000", - "description": "Viewport resolution", - "pattern": "^\\d+,\\d+$", - }, - }, - ) - +def test_key_value_widget_renders_numeric_and_pattern_constraints(): html = str(KeyValueWidget().render("config", {}, attrs={"id": "id_config"})) - assert '"minimum": 5' in html - assert '"maximum": 120' in html + assert '"minimum": 0' in html assert '"pattern": "^\\\\d+,\\\\d+$"' in html assert "Expected: " in html assert "Example: " in html @@ -66,21 +29,8 @@ def test_key_value_widget_renders_numeric_and_pattern_constraints(monkeypatch): assert "coerceValueForStorage_id_config" in html -def test_key_value_widget_accepts_common_boolean_spellings(monkeypatch): - monkeypatch.setattr( - KeyValueWidget, - "_get_config_options", - lambda self: { - "DEBUG": { - "plugin": "base", - "type": "boolean", - "default": False, - "description": "Enable debug mode", - }, - }, - ) - - html = str(KeyValueWidget().render("config", {"DEBUG": "True"}, attrs={"id": "id_config"})) +def test_key_value_widget_accepts_common_boolean_spellings(): + html = str(KeyValueWidget().render("config", {"CHECK_SSL_VALIDITY": "True"}, attrs={"id": "id_config"})) assert "enumValues = ['True', 'False']" in html assert "raw.toLowerCase()" in html @@ -88,27 +38,8 @@ def test_key_value_widget_accepts_common_boolean_spellings(monkeypatch): assert "lowered === 'false' || raw === '0'" in html -def test_key_value_widget_shows_array_and_object_examples_and_binary_rules(monkeypatch): - monkeypatch.setattr( - KeyValueWidget, - "_get_config_options", - lambda self: { - "WGET_ARGS_EXTRA": { - "plugin": "wget", - "type": "array", - "default": [], - "description": "Extra arguments to append to wget command", - }, - "WGET_BINARY": { - "plugin": "wget", - "type": "string", - "default": "wget", - "description": "Path to wget binary", - }, - }, - ) - - html = str(KeyValueWidget().render("config", {}, attrs={"id": "id_config"})) +def test_key_value_widget_shows_array_and_object_examples_and_binary_rules(): + html = str(KeyValueWidget().render("config", {"NODE_BINARY": "node"}, attrs={"id": "id_config"})) assert 'Example: ["--extra-arg"]' in html assert "Example: wget or /usr/bin/wget" in html @@ -117,20 +48,7 @@ def test_key_value_widget_shows_array_and_object_examples_and_binary_rules(monke assert "Binary paths cannot contain quotes" in html -def test_key_value_widget_falls_back_to_binary_validation_for_unknown_binary_keys(monkeypatch): - monkeypatch.setattr( - KeyValueWidget, - "_get_config_options", - lambda self: { - "CHROME_BINARY": { - "plugin": "base", - "type": "string", - "default": "", - "description": "Resolved Chromium/Chrome binary path shared across plugins", - }, - }, - ) - +def test_key_value_widget_falls_back_to_binary_validation_for_unknown_binary_keys(): html = str( KeyValueWidget().render( "config", diff --git a/archivebox/tests/test_ui_admin_links.py b/archivebox/tests/test_ui_admin_links.py index 7ace161a..da381ad7 100644 --- a/archivebox/tests/test_ui_admin_links.py +++ b/archivebox/tests/test_ui_admin_links.py @@ -1,6 +1,7 @@ import pytest import subprocess from datetime import datetime, timezone as dt_timezone +from importlib.resources import files from pathlib import Path from django.contrib.admin.sites import AdminSite from django.contrib.messages import get_messages @@ -10,10 +11,60 @@ from django.urls import reverse import html from uuid import uuid4 +from archivebox.tests.conftest import cli_env, run_archivebox_cmd +from archivebox.tests.conftest import install_real_binary + pytestmark = pytest.mark.django_db +@pytest.fixture +def real_hook_result(tmp_path): + from archivebox.core.models import ArchiveResult + from archivebox.plugins.hooks import extract_records_from_process, run_hook + + snapshot = _create_snapshot() + snap_dir = Path(snapshot.output_dir) + output_dir = snap_dir / "hashes" + output_dir.mkdir(parents=True, exist_ok=True) + (snap_dir / "source.txt").write_text("real admin link hook input", encoding="utf-8") + hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py"))) + process = run_hook( + hook_path, + output_dir, + config={ + "ABXPKG_LIB_DIR": str(tmp_path / "lib"), + "SNAP_DIR": str(snap_dir), + "SAFE_FLAG": "1", + "API_KEY": "super-secret-key", + "ACCESS_TOKEN": "super-secret-token", + "SHARED_SECRET": "super-secret-secret", + }, + timeout=30, + url=snapshot.url, + ) + process.refresh_from_db() + assert process.exit_code == 0, process.stderr + record = extract_records_from_process(process)[0] + hashes_file = output_dir / "hashes.json" + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin=record["plugin"], + hook_name=record["hook_name"], + process=process, + status=record["status"], + output_str=record["output_str"], + output_files={ + "hashes.json": { + "extension": "json", + "mimetype": "application/json", + "size": hashes_file.stat().st_size, + }, + }, + ) + return snapshot, process, result + + def _create_snapshot(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl @@ -85,22 +136,26 @@ def _admin_get_request(path="/"): @pytest.fixture -def running_process_record(): +def running_process_record(initialized_archive): from archivebox.machine.models import Machine, Process, psutil - cmd = ["/bin/sleep", "60"] - popen = subprocess.Popen( - cmd, - cwd=Path.cwd(), + cmd = ["archivebox", "manage", "shell"] + popen = run_archivebox_cmd( + ["manage", "shell"], + cwd=initialized_archive, + env=cli_env(live=True), + stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + capture_output=False, + wait=False, ) try: os_process = psutil.Process(popen.pid) process = Process.objects.create( machine=Machine.current(refresh=True), - process_type=Process.TypeChoices.HOOK, - pwd=str(Path.cwd()), + process_type=Process.TypeChoices.ORCHESTRATOR, + pwd=str(initialized_archive), cmd=cmd, pid=popen.pid, started_at=datetime.fromtimestamp(os_process.create_time(), tz=dt_timezone.utc), @@ -108,46 +163,27 @@ def running_process_record(): ) yield process finally: - if popen.poll() is None: - popen.terminate() - try: - popen.wait(timeout=5) - except subprocess.TimeoutExpired: - popen.kill() - popen.wait(timeout=5) + assert popen.stdin is not None + popen.stdin.close() + popen.wait(timeout=20) -def test_archiveresult_admin_links_plugin_and_process(): +def test_archiveresult_admin_links_plugin_and_process(real_hook_result): from archivebox.core.admin_archiveresults import ArchiveResultAdmin, render_archiveresults_list from archivebox.core.models import ArchiveResult - from archivebox.machine.models import Process - snapshot = _create_snapshot() - iface = _create_iface(_create_machine()) - process = Process.objects.create( - machine=iface.machine, - iface=iface, - process_type=Process.TypeChoices.HOOK, - pwd=str(snapshot.output_dir / "wget"), - cmd=["/tmp/on_Snapshot__06_wget.finite.bg.py", "--url=https://example.com"], - status=Process.StatusChoices.EXITED, - ) - result = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="wget", - hook_name="on_Snapshot__06_wget.finite.bg.py", - process=process, - status=ArchiveResult.StatusChoices.SUCCEEDED, - ) + snapshot, process, result = real_hook_result + iface = process.iface + assert iface is not None admin = ArchiveResultAdmin(ArchiveResult, AdminSite()) plugin_html = str(admin.plugin_with_icon(result)) process_html = str(admin.process_link(result)) - assert "/admin/environment/plugins/builtin.wget/" in plugin_html + assert "/admin/environment/plugins/builtin.hashes/" in plugin_html assert f"/admin/machine/process/{process.id}/change" in process_html - assert f"{str(process.id)[-8:]}" in process_html + assert f"{process.pid}" in process_html assert "-" not in process_html machine_html = str(admin.machine_link(result)) @@ -156,42 +192,21 @@ def test_archiveresult_admin_links_plugin_and_process(): inline_html = str(render_archiveresults_list(ArchiveResult.objects.filter(id=result.id))) assert f"/admin/machine/process/{process.id}/change" in inline_html - assert f">{str(process.id)[-8:]}" in inline_html + assert f">{process.pid}" in inline_html assert ">-" not in inline_html -def test_deleting_binary_and_process_records_preserves_results(): +@pytest.mark.django_db(transaction=True) +def test_deleting_binary_and_process_records_preserves_results(real_hook_result): from archivebox.core.admin_archiveresults import ArchiveResultAdmin, build_abx_dl_replay_command, render_archiveresults_list from archivebox.core.models import ArchiveResult from archivebox.machine.admin import ProcessAdmin - from archivebox.machine.models import Binary, Process + from archivebox.machine.models import Process - snapshot = _create_snapshot() - machine = _create_machine() - binary = Binary.objects.create( - machine=machine, - name="wget", - abspath="/usr/bin/wget", - version="1.21.2", - binprovider="env", - binproviders="env", - status=Binary.StatusChoices.INSTALLED, - ) - process = Process.objects.create( - machine=machine, - binary=binary, - process_type=Process.TypeChoices.HOOK, - pwd=str(snapshot.output_dir / "wget"), - cmd=["/tmp/on_Snapshot__06_wget.finite.bg.py", "--url=https://example.com"], - status=Process.StatusChoices.EXITED, - ) - result = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="wget", - hook_name="on_Snapshot__06_wget.finite.bg.py", - process=process, - status=ArchiveResult.StatusChoices.SUCCEEDED, - ) + snapshot, process, result = real_hook_result + binary = install_real_binary("python3", machine=process.machine) + process.binary = binary + process.save(update_fields=["binary"]) binary.delete() process.refresh_from_db() @@ -220,7 +235,7 @@ def test_deleting_binary_and_process_records_preserves_results(): assert admin.process_link(result) == "-" assert admin.machine_link(result) == "-" assert "cd " in build_abx_dl_replay_command(result) - assert "wget" in render_archiveresults_list(ArchiveResult.objects.filter(id=result.id)) + assert "hashes" in render_archiveresults_list(ArchiveResult.objects.filter(id=result.id)) def test_snapshot_admin_zip_links(): @@ -273,35 +288,12 @@ def test_archiveresult_admin_zip_links(): assert html.escape(zip_url, quote=True) in str(admin.admin_actions(result)) -def test_archiveresult_admin_copy_command_redacts_sensitive_env_keys(): +def test_archiveresult_admin_copy_command_redacts_sensitive_env_keys(real_hook_result): from archivebox.core.admin_archiveresults import ArchiveResultAdmin from archivebox.core.models import ArchiveResult - from archivebox.machine.models import Process - snapshot = _create_snapshot() - iface = _create_iface(_create_machine()) - process = Process.objects.create( - machine=iface.machine, - iface=iface, - process_type=Process.TypeChoices.HOOK, - pwd=str(snapshot.output_dir / "wget"), - cmd=["/tmp/on_Snapshot__06_wget.finite.bg.py", "--url=https://example.com"], - env={ - "SAFE_FLAG": "1", - "API_KEY": "super-secret-key", - "ACCESS_TOKEN": "super-secret-token", - "SHARED_SECRET": "super-secret-secret", - }, - status=Process.StatusChoices.EXITED, - url="https://example.com", - ) - result = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="wget", - hook_name="on_Snapshot__06_wget.finite.bg.py", - process=process, - status=ArchiveResult.StatusChoices.SUCCEEDED, - ) + _, process, result = real_hook_result + assert process.env["SAFE_FLAG"] == "1" admin = ArchiveResultAdmin(ArchiveResult, AdminSite()) admin.request = _admin_get_request() @@ -317,30 +309,17 @@ def test_archiveresult_admin_copy_command_redacts_sensitive_env_keys(): assert "super-secret-secret" not in cmd_html -def test_process_admin_links_binary_and_iface(): +@pytest.mark.django_db(transaction=True) +def test_process_admin_links_binary_and_iface(real_hook_result): from archivebox.machine.admin import ProcessAdmin - from archivebox.machine.models import Binary, Process + from archivebox.machine.models import Process - machine = _create_machine() - iface = _create_iface(machine) - binary = Binary.objects.create( - machine=machine, - name="wget", - abspath="/usr/local/bin/wget", - version="1.21.2", - binprovider="env", - binproviders="env", - status=Binary.StatusChoices.INSTALLED, - ) - process = Process.objects.create( - machine=machine, - iface=iface, - binary=binary, - process_type=Process.TypeChoices.HOOK, - pwd="/tmp/wget", - cmd=["/tmp/on_Snapshot__06_wget.finite.bg.py", "--url=https://example.com"], - status=Process.StatusChoices.EXITED, - ) + _, process, _ = real_hook_result + iface = process.iface + assert iface is not None + binary = install_real_binary("python3", machine=process.machine) + process.binary = binary + process.save(update_fields=["binary"]) admin = ProcessAdmin(Process, AdminSite()) @@ -358,9 +337,9 @@ def test_process_admin_kill_actions_only_terminate_running_processes(running_pro running = running_process_record exited = Process.objects.create( machine=Machine.current(), - process_type=Process.TypeChoices.HOOK, - pwd="/tmp/exited", - cmd=["/tmp/on_Snapshot__06_wget.finite.bg.py", "--url=https://example.com"], + process_type=Process.TypeChoices.ORCHESTRATOR, + pwd=running.pwd, + cmd=running.cmd, status=Process.StatusChoices.EXITED, ) @@ -377,17 +356,11 @@ def test_process_admin_kill_actions_only_terminate_running_processes(running_pro assert any("Skipped 1 process" in msg for msg in messages) -def test_process_admin_object_kill_action_redirects_and_skips_exited(): +def test_process_admin_object_kill_action_redirects_and_skips_exited(real_hook_result): from archivebox.machine.admin import ProcessAdmin - from archivebox.machine.models import Machine, Process + from archivebox.machine.models import Process - process = Process.objects.create( - machine=Machine.current(refresh=True), - process_type=Process.TypeChoices.HOOK, - pwd="/tmp/exited", - cmd=["/tmp/on_Snapshot__06_wget.finite.bg.py", "--url=https://example.com"], - status=Process.StatusChoices.EXITED, - ) + _, process, _ = real_hook_result admin = ProcessAdmin(Process, AdminSite()) request = _admin_post_request(f"/admin/machine/process/{process.pk}/change/") @@ -402,35 +375,17 @@ def test_process_admin_object_kill_action_redirects_and_skips_exited(): assert any("Skipped 1 process" in msg for msg in messages) -def test_process_admin_output_summary_uses_archiveresult_output_files(): - from archivebox.core.models import ArchiveResult +def test_process_admin_output_summary_uses_archiveresult_output_files(real_hook_result): from archivebox.machine.admin import ProcessAdmin from archivebox.machine.models import Process + from archivebox.misc.logging_util import printable_filesize - snapshot = _create_snapshot() - machine = _create_machine() - process = Process.objects.create( - machine=machine, - process_type=Process.TypeChoices.HOOK, - pwd=str(snapshot.output_dir / "wget"), - cmd=["/tmp/on_Snapshot__06_wget.finite.bg.py", "--url=https://example.com"], - status=Process.StatusChoices.EXITED, - ) - ArchiveResult.objects.create( - snapshot=snapshot, - plugin="wget", - hook_name="on_Snapshot__06_wget.finite.bg.py", - process=process, - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_files={ - "index.html": {"extension": "html", "mimetype": "text/html", "size": 1024}, - "title.txt": {"extension": "txt", "mimetype": "text/plain", "size": "512"}, - }, - ) + _, process, result = real_hook_result + expected_size = sum(int(metadata["size"]) for metadata in result.output_files.values()) admin = ProcessAdmin(Process, AdminSite()) output_html = str(admin.output_summary(process)) - assert "2 files" in output_html - assert "1.5 KB" in output_html + assert f"{len(result.output_files)} file" in output_html + assert printable_filesize(expected_size) in output_html diff --git a/archivebox/tests/test_ui_admin_machine.py b/archivebox/tests/test_ui_admin_machine.py index 4d53ec9c..45f8d80f 100644 --- a/archivebox/tests/test_ui_admin_machine.py +++ b/archivebox/tests/test_ui_admin_machine.py @@ -1,20 +1,113 @@ """Machine, binary, and process admin UI tests.""" import uuid +import asyncio +import os +from importlib.resources import files +from pathlib import Path import pytest from django.urls import reverse from django.utils import timezone from archivebox.tests.conftest import ADMIN_TEST_HOST +from archivebox.tests.conftest import install_real_binary +from archivebox.tests.conftest import resolve_abxpkg_binary_env -pytestmark = pytest.mark.django_db +pytestmark = pytest.mark.django_db(transaction=True) + + +@pytest.fixture +def real_exited_hook_process(tmp_path): + from archivebox.plugins.hooks import run_hook + + snap_dir = tmp_path / "snapshot" + output_dir = snap_dir / "hashes" + output_dir.mkdir(parents=True) + (snap_dir / "source.txt").write_text("real admin hook input", encoding="utf-8") + hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py"))) + process = run_hook( + hook_path, + output_dir, + config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)}, + timeout=30, + url="https://example.com/admin-hook", + ) + process.refresh_from_db() + assert process.exit_code == 0, process.stderr + assert (output_dir / "hashes.json").is_file() + return process + + +@pytest.fixture +def real_projected_hash_result(snapshot, cached_abxpkg_lib_dir): + from abx_dl.events import ProcessEvent, SnapshotEvent + from abx_dl.orchestrator import create_bus + from abx_dl.services.process_service import ProcessService as HookProcessService + from archivebox.core.models import ArchiveResult + from archivebox.machine.models import Process + from archivebox.services.archive_result_service import ArchiveResultService + from archivebox.services.process_service import ProcessService as PersistedProcessService + + plugin = "hashes" + hook_name = "on_Snapshot__93_hashes.py" + hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath(hook_name))) + hook_config = hook_path.parent / "config.json" + binary_env = resolve_abxpkg_binary_env(cached_abxpkg_lib_dir, deps_from=hook_config) + output_dir = snapshot.output_dir / plugin + output_dir.mkdir(parents=True, exist_ok=True) + (snapshot.output_dir / "source.txt").write_text("real admin projection input", encoding="utf-8") + bus = create_bus(name=f"test_admin_hashes_{snapshot.id}") + HookProcessService(bus, emit_jsonl=False, interactive_tty=False) + PersistedProcessService(bus) + ArchiveResultService(bus) + + async def run() -> None: + try: + snapshot_event = SnapshotEvent( + url=snapshot.url, + snapshot_id=str(snapshot.id), + output_dir=str(snapshot.output_dir), + ) + await bus.emit(snapshot_event).now() + await bus.emit( + ProcessEvent( + plugin_name=plugin, + hook_name=hook_name, + hook_path=str(hook_path), + hook_args=[f"--url={snapshot.url}"], + env={ + **binary_env, + "ABXPKG_LIB_DIR": str(cached_abxpkg_lib_dir), + "SNAP_DIR": str(snapshot.output_dir), + "PATH": f"{Path(os.sys.executable).parent}{os.pathsep}{os.environ['PATH']}", + }, + output_dir=str(output_dir), + timeout=60, + is_background=False, + url=snapshot.url, + process_type="hook", + worker_type="hook", + event_parent_id=snapshot_event.event_id, + ), + ).now() + await bus.wait_until_idle() + finally: + await bus.destroy(clear=False) + + asyncio.run(run()) + process = Process.objects.filter(pwd=str(output_dir)).order_by("-created_at").first() + assert process is not None + process.refresh_from_db() + assert process.exit_code == 0, (process.stdout, process.stderr) + result = ArchiveResult.objects.get(snapshot=snapshot, plugin=plugin, hook_name=hook_name) + return process, result class TestMachineAdmin: def test_binary_change_view_renders(self, client, admin_user, db): """Binary admin change form should load without FieldError.""" - from archivebox.machine.models import Machine, Binary + from archivebox.machine.models import Machine machine = Machine.objects.create( guid=f"test-guid-{uuid.uuid4()}", @@ -31,64 +124,36 @@ class TestMachineAdmin: os_kernel="test-kernel", stats={}, ) - binary = Binary.objects.create( - machine=machine, - name="gallery-dl", - binproviders="env", - binprovider="env", - abspath="/opt/homebrew/bin/gallery-dl", - version="1.26.9", - sha256="abc123", - status=Binary.StatusChoices.INSTALLED, - ) + binary = install_real_binary("python3", machine=machine) client.force_login(admin_user) url = f"/admin/machine/binary/{binary.pk}/change/" response = client.get(url, HTTP_HOST=ADMIN_TEST_HOST) assert response.status_code == 200 - assert b"gallery-dl" in response.content + assert binary.name.encode() in response.content + assert binary.version.encode() in response.content - def test_process_change_view_renders_copyable_cmd_env_and_readonly_runtime_fields(self, client, admin_user, db): + def test_process_change_view_renders_copyable_cmd_env_and_readonly_runtime_fields( + self, + client, + admin_user, + real_exited_hook_process, + ): from datetime import timedelta - from archivebox.machine.models import Machine, Process - machine = Machine.objects.create( - guid=f"test-guid-{uuid.uuid4()}", - hostname="test-host", - hw_in_docker=False, - hw_in_vm=False, - hw_manufacturer="Test", - hw_product="Test Product", - hw_uuid=f"test-hw-{uuid.uuid4()}", - os_arch="x86_64", - os_family="darwin", - os_platform="darwin", - os_release="test", - os_kernel="test-kernel", - stats={}, - ) - process = Process.objects.create( - machine=machine, - process_type=Process.TypeChoices.HOOK, - status=Process.StatusChoices.EXITED, - pwd="/tmp/archivebox", - cmd=["python", "/tmp/job.py", "--url=https://example.com"], - env={ + process = real_exited_hook_process + process.env.update( + { "ENABLED": True, "API_KEY": "super-secret-key", "ACCESS_TOKEN": "super-secret-token", "SHARED_SECRET": "super-secret-secret", }, - timeout=90, - pid=54321, - exit_code=0, - stdout="job stdout\nline 2", - stderr="job stderr\nline 2", - url="https://example.com/status", - started_at=timezone.now() - timedelta(seconds=52), - ended_at=timezone.now(), ) + process.started_at = timezone.now() - timedelta(seconds=52) + process.ended_at = timezone.now() + process.save(update_fields=["env", "started_at", "ended_at"]) client.force_login(admin_user) url = reverse("admin:machine_process_change", args=[process.pk]) @@ -96,10 +161,9 @@ class TestMachineAdmin: assert response.status_code == 200 assert b"Kill" in response.content - assert b"python /tmp/job.py --url=https://example.com" in response.content + assert b"on_Snapshot__93_hashes.py" in response.content assert b"ENABLED=True" in response.content - assert b"job stdout" in response.content - assert b"job stderr" in response.content + assert b"ArchiveResult" in response.content assert b"52s" in response.content assert b"API_KEY=" not in response.content assert b"ACCESS_TOKEN=" not in response.content @@ -117,16 +181,10 @@ class TestMachineAdmin: assert b'name="started_at"' not in response.content assert b'name="ended_at"' not in response.content - def test_process_kill_object_action_is_post_only(self, admin_client, db): - from archivebox.machine.models import Machine, Process + def test_process_kill_object_action_is_post_only(self, admin_client, real_exited_hook_process): + from archivebox.machine.models import Process - process = Process.objects.create( - machine=Machine.current(refresh=True), - process_type=Process.TypeChoices.HOOK, - pwd="/tmp/exited", - cmd=["/tmp/on_Snapshot__06_wget.finite.bg.py", "--url=https://example.com"], - status=Process.StatusChoices.EXITED, - ) + process = real_exited_hook_process action_url = reverse("admin:machine_process_actions", kwargs={"pk": process.pk, "tool": "kill_process"}) change_response = admin_client.get(reverse("admin:machine_process_change", args=[process.pk]), HTTP_HOST=ADMIN_TEST_HOST) @@ -143,46 +201,14 @@ class TestMachineAdmin: process.refresh_from_db() assert process.status == Process.StatusChoices.EXITED - def test_process_list_view_shows_duration_snapshot_and_crawl_columns(self, client, admin_user, snapshot, db): - from datetime import timedelta - from archivebox.core.models import ArchiveResult - from archivebox.machine.models import Machine, Process - - machine = Machine.objects.create( - guid=f"list-guid-{uuid.uuid4()}", - hostname="list-host", - hw_in_docker=False, - hw_in_vm=False, - hw_manufacturer="Test", - hw_product="Test Product", - hw_uuid=f"list-hw-{uuid.uuid4()}", - os_arch="x86_64", - os_family="darwin", - os_platform="darwin", - os_release="test", - os_kernel="test-kernel", - stats={}, - ) - process = Process.objects.create( - machine=machine, - process_type=Process.TypeChoices.HOOK, - status=Process.StatusChoices.EXITED, - pwd="/tmp/archivebox", - cmd=["python", "/tmp/job.py"], - env={}, - pid=12345, - exit_code=0, - started_at=timezone.now() - timedelta(milliseconds=10), - ended_at=timezone.now(), - ) - ArchiveResult.objects.create( - snapshot=snapshot, - process=process, - plugin="title", - hook_name="on_Snapshot__54_title", - status="succeeded", - output_str="Example Domain", - ) + def test_process_list_view_shows_duration_snapshot_and_crawl_columns( + self, + client, + admin_user, + snapshot, + real_projected_hash_result, + ): + process, result = real_projected_hash_result client.force_login(admin_user) response = client.get(reverse("admin:machine_process_changelist"), HTTP_HOST=ADMIN_TEST_HOST) @@ -191,7 +217,9 @@ class TestMachineAdmin: assert b"Duration" in response.content assert b"Snapshot" in response.content assert b"Crawl" in response.content - assert b"0.01s" in response.content + assert result.status == "succeeded" + assert process.started_at is not None + assert process.ended_at is not None changelist = response.context["cl"] row = next(obj for obj in changelist.result_list if obj.pk == process.pk) diff --git a/archivebox/tests/test_ui_admin_snapshot.py b/archivebox/tests/test_ui_admin_snapshot.py index 908b1df9..7d9e6c5e 100644 --- a/archivebox/tests/test_ui_admin_snapshot.py +++ b/archivebox/tests/test_ui_admin_snapshot.py @@ -1,8 +1,9 @@ """Snapshot model and admin UI tests.""" -import json +import shutil import warnings from pathlib import Path +from threading import Thread from types import SimpleNamespace import pytest @@ -10,11 +11,112 @@ from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME from django.core.paginator import UnorderedObjectListWarning from django.test import RequestFactory from django.urls import reverse -from django.utils import timezone from archivebox.tests.conftest import ADMIN_TEST_HOST +from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook -pytestmark = pytest.mark.django_db +pytestmark = pytest.mark.django_db(transaction=True) +REPO_ROOT = Path(__file__).resolve().parents[2] + + +@pytest.fixture +def real_hash_projection(snapshot, cached_abxpkg_lib_dir): + snapshot.output_dir.mkdir(parents=True, exist_ok=True) + (snapshot.output_dir / "source.txt").write_text("real snapshot admin input", encoding="utf-8") + return _run_shipped_snapshot_hook( + snapshot, + plugin="hashes", + hook_name="on_Snapshot__93_hashes.py", + lib_dir=cached_abxpkg_lib_dir, + ) + + +@pytest.fixture +def real_failed_title_projection(snapshot, cached_abxpkg_lib_dir): + return _run_shipped_snapshot_hook( + snapshot, + plugin="title", + hook_name="on_Snapshot__54_title.js", + lib_dir=cached_abxpkg_lib_dir, + expected_exit_codes=(1,), + ) + + +@pytest.fixture +def real_noresults_projection(snapshot, cached_abxpkg_lib_dir): + staticfile_dir = snapshot.output_dir / "staticfile" + staticfile_dir.mkdir(parents=True, exist_ok=True) + (staticfile_dir / "input.txt").write_text("plain text without links", encoding="utf-8") + return _run_shipped_snapshot_hook( + snapshot, + plugin="parse_txt_urls", + hook_name="on_Snapshot__71_parse_txt_urls.py", + lib_dir=cached_abxpkg_lib_dir, + ) + + +@pytest.fixture +def real_parse_projection(snapshot, cached_abxpkg_lib_dir): + staticfile_dir = snapshot.output_dir / "staticfile" + staticfile_dir.mkdir(parents=True, exist_ok=True) + (staticfile_dir / "input.txt").write_text("https://example.org/parsed\n", encoding="utf-8") + return _run_shipped_snapshot_hook( + snapshot, + plugin="parse_txt_urls", + hook_name="on_Snapshot__71_parse_txt_urls.py", + lib_dir=cached_abxpkg_lib_dir, + ) + + +@pytest.fixture +def running_wget_projection(snapshot, blocking_http_server): + from django.utils import timezone + + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl + from archivebox.services.runner import run_due_snapshot + + now = timezone.now() + Crawl.objects.filter(pk=snapshot.crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now, modified_at=now) + Snapshot.objects.filter(pk=snapshot.pk).update( + status=Snapshot.StatusChoices.QUEUED, + retry_at=now, + downloaded_at=None, + url=blocking_http_server.url, + ) + snapshot.refresh_from_db() + [result] = snapshot.create_pending_archiveresults(hooks=[("wget", "on_Snapshot__06_wget.finite.bg")]) + errors = [] + + def run_snapshot(): + try: + assert run_due_snapshot(snapshot, lock_seconds=60) is True + except BaseException as err: + errors.append(err) + finally: + blocking_http_server.request_started.set() + + runner = Thread(target=run_snapshot, name="archivebox-test-admin-wget-runner") + runner.start() + blocking_http_server.request_started.wait() + assert errors == [] + result.refresh_from_db() + assert result.status == ArchiveResult.StatusChoices.STARTED + yield result + blocking_http_server.release_response.set() + runner.join() + assert errors == [] + + +@pytest.fixture +def real_skipped_hash_projection(snapshot, cached_abxpkg_lib_dir): + return _run_shipped_snapshot_hook( + snapshot, + plugin="hashes", + hook_name="on_Snapshot__93_hashes.py", + lib_dir=cached_abxpkg_lib_dir, + env={"HASHES_ENABLED": "False"}, + ) def test_snapshot_changelist_uses_stable_ordering_without_unordered_paginator_warning(admin_client, snapshot): @@ -32,14 +134,13 @@ def test_snapshot_changelist_uses_stable_ordering_without_unordered_paginator_wa assert b"Searching matching snapshots..." in response.content -def test_snapshot_changelist_preview_uses_prefetched_output_files(admin_client, snapshot): +def test_snapshot_changelist_preview_uses_prefetched_output_files(admin_client, snapshot, real_hash_projection): from archivebox.core.models import ArchiveResult - ArchiveResult.objects.create( - snapshot=snapshot, + _process, result = real_hash_projection + ArchiveResult.objects.filter(pk=result.pk).update( plugin="screenshot", hook_name="on_Snapshot__40_screenshot.js", - status=ArchiveResult.StatusChoices.SUCCEEDED, output_size=128, output_files={"screenshot.png": {"size": 128, "root_relative": True}}, ) @@ -54,7 +155,7 @@ def test_snapshot_changelist_preview_uses_prefetched_output_files(admin_client, def test_snapshot_admin_tag_editor_escapes_tag_json_script_breakout(admin_client, snapshot): from archivebox.core.models import Tag - tag = Tag.objects.create(name="legacy-safe-tag") + tag = Tag.objects.create(name="safe-tag") snapshot.tags.add(tag) malicious_name = '' Tag.objects.filter(pk=tag.pk).update(name=malicious_name) @@ -69,17 +170,12 @@ def test_snapshot_admin_tag_editor_escapes_tag_json_script_breakout(admin_client assert b"</script><script id="archivebox-tag-xss">" in body -def test_snapshot_admin_archive_results_escape_extractor_output(admin_client, snapshot): +def test_snapshot_admin_archive_results_escape_extractor_output(admin_client, snapshot, real_hash_projection): from archivebox.core.models import ArchiveResult payload = '' - ArchiveResult.objects.create( - snapshot=snapshot, - plugin="title", - hook_name="on_Snapshot__54_title.js", - status=ArchiveResult.StatusChoices.SUCCEEDED, - output_str=payload, - ) + _process, result = real_hash_projection + ArchiveResult.objects.filter(pk=result.pk).update(output_str=payload) response = admin_client.get(reverse("admin:core_snapshot_change", args=[snapshot.pk]), HTTP_HOST=ADMIN_TEST_HOST) body = response.content @@ -90,67 +186,28 @@ def test_snapshot_admin_archive_results_escape_extractor_output(admin_client, sn assert b"<img src=x onerror="window.__archivebox_archiveresult_xss__=1">" in body -def test_snapshot_admin_archive_result_table_escapes_legacy_string_fields(admin_client, snapshot): - from uuid import uuid4 - +def test_snapshot_admin_archive_result_table_escapes_persisted_string_fields(admin_client, snapshot, real_hash_projection): from archivebox.core.models import ArchiveResult - from archivebox.machine.models import Binary, Machine, Process - machine = Machine.objects.create( - guid=f"xss-machine-{uuid4()}", - hostname='', - hw_in_docker=False, - hw_in_vm=False, - hw_manufacturer="Test", - hw_product="Test Product", - hw_uuid=f"xss-hw-{uuid4()}", - os_arch="arm64", - os_family="darwin", - os_platform="macOS", - os_release="14.0", - os_kernel="Darwin", - stats={}, - config={}, - ) - binary = Binary.objects.create( - machine=machine, - name="staticfile", - abspath="/usr/bin/staticfile", - version='v', - binprovider="env", - binproviders="env", - status=Binary.StatusChoices.INSTALLED, - ) - process = Process.objects.create( - machine=machine, - binary=binary, - process_type=Process.TypeChoices.HOOK, - pwd='/tmp/archivebox">', - cmd=["staticfile"], - status=Process.StatusChoices.EXITED, - ) - result = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="staticfile", - hook_name="on_Snapshot__00_staticfile.py", - process=process, - status=ArchiveResult.StatusChoices.SUCCEEDED, + process, result = real_hash_projection + machine = process.machine + type(machine).objects.filter(pk=machine.pk).update(hostname='') + type(process).objects.filter(pk=process.pk).update(pwd='/tmp/archivebox">') + ArchiveResult.objects.filter(pk=result.pk).update( output_files={'evil">.txt': {"size": 12, "mimetype": "text/plain"}}, - output_str='staticfile/evil">.txt', + output_str='hashes/evil">.txt', + plugin='', ) - ArchiveResult.objects.filter(pk=result.pk).update(plugin='') response = admin_client.get(reverse("admin:core_snapshot_change", args=[snapshot.pk]), HTTP_HOST=ADMIN_TEST_HOST) body = response.content assert response.status_code == 200 assert b'