mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-14 11:06:13 +05:00
7167c5dd8a
Add DATABASE_ENGINE=postgres (plus DATABASE_HOST/PORT/USER/PASSWORD/NAME) config and centralize all sqlite-vs-postgres branching in archivebox.misc.db: - get_database_settings() builds DATABASES for either backend; the sqlite path is unchanged (custom lock-retry backend, same PRAGMAs). - database_exists()/ensure_database_ready() replace index.sqlite3 file checks; init auto-creates the postgres database when missing. - approximate_row_counts() serves admin index counts from sqlite_stat1 or pg_class.reltuples; missing-table detection covers both vendors. - rebuild_models_from_migration_state() lets historical sqlite-only raw SQL migrations resync postgres schema from Django migration state at every divergence point (postgres can never hold legacy data, so affected tables are empty when these run). All raw-DDL and PRAGMA migrations are now vendor-gated with sqlite behavior byte-for-byte unchanged. - A pre_save clamp truncates CharField values to max_length: sqlite never enforced varchar(n) but postgres does (e.g. long crawl labels). - Collation-sensitive URL range scans branch to escaped LIKE on postgres (with a text_pattern_ops index) since linguistic collations break bytewise range tricks; the crawl-config JSON search wave gets a jsonb-text implementation. Verified on real PostgreSQL 16: fresh init applies the entire migration graph, schema matches models exactly (column-level parity check + makemigrations --check), and add/run/list/search/status/remove all work end-to-end. New test_postgres_backend.py suite boots a real throwaway postgres cluster (initdb + pg_ctl); CI workflows install postgres server binaries. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019YHSjZM6TstSAMN2PhgfUg
509 lines
18 KiB
YAML
509 lines
18 KiB
YAML
name: Parallel Tests
|
|
|
|
on:
|
|
workflow_call:
|
|
|
|
env:
|
|
PYTHONIOENCODING: utf-8
|
|
PYTHONLEGACYWINDOWSSTDIO: utf-8
|
|
USE_COLOR: False
|
|
UV_NO_SOURCES: "1"
|
|
CI_PYTHON_VERSION: "3.13.14"
|
|
UV_CACHE_DIR: ${{ github.workspace }}/.uv-cache
|
|
|
|
jobs:
|
|
discover-tests:
|
|
name: Discover test files
|
|
runs-on: ubuntu-24.04
|
|
outputs:
|
|
test-files: ${{ steps.set-matrix.outputs.test-files }}
|
|
plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
with:
|
|
submodules: true
|
|
fetch-depth: 1
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
|
with:
|
|
python-version: ${{ env.CI_PYTHON_VERSION }}
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
|
with:
|
|
version: "0.11.3"
|
|
enable-cache: false
|
|
|
|
- name: Resolve matrix tools through abxpkg
|
|
env:
|
|
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
shell: bash
|
|
run: |
|
|
set -Eeuo pipefail
|
|
ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
|
|
test -n "$ABXPKG_VERSION"
|
|
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
|
uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
|
|
--install \
|
|
--lib="$ABXPKG_LIB_DIR" \
|
|
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
|
>/dev/null
|
|
JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
|
|
PYTHON_BINARY="$ABXPKG_LIB_DIR/env/bin/python"
|
|
test -L "$JQ_BINARY"
|
|
test -x "$JQ_BINARY"
|
|
test -L "$PYTHON_BINARY"
|
|
test -x "$PYTHON_BINARY"
|
|
{
|
|
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
|
echo "JQ_BINARY=$JQ_BINARY"
|
|
echo "PYTHON_BINARY=$PYTHON_BINARY"
|
|
} >> "$GITHUB_ENV"
|
|
echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Discover test files
|
|
id: set-matrix
|
|
shell: bash
|
|
run: |
|
|
set -Eeuo pipefail
|
|
mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py archivebox)
|
|
test "${#output[@]}" -eq 2
|
|
json_array="${output[1]}"
|
|
echo "test-files=$json_array" >> "$GITHUB_OUTPUT"
|
|
echo "${output[0]}"
|
|
echo "$json_array" | "$JQ_BINARY" '.'
|
|
|
|
- name: Clone abx-plugins
|
|
run: bash .github/scripts/clone_abx_repo.sh abx-plugins
|
|
|
|
- name: Discover plugin tests
|
|
id: set-plugin-matrix
|
|
shell: bash
|
|
run: |
|
|
set -Eeuo pipefail
|
|
mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py plugins)
|
|
test "${#output[@]}" -eq 2
|
|
json_array="${output[1]}"
|
|
echo "plugin-tests=$json_array" >> "$GITHUB_OUTPUT"
|
|
echo "${output[0]}"
|
|
echo "$json_array" | "$JQ_BINARY" '.'
|
|
|
|
prepare-python-dependencies:
|
|
name: Prepare Python 3.13.14 dependency cache
|
|
runs-on: ubuntu-24.04
|
|
env:
|
|
PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
|
|
LIB_DIR: /tmp/abx-lib
|
|
ABXPKG_LIB_DIR: /tmp/abx-lib
|
|
|
|
steps:
|
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
with:
|
|
submodules: true
|
|
fetch-depth: 1
|
|
|
|
- name: Set up Python ${{ env.CI_PYTHON_VERSION }}
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
|
with:
|
|
python-version: ${{ env.CI_PYTHON_VERSION }}
|
|
architecture: x64
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
|
with:
|
|
version: "0.11.3"
|
|
enable-cache: false
|
|
|
|
- name: Clone abxpkg
|
|
run: bash .github/scripts/clone_abx_repo.sh abxpkg
|
|
|
|
- name: Clone abx-plugins
|
|
run: bash .github/scripts/clone_abx_repo.sh abx-plugins
|
|
|
|
- name: Clone abx-dl
|
|
run: bash .github/scripts/clone_abx_repo.sh abx-dl
|
|
|
|
- name: Restore or create the SHA-specific uv cache
|
|
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
|
with:
|
|
path: ${{ env.UV_CACHE_DIR }}
|
|
key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }}
|
|
|
|
- name: Bootstrap local abxpkg
|
|
run: |
|
|
set -Eeuo pipefail
|
|
uv venv
|
|
uv pip install -e ./abxpkg
|
|
|
|
- name: Resolve Linux build dependencies through abxpkg
|
|
run: |
|
|
set -Eeuo pipefail
|
|
mkdir -p "$LIB_DIR/env/bin"
|
|
echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
|
uv run --no-sync --no-sources abxpkg env \
|
|
--install \
|
|
--no-cache \
|
|
--json \
|
|
--lib="$ABXPKG_LIB_DIR" \
|
|
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries"
|
|
uv run --no-sync --no-sources abxpkg env \
|
|
--install \
|
|
--json \
|
|
--lib="$ABXPKG_LIB_DIR" \
|
|
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-env.json:required_binaries"
|
|
|
|
- name: Populate the full editable dependency cache
|
|
run: |
|
|
set -Eeuo pipefail
|
|
uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
|
|
uv run --no-sync --no-sources python -c 'import ldap; print(ldap.__version__)'
|
|
|
|
run-tests:
|
|
name: ${{ matrix.test.name }}
|
|
runs-on: ubuntu-24.04
|
|
needs: [discover-tests, prepare-python-dependencies]
|
|
env:
|
|
PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
|
|
CHROME_HEADLESS: "true"
|
|
PERSONAS_DIR: /tmp/abx-personas
|
|
CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
|
|
python: ["3.13"]
|
|
|
|
steps:
|
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
with:
|
|
submodules: true
|
|
fetch-depth: 1
|
|
|
|
- name: Set up Python ${{ matrix.python }}
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
|
with:
|
|
python-version: ${{ matrix.python }}
|
|
architecture: x64
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
|
with:
|
|
version: "0.11.3"
|
|
enable-cache: false
|
|
|
|
- name: Clone abxpkg
|
|
run: bash .github/scripts/clone_abx_repo.sh abxpkg
|
|
|
|
- name: Clone abx-plugins
|
|
run: bash .github/scripts/clone_abx_repo.sh abx-plugins
|
|
|
|
- name: Clone abx-dl
|
|
run: bash .github/scripts/clone_abx_repo.sh abx-dl
|
|
|
|
- name: Cache uv
|
|
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
|
with:
|
|
path: ${{ env.UV_CACHE_DIR }}
|
|
key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }}
|
|
fail-on-cache-miss: true
|
|
|
|
- name: Install dependencies with uv
|
|
run: |
|
|
set -Eeuo pipefail
|
|
uv venv
|
|
uv pip install --offline --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
|
|
|
|
- name: Resolve Node.js through abxpkg
|
|
env:
|
|
LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
run: |
|
|
set -Eeuo pipefail
|
|
mkdir -p "$LIB_DIR/env/bin"
|
|
export PATH="$LIB_DIR/env/bin:$PATH"
|
|
{
|
|
echo "LIB_DIR=$LIB_DIR"
|
|
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
|
} >> "$GITHUB_ENV"
|
|
echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
|
|
|
tooling_env="$(
|
|
uv run --no-sync --no-sources abxpkg env \
|
|
--install \
|
|
--json \
|
|
--lib="$LIB_DIR" \
|
|
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
|
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries"
|
|
)"
|
|
JQ_BINARY="$LIB_DIR/env/bin/jq"
|
|
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
|
<<< "$tooling_env" >> "$GITHUB_ENV"
|
|
|
|
node_binary="$LIB_DIR/env/bin/node"
|
|
{
|
|
echo "JQ_BINARY=$JQ_BINARY"
|
|
echo "NODE_BINARY=$node_binary"
|
|
} >> "$GITHUB_ENV"
|
|
test -L "$JQ_BINARY"
|
|
test -x "$JQ_BINARY"
|
|
test -L "$node_binary"
|
|
test -x "$node_binary"
|
|
"$node_binary" --version
|
|
|
|
- name: Resolve core test binaries through abxpkg
|
|
env:
|
|
LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p "$LIB_DIR/env/bin"
|
|
{
|
|
echo "LIB_DIR=$LIB_DIR"
|
|
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
|
} >> "$GITHUB_ENV"
|
|
echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
|
uv run --no-sync --no-sources abxpkg env \
|
|
--install \
|
|
--json \
|
|
--lib="$LIB_DIR" \
|
|
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/wget/config.json:required_binaries" \
|
|
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/git/config.json:required_binaries" \
|
|
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_ripgrep/config.json:required_binaries" \
|
|
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
|
>> "$GITHUB_ENV"
|
|
|
|
- name: Resolve Chrome through abxpkg
|
|
if: ${{ matrix.test.needs_chromium }}
|
|
env:
|
|
LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p "$LIB_DIR"
|
|
{
|
|
echo "LIB_DIR=$LIB_DIR"
|
|
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
|
} >> "$GITHUB_ENV"
|
|
|
|
uv run --no-sync --no-sources abxpkg env \
|
|
--install \
|
|
--json \
|
|
--lib="$LIB_DIR" \
|
|
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \
|
|
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
|
>> "$GITHUB_ENV"
|
|
|
|
- name: Resolve Sonic through abxpkg
|
|
if: ${{ matrix.test.needs_sonic }}
|
|
env:
|
|
LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p "$LIB_DIR"
|
|
{
|
|
echo "LIB_DIR=$LIB_DIR"
|
|
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
|
} >> "$GITHUB_ENV"
|
|
|
|
uv run --no-sync --no-sources abxpkg env \
|
|
--install \
|
|
--json \
|
|
--lib="$LIB_DIR" \
|
|
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \
|
|
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
|
>> "$GITHUB_ENV"
|
|
|
|
- name: Install PostgreSQL server binaries
|
|
run: |
|
|
set -Eeuo pipefail
|
|
if ! ls /usr/lib/postgresql/*/bin/initdb >/dev/null 2>&1 && ! command -v initdb >/dev/null 2>&1; then
|
|
sudo apt-get update && sudo apt-get install -y postgresql
|
|
fi
|
|
|
|
- name: Run test - ${{ matrix.test.name }}
|
|
env:
|
|
TEST_PATHS_JSON: ${{ toJson(matrix.test.paths) }}
|
|
run: |
|
|
set -Eeuo pipefail
|
|
mapfile -t test_paths < <("$JQ_BINARY" -r '.[]' <<<"$TEST_PATHS_JSON")
|
|
[[ "${#test_paths[@]}" -gt 0 ]]
|
|
mkdir -p tests/out
|
|
uv run --no-sync --no-sources pytest -vs "${test_paths[@]}" --basetemp="$GITHUB_WORKSPACE/tests/out"
|
|
|
|
plugin-tests:
|
|
name: ${{ matrix.plugin.name }}
|
|
runs-on: ubuntu-24.04
|
|
needs: [discover-tests, prepare-python-dependencies]
|
|
env:
|
|
PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
|
|
CHROME_HEADLESS: "true"
|
|
PERSONAS_DIR: /tmp/abx-personas
|
|
CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
with:
|
|
submodules: true
|
|
fetch-depth: 1
|
|
|
|
- name: Set up Python ${{ env.CI_PYTHON_VERSION }}
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
|
with:
|
|
python-version: ${{ env.CI_PYTHON_VERSION }}
|
|
architecture: x64
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
|
with:
|
|
version: "0.11.3"
|
|
enable-cache: false
|
|
|
|
- name: Clone abxpkg
|
|
run: bash .github/scripts/clone_abx_repo.sh abxpkg
|
|
|
|
- name: Clone abx-plugins
|
|
run: bash .github/scripts/clone_abx_repo.sh abx-plugins
|
|
|
|
- name: Clone abx-dl
|
|
run: bash .github/scripts/clone_abx_repo.sh abx-dl
|
|
|
|
- name: Cache uv
|
|
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
|
with:
|
|
path: ${{ env.UV_CACHE_DIR }}
|
|
key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }}
|
|
fail-on-cache-miss: true
|
|
|
|
- name: Install dependencies with uv
|
|
run: |
|
|
set -Eeuo pipefail
|
|
uv venv
|
|
uv pip install --offline --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
|
|
|
|
- name: Resolve Node.js through abxpkg
|
|
env:
|
|
LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
run: |
|
|
set -Eeuo pipefail
|
|
mkdir -p "$LIB_DIR/env/bin"
|
|
export PATH="$LIB_DIR/env/bin:$PATH"
|
|
{
|
|
echo "LIB_DIR=$LIB_DIR"
|
|
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
|
} >> "$GITHUB_ENV"
|
|
echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
|
|
|
tooling_env="$(
|
|
uv run --no-sync --no-sources abxpkg env \
|
|
--install \
|
|
--json \
|
|
--lib="$LIB_DIR" \
|
|
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
|
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries"
|
|
)"
|
|
JQ_BINARY="$LIB_DIR/env/bin/jq"
|
|
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
|
<<< "$tooling_env" >> "$GITHUB_ENV"
|
|
|
|
node_binary="$LIB_DIR/env/bin/node"
|
|
{
|
|
echo "JQ_BINARY=$JQ_BINARY"
|
|
echo "NODE_BINARY=$node_binary"
|
|
} >> "$GITHUB_ENV"
|
|
test -L "$JQ_BINARY"
|
|
test -x "$JQ_BINARY"
|
|
test -L "$node_binary"
|
|
test -x "$node_binary"
|
|
"$node_binary" --version
|
|
|
|
- name: Resolve plugin dependencies through abxpkg
|
|
env:
|
|
LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p "$LIB_DIR/env/bin"
|
|
{
|
|
echo "LIB_DIR=$LIB_DIR"
|
|
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
|
} >> "$GITHUB_ENV"
|
|
echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
|
uv run --no-sync --no-sources abxpkg env \
|
|
--install \
|
|
--json \
|
|
--lib="$LIB_DIR" \
|
|
--deps-from="$GITHUB_WORKSPACE/${{ matrix.plugin.config_path }}:required_binaries" \
|
|
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
|
>> "$GITHUB_ENV"
|
|
|
|
- name: Resolve Chrome through abxpkg
|
|
if: ${{ matrix.plugin.needs_chromium }}
|
|
env:
|
|
LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p "$LIB_DIR"
|
|
{
|
|
echo "LIB_DIR=$LIB_DIR"
|
|
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
|
} >> "$GITHUB_ENV"
|
|
|
|
uv run --no-sync --no-sources abxpkg env \
|
|
--install \
|
|
--json \
|
|
--lib="$LIB_DIR" \
|
|
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \
|
|
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
|
>> "$GITHUB_ENV"
|
|
|
|
- name: Resolve Sonic through abxpkg
|
|
if: ${{ matrix.plugin.needs_sonic }}
|
|
env:
|
|
LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p "$LIB_DIR"
|
|
{
|
|
echo "LIB_DIR=$LIB_DIR"
|
|
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
|
} >> "$GITHUB_ENV"
|
|
|
|
uv run --no-sync --no-sources abxpkg env \
|
|
--install \
|
|
--json \
|
|
--lib="$LIB_DIR" \
|
|
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \
|
|
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
|
>> "$GITHUB_ENV"
|
|
|
|
- name: Run plugin tests - ${{ matrix.plugin.name }}
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
|
TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
|
|
API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
|
|
run: |
|
|
set -Eeuo pipefail
|
|
test_path="$GITHUB_WORKSPACE/${{ matrix.plugin.test_path }}"
|
|
test -d "$test_path"
|
|
test_count="$("$LIB_DIR/env/bin/python" -c 'import pathlib, sys; print(sum(1 for _ in pathlib.Path(sys.argv[1]).rglob("test_*.py")))' "$test_path")"
|
|
test "$test_count" -gt 0
|
|
DATA_DIR="$(mktemp -d -t archivebox_plugin_tests.XXXXXX)"
|
|
export DATA_DIR
|
|
plugin_tmpdir="$(mktemp -d -t archivebox_plugin_run.XXXXXX)"
|
|
cd "$plugin_tmpdir"
|
|
uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources python -m pytest \
|
|
"$test_path" -p no:django -v --tb=short
|