ArchiveBox/.github/workflows/test.yml

name: Install and CLI Compatibility

on:
  workflow_call:

env:
  DOCKER_IMAGE: archivebox-ci
  ARCHIVEBOX_IMAGE: archivebox-ci
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
  UV_NO_SOURCES: "1"

jobs:
  install_cli_platform:
    name: CLI install / ${{ matrix.os_name }} / Python ${{ matrix.python }}
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-24.04
            os_name: linux
            python: "3.14"
          - os: macos-15
            os_name: macOS
            python: "3.13"
          - os: macos-15
            os_name: macOS
            python: "3.14"

    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
        with:
          submodules: true
          fetch-depth: 1

      ### Set up Python
      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: ${{ matrix.python }}

      - name: Install uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
        with:
          version: "0.11.3"

      - name: Prepare abxpkg environment
        env:
          LIB_DIR: ${{ runner.temp }}/abx-lib
          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
        run: |
          set -Eeuo pipefail
          ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
          test -n "$ABXPKG_VERSION"
          mkdir -p "$LIB_DIR/env/bin"
          {
            echo "ABXPKG_VERSION=$ABXPKG_VERSION"
            echo "LIB_DIR=$LIB_DIR"
            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
          } >> "$GITHUB_ENV"
          echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"          

      - name: Resolve CI tools through abxpkg
        env:
          LIB_DIR: ${{ runner.temp }}/abx-lib
          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
        run: |
          set -Eeuo pipefail
          export PATH="$LIB_DIR/env/bin:$PATH"
          tooling_env="$(
            uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
              --install \
              --json \
              --lib="$LIB_DIR" \
              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
          )"
          JQ_BINARY="$LIB_DIR/env/bin/jq"
          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
            <<< "$tooling_env" >> "$GITHUB_ENV"
          echo "JQ_BINARY=$JQ_BINARY" >> "$GITHUB_ENV"
          test -L "$JQ_BINARY"
          test -x "$JQ_BINARY"          

      - name: Resolve Linux build dependencies through abxpkg
        if: runner.os == 'Linux'
        env:
          LIB_DIR: ${{ runner.temp }}/abx-lib
          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
        run: |
          set -Eeuo pipefail
          uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
            --install \
            --no-cache \
            --json \
            --lib="$LIB_DIR" \
            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries"          

      - name: Resolve host Homebrew through abxpkg
        if: runner.os == 'macOS'
        env:
          LIB_DIR: ${{ runner.temp }}/abx-lib
          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
        run: |
          set -Eeuo pipefail
          brew_env="$(
            uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
              --install \
              --json \
              --lib="$LIB_DIR" \
              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries"
          )"
          brew_binary="$("$JQ_BINARY" -r '.BREW_BINARY' <<< "$brew_env")"
          test -L "$brew_binary"
          echo "BREW_BINARY=$brew_binary" >> "$GITHUB_ENV"          

      - name: Resolve macOS build dependencies through abxpkg
        if: runner.os == 'macOS'
        env:
          LIB_DIR: ${{ runner.temp }}/abx-lib
          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
        run: |
          set -Eeuo pipefail
          brew_target="$(readlink "$BREW_BINARY")"
          test -x "$brew_target"
          brew_root="$(dirname "$(dirname "$brew_target")")"
          export ABXPKG_BREW_ROOT="$brew_root"

          uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
            --install \
            --json \
            --lib="$LIB_DIR" \
            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-build.json:required_binaries"

          PATH="$brew_root/opt/openldap/bin:$PATH" \
            uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
              --install \
              --json \
              --lib="$LIB_DIR" \
              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-openldap-env.json:required_binaries" \
            | "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
              >> "$GITHUB_ENV"

          ldapvc_target="$(readlink "$LIB_DIR/env/bin/ldapvc")"
          while [[ -L "$ldapvc_target" ]]; do
            next_ldapvc_target="$(readlink "$ldapvc_target")"
            if [[ "$next_ldapvc_target" == /* ]]; then
              ldapvc_target="$next_ldapvc_target"
            else
              ldapvc_target="$(dirname "$ldapvc_target")/$next_ldapvc_target"
            fi
          done
          test -x "$ldapvc_target"
          openldap_prefix="$(dirname "$(dirname "$ldapvc_target")")"
          test -f "$openldap_prefix/include/ldap.h"
          test -f "$openldap_prefix/lib/libldap.dylib"
          {
            echo "ABXPKG_BREW_ROOT=$brew_root"
            echo "CPPFLAGS=-I$openldap_prefix/include${CPPFLAGS:+ $CPPFLAGS}"
            echo "LDFLAGS=-L$openldap_prefix/lib${LDFLAGS:+ $LDFLAGS}"
            echo "PKG_CONFIG_PATH=$openldap_prefix/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}"
          } >> "$GITHUB_ENV"          

      - name: Install dependencies with uv
        run: uv sync --locked --dev --all-extras

      - name: ArchiveBox full install check
        run: |
          set -Eeuo pipefail
          DATA_DIR="$PWD/tests/out/data"
          mkdir -p "$DATA_DIR"
          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox init
          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox install
          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox version
          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status          

  docker_tests:
    name: Docker CLI and mount compatibility
    runs-on: ubuntu-24.04

    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
        with:
          submodules: true
          fetch-depth: 1

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: "3.13"

      - name: Install uv
        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
        with:
          version: "0.11.3"

      - name: Resolve Docker and curl through abxpkg
        env:
          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
        shell: bash
        run: |
          set -Eeuo pipefail
          ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
          test -n "$ABXPKG_VERSION"
          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
          echo "ABXPKG_VERSION=$ABXPKG_VERSION" >> "$GITHUB_ENV"
          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"

          tooling_env="$(
            uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
              --install \
              --json \
              --lib="$ABXPKG_LIB_DIR" \
              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" \
              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries"
          )"
          JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
            <<< "$tooling_env" >> "$GITHUB_ENV"

          for binary in "$ABXPKG_LIB_DIR/env/bin/docker" "$ABXPKG_LIB_DIR/env/bin/curl"; do
            test -L "$binary"
            test -x "$binary"
          done
          {
            echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
            echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
            echo "JQ_BINARY=$JQ_BINARY"
          } >> "$GITHUB_ENV"          

      - name: Build image
        run: |
          "$DOCKER_BINARY" build . -t "$DOCKER_IMAGE"          

      - name: Resolve real mount test dependencies through abxpkg
        run: |
          set -Eeuo pipefail
          mount_env="$(
            uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
              --install \
              --json \
              --lib="$ABXPKG_LIB_DIR" \
              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries"
          )"
          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
            <<< "$mount_env" >> "$GITHUB_ENV"

          for binary_name in bash chmod exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do
            binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
            test -L "$binary"
            test -x "$binary"
          done          

      - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
        shell: bash
        run: |
          set -Eeuo pipefail
          NFS_SOURCE="/srv/archivebox-ci-nfs"
          SMB_SOURCE="/srv/archivebox-ci-smb"
          NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
          SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
          mounted_nfs=0
          mounted_smb=0

          cleanup() {
            local original_status=$? cleanup_status=0
            trap - EXIT
            if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
              cleanup_status=1
            fi
            if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
              cleanup_status=1
            fi
            if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
              cleanup_status=1
            fi
            if [[ "$original_status" != "0" ]]; then
              exit "$original_status"
            fi
            exit "$cleanup_status"
          }
          trap cleanup EXIT

          "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
          "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
          "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
          "$SUDO_BINARY" "$BASH_BINARY" -c \
            "printf '%s *(rw,sync,no_subtree_check,no_root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
          "$SUDO_BINARY" "$BASH_BINARY" -c \
            "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"

          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
          "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
          "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
          "$SYSTEMCTL_BINARY" is-active --quiet smbd.service

          "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
          mounted_nfs=1
          "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
            -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
          mounted_smb=1

          NFS_TEST_DIR="$NFS_MOUNT" \
          SMB_TEST_DIR="$SMB_MOUNT" \
          IMAGE="$DOCKER_IMAGE" \
            "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only          

      - name: Init data dir
        run: |
          "$MKDIR_BINARY" "${{ github.workspace }}/data"
          "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init          

      - name: Run test server
        run: |
          "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
          "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e          

      - name: Add link
        run: |
          "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local          

      - name: Add stdin link
        run: |
          echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add          

      - name: List links
        run: |
          archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$DOCKER_IMAGE" list)"
          [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
          [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }          

      - name: Start docker-compose stack
        run: |
          "$DOCKER_BINARY" compose run --rm archivebox init
          "$DOCKER_BINARY" compose up -d --wait
          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')"
          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js')"
          [[ "$index_html" == *ArchiveBox* ]]
          [[ "$admin_js" == *window.django* ]]          

      - name: Check added urls show up in index
        run: |
          "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://example.com/#test_docker'
          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')"
          [[ "$index_html" == *'http://example.com/#test_docker'* ]]
          "$DOCKER_BINARY" compose down