name: Parallel Tests

on:
  workflow_dispatch:
    inputs:
      test_file:
        description: Run one discovered test file
        required: true
        default: archivebox/tests/test_cli_list.py
        type: string
  workflow_call:
    inputs:
      test_file:
        description: Run one discovered test file; empty runs the complete matrix
        required: false
        default: ""
        type: string

env:
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
  UV_NO_SOURCES: "1"
  CI_PYTHON_VERSION: "3.13"

jobs:
  discover-tests:
    name: Discover every test
    # Selection must run on GitHub so an offline NAS cannot block the fallback.
    runs-on: ubuntu-24.04
    outputs:
      tests: ${{ steps.discover.outputs.tests }}
      runner: ${{ steps.runner.outputs.use-runner || '["ubuntu-24.04"]' }}

    steps:
      - name: Select available CI capacity
        id: runner
        if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
        uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
        with:
          github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}

      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
        with:
          submodules: true
          fetch-depth: 1

      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: ${{ env.CI_PYTHON_VERSION }}

      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
        with:
          version: "0.11.3"
          enable-cache: false
          cache-dependency-glob: uv.lock

      - name: Resolve discovery tools through abxpkg
        env:
          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
        run: |
          set -Eeuo pipefail
          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
          test -n "$ABXPKG_SPEC"
          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
          export PYTHON_BINARY="$pythonLocation/bin/python"
          test -x "$PYTHON_BINARY"
          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
            --install \
            --lib="$ABXPKG_LIB_DIR" \
            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
          {
            echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
          } >> "$GITHUB_ENV"
          test -L "$ABXPKG_LIB_DIR/env/bin/python"
          test -x "$ABXPKG_LIB_DIR/env/bin/python"
          test -L "$ABXPKG_LIB_DIR/env/bin/jq"
          test -x "$ABXPKG_LIB_DIR/env/bin/jq"

      - name: Discover every test exactly once
        id: discover
        env:
          TEST_FILE: ${{ inputs.test_file }}
          UGNAS_CI_MAX_JOBS: ${{ vars.UGNAS_CI_MAX_JOBS || 3 }}
        run: |
          set -Eeuo pipefail
          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
          test "${#output[@]}" -eq 2
          if [[ -n "$TEST_FILE" ]]; then
            output[1]="$("$JQ_BINARY" -ce '[.[] | select(.paths == [env.TEST_FILE])] | select(length == 1)' <<< "${output[1]}")"
          fi
          echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
          echo "${output[0]}"
          "$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"

  tests:
    name: ${{ matrix.test.name }}
    needs: discover-tests
    # Opt in trusted branches only; PRs always use GitHub-hosted runners.
    # Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
    # The hosted discovery job falls back when workers are offline or the
    # availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
    runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
    env:
      CHROME_HEADLESS: "true"

    strategy:
      fail-fast: false
      matrix:
        test: ${{ fromJson(needs.discover-tests.outputs.tests) }}

    steps:
      - name: Set job-local runtime paths
        env:
          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
        run: |
          workers=2
          cache_suffix=''
          if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
            workers="$(nproc)"
            cache_suffix="-workers-$workers"
            # Avoid the hosted image's cold Chromium executable page faults.
            # abxpkg owns installation and reuse in the existing runtime cache.
            echo "CHROME_BINPROVIDERS=playwright,puppeteer" >> "$GITHUB_ENV"
          fi
          {
            echo "CI_TEST_WORKERS=$workers"
            echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
            echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
            echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
            echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
            echo "CHROME_USER_DATA_DIR=$RUNNER_TEMP/abx-personas/Default/chrome_profile"
          } >> "$GITHUB_ENV"

      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
        with:
          submodules: true
          fetch-depth: 1

      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: ${{ env.CI_PYTHON_VERSION }}
          architecture: x64
        id: python

      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
        with:
          version: "0.11.3"
          enable-cache: false
          cache-dependency-glob: uv.lock

      # Self-hosted tool storage survives jobs. Normal abxpkg validation and
      # install locks allow reuse across files; cold-install tests use tmp_path.
      - name: Reuse persistent runner tools
        if: runner.environment == 'self-hosted'
        env:
          PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
        run: |
          mkdir -p "$PERSISTENT_ABXPKG_LIB"
          echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"

      # Restored binaries still pass normal abxpkg resolution/version checks.
      # Installation and isolation tests explicitly choose fresh temporary libs.
      - name: Identify runtime dependency cache
        shell: bash
        run: |
          uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
          import hashlib
          import json
          import tomllib
          from pathlib import Path

          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
          versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
          print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
          PYTHON

      - name: Reuse validated runtime dependencies
        if: runner.environment != 'self-hosted'
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
        with:
          path: ${{ runner.temp }}/archivebox-lib
          key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
          restore-keys: |
            abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-

      - name: Resolve optional build dependencies through abxpkg
        if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
        env:
          TEST_EXTRA: ${{ matrix.test.extra }}
          TEST_PATHS: ${{ matrix.test.paths_arg }}
        run: |
          set -Eeuo pipefail
          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
          test -n "$ABXPKG_SPEC"
          dependency_sources=()
          if [[ -n "$TEST_EXTRA" ]]; then
            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
          fi
          if [[ "$TEST_PATHS" == *test_postgres_backend* ]]; then
            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:postgres_binaries")
          fi
          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
            --install \
            --no-cache \
            --json \
            --lib="$ABXPKG_LIB_DIR" \
            "${dependency_sources[@]}"

      - name: Install test dependencies
        env:
          TEST_EXTRA: ${{ matrix.test.extra }}
        run: |
          set -Eeuo pipefail
          extra_args=()
          if [[ -n "$TEST_EXTRA" ]]; then
            extra_args=(--extra "$TEST_EXTRA")
          fi
          uv sync --no-cache --locked --dev "${extra_args[@]}"
      - name: Run ${{ matrix.test.name }}
        env:
          TEST_PATHS: ${{ matrix.test.paths_arg }}
          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
        run: |
          set -Eeuo pipefail
          echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
          read -r -a test_paths <<< "$TEST_PATHS"
          test_args=()
          for test_path in "${test_paths[@]}"; do
            test -f "$test_path"
            test_args+=("$GITHUB_WORKSPACE/$test_path")
          done
          # Each test still gets isolated DB/filesystem state from conftest.
          # Keep one worker pool per batch instead of starting it for every file.
          mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest"
          uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "${test_args[@]}" --durations=20 \
            --basetemp="$RUNNER_TEMP/pytest" \
            --junitxml="$RUNNER_TEMP/test-results/results.xml"
      - name: Save test results
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: test-results-${{ strategy.job-index }}
          path: ${{ runner.temp }}/test-results/*.xml
          if-no-files-found: error
          retention-days: 14
      - name: Save failed runtime diagnostics
        if: failure()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: runtime-logs-${{ strategy.job-index }}
          path: ${{ runner.temp }}/pytest/**/*.log
          if-no-files-found: ignore
          retention-days: 14