cozystack/.github/workflows/pull-requests.yaml

name: Pull Request

env:
  # TODO: unhardcode this
  REGISTRY: iad.ocir.io/idyksih5sir9/cozystack
on:
  pull_request:
    types: [opened, synchronize, reopened]

# Cancel in‑flight runs for the same PR when a new push arrives.
concurrency:
  group: pr-${{ github.workflow }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  detect-changes:
    name: Detect changes
    runs-on: ubuntu-latest
    outputs:
      code: ${{ steps.filter.outputs.code }}
    steps:
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            code:
              - '!docs/**'

  build:
    name: Build
    # Run each Build on its own ephemeral VM instead of the shared self-hosted
    # runner. The shared host runs 4 runner agents against one dockerd-embedded
    # buildkit; concurrent build jobs serialize on buildkit's single-writer bbolt
    # cache lock + exporter mutex and stall to the 30-min timeout (proven via a
    # live SIGUSR1 goroutine dump). One VM per job => one buildkit per job => that
    # contention is severed by construction. Phase 0.5 of #2937.
    #
    # 24cpu/96gb rather than the 4cpu/16gb this used to run on: `make build` is
    # still serial across images, but this branch is the only one that still
    # builds ubuntu-container-disk, and it does so once per Kubernetes minor with
    # no shared build cache to warm it (main dropped the image in 39cdc8866, six
    # days before release-1.5 was cut, so nothing writes its :*-buildcache refs).
    # Those six builds now run concurrently, and each drives its own libguestfs
    # appliance -- ~9gb of RAM and a core apiece, which the small shape cannot
    # give. The `debug` label still routes to self-hosted for the breakpoint path.
    runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-24cpu-96gb-x86-64' }}
    timeout-minutes: 30
    permissions:
      contents: read
      packages: write

    needs: ["detect-changes"]
    # Never run when the PR carries the "release" label or only docs changed.
    if: |
      needs.detect-changes.outputs.code == 'true'
      && !contains(github.event.pull_request.labels.*.name, 'release')

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      # Ephemeral runners don't carry two tools the persistent self-hosted
      # runner accreted: the helm-unittest plugin (make unit-tests) and the
      # flux CLI (the installer's `flux push artifact` publishes the
      # cozystack-packages OCI artifact the operator pulls). Install if absent
      # (idempotent; the debug/self-hosted path already has them). Durable fix:
      # bake both into the oracle-vm runner image, then drop this step.
      - name: Set up build toolchain
        run: |
          helm plugin list 2>/dev/null | grep -q unittest \
            || helm plugin install https://github.com/helm-unittest/helm-unittest
          command -v flux >/dev/null \
            || curl -s https://fluxcd.io/install.sh | sudo bash

      - name: Run unit tests
        run: make unit-tests

      - name: Run controller Go tests
        run: make test-controllers

      - name: Set up Docker config
        run: |
          if [ -d ~/.docker ]; then
            cp -r ~/.docker "${{ runner.temp }}/.docker"
          fi

      - name: Login to GitHub Container Registry
        if: ${{ !github.event.pull_request.head.repo.fork }}
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.OCIR_USER}}
          password: ${{ secrets.OCIR_TOKEN }}
          registry: iad.ocir.io
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker

      - name: Build
        run: make build
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          # Unique per (PR, commit) so concurrent PR builds never push the same
          # tag. PUBLISH_VERSIONED/PUBLISH_FLOATING stay off — PR images are
          # consumed by digest, and :latest belongs to release builds.
          IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
          PUBLISH_VERSIONED: '0'
          PUBLISH_FLOATING: '0'

      - name: Build Talos image
        run: make -C packages/core/talos talos-nocloud

      - name: Save git diff as patch
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        run: git diff HEAD > _out/assets/pr.patch

      - name: Upload git diff patch
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        uses: actions/upload-artifact@v4
        with:
          name: pr-patch
          path: _out/assets/pr.patch
     
      - name: Upload Talos image
        uses: actions/upload-artifact@v4
        with:
          name: talos-image
          path: _out/assets/nocloud-amd64.raw.xz

  resolve_assets:
    name: "Resolve assets"
    runs-on: ubuntu-latest
    if: contains(github.event.pull_request.labels.*.name, 'release')
    outputs:
      disk_id:      ${{ steps.fetch_assets.outputs.disk_id }}

    steps:
      - name: Generate GitHub App token
        id: app-token
        uses: actions/create-github-app-token@v1
        with:
          app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}
          private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}
          owner: cozystack

      - name: Checkout code
        if: contains(github.event.pull_request.labels.*.name, 'release')
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Extract tag from PR branch (release PR)
        if: contains(github.event.pull_request.labels.*.name, 'release')
        id: get_tag
        uses: actions/github-script@v7
        with:
          script: |
            const branch = context.payload.pull_request.head.ref;
            const m = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/);
            if (!m) {
              core.setFailed(`❌ Branch '${branch}' does not match 'release-X.Y.Z[-suffix]'`);
              return;
            }
            core.setOutput('tag', `v${m[1]}`);

      - name: Find draft release & asset IDs (release PR)
        if: contains(github.event.pull_request.labels.*.name, 'release')
        id: fetch_assets
        uses: actions/github-script@v7
        with:
          github-token: ${{ steps.app-token.outputs.token }}
          script: |
            const tag = '${{ steps.get_tag.outputs.tag }}';
            const releases = await github.rest.repos.listReleases({
              owner: context.repo.owner,
              repo:  context.repo.repo,
              per_page: 100
            });
            const draft = releases.data.find(r => r.tag_name === tag && r.draft);
            if (!draft) {
              core.setFailed(`Draft release '${tag}' not found`);
              return;
            }
            const find = (n) => draft.assets.find(a => a.name === n)?.id;
            const diskId     = find('nocloud-amd64.raw.xz');
            if (!diskId) {
              core.setFailed('Required assets missing in draft release');
              return;
            }
            core.setOutput('disk_id',     diskId);


  e2e:
    name: "E2E Tests"
    runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-24cpu-96gb-x86-64' }}
    #runs-on: [oracle-vm-32cpu-128gb-x86-64]
    timeout-minutes: 180
    permissions:
      contents: read
      packages: read
      checks: write
    needs: ["build", "resolve_assets"]
    if: ${{ always() && (needs.build.result == 'success' || needs.resolve_assets.result == 'success') }}

    steps:
      - name: Generate GitHub App token
        if: contains(github.event.pull_request.labels.*.name, 'release')
        id: app-token
        uses: actions/create-github-app-token@v1
        with:
          app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}
          private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}
          owner: cozystack

      # ▸ Checkout and prepare the codebase
      # fetch-depth: 0 is required by the "Select E2E tests" step so that
      # `git diff origin/${BASE_REF}...HEAD` can resolve the base ref.
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # ▸ Regular PR path – download artefacts produced by the *build* job
      - name: "Download Talos image (regular PR)"
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        uses: actions/download-artifact@v4
        with:
          name: talos-image
          path: _out/assets

      - name: Download PR patch
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        uses: actions/download-artifact@v4
        with:
          name: pr-patch
          path: _out/assets

      - name: Apply patch
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        run: |
          git apply _out/assets/pr.patch

      # ▸ Release PR path – fetch artefacts from the corresponding draft release
      - name: Download assets from draft release (release PR)
        if: contains(github.event.pull_request.labels.*.name, 'release')
        run: |
          mkdir -p _out/assets
          curl -sSL -H "Authorization: token ${APP_TOKEN}" -H "Accept: application/octet-stream" \
            -o _out/assets/nocloud-amd64.raw.xz \
            "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${{ needs.resolve_assets.outputs.disk_id }}"
        env:
          APP_TOKEN: ${{ steps.app-token.outputs.token }}

      - name: Set sandbox ID
        run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> $GITHUB_ENV

      # ▸ Prepare environment
      - name: Prepare workspace
        run: |
          rm -rf /tmp/$SANDBOX_NAME
          cp -r ${{ github.workspace }} /tmp/$SANDBOX_NAME

      - name: Prepare environment
        run: |
          cd /tmp/$SANDBOX_NAME
          attempt=0
          until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do
            attempt=$((attempt + 1))
            if [ $attempt -ge 3 ]; then
              echo "Attempt $attempt failed, exiting..."
              exit 1
            fi
            echo "Attempt $attempt failed, retrying..."
          done
          echo "Prepare environment completed after $((attempt + 1)) attempts"

      # ▸ Install Cozystack
      - name: Install Cozystack into sandbox
        run: |
          cd /tmp/$SANDBOX_NAME
          if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then
            echo "❌ Install failed (no retry — see diagnostics below)"
            echo "::group::Diagnostics: HelmRelease status"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true
            echo "::endgroup::"
            echo "::group::Diagnostics: not-Ready HelmReleases (describe)"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \
              | while read -r ns name; do
                  echo "--- $ns/$name ---"
                  docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true
                done
            echo "::endgroup::"
            echo "::group::Diagnostics: recent events"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -50' || true
            echo "::endgroup::"
            exit 1
          fi

      # Test Impact Analysis (TIA): walk the PackageSource graph for the PR
      # diff and select only the bats files affected. Runs by default; opt
      # out via the `full-e2e` label to run the full suite.
      # `skip=true` means nothing to test (docs-only PR).
      - name: Select E2E tests
        id: select
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
        env:
          BASE_REF: ${{ github.base_ref }}
        run: |
          cd /tmp/$SANDBOX_NAME
          git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt
          apps=$(./hack/select-e2e.sh /tmp/changed.txt)
          echo "Selected apps: ${apps:-<none>}"
          echo "apps=${apps}" >> $GITHUB_OUTPUT
          if [ -z "${apps}" ]; then
            echo "skip=true" >> $GITHUB_OUTPUT
          else
            echo "skip=false" >> $GITHUB_OUTPUT
          fi

      - name: Run OpenAPI tests
        run: |
          cd /tmp/$SANDBOX_NAME
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi

      # Run E2E tests. With `full-e2e` label the selector is skipped, its
      # outputs are empty, the != 'true' guard lets this step run, and the
      # full bats list is used. Without the label, the selector decides;
      # an empty selection (docs-only) skips this step entirely.
      - name: Run E2E tests
        id: e2e_tests
        if: ${{ steps.select.outputs.skip != 'true' }}
        env:
          SELECTED_APPS: ${{ steps.select.outputs.apps }}
          FULL_E2E: ${{ contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
        run: |
          cd /tmp/$SANDBOX_NAME
          if [ "$FULL_E2E" = "true" ] || [ -z "$SELECTED_APPS" ]; then
            apps_list=$(ls hack/e2e-apps/*.bats | xargs -n1 basename | cut -d. -f1)
          else
            apps_list="$SELECTED_APPS"
          fi
          failed_tests=""
          for app in $apps_list; do
            echo "::group::Testing $app"
            if make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-apps-$app; then
              echo "::endgroup::"
              echo "✅ Test $app completed successfully"
            else
              echo "::endgroup::"
              echo "❌ Test $app failed (no retry — see diagnostics below)"
              failed_tests="$failed_tests $app"
              echo "::group::Diagnostics for $app"
              docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true
              docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true
              # COSI state at the moment of failure — the CRDs ship no printer
              # columns, so pull the readiness fields explicitly (cozyreport
              # captures the same at end-of-job, but bucket states move between
              # a test failing and the report being taken).
              docker exec "$SANDBOX_NAME" sh -c '
                kubectl get bucketclaims.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.bucketReady,BUCKET:.status.bucketName" 2>&1
                kubectl get buckets.objectstorage.k8s.io -o custom-columns="NAME:.metadata.name,READY:.status.bucketReady,ID:.status.bucketID,CLAIMNS:.spec.bucketClaim.namespace" 2>&1
                kubectl get bucketaccesses.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,GRANTED:.status.accessGranted,CLAIM:.spec.bucketClaimName" 2>&1
              ' || true
              echo "::endgroup::"
            fi
          done
          if [ -n "$failed_tests" ]; then
            echo "❌ Failed tests:$failed_tests"
            exit 1
          fi
          echo "✅ All E2E tests passed"

      # ▸ Collect debug information (always runs)
      - name: Collect report
        if: always()
        run: |
          cd /tmp/$SANDBOX_NAME
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true

      - name: Upload cozyreport.tgz
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: cozyreport
          path: /tmp/${{ env.SANDBOX_NAME }}/_out/cozyreport.tgz

      - name: Collect images list
        if: always()
        run: |
          cd /tmp/$SANDBOX_NAME
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-images || true

      - name: Upload image list
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: image-list
          path: /tmp/${{ env.SANDBOX_NAME }}/_out/images.txt

      # ▸ Open an SSH breakpoint to the failing sandbox so maintainers can attach,
      #   inspect Talos/Cozystack state and resume with `breakpoint resume`.
      #
      #   Gated by the `debug` or `release` label (release PRs are bot-created,
      #   so their presence implies maintainer authorship). authorized-users is
      #   kept as a second defense layer at the breakpoint level.
      #
      #   Uses cozystack/breakpoint-action (fork of namespacelabs/breakpoint-action)
      #   pinned by SHA. The fork adds: pause-idle mode (initial grace period for
      #   the first SSH connection, idle-aware exit afterwards), endpoint output
      #   and ::notice:: annotation, and a dedicated Check Run "Breakpoint Open"
      #   that carries the SSH endpoint in output.summary while the breakpoint is
      #   paused (conclusion=failure → standard ✗ in `gh pr checks`; updated to
      #   conclusion=success when the breakpoint exits).
      #
      #   Configured via the BREAKPOINT_ENDPOINT repository variable — in fork
      #   PRs repository variables are not exposed, so the step is skipped and
      #   forks cannot reach the self-hosted rendezvous server.
      - name: Breakpoint on E2E failure
        if: |
          failure() &&
          vars.BREAKPOINT_ENDPOINT != '' &&
          (contains(github.event.pull_request.labels.*.name, 'debug')
            || contains(github.event.pull_request.labels.*.name, 'release'))
        # cozystack/breakpoint-action v2-cozy.1
        # mode: pause-idle defaults: grace-period=20m, idle-timeout=10m
        uses: cozystack/breakpoint-action@a6f3a6f87be398ad63b6577351e3398e53f578e4
        with:
          mode: pause-idle
          endpoint: ${{ vars.BREAKPOINT_ENDPOINT }}
          authorized-users: androndo, Arsolitt, IvanHunters, kvaps, lexfrei, lllamnyp, mattia-eleuteri, matthieu-robin, myasnikovdaniil, sircthulhu, tym83
          check-run-name: "Breakpoint Open"
          github-token: ${{ github.token }}
          check-run-summary-template: |
            ## 🔴 SSH breakpoint open — paused for debug

            ```
            {endpoint}
            ```

            Enter the e2e sandbox after SSH:
            ```
            docker exec -ti $(docker ps --filter name=cozy-e2e-sandbox -q | head -1) bash
            export KUBECONFIG=/workspace/kubeconfig
            ```

            Resume from inside: `breakpoint resume`. Otherwise the breakpoint
            exits 10 minutes after the last SSH session disconnects.

      # ▸ Tear down environment (always runs)
      - name: Tear down sandbox
        if: always()
        run: make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true

      - name: Remove workspace
        if: always()
        run: rm -rf /tmp/$SANDBOX_NAME