cozystack/.github/workflows/pull-requests.yaml

name: Pull Request

env:
  # TODO: unhardcode this
  REGISTRY: iad.ocir.io/idyksih5sir9/cozystack
on:
  pull_request:
    # `labeled` makes the promote PR's manual full-e2e opt-in actionable after
    # the bot opens it. The plan/resolve guards below discard every other label
    # event before it can launch build or E2E work.
    types: [opened, synchronize, reopened, labeled]

# Cancel in‑flight runs for the same PR when a new push arrives.
concurrency:
  group: pr-${{ github.workflow }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

# Top-level token defaults to read-only; jobs request the minimum extra scopes.
permissions:
  contents: read

jobs:
  # Compute the docs-only gate and the per-package build matrix from the PR
  # diff. hack/build-matrix.sh parses the root Makefile `build:` target (the
  # single source of truth) and emits only the units whose package dir changed,
  # escalating to the full set on a shared-dependency change. talos + installer
  # are handled by dedicated jobs (see build-matrix.sh for why).
  plan:
    name: Plan build
    runs-on: ubuntu-latest
    if: github.event.action != 'labeled' || github.event.label.name == 'full-e2e'
    permissions:
      contents: read
    outputs:
      code: ${{ steps.p.outputs.code }}
      trigger_map: ${{ steps.p.outputs.trigger_map }}
      matrix: ${{ steps.p.outputs.matrix }}
      any: ${{ steps.p.outputs.any }}
      touched: ${{ steps.p.outputs.touched }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
      - name: Compute changed files and build matrix
        id: p
        env:
          BASE_REF: ${{ github.base_ref }}
        run: |
          git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt
          echo "Changed files:"; sed 's/^/  /' /tmp/changed.txt
          if grep -qvE '^docs/' /tmp/changed.txt; then
            echo "code=true" >> "$GITHUB_OUTPUT"
          else
            echo "code=false" >> "$GITHUB_OUTPUT"
          fi
          # docs/agents/contributing.md carries the downstream trigger map, which
          # hack/downstream-trigger-map.bats asserts against: every path the map
          # cites still exists, and its repository list still matches the PR
          # template. Editing the map alone is a docs-only PR, which would skip the
          # unit tests and exempt the map from its own guard. This flag lets the
          # unit-test job opt back in, and only that job: it deliberately does not
          # feed `code`, so a docs PR still builds nothing.
          #
          # Detect the map by its old path too: a rename is reported as the new path
          # alone, so a PR that moves the map would skip the very test that pins the
          # literal below to it, and break for whoever pushes next.
          git diff --name-only --no-renames "origin/${BASE_REF}...HEAD" > /tmp/changed_norenames.txt
          if grep -qxF 'docs/agents/contributing.md' /tmp/changed_norenames.txt; then
            echo "trigger_map=true" >> "$GITHUB_OUTPUT"
          else
            echo "trigger_map=false" >> "$GITHUB_OUTPUT"
          fi
          matrix="$(hack/build-matrix.sh /tmp/changed.txt)"
          echo "Build matrix: $matrix"
          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
          # Package dirs the PR edited (packages/<area>/<name>). Their committed
          # refs are the PR's intent and must survive the current-main overlay —
          # otherwise an upstream image bump in a non-build-unit package would be
          # silently reverted in e2e/the installer artifact (see
          # hack/overlay-main-images.sh). Whitespace-separated, no trailing slash.
          touched="$(sed -nE 's#^(packages/[^/]+/[^/]+)/.*#\1#p' /tmp/changed.txt | sort -u | tr '\n' ' ')"
          echo "Touched packages: $touched"
          echo "touched=$touched" >> "$GITHUB_OUTPUT"
          if [ "$matrix" = "[]" ]; then
            echo "any=false" >> "$GITHUB_OUTPUT"
          else
            echo "any=true" >> "$GITHUB_OUTPUT"
          fi

  # Helm-unittest + Go controller tests. Decoupled from the image build and from
  # e2e so they give fast, independent signal; they gate the PR as their own
  # required check rather than blocking e2e from starting.
  checks:
    name: Unit & controller tests
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 30
    permissions:
      contents: read
    needs: ["plan"]
    # trigger_map opts a docs-only PR back in when it edits the downstream trigger
    # map, so hack/downstream-trigger-map.bats still guards it. No other job reads
    # that flag: a docs PR runs the unit tests and nothing heavier.
    if: |
      (needs.plan.outputs.code == 'true' || needs.plan.outputs.trigger_map == 'true')
      && !contains(github.event.pull_request.labels.*.name, 'release')
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
      # Ephemeral runners don't carry the helm-unittest plugin the persistent
      # self-hosted runner accreted. Install if absent (idempotent).
      - name: Set up test toolchain
        run: |
          helm plugin list 2>/dev/null | grep -q unittest \
            || helm plugin install https://github.com/helm-unittest/helm-unittest
      - name: Run unit tests
        run: make unit-tests
      - name: Run controller Go tests
        run: make test-controllers

  # One image (package) per job, fanned out across the ephemeral pool. Each job
  # gets its own buildx (docker-container driver) so the shared-buildkit
  # contention that wedged the old single Build job (#2937) is severed by
  # construction. Each job patches only its own package's values.yaml/.tag files
  # and uploads that diff as a patch fragment; `finalize` reassembles them.
  build:
    name: Build ${{ matrix.pkg }}
    needs: ["plan"]
    if: |
      needs.plan.outputs.code == 'true'
      && needs.plan.outputs.any == 'true'
      && !contains(github.event.pull_request.labels.*.name, 'release')
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 30
    permissions:
      contents: read
      packages: write
    strategy:
      fail-fast: false
      matrix:
        pkg: ${{ fromJSON(needs.plan.outputs.matrix) }}
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true
      # installer is built in finalize, but some package image targets shell out
      # to flux; install it if absent (idempotent, cheap).
      - name: Set up build toolchain
        run: |
          if ! command -v flux >/dev/null; then
            install_script="$(mktemp)"
            curl -fsSL https://fluxcd.io/install.sh -o "$install_script"
            sudo bash "$install_script"
            rm -f "$install_script"
          fi
      - name: Set up Docker config
        run: |
          if [ -d ~/.docker ]; then
            cp -r ~/.docker "${{ runner.temp }}/.docker"
          fi
      - name: Login to GitHub Container Registry
        if: ${{ !github.event.pull_request.head.repo.fork }}
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          username: ${{ secrets.OCIR_USER }}
          password: ${{ secrets.OCIR_TOKEN }}
          registry: iad.ocir.io
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # docker-container driver: isolated buildkit per job (no shared-buildkit
      # contention) and a prerequisite for the mode=max cache reads (#2938).
      # Its buildkitd also routes docker.io base-image pulls through
      # mirror.gcr.io (hack/buildkitd.toml): the Build step authenticates only
      # to the push registry (OCIR), so base images (golang/ubuntu/nginx/...)
      # would otherwise be pulled from Docker Hub anonymously and intermittently
      # trip its anonymous-pull rate limit; the mirror sidesteps that and falls
      # back to Docker Hub on a miss/outage. Create the builder under the SAME
      # DOCKER_CONFIG the Build step reads (runner.temp/.docker, holding the
      # OCIR auth) so buildx forwards push/cache auth to the container builder.
      - name: Set up Buildx (docker-container driver, docker.io mirror)
        id: buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
        with:
          driver: docker-container
          buildkitd-config: hack/buildkitd.toml
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      - name: Build ${{ matrix.pkg }}
        run: make -C "$PKG" image
        env:
          PKG: ${{ matrix.pkg }}
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          # Build on the docker-container builder created above so base-image
          # pulls go through the docker.io mirror (BUILDX_ARGS adds
          # --builder=$(BUILDER)).
          BUILDER: ${{ steps.buildx.outputs.name }}
          # Unique per (PR, commit) so concurrent PR builds never push the same
          # tag. PR images are consumed by digest; :latest belongs to releases.
          IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
          PUBLISH_VERSIONED: '0'
          PUBLISH_FLOATING: '0'
      - name: Capture digest patch fragment
        id: frag
        env:
          PKG: ${{ matrix.pkg }}
        run: |
          safe="$(echo "$PKG" | tr '/' '-')"
          mkdir -p _out/fragments
          # --binary so vendored binary provider bundles (components*.gz),
          # which some package image targets regenerate, survive reassembly
          # in finalize (a plain "Binary files differ" stub is unappliable).
          git diff --binary HEAD > "_out/fragments/${safe}.patch"
          echo "safe=${safe}" >> "$GITHUB_OUTPUT"
          echo "Fragment for ${PKG}:"; git --no-pager diff --stat HEAD
      - name: Upload patch fragment
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-patch-fragment-${{ steps.frag.outputs.safe }}
          path: _out/fragments/${{ steps.frag.outputs.safe }}.patch
          if-no-files-found: error

  # Talos is its own leg: the nocloud disk and the installer tarball are heavy
  # and shared through _out/assets, so the talos image and the disk are built in
  # one job. Always runs on non-docs PRs — e2e needs the disk regardless of the
  # per-package matrix scope.
  build-talos:
    name: Build Talos
    needs: ["plan"]
    if: |
      needs.plan.outputs.code == 'true'
      && !contains(github.event.pull_request.labels.*.name, 'release')
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 40
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - name: Set up Docker config
        run: |
          if [ -d ~/.docker ]; then
            cp -r ~/.docker "${{ runner.temp }}/.docker"
          fi
      - name: Login to GitHub Container Registry
        if: ${{ !github.event.pull_request.head.repo.fork }}
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          username: ${{ secrets.OCIR_USER }}
          password: ${{ secrets.OCIR_TOKEN }}
          registry: iad.ocir.io
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
      # (hack/buildkitd.toml) so the talos image build dodges Docker Hub's
      # anonymous-pull rate limit; see the build job for the full rationale.
      - name: Set up Buildx (docker-container driver, docker.io mirror)
        id: buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
        with:
          driver: docker-container
          buildkitd-config: hack/buildkitd.toml
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      - name: Build Talos image (matchbox + installer tarball)
        run: make -C packages/core/talos image
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          BUILDER: ${{ steps.buildx.outputs.name }}
          IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
          PUBLISH_VERSIONED: '0'
          PUBLISH_FLOATING: '0'
      - name: Build Talos nocloud disk
        run: make -C packages/core/talos talos-nocloud
      - name: Capture Talos patch fragment
        run: |
          mkdir -p _out/fragments
          git diff --binary HEAD > _out/fragments/talos.patch
          echo "Talos fragment:"; git --no-pager diff --stat HEAD
      - name: Upload Talos patch fragment
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-patch-fragment-talos
          path: _out/fragments/talos.patch
          if-no-files-found: error
      - name: Upload Talos image
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: talos-image
          path: _out/assets/nocloud-amd64.raw.xz
          if-no-files-found: error

  # Reassemble the unified pr.patch from every fragment, then build the installer
  # LAST: `make -C packages/core/installer image` runs `flux push artifact
  # --path=packages`, which bundles the WHOLE digest-patched tree into the OCI
  # artifact the operator pulls — so every other package's digest edit must
  # already be applied here. e2e consumes the resulting pr.patch unchanged.
  finalize:
    name: Finalize (merge patches + installer)
    # Depends on `checks` too: gating finalize on unit/controller tests cascades
    # to e2e (which needs finalize), so a red unit test skips the expensive
    # install+e2e path instead of wasting ~1h on it. Restores the gating the old
    # in-build-job tests gave, without re-coupling the cheap parallel builds.
    needs: ["plan", "checks", "build", "build-talos"]
    if: |
      !cancelled()
      && needs.plan.outputs.code == 'true'
      && !contains(github.event.pull_request.labels.*.name, 'release')
      && needs.checks.result == 'success'
      && (needs.build.result == 'success' || needs.build.result == 'skipped')
      && needs.build-talos.result == 'success'
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 30
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - name: Set up build toolchain
        run: |
          if ! command -v flux >/dev/null; then
            install_script="$(mktemp)"
            curl -fsSL https://fluxcd.io/install.sh -o "$install_script"
            sudo bash "$install_script"
            rm -f "$install_script"
          fi
      - name: Download patch fragments
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          pattern: pr-patch-fragment-*
          merge-multiple: true
          path: _out/fragments
      - name: Apply digest fragments
        run: |
          applied=0
          # Each fragment patches only its own package's files; --3way lets two
          # fragments that touch adjacent lines of a shared file (e.g. a package
          # Makefile that writes a sibling package's values.yaml) merge cleanly
          # instead of the second apply aborting the whole finalize.
          for f in _out/fragments/*.patch; do
            [ -s "$f" ] || continue
            echo "Applying $f"
            git apply --3way "$f"
            applied=$((applied + 1))
          done
          echo "Applied $applied fragment(s)"
          git --no-pager diff --stat HEAD
      - name: Set up Docker config
        run: |
          if [ -d ~/.docker ]; then
            cp -r ~/.docker "${{ runner.temp }}/.docker"
          fi
      - name: Login to GitHub Container Registry
        if: ${{ !github.event.pull_request.head.repo.fork }}
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          username: ${{ secrets.OCIR_USER }}
          password: ${{ secrets.OCIR_TOKEN }}
          registry: iad.ocir.io
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # Overlay current-main image refs onto every package this PR did NOT
      # rebuild. Without it, unbuilt packages fall back to the last-release refs
      # committed in the tree (e.g. v1.5.0) even when main is hundreds of commits
      # ahead — so e2e and the installer's packages artifact would ship stale
      # images for everything outside this PR's build matrix. Source of truth is
      # the cozystack-packages:main OCI artifact build-main.yaml already
      # publishes (the whole current-main packages tree, refs digest-pinned to
      # main's images). Runs after login (creds available) and before the
      # installer build below so the flux-pushed packages tree — and the
      # pr.patch e2e applies — already reflect the overlay. A missing artifact
      # (build-main not yet green) is a no-op: packages keep their committed
      # refs (prior behaviour). See hack/overlay-main-images.sh.
      - name: Pull current-main packages tree
        run: |
          rm -rf _out/mainpkgs
          mkdir -p _out/mainpkgs
          flux pull artifact "oci://${REGISTRY}/cozystack-packages:main" \
            --output _out/mainpkgs \
            || { echo "cozystack-packages:main pull failed — keeping committed refs"; rm -rf _out/mainpkgs; }
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      - name: Overlay current-main refs for unbuilt packages
        # Best-effort: this only improves WHICH images e2e and the installer use,
        # so an unexpected failure must degrade to the committed refs (prior
        # behaviour), never block the PR. overlay-main-images.sh already handles
        # the expected paths (missing artifact, drift, absent unit) internally.
        run: |
          hack/overlay-main-images.sh _out/mainpkgs "$BUILD_MATRIX" "$PR_TOUCHED" \
            || echo "::warning::overlay-main-images failed; unbuilt packages keep committed refs"
        env:
          # The set of packages THIS PR rebuilt; their pr-<N>-<sha> refs are
          # authoritative, so overlay-main-images.sh skips them. Computed by the
          # plan job from the Makefile (a JSON array of package dirs), not from
          # untrusted PR input — passed via env per workflow-injection hygiene.
          BUILD_MATRIX: ${{ needs.plan.outputs.matrix }}
          # Packages THIS PR edited (changed-file dirs from the plan job's git
          # diff). Their committed refs are the PR's intent — an upstream image
          # bump in a non-build-unit package would otherwise be silently reverted
          # by the overlay — so overlay-main-images.sh preserves them. Also
          # env-passed, derived from the diff, not from untrusted PR input.
          PR_TOUCHED: ${{ needs.plan.outputs.touched }}
      # buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
      # (hack/buildkitd.toml) so the installer build dodges Docker Hub's
      # anonymous-pull rate limit; see the build job for the full rationale.
      - name: Set up Buildx (docker-container driver, docker.io mirror)
        id: buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
        with:
          driver: docker-container
          buildkitd-config: hack/buildkitd.toml
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # Builds the operator image AND flux-pushes the now-fully-patched packages
      # tree, then patches the installer's platformSourceRef to that artifact.
      - name: Build installer (operator + packages artifact)
        run: make -C packages/core/installer image
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          BUILDER: ${{ steps.buildx.outputs.name }}
          IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
          PUBLISH_VERSIONED: '0'
          PUBLISH_FLOATING: '0'
      - name: Assemble pr.patch
        run: |
          mkdir -p _out/assets
          # --binary so regenerated provider bundles (components*.gz) stay in
          # the patch e2e applies; mirrors main's pr.patch binary fix (#e566992).
          git diff --binary HEAD > _out/assets/pr.patch
          echo "Final pr.patch:"; git --no-pager diff --stat HEAD
      - name: Upload git diff patch
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-patch
          path: _out/assets/pr.patch
          if-no-files-found: error

  resolve_assets:
    name: "Resolve assets"
    runs-on: ubuntu-latest
    if: |
      contains(github.event.pull_request.labels.*.name, 'release')
      && (github.event.action != 'labeled' || github.event.label.name == 'full-e2e')
    outputs:
      disk_id:      ${{ steps.fetch_assets.outputs.disk_id }}

    permissions:
      contents: read
    steps:
      - name: Generate GitHub App token
        id: app-token
        uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1
        with:
          app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}
          private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}
          owner: cozystack
          repositories: ${{ github.event.repository.name }}
          permission-contents: read

      - name: Checkout code
        if: contains(github.event.pull_request.labels.*.name, 'release')
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Extract tag from PR branch (release PR)
        if: contains(github.event.pull_request.labels.*.name, 'release')
        id: get_tag
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        with:
          script: |
            const branch = context.payload.pull_request.head.ref;
            const m = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/);
            if (!m) {
              core.setFailed(`❌ Branch '${branch}' does not match 'release-X.Y.Z[-suffix]'`);
              return;
            }
            core.setOutput('tag', `v${m[1]}`);

      - name: Find draft release & asset IDs (release PR)
        if: contains(github.event.pull_request.labels.*.name, 'release')
        id: fetch_assets
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        with:
          github-token: ${{ steps.app-token.outputs.token }}
          script: |
            const tag = '${{ steps.get_tag.outputs.tag }}';
            const releases = await github.rest.repos.listReleases({
              owner: context.repo.owner,
              repo:  context.repo.repo,
              per_page: 100
            });
            const draft = releases.data.find(r => r.tag_name === tag && r.draft);
            if (!draft) {
              core.setFailed(`Draft release '${tag}' not found`);
              return;
            }
            const find = (n) => draft.assets.find(a => a.name === n)?.id;
            const diskId     = find('nocloud-amd64.raw.xz');
            if (!diskId) {
              core.setFailed('Required assets missing in draft release');
              return;
            }
            core.setOutput('disk_id',     diskId);


  e2e:
    name: "E2E Tests"
    # The sandbox boots three QEMU guests (hack/e2e-prepare-cluster.bats), each
    # with `-smp 8 -m 24576`, so the guests alone demand 24 vCPU and 72 GiB.
    # On the 24-vCPU runner that is a 1:1 CPU commit with zero host headroom for
    # QEMU's own emulation/IO threads, vhost-net, the host iptables MASQUERADE
    # every guest image pull traverses, containerd, the kernel and the runner
    # agent. During "Install Cozystack" (all platform operators + image pulls
    # across three nodes at once) the guest vCPUs contend with those host threads
    # and lose cycles to CPU steal, so in-guest kubelet/apiserver readiness and
    # liveness probes miss their deadlines ("context deadline exceeded"). A
    # HelmRelease then flips InProgress -> Failed and its dependents cascade to
    # DependencyNotReady — on a different component each run, the signature of
    # resource thrash rather than a product bug. The 32-vCPU / 128 GiB class
    # keeps the guests' 24 vCPU / 72 GiB unchanged (no slower install) while
    # leaving 8 vCPU (25%) and 56 GiB of host headroom, which removes the
    # oversubscription that starves the probes.
    runs-on: oracle-vm-32cpu-128gb-x86-64
    timeout-minutes: 180
    permissions:
      contents: read
      packages: read
      checks: write
    needs: ["finalize", "build-talos", "resolve_assets"]
    # Regular PRs still enter through finalize. A release-labelled promote PR
    # enters through resolve_assets only when a maintainer explicitly adds
    # full-e2e; without that label the immutable rc's already-green suite is not
    # repeated. The labeled trigger above makes that post-open opt-in start a run.
    if: |
      !cancelled()
      && (
        needs.finalize.result == 'success'
        || (
          needs.resolve_assets.result == 'success'
          && contains(github.event.pull_request.labels.*.name, 'release')
          && contains(github.event.pull_request.labels.*.name, 'full-e2e')
        )
      )

    steps:
      - name: Generate GitHub App token
        if: contains(github.event.pull_request.labels.*.name, 'release')
        id: app-token
        uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1
        with:
          app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}
          private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}
          owner: cozystack
          repositories: ${{ github.event.repository.name }}
          permission-contents: read

      # ▸ Checkout and prepare the codebase
      # fetch-depth: 0 is required by the "Select E2E tests" step so that
      # `git diff origin/${BASE_REF}...HEAD` can resolve the base ref.
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0

      # ▸ Regular PR path – download the Talos disk from build-talos and pr.patch from finalize
      - name: "Download Talos image (regular PR)"
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: talos-image
          path: _out/assets

      - name: Download PR patch
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: pr-patch
          path: _out/assets

      - name: Apply patch
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        run: |
          git apply _out/assets/pr.patch

      # ▸ Release PR path – fetch artefacts from the corresponding draft release
      - name: Download assets from draft release (release PR)
        if: contains(github.event.pull_request.labels.*.name, 'release')
        run: |
          mkdir -p _out/assets
          curl -sSL -H "Authorization: token ${APP_TOKEN}" -H "Accept: application/octet-stream" \
            -o _out/assets/nocloud-amd64.raw.xz \
            "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${{ needs.resolve_assets.outputs.disk_id }}"
        env:
          APP_TOKEN: ${{ steps.app-token.outputs.token }}

      - name: Set sandbox ID
        run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> $GITHUB_ENV

      # ▸ Prepare environment
      - name: Prepare environment
        run: |
          attempt=0
          until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do
            attempt=$((attempt + 1))
            if [ $attempt -ge 3 ]; then
              echo "Attempt $attempt failed, exiting..."
              exit 1
            fi
            echo "Attempt $attempt failed, retrying..."
          done
          echo "Prepare environment completed after $((attempt + 1)) attempts"

      # ▸ Install Cozystack
      - name: Install Cozystack into sandbox
        run: |
          if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then
            echo "❌ Install failed (no retry — see diagnostics below)"
            echo "::group::Diagnostics: HelmRelease status"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true
            echo "::endgroup::"
            echo "::group::Diagnostics: not-Ready HelmReleases (describe)"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \
              | while read -r ns name; do
                  echo "--- $ns/$name ---"
                  docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true
                done
            echo "::endgroup::"
            echo "::group::Diagnostics: recent events"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -50' || true
            echo "::endgroup::"
            exit 1
          fi

      # Test Impact Analysis (TIA): walk the PackageSource graph for the PR
      # diff and select only the Chainsaw suites affected. Runs by default; opt
      # out via the `full-e2e` label to run the full suite.
      # `skip=true` means nothing to test (docs-only PR).
      - name: Select E2E tests
        id: select
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
        env:
          BASE_REF: ${{ github.base_ref }}
        run: |
          git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt
          apps=$(./hack/select-e2e.sh /tmp/changed.txt)
          echo "Selected apps: ${apps:-<none>}"
          echo "apps=${apps}" >> $GITHUB_OUTPUT
          if [ -z "${apps}" ]; then
            echo "skip=true" >> $GITHUB_OUTPUT
          else
            echo "skip=false" >> $GITHUB_OUTPUT
          fi

      - name: Run OpenAPI tests
        run: |
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi

      # ▸ Run E2E tests (Chainsaw, scoped by Test Impact Analysis)
      #
      #   The "Select E2E tests" step above picks the suites impacted by the PR
      #   diff. With the `full-e2e` label that step is skipped, SELECTED_APPS is
      #   empty, and we fall back to the whole suite; an empty selection on a
      #   docs-only PR skips this step entirely (the skip != 'true' guard).
      #
      #   Chainsaw discovers each named suite under hack/e2e-chainsaw/ and polls
      #   every assertion to its own timeout, so the old per-app 3-retry loop is
      #   gone — a failure is reported as a structured assertion diff, with
      #   cluster-state diagnostics dumped below on a non-zero exit.
      - name: Run E2E tests
        id: e2e_tests
        if: ${{ steps.select.outputs.skip != 'true' }}
        env:
          SELECTED_APPS: ${{ steps.select.outputs.apps }}
          FULL_E2E: ${{ contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
        run: |
          if [ "$FULL_E2E" = "true" ] || [ -z "$SELECTED_APPS" ]; then
            suites=""        # empty => test-chainsaw runs the whole suite
            echo "Running full chainsaw suite"
          else
            suites="$SELECTED_APPS"
            echo "Running selected chainsaw suites: $suites"
          fi
          if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES="$suites"; then
            echo "❌ Chainsaw E2E failed (see the assertion diffs above and diagnostics below)"
            echo "::group::Diagnostics: HelmRelease status"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true
            echo "::endgroup::"
            echo "::group::Diagnostics: recent events"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true
            echo "::endgroup::"
            # COSI state at the moment of failure — the CRDs ship no printer
            # columns, so pull the readiness fields explicitly (cozyreport
            # captures the same at end-of-job, but bucket states move between
            # a test failing and the report being taken).
            echo "::group::Diagnostics: COSI bucket state"
            docker exec "$SANDBOX_NAME" sh -c '
              kubectl get bucketclaims.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.bucketReady,BUCKET:.status.bucketName" 2>&1
              kubectl get buckets.objectstorage.k8s.io -o custom-columns="NAME:.metadata.name,READY:.status.bucketReady,ID:.status.bucketID,CLAIMNS:.spec.bucketClaim.namespace" 2>&1
              kubectl get bucketaccesses.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,GRANTED:.status.accessGranted,CLAIM:.spec.bucketClaimName" 2>&1
            ' || true
            echo "::endgroup::"
            exit 1
          fi
          echo "✅ All selected E2E tests passed"

      - name: Collect chainsaw report
        if: always()
        run: |
          mkdir -p _out
          docker cp "$SANDBOX_NAME":/workspace/hack/e2e-chainsaw/chainsaw-report.xml \
            _out/chainsaw-report.xml || true

      - name: Upload chainsaw report
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: chainsaw-report
          path: _out/chainsaw-report.xml
          if-no-files-found: ignore

      # ▸ Collect debug information (always runs)
      - name: Collect report
        if: always()
        run: |
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true

      - name: Upload cozyreport.tgz
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: cozyreport
          path: _out/cozyreport.tgz

      - name: Collect images list
        if: always()
        run: |
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-images || true

      - name: Upload image list
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: image-list
          path: _out/images.txt

      # ▸ Open an SSH breakpoint to the failing sandbox so maintainers can attach,
      #   inspect Talos/Cozystack state and resume with `breakpoint resume`.
      #
      #   Gated by the `debug` or `release` label (release PRs are bot-created,
      #   so their presence implies maintainer authorship). authorized-users is
      #   kept as a second defense layer at the breakpoint level.
      #
      #   Uses cozystack/breakpoint-action (fork of namespacelabs/breakpoint-action)
      #   pinned by SHA. The fork adds: pause-idle mode (initial grace period for
      #   the first SSH connection, idle-aware exit afterwards), endpoint output
      #   and ::notice:: annotation, and a dedicated Check Run "Breakpoint Open"
      #   that carries the SSH endpoint in output.summary while the breakpoint is
      #   paused (conclusion=failure → standard ✗ in `gh pr checks`; updated to
      #   conclusion=success when the breakpoint exits).
      #
      #   Configured via the BREAKPOINT_ENDPOINT repository variable — in fork
      #   PRs repository variables are not exposed, so the step is skipped and
      #   forks cannot reach the self-hosted rendezvous server.
      - name: Breakpoint on E2E failure
        if: |
          failure() &&
          vars.BREAKPOINT_ENDPOINT != '' &&
          (contains(github.event.pull_request.labels.*.name, 'debug')
            || contains(github.event.pull_request.labels.*.name, 'release'))
        # cozystack/breakpoint-action v2-cozy.1
        # mode: pause-idle defaults: grace-period=20m, idle-timeout=10m
        uses: cozystack/breakpoint-action@a6f3a6f87be398ad63b6577351e3398e53f578e4
        with:
          mode: pause-idle
          endpoint: ${{ vars.BREAKPOINT_ENDPOINT }}
          authorized-users: androndo, Arsolitt, IvanHunters, kvaps, lexfrei, lllamnyp, mattia-eleuteri, matthieu-robin, myasnikovdaniil, sircthulhu, tym83
          check-run-name: "Breakpoint Open"
          github-token: ${{ github.token }}
          check-run-summary-template: |
            ## 🔴 SSH breakpoint open — paused for debug

            ```
            {endpoint}
            ```

            Enter the e2e sandbox after SSH:
            ```
            docker exec -ti $(docker ps --filter name=cozy-e2e-sandbox -q | head -1) bash
            export KUBECONFIG=/workspace/kubeconfig
            ```

            Resume from inside: `breakpoint resume`. Otherwise the breakpoint
            exits 10 minutes after the last SSH session disconnects.

      # ▸ Tear down environment (always runs)
      - name: Tear down sandbox
        if: always()
        run: make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true