cozystack/.github/workflows/pull-requests.yaml

name: Pull Request

env:
  # TODO: unhardcode this
  REGISTRY: iad.ocir.io/idyksih5sir9/cozystack
on:
  pull_request:
    # `labeled` makes the promote PR's manual full-e2e opt-in actionable after
    # the bot opens it. The plan/resolve guards below discard every other label
    # event before it can launch build or E2E work.
    types: [opened, synchronize, reopened, labeled]

# Cancel in‑flight runs for the same PR when a new push arrives.
#
# A `labeled` event that `plan` discards gets its OWN group, so labelling an
# open PR cannot cancel a live opened/synchronize run. Any label other than
# full-e2e makes `plan` skip, and `e2e-report` skips with it, so without the
# split the pair "cancel the real run, then skip" left the head SHA with no
# "E2E Tests" status at all — and an unposted required status stays "Expected",
# blocking the merge until someone pushes again or re-runs the cancelled run.
# Observed on live CI: run 30614258694 (cancelled) followed by 30614280603
# (skipped) on the same SHA, 24s apart, from a maintainer adding kind/* labels
# just after a push. (`pr-labeler` cannot cause this — it labels with the
# default GITHUB_TOKEN, which by design triggers no further workflow runs.)
#
# The condition is the exact complement of `plan`'s guard, so only a run that
# will publish NOTHING is moved aside. A `labeled` run carrying full-e2e does
# publish, so it stays in the main group and supersedes the head run the way a
# push does. That keeps exactly one publisher of the `E2E Tests` context alive
# per head SHA, which the status depends on: two live publishers are latest-wins
# per context, and the full suite is the superset, so a full-e2e run that fails
# early — a build flake, a suite failing at minute twenty — would finish before
# the narrower head run it was meant to widen, and that run's later green would
# erase the failure. Cancelling also means opting into the full suite costs one
# suite rather than two, and a second label landing afterwards cannot cancel the
# full-e2e run, because it is no longer in the same group as it.
concurrency:
  group: pr-${{ github.workflow }}-${{ github.event.pull_request.number }}${{ (github.event.action == 'labeled' && github.event.label.name != 'full-e2e') && '-label' || '' }}
  cancel-in-progress: true

# Top-level token defaults to read-only. A job-level block replaces this one
# rather than adding to it, so each job restates every scope it needs.
permissions:
  contents: read

jobs:
  # Compute the docs-only gate and the per-package build matrix from the PR
  # diff. hack/build-matrix.sh parses the root Makefile `build:` target (the
  # single source of truth) and emits only the units whose package dir changed,
  # escalating to the full set on a shared-dependency change. talos + installer
  # are handled by dedicated jobs (see build-matrix.sh for why).
  plan:
    name: Plan build
    runs-on: ubuntu-latest
    if: github.event.action != 'labeled' || github.event.label.name == 'full-e2e'
    permissions:
      contents: read
      # For the opening `E2E Tests` status below; e2e-report writes the terminal
      # one with the same scope.
      statuses: write
    outputs:
      code: ${{ steps.p.outputs.code }}
      trigger_map: ${{ steps.p.outputs.trigger_map }}
      matrix: ${{ steps.p.outputs.matrix }}
      any: ${{ steps.p.outputs.any }}
      touched: ${{ steps.p.outputs.touched }}
    steps:
      # Claim the required context for THIS run before anything can be merged on
      # the strength of the last one. Commit statuses are latest-wins per (sha,
      # context) and nothing expires them, so a head SHA that already went green
      # keeps reading green while a newly started run works — re-run all jobs, a
      # full-e2e label, a reopen — and the PR stays mergeable through a suite
      # that may be about to fail. While the gate was a job, GitHub created a
      # fresh non-success check run each time and that window did not exist.
      # Overwriting with `pending` here restores it: the run that owns the
      # status says so at the start and concludes it in e2e-report.
      #
      # Same-repo only. A fork's `pull_request` token is read-only, so this call
      # would 403 and fail `plan` on every fork PR, and that read-only token is
      # exactly what makes the whole commit-status design safe (§10). Fork PRs
      # get their pending from e2e-fork.yaml's `resolve` instead, which is
      # privileged.
      #
      # Note what that costs a fork. `resolve` cannot start until this entire
      # run has finished, so the window this step shuts in seconds stays open
      # for the length of the build on the fork lane. Structural rather than an
      # oversight, since a read-only token cannot post a status at any point in
      # the run, and bounded by the build. §10 records it.
      - name: Open the "E2E Tests" status for this run
        if: ${{ !github.event.pull_request.head.repo.fork }}
        # A transient status-API failure here must not take the run with it.
        # This step is the first in `plan`, and `plan` is what `checks`,
        # `build`, `build-talos`, `finalize` and `e2e` all hang off, so a
        # failure costs a full rebuild for nothing. Skipping the opener only
        # degrades to the behaviour that preceded it — the terminal status
        # still gets posted by `e2e-report`, and a run that dies before that
        # leaves the merge blocked either way.
        continue-on-error: true
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        with:
          script: |
            const { HEAD_SHA, RUN_URL } = process.env;
            await github.rest.repos.createCommitStatus({
              owner: context.repo.owner,
              repo: context.repo.repo,
              sha: HEAD_SHA,
              state: 'pending',
              context: 'E2E Tests',
              description: 'e2e in progress — this run has not reported yet',
              target_url: RUN_URL,
            });
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
      - name: Compute changed files and build matrix
        id: p
        env:
          BASE_REF: ${{ github.base_ref }}
        run: |
          git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt
          # The docs-only decision reads a RENAME-BLIND list. `git diff
          # --name-only` applies rename detection and reports a rename as its
          # new path alone, so moving a code file into docs/ reads as a
          # docs-only change: nothing builds, no e2e runs, and the removal of
          # that code path ships untested. It also deadlocks the fork lane,
          # which answers the same question from `pulls.listFiles` and does see
          # the old path through `previous_filename`. That lane concludes e2e is
          # required, then fails closed on a pr-patch artifact this lane never
          # produced — an unfixable red on a legitimate contribution, with no
          # label override and no push that clears it. Both lanes have to answer
          # "is this docs-only" the same way, and the only answer that is also
          # correct keeps both sides of a rename.
          #
          # The build matrix still reads the rename-detected list, which is not
          # a claim that doing so is right. Moving a file between packages
          # leaves the source package changed and unselected, so the matrix
          # under-builds on a cross-package rename. That behaviour predates this
          # line and is not what the docs-only decision is for, so it is left
          # alone here rather than widened by a side effect.
          git diff --name-only --no-renames "origin/${BASE_REF}...HEAD" > /tmp/changed_norenames.txt
          echo "Changed files:"; sed 's/^/  /' /tmp/changed_norenames.txt
          if grep -qvE '^docs/' /tmp/changed_norenames.txt; then
            echo "code=true" >> "$GITHUB_OUTPUT"
          else
            echo "code=false" >> "$GITHUB_OUTPUT"
          fi
          # docs/agents/contributing.md carries the downstream trigger map, which
          # hack/downstream-trigger-map.bats asserts against: every path the map
          # cites still exists, and its repository list still matches the PR
          # template. Editing the map alone is a docs-only PR, which would skip the
          # unit tests and exempt the map from its own guard. This flag lets the
          # unit-test job opt back in, and only that job: it deliberately does not
          # feed `code`, so a docs PR still builds nothing.
          #
          # Detect the map by its old path too, which is why this reads the
          # rename-blind list above: a PR that moves the map would otherwise skip
          # the very test that pins the literal below to it, and break for
          # whoever pushes next.
          if grep -qxF 'docs/agents/contributing.md' /tmp/changed_norenames.txt; then
            echo "trigger_map=true" >> "$GITHUB_OUTPUT"
          else
            echo "trigger_map=false" >> "$GITHUB_OUTPUT"
          fi
          matrix="$(hack/build-matrix.sh /tmp/changed.txt)"
          echo "Build matrix: $matrix"
          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
          # Package dirs the PR edited (packages/<area>/<name>). Their committed
          # refs are the PR's intent and must survive the current-main overlay —
          # otherwise an upstream image bump in a non-build-unit package would be
          # silently reverted in e2e/the installer artifact (see
          # hack/overlay-main-images.sh). Whitespace-separated, no trailing slash.
          touched="$(sed -nE 's#^(packages/[^/]+/[^/]+)/.*#\1#p' /tmp/changed.txt | sort -u | tr '\n' ' ')"
          echo "Touched packages: $touched"
          echo "touched=$touched" >> "$GITHUB_OUTPUT"
          if [ "$matrix" = "[]" ]; then
            echo "any=false" >> "$GITHUB_OUTPUT"
          else
            echo "any=true" >> "$GITHUB_OUTPUT"
          fi

  # Helm-unittest + Go controller tests. Decoupled from the image build and from
  # e2e so they give fast, independent signal; they gate the PR as their own
  # required check rather than blocking e2e from starting.
  checks:
    name: Unit & controller tests
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 30
    permissions:
      contents: read
    needs: ["plan"]
    # trigger_map opts a docs-only PR back in when it edits the downstream trigger
    # map, so hack/downstream-trigger-map.bats still guards it. No other job reads
    # that flag: a docs PR runs the unit tests and nothing heavier.
    if: |
      (needs.plan.outputs.code == 'true' || needs.plan.outputs.trigger_map == 'true')
      && !contains(github.event.pull_request.labels.*.name, 'release')
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
      # Ephemeral runners don't carry the helm-unittest plugin the persistent
      # self-hosted runner accreted. Install if absent (idempotent).
      - name: Set up test toolchain
        run: |
          helm plugin list 2>/dev/null | grep -q unittest \
            || helm plugin install https://github.com/helm-unittest/helm-unittest
      - name: Run unit tests
        run: make unit-tests
      - name: Run controller Go tests
        run: make test-controllers

  # One image (package) per job, fanned out across the ephemeral pool. Each job
  # gets its own buildx (docker-container driver) so the shared-buildkit
  # contention that wedged the old single Build job (#2937) is severed by
  # construction. Each job patches only its own package's values.yaml/.tag files
  # and uploads that diff as a patch fragment; `finalize` reassembles them.
  build:
    name: Build ${{ matrix.pkg }}
    needs: ["plan"]
    if: |
      needs.plan.outputs.code == 'true'
      && needs.plan.outputs.any == 'true'
      && !contains(github.event.pull_request.labels.*.name, 'release')
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 30
    permissions:
      contents: read
      packages: write
    strategy:
      fail-fast: false
      matrix:
        pkg: ${{ fromJSON(needs.plan.outputs.matrix) }}
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true
      # installer is built in finalize, but some package image targets shell out
      # to flux; install it if absent (idempotent, cheap).
      - name: Set up build toolchain
        run: |
          if ! command -v flux >/dev/null; then
            install_script="$(mktemp)"
            curl -fsSL https://fluxcd.io/install.sh -o "$install_script"
            sudo bash "$install_script"
            rm -f "$install_script"
          fi
      - name: Set up Docker config
        run: |
          if [ -d ~/.docker ]; then
            cp -r ~/.docker "${{ runner.temp }}/.docker"
          fi
      - name: Login to GitHub Container Registry
        if: ${{ !github.event.pull_request.head.repo.fork }}
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          username: ${{ secrets.OCIR_USER }}
          password: ${{ secrets.OCIR_TOKEN }}
          registry: iad.ocir.io
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # docker-container driver: isolated buildkit per job (no shared-buildkit
      # contention) and a prerequisite for the mode=max cache reads (#2938).
      # Its buildkitd also routes docker.io base-image pulls through
      # mirror.gcr.io (hack/buildkitd.toml): the Build step authenticates only
      # to the push registry (OCIR), so base images (golang/ubuntu/nginx/...)
      # would otherwise be pulled from Docker Hub anonymously and intermittently
      # trip its anonymous-pull rate limit; the mirror sidesteps that and falls
      # back to Docker Hub on a miss/outage. Create the builder under the SAME
      # DOCKER_CONFIG the Build step reads (runner.temp/.docker, holding the
      # OCIR auth) so buildx forwards push/cache auth to the container builder.
      - name: Set up Buildx (docker-container driver, docker.io mirror)
        id: buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
        with:
          driver: docker-container
          buildkitd-config: hack/buildkitd.toml
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      - name: Build ${{ matrix.pkg }}
        run: |
          mkdir -p _out/oci
          make -C "$PKG" image
        env:
          PKG: ${{ matrix.pkg }}
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          # Build on the docker-container builder created above so base-image
          # pulls go through the docker.io mirror (BUILDX_ARGS adds
          # --builder=$(BUILDER)).
          BUILDER: ${{ steps.buildx.outputs.name }}
          # Unique per (PR, commit) so concurrent PR builds never push the same
          # tag. PR images are consumed by digest; :latest belongs to releases.
          IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
          PUBLISH_VERSIONED: '0'
          PUBLISH_FLOATING: '0'
          # Fork PRs get no registry credentials, so instead of pushing, build
          # each image to <workspace>/_out/oci/<name>.oci.tar and upload it below;
          # the privileged e2e-fork.yaml workflow_run pushes them by digest. Empty
          # on same-repo PRs → normal push path. Setting it also forces PUSH/LOAD
          # off in common-envs.mk (#3257). Must be ABSOLUTE (workspace-rooted):
          # the build runs `make -C <pkg>`, so buildx resolves a relative dest
          # against the package dir — the archive would land in <pkg>/_out/oci and
          # miss the repo-root _out/oci the upload reads.
          OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}
      - name: Capture digest patch fragment
        id: frag
        env:
          PKG: ${{ matrix.pkg }}
        run: |
          safe="$(echo "$PKG" | tr '/' '-')"
          mkdir -p _out/fragments
          # --binary so vendored binary provider bundles (components*.gz),
          # which some package image targets regenerate, survive reassembly
          # in finalize (a plain "Binary files differ" stub is unappliable).
          git diff --binary HEAD > "_out/fragments/${safe}.patch"
          echo "safe=${safe}" >> "$GITHUB_OUTPUT"
          echo "Fragment for ${PKG}:"; git --no-pager diff --stat HEAD
      - name: Upload patch fragment
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-patch-fragment-${{ steps.frag.outputs.safe }}
          path: _out/fragments/${{ steps.frag.outputs.safe }}.patch
          if-no-files-found: error
      # Fork PR only: the images were exported to OCI archives instead of pushed
      # (OCI_EXPORT_DIR above). Upload them for e2e-fork.yaml to push by digest.
      - name: Upload OCI image archives (fork PR)
        if: ${{ github.event.pull_request.head.repo.fork }}
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-oci-${{ steps.frag.outputs.safe }}
          path: _out/oci/*.oci.tar
          if-no-files-found: error

  # Talos is its own leg: the nocloud disk and the installer tarball are heavy
  # and shared through _out/assets, so the talos image and the disk are built in
  # one job. Always runs on non-docs PRs — e2e needs the disk regardless of the
  # per-package matrix scope.
  build-talos:
    name: Build Talos
    needs: ["plan"]
    if: |
      needs.plan.outputs.code == 'true'
      && !contains(github.event.pull_request.labels.*.name, 'release')
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 40
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - name: Set up Docker config
        run: |
          if [ -d ~/.docker ]; then
            cp -r ~/.docker "${{ runner.temp }}/.docker"
          fi
      - name: Login to GitHub Container Registry
        if: ${{ !github.event.pull_request.head.repo.fork }}
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          username: ${{ secrets.OCIR_USER }}
          password: ${{ secrets.OCIR_TOKEN }}
          registry: iad.ocir.io
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
      # (hack/buildkitd.toml) so the talos image build dodges Docker Hub's
      # anonymous-pull rate limit; see the build job for the full rationale.
      - name: Set up Buildx (docker-container driver, docker.io mirror)
        id: buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
        with:
          driver: docker-container
          buildkitd-config: hack/buildkitd.toml
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      - name: Build Talos image (matchbox + installer tarball)
        run: |
          mkdir -p _out/oci
          make -C packages/core/talos image
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          BUILDER: ${{ steps.buildx.outputs.name }}
          IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
          PUBLISH_VERSIONED: '0'
          PUBLISH_FLOATING: '0'
          # Fork PR: export to OCI archive instead of pushing (see build job).
          OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}
      - name: Build Talos nocloud disk
        run: make -C packages/core/talos talos-nocloud
      - name: Capture Talos patch fragment
        run: |
          mkdir -p _out/fragments
          git diff --binary HEAD > _out/fragments/talos.patch
          echo "Talos fragment:"; git --no-pager diff --stat HEAD
      - name: Upload Talos patch fragment
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-patch-fragment-talos
          path: _out/fragments/talos.patch
          if-no-files-found: error
      # Fork PR only: upload the Talos image OCI archive for e2e-fork.yaml.
      - name: Upload Talos OCI image archive (fork PR)
        if: ${{ github.event.pull_request.head.repo.fork }}
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-oci-talos
          path: _out/oci/*.oci.tar
          if-no-files-found: error
      - name: Upload Talos image
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: talos-image
          path: _out/assets/nocloud-amd64.raw.xz
          if-no-files-found: error

  # Reassemble the unified pr.patch from every fragment, then build the installer
  # LAST: `make -C packages/core/installer image` runs `flux push artifact
  # --path=packages`, which bundles the WHOLE digest-patched tree into the OCI
  # artifact the operator pulls — so every other package's digest edit must
  # already be applied here. e2e consumes the resulting pr.patch unchanged.
  finalize:
    name: Finalize (merge patches + installer)
    # Depends on `checks` too: gating finalize on unit/controller tests cascades
    # to e2e (which needs finalize), so a red unit test skips the expensive
    # install+e2e path instead of wasting ~1h on it. Restores the gating the old
    # in-build-job tests gave, without re-coupling the cheap parallel builds.
    needs: ["plan", "checks", "build", "build-talos"]
    if: |
      !cancelled()
      && needs.plan.outputs.code == 'true'
      && !contains(github.event.pull_request.labels.*.name, 'release')
      && needs.checks.result == 'success'
      && (needs.build.result == 'success' || needs.build.result == 'skipped')
      && needs.build-talos.result == 'success'
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 30
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - name: Set up build toolchain
        run: |
          if ! command -v flux >/dev/null; then
            install_script="$(mktemp)"
            curl -fsSL https://fluxcd.io/install.sh -o "$install_script"
            sudo bash "$install_script"
            rm -f "$install_script"
          fi
      - name: Download patch fragments
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          pattern: pr-patch-fragment-*
          merge-multiple: true
          path: _out/fragments
      - name: Apply digest fragments
        run: |
          applied=0
          # Each fragment patches only its own package's files; --3way lets two
          # fragments that touch adjacent lines of a shared file (e.g. a package
          # Makefile that writes a sibling package's values.yaml) merge cleanly
          # instead of the second apply aborting the whole finalize.
          for f in _out/fragments/*.patch; do
            [ -s "$f" ] || continue
            echo "Applying $f"
            git apply --3way "$f"
            applied=$((applied + 1))
          done
          echo "Applied $applied fragment(s)"
          git --no-pager diff --stat HEAD
      - name: Set up Docker config
        run: |
          if [ -d ~/.docker ]; then
            cp -r ~/.docker "${{ runner.temp }}/.docker"
          fi
      - name: Login to GitHub Container Registry
        if: ${{ !github.event.pull_request.head.repo.fork }}
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          username: ${{ secrets.OCIR_USER }}
          password: ${{ secrets.OCIR_TOKEN }}
          registry: iad.ocir.io
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # Overlay current-main image refs onto every package this PR did NOT
      # rebuild. Without it, unbuilt packages fall back to the last-release refs
      # committed in the tree (e.g. v1.5.0) even when main is hundreds of commits
      # ahead — so e2e and the installer's packages artifact would ship stale
      # images for everything outside this PR's build matrix. Source of truth is
      # the cozystack-packages:main OCI artifact build-main.yaml already
      # publishes (the whole current-main packages tree, refs digest-pinned to
      # main's images). Runs after login (creds available) and before the
      # installer build below so the flux-pushed packages tree — and the
      # pr.patch e2e applies — already reflect the overlay. A missing artifact
      # (build-main not yet green) is a no-op: packages keep their committed
      # refs (prior behaviour). See hack/overlay-main-images.sh.
      #
      # Read the artifact for the PR's OWN base branch, not always main. Overlaying
      # main's images onto a release-line PR mixes generations: main's binaries
      # against that line's charts. #3437 (a one-line change on release-1.6) failed
      # install deterministically that way, because main's cozystack-controller
      # served an aggregated OpenAPI release-1.6's charts could not validate
      # against (`unknown model in reference: …v1alpha1.OptionSpec`) — a broken
      # lane reading as a broken PR. build-release.yaml publishes
      # cozystack-packages:<line> for every maintained release-<major>.<minor>
      # branch, so each base branch has its own artifact to read.
      #
      # A missing artifact stays a no-op (packages keep their committed refs), which
      # also covers a line whose first build-release run has not landed yet.
      - name: Pull base-branch packages tree
        run: |
          rm -rf _out/mainpkgs
          mkdir -p _out/mainpkgs
          if ! flux pull artifact "oci://${REGISTRY}/cozystack-packages:${BASE_REF}" \
                 --output _out/mainpkgs; then
            rm -rf _out/mainpkgs
            # Degrading to committed refs is safe, but SAY so: on a release line a
            # missing artifact means build-release.yaml has not published this line
            # yet (first build pending, or its branch filter does not match this
            # branch), and the PR is then testing the line's last release instead of
            # its tip. Silent would look identical to a working overlay.
            if [ "${BASE_REF}" = "main" ]; then
              echo "cozystack-packages:${BASE_REF} pull failed — keeping committed refs"
            else
              echo "::warning title=No packages artifact for ${BASE_REF}::Keeping committed refs, so unbuilt packages use this line's last release rather than its tip. Check that build-release.yaml has run for ${BASE_REF}."
            fi
          fi
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          # Env-passed, not interpolated into the script: base_ref is repo-controlled
          # here (a branch name in this repo), but the workflow-injection hygiene
          # applies uniformly.
          BASE_REF: ${{ github.base_ref }}
      - name: Overlay base-branch refs for unbuilt packages
        # Best-effort: this only improves WHICH images e2e and the installer use,
        # so an unexpected failure must degrade to the committed refs (prior
        # behaviour), never block the PR. overlay-main-images.sh already handles
        # the expected paths (missing artifact, drift, absent unit) internally —
        # including the empty directory the step above leaves when a line has no
        # artifact yet.
        run: |
          hack/overlay-main-images.sh _out/mainpkgs "$BUILD_MATRIX" "$PR_TOUCHED" \
            || echo "::warning::overlay-main-images failed; unbuilt packages keep committed refs"
        env:
          # The set of packages THIS PR rebuilt; their pr-<N>-<sha> refs are
          # authoritative, so overlay-main-images.sh skips them. Computed by the
          # plan job from the Makefile (a JSON array of package dirs), not from
          # untrusted PR input — passed via env per workflow-injection hygiene.
          BUILD_MATRIX: ${{ needs.plan.outputs.matrix }}
          # Packages THIS PR edited (changed-file dirs from the plan job's git
          # diff). Their committed refs are the PR's intent — an upstream image
          # bump in a non-build-unit package would otherwise be silently reverted
          # by the overlay — so overlay-main-images.sh preserves them. Also
          # env-passed, derived from the diff, not from untrusted PR input.
          PR_TOUCHED: ${{ needs.plan.outputs.touched }}
      # buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
      # (hack/buildkitd.toml) so the installer build dodges Docker Hub's
      # anonymous-pull rate limit; see the build job for the full rationale.
      - name: Set up Buildx (docker-container driver, docker.io mirror)
        id: buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
        with:
          driver: docker-container
          buildkitd-config: hack/buildkitd.toml
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # Same-repo PR: build the operator image AND flux-push the now-fully-patched
      # packages tree, then patch the installer's platformSourceRef to that
      # artifact. Fork PR: no registry creds — build ONLY the operator image to an
      # OCI archive (image-packages/chart, which push, are skipped) and upload it;
      # e2e-fork.yaml pushes the operator + flux-pushes the packages tree and
      # bakes platformSourceRef there. The overlay above already ran, so the
      # pr.patch assembled below carries the fully digest-pinned tree either way.
      - name: Build installer (operator + packages artifact)
        run: |
          mkdir -p _out/oci
          if [ "$IS_FORK" = "true" ]; then
            make -C packages/core/installer pre-checks image-operator
          else
            make -C packages/core/installer image
          fi
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          BUILDER: ${{ steps.buildx.outputs.name }}
          IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
          PUBLISH_VERSIONED: '0'
          PUBLISH_FLOATING: '0'
          IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
          OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}
      # Fork PR only: upload the operator image OCI archive for e2e-fork.yaml.
      - name: Upload installer OCI image archive (fork PR)
        if: ${{ github.event.pull_request.head.repo.fork }}
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-oci-installer
          path: _out/oci/*.oci.tar
          if-no-files-found: error
      - name: Assemble pr.patch
        run: |
          mkdir -p _out/assets
          # --binary so regenerated provider bundles (components*.gz) stay in
          # the patch e2e applies; mirrors main's pr.patch binary fix (#e566992).
          git diff --binary HEAD > _out/assets/pr.patch
          echo "Final pr.patch:"; git --no-pager diff --stat HEAD
      - name: Upload git diff patch
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-patch
          path: _out/assets/pr.patch
          if-no-files-found: error

  resolve_assets:
    name: "Resolve assets"
    runs-on: ubuntu-latest
    if: |
      contains(github.event.pull_request.labels.*.name, 'release')
      && (github.event.action != 'labeled' || github.event.label.name == 'full-e2e')
    outputs:
      disk_id:      ${{ steps.fetch_assets.outputs.disk_id }}

    permissions:
      contents: read
    steps:
      - name: Generate GitHub App token
        id: app-token
        uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1
        with:
          app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}
          private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}
          owner: cozystack
          repositories: ${{ github.event.repository.name }}
          permission-contents: read

      - name: Checkout code
        if: contains(github.event.pull_request.labels.*.name, 'release')
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Extract tag from PR branch (release PR)
        if: contains(github.event.pull_request.labels.*.name, 'release')
        id: get_tag
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        with:
          script: |
            const branch = context.payload.pull_request.head.ref;
            const m = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/);
            if (!m) {
              core.setFailed(`❌ Branch '${branch}' does not match 'release-X.Y.Z[-suffix]'`);
              return;
            }
            core.setOutput('tag', `v${m[1]}`);

      - name: Find draft release & asset IDs (release PR)
        if: contains(github.event.pull_request.labels.*.name, 'release')
        id: fetch_assets
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        with:
          github-token: ${{ steps.app-token.outputs.token }}
          script: |
            const tag = '${{ steps.get_tag.outputs.tag }}';
            const releases = await github.rest.repos.listReleases({
              owner: context.repo.owner,
              repo:  context.repo.repo,
              per_page: 100
            });
            const draft = releases.data.find(r => r.tag_name === tag && r.draft);
            if (!draft) {
              core.setFailed(`Draft release '${tag}' not found`);
              return;
            }
            const find = (n) => draft.assets.find(a => a.name === n)?.id;
            const diskId     = find('nocloud-amd64.raw.xz');
            if (!diskId) {
              core.setFailed('Required assets missing in draft release');
              return;
            }
            core.setOutput('disk_id',     diskId);


  # In-tree e2e for SAME-REPO PRs. Deliberately NOT named "E2E Tests": the
  # required "E2E Tests" context is a commit status posted by the `e2e-report`
  # job (non-fork) or by e2e-fork.yaml (fork). A required *job* that skips
  # counts as PASSED by branch protection, which is exactly how fork PRs used
  # to merge with e2e never run (#3257); routing the required context through
  # a status removes that skip-to-green path. Fork PRs skip this job
  # (they have no pushed images to pull) and run e2e in the privileged
  # workflow_run instead.
  e2e:
    name: "E2E (in-tree)"
    # The sandbox boots three QEMU guests (hack/e2e-prepare-cluster.bats), each
    # with `-smp 8 -m 24576`, so the guests alone demand 24 vCPU and 72 GiB.
    # On the 24-vCPU runner that is a 1:1 CPU commit with zero host headroom for
    # QEMU's own emulation/IO threads, vhost-net, the host iptables MASQUERADE
    # every guest image pull traverses, containerd, the kernel and the runner
    # agent. During "Install Cozystack" (all platform operators + image pulls
    # across three nodes at once) the guest vCPUs contend with those host threads
    # and lose cycles to CPU steal, so in-guest kubelet/apiserver readiness and
    # liveness probes miss their deadlines ("context deadline exceeded"). A
    # HelmRelease then flips InProgress -> Failed and its dependents cascade to
    # DependencyNotReady — on a different component each run, the signature of
    # resource thrash rather than a product bug. The 32-vCPU / 128 GiB class
    # keeps the guests' 24 vCPU / 72 GiB unchanged (no slower install) while
    # leaving 8 vCPU (25%) and 56 GiB of host headroom, which removes the
    # oversubscription that starves the probes.
    runs-on: oracle-vm-32cpu-128gb-x86-64
    timeout-minutes: 180
    permissions:
      contents: read
      packages: read
      checks: write
    needs: ["finalize", "build-talos", "resolve_assets"]
    # Regular PRs still enter through finalize. A release-labelled promote PR
    # enters through resolve_assets only when a maintainer explicitly adds
    # full-e2e; without that label the immutable rc's already-green suite is not
    # repeated. The labeled trigger above makes that post-open opt-in start a run.
    #
    # Fork PRs never enter here: their build exports OCI archives instead of
    # pushing images, so there is no closure for this job to test. e2e-fork.yaml
    # runs the suite for them from the privileged workflow_run and concludes the
    # required "E2E Tests" status; for same-repo PRs e2e-report concludes it.
    if: |
      !cancelled()
      && !github.event.pull_request.head.repo.fork
      && (
        needs.finalize.result == 'success'
        || (
          needs.resolve_assets.result == 'success'
          && contains(github.event.pull_request.labels.*.name, 'release')
          && contains(github.event.pull_request.labels.*.name, 'full-e2e')
        )
      )

    steps:
      - name: Generate GitHub App token
        if: contains(github.event.pull_request.labels.*.name, 'release')
        id: app-token
        uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1
        with:
          app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}
          private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}
          owner: cozystack
          repositories: ${{ github.event.repository.name }}
          permission-contents: read

      # ▸ Checkout and prepare the codebase
      # fetch-depth: 0 is required by the "Select E2E tests" step so that
      # `git diff origin/${BASE_REF}...HEAD` can resolve the base ref.
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0

      # ▸ Regular PR path – download the Talos disk from build-talos and pr.patch from finalize
      - name: "Download Talos image (regular PR)"
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: talos-image
          path: _out/assets

      - name: Download PR patch
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: pr-patch
          path: _out/assets

      - name: Apply patch
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        run: |
          git apply _out/assets/pr.patch

      # ▸ Release PR path – fetch artefacts from the corresponding draft release
      - name: Download assets from draft release (release PR)
        if: contains(github.event.pull_request.labels.*.name, 'release')
        run: |
          mkdir -p _out/assets
          curl -sSL -H "Authorization: token ${APP_TOKEN}" -H "Accept: application/octet-stream" \
            -o _out/assets/nocloud-amd64.raw.xz \
            "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${{ needs.resolve_assets.outputs.disk_id }}"
        env:
          APP_TOKEN: ${{ steps.app-token.outputs.token }}

      - name: Set sandbox ID
        run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> $GITHUB_ENV

      # ▸ Prepare environment
      - name: Prepare environment
        run: |
          attempt=0
          until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do
            attempt=$((attempt + 1))
            if [ $attempt -ge 3 ]; then
              echo "Attempt $attempt failed, exiting..."
              exit 1
            fi
            echo "Attempt $attempt failed, retrying..."
          done
          echo "Prepare environment completed after $((attempt + 1)) attempts"

      # ▸ Install Cozystack
      - name: Install Cozystack into sandbox
        run: |
          if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then
            echo "❌ Install failed (no retry — see diagnostics below)"
            echo "::group::Diagnostics: HelmRelease status"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true
            echo "::endgroup::"
            echo "::group::Diagnostics: not-Ready HelmReleases (describe)"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \
              | while read -r ns name; do
                  echo "--- $ns/$name ---"
                  docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true
                done
            echo "::endgroup::"
            echo "::group::Diagnostics: recent events"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -50' || true
            echo "::endgroup::"
            exit 1
          fi

      # Test Impact Analysis (TIA): walk the PackageSource graph for the PR
      # diff and select only the Chainsaw suites affected. Runs by default; opt
      # out via the `full-e2e` label to run the full suite.
      #
      # `skip=true` means every changed path was matched by one of select-e2e.sh's
      # rules that select nothing — docs and dashboards, a disabled Chainsaw
      # suite, a backups example with no suite, or the inert repo-meta pattern
      # — so skipping Chainsaw and reporting green is the correct verdict. A
      # path none of the rules recognise escalates to the full suite instead of
      # returning empty, so `skip=true` can no longer mean "TIA had no idea"
      # (#3392). What it still cannot rule out is a workflow nobody classified:
      # .github/ is inert as a directory and the e2e workflows are exempted
      # from it by name, so a workflow added under .github/workflows/ reads as
      # inert until someone puts it on that list, and the fall-through never
      # sees it.
      - name: Select E2E tests
        id: select
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
        env:
          BASE_REF: ${{ github.base_ref }}
        run: |
          git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt
          apps=$(./hack/select-e2e.sh /tmp/changed.txt)
          echo "Selected apps: ${apps:-<none>}"
          echo "apps=${apps}" >> $GITHUB_OUTPUT
          if [ -z "${apps}" ]; then
            echo "skip=true" >> $GITHUB_OUTPUT
          else
            echo "skip=false" >> $GITHUB_OUTPUT
          fi

      - name: Run OpenAPI tests
        run: |
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi

      # ▸ Run E2E tests (Chainsaw, scoped by Test Impact Analysis)
      #
      #   The "Select E2E tests" step above picks the suites impacted by the PR
      #   diff. With the `full-e2e` label that step is skipped, so FULL_E2E is
      #   the only way to reach this step with no selection and we run the whole
      #   suite; an all-inert selection skips this step entirely (the
      #   skip != 'true' guard).
      #
      #   Chainsaw discovers each named suite under hack/e2e-chainsaw/ and polls
      #   every assertion to its own timeout, so the old per-app 3-retry loop is
      #   gone — a failure is reported as a structured assertion diff, with
      #   cluster-state diagnostics dumped below on a non-zero exit.
      - name: Run E2E tests
        id: e2e_tests
        if: ${{ steps.select.outputs.skip != 'true' }}
        env:
          SELECTED_APPS: ${{ steps.select.outputs.apps }}
          FULL_E2E: ${{ contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
        run: |
          # Only the full-e2e label can get here with an empty selection: when
          # `select` runs, an empty result sets skip=true and this step never
          # starts. Testing FULL_E2E alone keeps that invariant visible.
          if [ "$FULL_E2E" = "true" ]; then
            suites=""        # empty => test-chainsaw runs the whole suite
            echo "Running full chainsaw suite"
          else
            suites="$SELECTED_APPS"
            echo "Running selected chainsaw suites: $suites"
          fi
          if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES="$suites"; then
            echo "❌ Chainsaw E2E failed (see the assertion diffs above and diagnostics below)"
            echo "::group::Diagnostics: HelmRelease status"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true
            echo "::endgroup::"
            echo "::group::Diagnostics: recent events"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true
            echo "::endgroup::"
            # COSI state at the moment of failure — the CRDs ship no printer
            # columns, so pull the readiness fields explicitly (cozyreport
            # captures the same at end-of-job, but bucket states move between
            # a test failing and the report being taken).
            echo "::group::Diagnostics: COSI bucket state"
            docker exec "$SANDBOX_NAME" sh -c '
              kubectl get bucketclaims.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.bucketReady,BUCKET:.status.bucketName" 2>&1
              kubectl get buckets.objectstorage.k8s.io -o custom-columns="NAME:.metadata.name,READY:.status.bucketReady,ID:.status.bucketID,CLAIMNS:.spec.bucketClaim.namespace" 2>&1
              kubectl get bucketaccesses.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,GRANTED:.status.accessGranted,CLAIM:.spec.bucketClaimName" 2>&1
            ' || true
            echo "::endgroup::"
            exit 1
          fi
          echo "✅ All selected E2E tests passed"

      - name: Collect chainsaw report
        if: always()
        run: |
          mkdir -p _out
          docker cp "$SANDBOX_NAME":/workspace/hack/e2e-chainsaw/chainsaw-report.xml \
            _out/chainsaw-report.xml || true

      - name: Upload chainsaw report
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: chainsaw-report
          path: _out/chainsaw-report.xml
          if-no-files-found: ignore

      # ▸ Collect debug information (always runs)
      #
      #   A ceiling of its own, not the job's 180 minutes. `Upload
      #   cozyreport.tgz` is the next step and the collector writes its tarball
      #   at the very end of its run, so a read that never returns spends what
      #   is left of the job budget, the job is cancelled, and the upload never
      #   runs -- the artifact is lost rather than truncated, on exactly the run
      #   where it is the only evidence left. `|| true` guards against a
      #   non-zero exit, not against a read that does not return.
      #
      #   30 minutes is measured against the step: it takes 3-42s across sampled
      #   runs, on failed e2e jobs as well as green ones. It RESERVES nothing --
      #   `timeout-minutes` caps a step, it does not carve budget out of the job,
      #   so a suite that has already burned 175 of the 180 leaves this step five
      #   minutes and the upload is cancelled with everything else. What the cap
      #   buys is that a hung read cannot be the reason.
      #   `continue-on-error` because the ceiling must not change what the job
      #   REPORTS. `|| true` on the make call is this step's standing statement
      #   that collecting diagnostics is never fatal, and a step killed at its
      #   ceiling never reaches that `|| true` -- the runner ends the process. So
      #   without this, a passing E2E run whose collection ran long is published
      #   as a failed one, and every later `failure()` in this job starts meaning
      #   "the tests failed OR the report was slow".
      - name: Collect report
        id: collect_report
        if: always()
        timeout-minutes: 30
        continue-on-error: true
        run: |
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true

      # `continue-on-error` keeps the overrun out of the job's result, which also
      # keeps it out of everyone's view: a step that ended at its ceiling renders
      # green and the only trace is in the raw log. This branch's own standard is
      # that a bound which fires is recorded, and that has to hold for the outer
      # bound too.
      #
      # The wording says what `outcome` establishes and no more: a step can end
      # non-zero without any ceiling firing, so naming the timeout as the cause
      # would be the false-mechanism claim the collector refuses to make for a 137.
      - name: Report collection overran its ceiling
        if: always() && steps.collect_report.outcome == 'failure'
        run: |
          echo "::warning title=cozyreport::Collect report ended non-zero -- its timeout-minutes fired, or the step failed before the collector ran; cozyreport.tgz is missing or partial for this run"

      - name: Upload cozyreport.tgz
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: cozyreport
          path: _out/cozyreport.tgz

      - name: Collect images list
        if: always()
        run: |
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-images || true

      - name: Upload image list
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: image-list
          path: _out/images.txt

      # ▸ Open an SSH breakpoint to the failing sandbox so maintainers can attach,
      #   inspect Talos/Cozystack state and resume with `breakpoint resume`.
      #
      #   Opens when the E2E job fails AND any of: the `debug` label, the
      #   `release` label (release PRs are bot-created, so their presence
      #   implies maintainer authorship), or the PR author is a maintainer
      #   (author_association is OWNER/MEMBER/COLLABORATOR). The maintainer-
      #   authorship path needs no label and does NOT move the jobs onto
      #   self-hosted runners — only the `debug` label flips `runs-on`, so a
      #   maintainer PR opens the breakpoint on the default oracle-vm runner the
      #   job already uses and simply holds that ephemeral runner. authorized-users
      #   is kept as a second defense layer controlling who may actually SSH in.
      #
      #   Uses cozystack/breakpoint-action (fork of namespacelabs/breakpoint-action)
      #   pinned by SHA. The fork adds: pause-idle mode (initial grace period for
      #   the first SSH connection, idle-aware exit afterwards), endpoint output
      #   and ::notice:: annotation, and a dedicated Check Run "Breakpoint Open"
      #   that carries the SSH endpoint in output.summary while the breakpoint is
      #   paused (conclusion=failure → standard ✗ in `gh pr checks`; updated to
      #   conclusion=success when the breakpoint exits).
      #
      #   Configured via the BREAKPOINT_ENDPOINT repository variable — in fork
      #   PRs repository variables are not exposed, so the step is skipped and
      #   forks cannot reach the self-hosted rendezvous server.
      - name: Breakpoint on E2E failure
        if: |
          failure() &&
          vars.BREAKPOINT_ENDPOINT != '' &&
          (contains(github.event.pull_request.labels.*.name, 'debug')
            || contains(github.event.pull_request.labels.*.name, 'release')
            || contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))
        # cozystack/breakpoint-action v2-cozy.1
        # mode: pause-idle defaults: grace-period=20m, idle-timeout=10m
        uses: cozystack/breakpoint-action@a6f3a6f87be398ad63b6577351e3398e53f578e4
        with:
          mode: pause-idle
          endpoint: ${{ vars.BREAKPOINT_ENDPOINT }}
          authorized-users: androndo, Arsolitt, IvanHunters, kvaps, lexfrei, lllamnyp, mattia-eleuteri, matthieu-robin, myasnikovdaniil, sircthulhu, tym83
          check-run-name: "Breakpoint Open"
          github-token: ${{ github.token }}
          check-run-summary-template: |
            ## 🔴 SSH breakpoint open — paused for debug

            ```
            {endpoint}
            ```

            Enter the e2e sandbox after SSH:
            ```
            docker exec -ti $(docker ps --filter name=cozy-e2e-sandbox -q | head -1) bash
            export KUBECONFIG=/workspace/kubeconfig
            ```

            Resume from inside: `breakpoint resume`. Otherwise the breakpoint
            exits 10 minutes after the last SSH session disconnects.

      # ▸ Tear down environment (always runs)
      - name: Tear down sandbox
        if: always()
        run: make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true

  # Report the required "E2E Tests" gate as a commit status rather than relying
  # on a job named "E2E Tests". A required *job* that is skipped counts
  # as PASSED by branch protection — desired for docs-only PRs, but the exact
  # hole that let fork PRs merge with e2e never run (#3257). This job always runs
  # on same-repo PRs and concludes "E2E Tests" deterministically:
  #   • plan failed              → failure
  #   • docs-only PR (code!=true)→ success (e2e not required)
  #   • in-tree e2e succeeded     → success
  #   • otherwise (skipped/failed/cancelled with code changes) → failure
  # Fork PRs skip this job; e2e-fork.yaml concludes "E2E Tests" for them, so a
  # fork PR stays "Expected"/not-mergeable until the privileged run reports.
  #
  # The required "E2E Tests" merge gate is published as a COMMIT STATUS with the
  # default GITHUB_TOKEN (+ statuses:write) — NOT as a job, and NOT as a
  # check-run.
  #   - Not a job: a *skipped* required job counts as passed by branch
  #     protection, which is how fork PRs used to merge with e2e never run
  #     (#3257). A status is "success" only when this job posts it; a
  #     never-posted required status stays "Expected" and blocks the merge.
  #   - Not a check-run: a check-run made with the default token floats into an
  #     arbitrary GitHub Actions check-suite (it rendered under "PR size label"),
  #     and an app-token check-run needs the app granted checks:write. A commit
  #     status has neither problem — branch protection matches it by context name
  #     and it carries no check-suite grouping. Safe with the default token: a
  #     fork's pull_request token is read-only and cannot post a status, so only
  #     same-repo runs (this job, guarded !fork) and the privileged workflow_run
  #     in e2e-fork.yaml can set it.
  # REQUIRES main branch protection to require the "E2E Tests" status context —
  # see docs/agents/e2e-testing.md §10.
  #
  # The `labeled` guard must stay in lockstep with `plan`'s: this job runs if and
  # only if `plan` runs. `labeled` events other than full-e2e are discarded there,
  # so without the same guard this job would run with plan *skipped*, read
  # PLAN_RESULT != 'success', and post "E2E Tests" = failure — clobbering an
  # already-green required status every time anyone labels an open PR.
  e2e-report:
    name: "Report E2E status"
    needs: ["plan", "e2e"]
    # !cancelled() rather than always(). A cancelled run no longer owns the
    # status for its head SHA, because something superseded it and that
    # something is now responsible. Posting from here would stamp a red on a
    # suite that never finished, and the run that replaced it may be one that
    # deliberately posts nothing, leaving nothing to flip it back. A cancelled
    # run must report nothing.
    #
    # Cancellation still reaches this job from a push, from a `full-e2e` label
    # run superseding the head run, and from a manual cancel. It no longer
    # reaches it from a label the workflow discards: those run in their own
    # concurrency group (see the key at the top of this file) precisely so they
    # cannot cancel a live run.
    if: ${{ !cancelled() && !github.event.pull_request.head.repo.fork && (github.event.action != 'labeled' || github.event.label.name == 'full-e2e') }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      statuses: write
    steps:
      - name: Report "E2E Tests" commit status
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        env:
          PLAN_RESULT: ${{ needs.plan.result }}
          CODE: ${{ needs.plan.outputs.code }}
          E2E_RESULT: ${{ needs.e2e.result }}
          IS_RELEASE: ${{ contains(github.event.pull_request.labels.*.name, 'release') }}
          IS_FULL_E2E: ${{ contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        with:
          script: |
            const { PLAN_RESULT, CODE, E2E_RESULT, IS_RELEASE, IS_FULL_E2E, HEAD_SHA, RUN_URL } = process.env;
            let state, description;
            if (PLAN_RESULT !== 'success') {
              state = 'failure';
              description = `plan job did not succeed (result: ${PLAN_RESULT})`;
            } else if (CODE !== 'true') {
              state = 'success';
              description = 'docs-only PR — e2e not required';
            } else if (IS_RELEASE === 'true' && IS_FULL_E2E !== 'true') {
              // Release promotion PRs deliberately do not repeat e2e: the rc was
              // already verified against the immutable candidate whose digests
              // this tree pins, which is the whole point of the rc gate. The `e2e`
              // job skips for exactly this case, and while the gate was a job that
              // skip satisfied branch protection. Now that the gate is a status,
              // the skip has to be reported explicitly or every promotion needs a
              // full suite re-run or an admin override.
              state = 'success';
              description = 'release promotion — verified at rc, e2e not repeated (add full-e2e to force a run)';
            } else if (E2E_RESULT === 'success') {
              state = 'success';
              description = 'in-tree e2e passed';
            } else {
              state = 'failure';
              description = `in-tree e2e did not pass (result: ${E2E_RESULT})`;
            }
            try {
              await github.rest.repos.createCommitStatus({
                owner: context.repo.owner,
                repo: context.repo.repo,
                sha: HEAD_SHA,
                state,
                context: 'E2E Tests',
                description,
                target_url: RUN_URL,
              });
            } catch (err) {
              // The 'E2E Tests' status is the required merge gate. If we cannot
              // publish it, fail loudly and fail-closed: setFailed keeps the job
              // red and the required status unreported ("Expected"), so the PR
              // stays non-mergeable.
              core.setFailed(`Failed to report E2E Tests status: ${err}`);
              return;
            }
            core.info(`E2E Tests → ${state}: ${description}`);