cozystack/.github/workflows/pull-requests.yaml

name: Pull Request

env:
  # TODO: unhardcode this
  REGISTRY: iad.ocir.io/idyksih5sir9/cozystack
on:
  pull_request:
    types: [opened, synchronize, reopened]

# Cancel in‑flight runs for the same PR when a new push arrives.
concurrency:
  group: pr-${{ github.workflow }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

# Top-level token defaults to read-only; jobs request the minimum extra scopes.
permissions:
  contents: read

jobs:
  # Compute the docs-only gate and the per-package build matrix from the PR
  # diff. hack/build-matrix.sh parses the root Makefile `build:` target (the
  # single source of truth) and emits only the units whose package dir changed,
  # escalating to the full set on a shared-dependency change. talos + installer
  # are handled by dedicated jobs (see build-matrix.sh for why).
  plan:
    name: Plan build
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      code: ${{ steps.p.outputs.code }}
      trigger_map: ${{ steps.p.outputs.trigger_map }}
      matrix: ${{ steps.p.outputs.matrix }}
      any: ${{ steps.p.outputs.any }}
      touched: ${{ steps.p.outputs.touched }}
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
      - name: Compute changed files and build matrix
        id: p
        env:
          BASE_REF: ${{ github.base_ref }}
        run: |
          git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt
          echo "Changed files:"; sed 's/^/  /' /tmp/changed.txt
          if grep -qvE '^docs/' /tmp/changed.txt; then
            echo "code=true" >> "$GITHUB_OUTPUT"
          else
            echo "code=false" >> "$GITHUB_OUTPUT"
          fi
          # docs/agents/contributing.md carries the downstream trigger map, which
          # hack/downstream-trigger-map.bats asserts against: every path the map
          # cites still exists, and its repository list still matches the PR
          # template. Editing the map alone is a docs-only PR, which would skip the
          # unit tests and exempt the map from its own guard. This flag lets the
          # unit-test job opt back in, and only that job: it deliberately does not
          # feed `code`, so a docs PR still builds nothing.
          #
          # Detect the map by its old path too: a rename is reported as the new path
          # alone, so a PR that moves the map would skip the very test that pins the
          # literal below to it, and break for whoever pushes next.
          git diff --name-only --no-renames "origin/${BASE_REF}...HEAD" > /tmp/changed_norenames.txt
          if grep -qxF 'docs/agents/contributing.md' /tmp/changed_norenames.txt; then
            echo "trigger_map=true" >> "$GITHUB_OUTPUT"
          else
            echo "trigger_map=false" >> "$GITHUB_OUTPUT"
          fi
          matrix="$(hack/build-matrix.sh /tmp/changed.txt)"
          echo "Build matrix: $matrix"
          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
          # Package dirs the PR edited (packages/<area>/<name>). Their committed
          # refs are the PR's intent and must survive the current-main overlay —
          # otherwise an upstream image bump in a non-build-unit package would be
          # silently reverted in e2e/the installer artifact (see
          # hack/overlay-main-images.sh). Whitespace-separated, no trailing slash.
          touched="$(sed -nE 's#^(packages/[^/]+/[^/]+)/.*#\1#p' /tmp/changed.txt | sort -u | tr '\n' ' ')"
          echo "Touched packages: $touched"
          echo "touched=$touched" >> "$GITHUB_OUTPUT"
          if [ "$matrix" = "[]" ]; then
            echo "any=false" >> "$GITHUB_OUTPUT"
          else
            echo "any=true" >> "$GITHUB_OUTPUT"
          fi

  # Verify GitHub's prospective merge tree here, so a `packages/` edit pushed to
  # this PR fails this check before it can merge.
  #
  # Checking the MERGE tree rather than the head is what makes base drift
  # visible when this job runs — but nothing re-runs it when the base moves:
  # GitHub does not fire `pull_request` on the base branch advancing, it only
  # recomputes refs/pull/N/merge. So a `packages/` change landing on the base
  # after this went green is never re-verified here, and the PR stays mergeable
  # on the stale result. Finalize repeats the check against the real merge
  # commit before creating the write-once stable tag, which is where that case
  # is actually caught — loudly, and before any stable name exists.
  #
  # The trigger predicate is the promote PR's author plus its head-branch shape,
  # NOT the `release` label main keys on. This branch's `on.pull_request.types`
  # has no `labeled` event, and `gh pr create --label release` applies the label
  # in a separate call one second after the PR opens (observed on the v1.6.1
  # promote PR), so the `opened` payload carries no labels at all and a
  # label-keyed guard would never fire here. Every `cozystack-ci[bot]` PR whose
  # head starts with `release-` is a promote PR, so this is if anything the
  # narrower predicate — and it re-runs on `synchronize`, which is what makes a
  # `packages/` edit pushed to the PR visible. A prerelease staging head
  # (release-X.Y.Z-rc.N) never gets a PR under the checks-at-rc flow; if one ever
  # did, the verifier's own X.Y.Z assertion fails it closed rather than passing.
  verify-release-candidate:
    name: Verify release packages candidate
    runs-on: ubuntu-latest
    # Matching `checks` and `finalize`. Without a ceiling the job inherits the
    # 6-hour default, and a stalled `flux pull artifact` would hold this check
    # pending for all of it.
    timeout-minutes: 30
    if: |
      github.event.pull_request.user.login == 'cozystack-ci[bot]'
      && startsWith(github.head_ref, 'release-')
    permissions:
      contents: read
      packages: read
    steps:
      # The default pull_request checkout is GitHub's prospective merge commit,
      # not just the head branch. That makes base-branch package drift visible.
      - name: Checkout prospective merge
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0

      # Execute the verifier from the trusted base SHA. An older rc-derived head
      # may not contain it, and a PR must not be able to weaken its own guard.
      - name: Checkout release tooling from base
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          path: .release-tooling
          sparse-checkout: hack
          persist-credentials: false

      # Both tools are version-pinned. This job decides whether a stable release
      # may be created, so what it runs has to be a fixed input: `latest/download`
      # resolves to whatever mikefarah shipped this morning, and a yq that changed
      # how it emits or compares a value turns a release gate into a moving one.
      # The version tracks packages/core/testing/images/e2e-sandbox/Dockerfile.
      # yq is additionally pinned by content; the Flux installer is not, because
      # it checksums the binary it fetches and the mutable part is its own
      # bootstrap script.
      - name: Set up promotion toolchain (flux, yq)
        env:
          FLUX_VERSION: "2.8.6"
          YQ_VERSION: "4.53.3"
          YQ_SHA256: "fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4"
        run: |
          if ! flux version --client 2>/dev/null | grep -qx "flux: v${FLUX_VERSION}"; then
            install_script="$(mktemp)"
            curl -fsSL https://fluxcd.io/install.sh -o "$install_script"
            sudo env "FLUX_VERSION=$FLUX_VERSION" bash "$install_script"
            rm -f "$install_script"
          fi
          if ! yq --version 2>/dev/null | grep -q "mikefarah.* version v${YQ_VERSION}\$"; then
            yq_bin="$(mktemp)"
            curl -fsSL -o "$yq_bin" \
              "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64"
            printf '%s  %s\n' "$YQ_SHA256" "$yq_bin" | sha256sum -c -
            sudo install -m 0755 "$yq_bin" /usr/local/bin/yq
            rm -f "$yq_bin"
          fi

      - name: Login to registry (GHCR)
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
          registry: ghcr.io
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker

      - name: Verify stable packages candidate
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          HEAD_BRANCH: ${{ github.head_ref }}
        run: |
          STABLE_VERSION="${HEAD_BRANCH#release-}"
          .release-tooling/hack/verify-promoted-packages.sh "$STABLE_VERSION"

  # Helm-unittest + Go controller tests. Decoupled from the image build and from
  # e2e so they give fast, independent signal; they gate the PR as their own
  # required check rather than blocking e2e from starting.
  checks:
    name: Unit & controller tests
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 30
    permissions:
      contents: read
    needs: ["plan"]
    # trigger_map opts a docs-only PR back in when it edits the downstream trigger
    # map, so hack/downstream-trigger-map.bats still guards it. No other job reads
    # that flag: a docs PR runs the unit tests and nothing heavier.
    if: |
      (needs.plan.outputs.code == 'true' || needs.plan.outputs.trigger_map == 'true')
      && !contains(github.event.pull_request.labels.*.name, 'release')
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
      # Ephemeral runners don't carry the helm-unittest plugin the persistent
      # self-hosted runner accreted. Install if absent (idempotent).
      - name: Set up test toolchain
        run: |
          helm plugin list 2>/dev/null | grep -q unittest \
            || helm plugin install https://github.com/helm-unittest/helm-unittest
      - name: Run unit tests
        run: make unit-tests
      - name: Run controller Go tests
        run: make test-controllers

  # One image (package) per job, fanned out across the ephemeral pool. Each job
  # gets its own buildx (docker-container driver) so the shared-buildkit
  # contention that wedged the old single Build job (#2937) is severed by
  # construction. Each job patches only its own package's values.yaml/.tag files
  # and uploads that diff as a patch fragment; `finalize` reassembles them.
  build:
    name: Build ${{ matrix.pkg }}
    needs: ["plan"]
    if: |
      needs.plan.outputs.code == 'true'
      && needs.plan.outputs.any == 'true'
      && !contains(github.event.pull_request.labels.*.name, 'release')
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 30
    permissions:
      contents: read
      packages: write
    strategy:
      fail-fast: false
      matrix:
        pkg: ${{ fromJSON(needs.plan.outputs.matrix) }}
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true
      # installer is built in finalize, but some package image targets shell out
      # to flux; install it if absent (idempotent, cheap).
      - name: Set up build toolchain
        run: |
          if ! command -v flux >/dev/null; then
            install_script="$(mktemp)"
            curl -fsSL https://fluxcd.io/install.sh -o "$install_script"
            sudo bash "$install_script"
            rm -f "$install_script"
          fi
      - name: Set up Docker config
        run: |
          if [ -d ~/.docker ]; then
            cp -r ~/.docker "${{ runner.temp }}/.docker"
          fi
      - name: Login to GitHub Container Registry
        if: ${{ !github.event.pull_request.head.repo.fork }}
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          username: ${{ secrets.OCIR_USER }}
          password: ${{ secrets.OCIR_TOKEN }}
          registry: iad.ocir.io
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # docker-container driver: isolated buildkit per job (no shared-buildkit
      # contention) and a prerequisite for the mode=max cache reads (#2938).
      # Its buildkitd also routes docker.io base-image pulls through
      # mirror.gcr.io (hack/buildkitd.toml): the Build step authenticates only
      # to the push registry (OCIR), so base images (golang/ubuntu/nginx/...)
      # would otherwise be pulled from Docker Hub anonymously and intermittently
      # trip its anonymous-pull rate limit; the mirror sidesteps that and falls
      # back to Docker Hub on a miss/outage. Create the builder under the SAME
      # DOCKER_CONFIG the Build step reads (runner.temp/.docker, holding the
      # OCIR auth) so buildx forwards push/cache auth to the container builder.
      - name: Set up Buildx (docker-container driver, docker.io mirror)
        id: buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
        with:
          driver: docker-container
          buildkitd-config: hack/buildkitd.toml
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      - name: Build ${{ matrix.pkg }}
        run: make -C "$PKG" image
        env:
          PKG: ${{ matrix.pkg }}
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          # Build on the docker-container builder created above so base-image
          # pulls go through the docker.io mirror (BUILDX_ARGS adds
          # --builder=$(BUILDER)).
          BUILDER: ${{ steps.buildx.outputs.name }}
          # Unique per (PR, commit) so concurrent PR builds never push the same
          # tag. PR images are consumed by digest; :latest belongs to releases.
          IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
          PUBLISH_VERSIONED: '0'
          PUBLISH_FLOATING: '0'
      - name: Capture digest patch fragment
        id: frag
        env:
          PKG: ${{ matrix.pkg }}
        run: |
          safe="$(echo "$PKG" | tr '/' '-')"
          mkdir -p _out/fragments
          # --binary so vendored binary provider bundles (components*.gz),
          # which some package image targets regenerate, survive reassembly
          # in finalize (a plain "Binary files differ" stub is unappliable).
          git diff --binary HEAD > "_out/fragments/${safe}.patch"
          echo "safe=${safe}" >> "$GITHUB_OUTPUT"
          echo "Fragment for ${PKG}:"; git --no-pager diff --stat HEAD
      - name: Upload patch fragment
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-patch-fragment-${{ steps.frag.outputs.safe }}
          path: _out/fragments/${{ steps.frag.outputs.safe }}.patch
          if-no-files-found: error

  # Talos is its own leg: the installer tarball is heavy and the talos image is
  # built from it, so both live in one job. No nocloud disk is built here any
  # more, and e2e no longer downloads one -- the container lane boots the Talos
  # image directly. `images/talos/profiles/nocloud.yaml` is therefore compiled
  # by the tag build's `make assets` rather than per PR; what a PR still
  # compiles is installer.yaml, which shares every input with it bar the
  # platform and the output format. Always runs on non-docs PRs, because the
  # patch fragment and the installer chain need it regardless of the
  # per-package matrix scope.
  build-talos:
    name: Build Talos
    needs: ["plan"]
    if: |
      needs.plan.outputs.code == 'true'
      && !contains(github.event.pull_request.labels.*.name, 'release')
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 40
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - name: Set up Docker config
        run: |
          if [ -d ~/.docker ]; then
            cp -r ~/.docker "${{ runner.temp }}/.docker"
          fi
      - name: Login to GitHub Container Registry
        if: ${{ !github.event.pull_request.head.repo.fork }}
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          username: ${{ secrets.OCIR_USER }}
          password: ${{ secrets.OCIR_TOKEN }}
          registry: iad.ocir.io
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
      # (hack/buildkitd.toml) so the talos image build dodges Docker Hub's
      # anonymous-pull rate limit; see the build job for the full rationale.
      - name: Set up Buildx (docker-container driver, docker.io mirror)
        id: buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
        with:
          driver: docker-container
          buildkitd-config: hack/buildkitd.toml
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      - name: Build Talos image (matchbox + installer tarball)
        run: make -C packages/core/talos image
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          BUILDER: ${{ steps.buildx.outputs.name }}
          IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
          PUBLISH_VERSIONED: '0'
          PUBLISH_FLOATING: '0'
      - name: Capture Talos patch fragment
        run: |
          mkdir -p _out/fragments
          git diff --binary HEAD > _out/fragments/talos.patch
          echo "Talos fragment:"; git --no-pager diff --stat HEAD
      - name: Upload Talos patch fragment
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-patch-fragment-talos
          path: _out/fragments/talos.patch
          if-no-files-found: error

  # Reassemble the unified pr.patch from every fragment, then build the installer
  # LAST: `make -C packages/core/installer image` runs `flux push artifact
  # --path=packages`, which bundles the WHOLE digest-patched tree into the OCI
  # artifact the operator pulls — so every other package's digest edit must
  # already be applied here. e2e consumes the resulting pr.patch unchanged.
  finalize:
    name: Finalize (merge patches + installer)
    # Depends on `checks` too: gating finalize on unit/controller tests cascades
    # to e2e (which needs finalize), so a red unit test skips the expensive
    # install+e2e path instead of wasting ~1h on it. Restores the gating the old
    # in-build-job tests gave, without re-coupling the cheap parallel builds.
    needs: ["plan", "checks", "build", "build-talos"]
    if: |
      !cancelled()
      && needs.plan.outputs.code == 'true'
      && !contains(github.event.pull_request.labels.*.name, 'release')
      && needs.checks.result == 'success'
      && (needs.build.result == 'success' || needs.build.result == 'skipped')
      && needs.build-talos.result == 'success'
    runs-on: oracle-vm-4cpu-16gb-x86-64
    timeout-minutes: 30
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - name: Set up build toolchain
        run: |
          if ! command -v flux >/dev/null; then
            install_script="$(mktemp)"
            curl -fsSL https://fluxcd.io/install.sh -o "$install_script"
            sudo bash "$install_script"
            rm -f "$install_script"
          fi
      - name: Download patch fragments
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          pattern: pr-patch-fragment-*
          merge-multiple: true
          path: _out/fragments
      - name: Apply digest fragments
        run: |
          applied=0
          # Each fragment patches only its own package's files; --3way lets two
          # fragments that touch adjacent lines of a shared file (e.g. a package
          # Makefile that writes a sibling package's values.yaml) merge cleanly
          # instead of the second apply aborting the whole finalize.
          for f in _out/fragments/*.patch; do
            [ -s "$f" ] || continue
            echo "Applying $f"
            git apply --3way "$f"
            applied=$((applied + 1))
          done
          echo "Applied $applied fragment(s)"
          git --no-pager diff --stat HEAD
      - name: Set up Docker config
        run: |
          if [ -d ~/.docker ]; then
            cp -r ~/.docker "${{ runner.temp }}/.docker"
          fi
      - name: Login to GitHub Container Registry
        if: ${{ !github.event.pull_request.head.repo.fork }}
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
        with:
          username: ${{ secrets.OCIR_USER }}
          password: ${{ secrets.OCIR_TOKEN }}
          registry: iad.ocir.io
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # Overlay current-main image refs onto every package this PR did NOT
      # rebuild. Without it, unbuilt packages fall back to the last-release refs
      # committed in the tree (e.g. v1.5.0) even when main is hundreds of commits
      # ahead — so e2e and the installer's packages artifact would ship stale
      # images for everything outside this PR's build matrix. Source of truth is
      # the cozystack-packages:main OCI artifact build-main.yaml already
      # publishes (the whole current-main packages tree, refs digest-pinned to
      # main's images). Runs after login (creds available) and before the
      # installer build below so the flux-pushed packages tree — and the
      # pr.patch e2e applies — already reflect the overlay. A missing artifact
      # (build-main not yet green) is a no-op: packages keep their committed
      # refs (prior behaviour). See hack/overlay-main-images.sh.
      #
      # Read the artifact for the PR's OWN base branch, not always main. Overlaying
      # main's images onto a release-line PR mixes generations: main's binaries
      # against that line's charts. #3437 (a one-line change on release-1.6) failed
      # install deterministically that way, because main's cozystack-controller
      # served an aggregated OpenAPI release-1.6's charts could not validate
      # against (`unknown model in reference: …v1alpha1.OptionSpec`) — a broken
      # lane reading as a broken PR. build-release.yaml publishes
      # cozystack-packages:<line> for every maintained release-<major>.<minor>
      # branch, so each base branch has its own artifact to read.
      #
      # A missing artifact stays a no-op (packages keep their committed refs), which
      # also covers a line whose first build-release run has not landed yet.
      - name: Pull base-branch packages tree
        run: |
          rm -rf _out/mainpkgs
          mkdir -p _out/mainpkgs
          if ! flux pull artifact "oci://${REGISTRY}/cozystack-packages:${BASE_REF}" \
                 --output _out/mainpkgs; then
            rm -rf _out/mainpkgs
            # Degrading to committed refs is safe, but SAY so: on a release line a
            # missing artifact means build-release.yaml has not published this line
            # yet (first build pending, or its branch filter does not match this
            # branch), and the PR is then testing the line's last release instead of
            # its tip. Silent would look identical to a working overlay.
            if [ "${BASE_REF}" = "main" ]; then
              echo "cozystack-packages:${BASE_REF} pull failed — keeping committed refs"
            else
              echo "::warning title=No packages artifact for ${BASE_REF}::Keeping committed refs, so unbuilt packages use this line's last release rather than its tip. Check that build-release.yaml has run for ${BASE_REF}."
            fi
          fi
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          # Env-passed, not interpolated into the script: base_ref is repo-controlled
          # here (a branch name in this repo), but the workflow-injection hygiene
          # applies uniformly.
          BASE_REF: ${{ github.base_ref }}
      - name: Overlay base-branch refs for unbuilt packages
        # Best-effort: this only improves WHICH images e2e and the installer use,
        # so an unexpected failure must degrade to the committed refs (prior
        # behaviour), never block the PR. overlay-main-images.sh already handles
        # the expected paths (missing artifact, drift, absent unit) internally —
        # including the empty directory the step above leaves when a line has no
        # artifact yet.
        run: |
          hack/overlay-main-images.sh _out/mainpkgs "$BUILD_MATRIX" "$PR_TOUCHED" \
            || echo "::warning::overlay-main-images failed; unbuilt packages keep committed refs"
        env:
          # The set of packages THIS PR rebuilt; their pr-<N>-<sha> refs are
          # authoritative, so overlay-main-images.sh skips them. Computed by the
          # plan job from the Makefile (a JSON array of package dirs), not from
          # untrusted PR input — passed via env per workflow-injection hygiene.
          BUILD_MATRIX: ${{ needs.plan.outputs.matrix }}
          # Packages THIS PR edited (changed-file dirs from the plan job's git
          # diff). Their committed refs are the PR's intent — an upstream image
          # bump in a non-build-unit package would otherwise be silently reverted
          # by the overlay — so overlay-main-images.sh preserves them. Also
          # env-passed, derived from the diff, not from untrusted PR input.
          PR_TOUCHED: ${{ needs.plan.outputs.touched }}
      # buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
      # (hack/buildkitd.toml) so the installer build dodges Docker Hub's
      # anonymous-pull rate limit; see the build job for the full rationale.
      - name: Set up Buildx (docker-container driver, docker.io mirror)
        id: buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
        with:
          driver: docker-container
          buildkitd-config: hack/buildkitd.toml
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
      # Builds the operator image AND flux-pushes the now-fully-patched packages
      # tree, then patches the installer's platformSourceRef to that artifact.
      - name: Build installer (operator + packages artifact)
        run: make -C packages/core/installer image
        env:
          DOCKER_CONFIG: ${{ runner.temp }}/.docker
          BUILDER: ${{ steps.buildx.outputs.name }}
          IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
          PUBLISH_VERSIONED: '0'
          PUBLISH_FLOATING: '0'
      - name: Assemble pr.patch
        run: |
          mkdir -p _out/assets
          # --binary so regenerated provider bundles (components*.gz) stay in
          # the patch e2e applies; mirrors main's pr.patch binary fix (#e566992).
          git diff --binary HEAD > _out/assets/pr.patch
          echo "Final pr.patch:"; git --no-pager diff --stat HEAD
      - name: Upload git diff patch
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: pr-patch
          path: _out/assets/pr.patch
          if-no-files-found: error

  resolve_assets:
    name: "Resolve assets"
    runs-on: ubuntu-latest
    if: contains(github.event.pull_request.labels.*.name, 'release')
    permissions:
      contents: read
    steps:
      - name: Generate GitHub App token
        id: app-token
        uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1
        with:
          app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}
          private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}
          owner: cozystack
          repositories: ${{ github.event.repository.name }}
          permission-contents: read

      - name: Checkout code
        if: contains(github.event.pull_request.labels.*.name, 'release')
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: Extract tag from PR branch (release PR)
        if: contains(github.event.pull_request.labels.*.name, 'release')
        id: get_tag
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        with:
          script: |
            const branch = context.payload.pull_request.head.ref;
            const m = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/);
            if (!m) {
              core.setFailed(`❌ Branch '${branch}' does not match 'release-X.Y.Z[-suffix]'`);
              return;
            }
            core.setOutput('tag', `v${m[1]}`);

      # The draft release has to exist for a release PR's e2e run to mean
      # anything, and this job is what proves it. It used to also resolve the
      # nocloud disk asset and export its id; the container lane boots no disk,
      # so nothing in this workflow reads that id any more and resolving it only
      # added a way to fail a release PR over an asset it does not use. The
      # asset is still resolved where it is still consumed, in e2e-tag.yaml.
      - name: Verify draft release exists (release PR)
        if: contains(github.event.pull_request.labels.*.name, 'release')
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        with:
          github-token: ${{ steps.app-token.outputs.token }}
          script: |
            const tag = '${{ steps.get_tag.outputs.tag }}';
            const releases = await github.rest.repos.listReleases({
              owner: context.repo.owner,
              repo:  context.repo.repo,
              per_page: 100
            });
            const draft = releases.data.find(r => r.tag_name === tag && r.draft);
            if (!draft) {
              core.setFailed(`Draft release '${tag}' not found`);
              return;
            }


  # In-tree e2e, on Talos nodes running as CONTAINERS. The QEMU substrate this
  # lane used to boot is off the per-PR path only: it is still wired into
  # e2e-tag.yaml, which runs `make prepare-env` -> `make -C packages/core/testing
  # prepare-cluster`. That target and hack/e2e-prepare-cluster.bats are
  # therefore live code, not leftovers, and deleting them takes every
  # release-candidate run with them. They carry the coverage containers cannot:
  # DRBD, and with it the `replicated` StorageClass, its Immediate binding mode
  # and the tenant StorageClass propagation that only applies to
  # remotely-accessible classes.
  e2e:
    name: "E2E Tests"
    # hack/e2e-compose.yaml gives each of the three nodes `cpus: 8` and
    # `mem_limit: 24576m`, matching what the QEMU guests were given, so the nodes
    # still demand 24 vCPU and 72 GiB between them. The same headroom argument
    # therefore still decides the class: on a 24-vCPU runner that is a 1:1 commit
    # with nothing left for containerd, the kernel, the host iptables MASQUERADE
    # every image pull traverses and the runner agent, and during "Install
    # Cozystack" the nodes contend with those and lose cycles. Readiness and
    # liveness probes then miss their deadlines, a HelmRelease flips InProgress ->
    # Failed and its dependents cascade to DependencyNotReady -- on a different
    # component each run, the signature of resource thrash rather than a product
    # bug. The 32-vCPU / 128 GiB class leaves 8 vCPU (25%) and 56 GiB of headroom
    # and removes that oversubscription.
    #
    # One thing this class does NOT fix, and it is worth knowing before reading a
    # red run: a container node reports the HOST's CPU and memory as its capacity,
    # and hack/e2e-container-up.sh counterbalances that with kubelet
    # systemReserved so the scheduler sees 8 CPU / 24 GiB. That corrects the
    # scheduler and not what a pod reads from /proc, so a workload that sizes
    # itself from /proc/cpuinfo or /proc/meminfo is configured for the whole host.
    runs-on: oracle-vm-32cpu-128gb-x86-64
    timeout-minutes: 180
    permissions:
      contents: read
      packages: read
      checks: write
    needs: ["finalize", "resolve_assets"]
    if: ${{ !cancelled() && (needs.finalize.result == 'success' || needs.resolve_assets.result == 'success') }}

    steps:
      # ▸ Checkout and prepare the codebase
      # fetch-depth: 0 is required by the "Select E2E tests" step so that
      # `git diff origin/${BASE_REF}...HEAD` can resolve the base ref.
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0

      # ▸ Regular PR path – pr.patch from finalize. No Talos disk: the nodes boot
      #   from ghcr.io/siderolabs/talos directly (hack/e2e-compose.yaml), so this
      #   lane consumes no nocloud asset and waits on no Talos build.
      - name: Download PR patch
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: pr-patch
          path: _out/assets

      - name: Apply patch
        if: "!contains(github.event.pull_request.labels.*.name, 'release')"
        run: |
          git apply _out/assets/pr.patch

      - name: Set sandbox ID
        run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> $GITHUB_ENV

      # ▸ Host kernel modules. MANDATORY and up front, because nothing later can
      #   supply them: the nodes take their kernel from this host, and Talos
      #   cannot load modules in container mode -- `machine.kernel.modules` is a
      #   silent no-op there, `kernel_module_spec.go` returns early on
      #   ModeContainer with no error and no event.
      #
      #   A module missing here is missing for every node for the whole run, and
      #   it surfaces in disguise: no `openvswitch` and kube-ovn comes up with no
      #   dataplane, no `zfs` and every LINSTOR pool creation fails. Both read as
      #   a CNI or a storage regression an hour later. So this loads both and
      #   then asserts both, and a failure here names the host rather than the
      #   product.
      - name: Prepare container-lane host
        run: |
          set -eu
          echo "::group::Loading kernel modules required by the container lane"
          for m in openvswitch zfs; do
            if grep -q "^${m} " /proc/modules; then
              echo "${m}: already loaded"
              continue
            fi
            if [ "${m}" = zfs ]; then
              sudo apt-get update -qq
              sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq zfsutils-linux
            fi
            echo "${m}: loading"
            sudo modprobe "${m}" || echo "${m}: modprobe failed"
          done
          echo "::endgroup::"
          missing=''
          for m in openvswitch zfs; do
            grep -q "^${m} " /proc/modules || missing="${missing} ${m}"
          done
          if [ -n "${missing}" ]; then
            echo "::error title=container-lane host::kernel module(s) could not be loaded:${missing}"
            echo "This is a host problem, not a product failure. The e2e nodes run as" >&2
            echo "containers on this runner's kernel and Talos cannot load modules in" >&2
            echo "container mode, so what is not here now is unavailable for the whole" >&2
            echo "run: openvswitch backs the kube-ovn dataplane, zfs backs every" >&2
            echo "LINSTOR pool. Failing here rather than an hour later is the point." >&2
            exit 1
          fi
          grep -E '^(openvswitch|zfs) ' /proc/modules

      # ▸ Prepare environment
      - name: Prepare workspace
        run: |
          rm -rf /tmp/$SANDBOX_NAME
          cp -r ${{ github.workspace }} /tmp/$SANDBOX_NAME

      # Two make targets rather than one: the nodes are siblings on the runner's
      # docker daemon (the /dev and /run/udev binds in hack/e2e-compose.yaml must
      # resolve against the real host devtmpfs, which is what lets this lane
      # carry LINSTOR at all), so the host-side half needs a docker socket the
      # sandbox does not have.
      - name: Prepare environment
        run: |
          cd /tmp/$SANDBOX_NAME
          attempt=0
          until make SANDBOX_NAME=$SANDBOX_NAME prepare-env-container; do
            attempt=$((attempt + 1))
            if [ $attempt -ge 3 ]; then
              echo "Attempt $attempt failed, exiting..."
              exit 1
            fi
            echo "Attempt $attempt failed, retrying..."
            make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete-cluster-container || true
          done
          echo "Prepare environment completed after $((attempt + 1)) attempts"

      # ▸ Install Cozystack
      - name: Install Cozystack into sandbox
        env:
          # This lane has no ARP VIP: .11 is a plain control-plane node standing
          # in for the QEMU lane's 192.168.123.10.
          COZY_APISERVER_ENDPOINT: https://192.168.123.11:6443
          # Talos containers share the runner kernel and cannot load DRBD, so the
          # harness removes the DRBD-only satellite sidecar, registers the host's
          # pre-created zpools and keeps LINSTOR's local ZFS path.
          COZY_LINSTOR_DRBD_ENABLED: "false"
          # Root monitoring is the one install-time workload whose chart selects
          # `replicated` explicitly instead of inheriting the default class.
          COZY_E2E_STORAGE_CLASS: local
        run: |
          cd /tmp/$SANDBOX_NAME
          if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then
            echo "❌ Install failed (no retry — see diagnostics below)"
            echo "::group::Diagnostics: HelmRelease status"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true
            echo "::endgroup::"
            echo "::group::Diagnostics: not-Ready HelmReleases (describe)"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \
              | while read -r ns name; do
                  echo "--- $ns/$name ---"
                  docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true
                done
            echo "::endgroup::"
            echo "::group::Diagnostics: recent events"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -50' || true
            echo "::endgroup::"
            exit 1
          fi

      # Test Impact Analysis (TIA): walk the PackageSource graph for the PR
      # diff and select only the Chainsaw suites affected. Runs by default; opt
      # out via the `full-e2e` label to run the full suite.
      # `skip=true` means nothing to test (docs-only PR).
      - name: Select E2E tests
        id: select
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
        env:
          BASE_REF: ${{ github.base_ref }}
        run: |
          cd /tmp/$SANDBOX_NAME
          git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt
          apps=$(./hack/select-e2e.sh /tmp/changed.txt)
          echo "Selected apps: ${apps:-<none>}"
          echo "apps=${apps}" >> $GITHUB_OUTPUT
          if [ -z "${apps}" ]; then
            echo "skip=true" >> $GITHUB_OUTPUT
          else
            echo "skip=false" >> $GITHUB_OUTPUT
          fi

      - name: Run OpenAPI tests
        run: |
          cd /tmp/$SANDBOX_NAME
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi

      # ▸ Run E2E tests (Chainsaw, scoped by Test Impact Analysis)
      #
      #   The "Select E2E tests" step above picks the suites impacted by the PR
      #   diff. With the `full-e2e` label that step is skipped, SELECTED_APPS is
      #   empty, and we fall back to the whole suite; an empty selection on a
      #   docs-only PR skips this step entirely (the skip != 'true' guard).
      #
      #   Chainsaw discovers each named suite under hack/e2e-chainsaw/ and polls
      #   every assertion to its own timeout, so the old per-app 3-retry loop is
      #   gone — a failure is reported as a structured assertion diff, with
      #   cluster-state diagnostics dumped below on a non-zero exit.
      - name: Run E2E tests
        id: e2e_tests
        if: ${{ steps.select.outputs.skip != 'true' }}
        env:
          # No DRBD-backed class exists here. Chainsaw templates the storage
          # fixtures with `local`, and the Kubernetes helper omits only the
          # remote-propagation and RWX assertions while running the same suites.
          COZY_E2E_STORAGE_CLASS: local
          SELECTED_APPS: ${{ steps.select.outputs.apps }}
          FULL_E2E: ${{ contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
        run: |
          cd /tmp/$SANDBOX_NAME
          if [ "$FULL_E2E" = "true" ] || [ -z "$SELECTED_APPS" ]; then
            suites=""        # empty => test-chainsaw runs the whole suite
            echo "Running full chainsaw suite"
          else
            suites="$SELECTED_APPS"
            echo "Running selected chainsaw suites: $suites"
          fi
          if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES="$suites"; then
            echo "❌ Chainsaw E2E failed (see the assertion diffs above and diagnostics below)"
            echo "::group::Diagnostics: HelmRelease status"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true
            echo "::endgroup::"
            echo "::group::Diagnostics: recent events"
            docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true
            echo "::endgroup::"
            # COSI state at the moment of failure — the CRDs ship no printer
            # columns, so pull the readiness fields explicitly (cozyreport
            # captures the same at end-of-job, but bucket states move between
            # a test failing and the report being taken).
            echo "::group::Diagnostics: COSI bucket state"
            docker exec "$SANDBOX_NAME" sh -c '
              kubectl get bucketclaims.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.bucketReady,BUCKET:.status.bucketName" 2>&1
              kubectl get buckets.objectstorage.k8s.io -o custom-columns="NAME:.metadata.name,READY:.status.bucketReady,ID:.status.bucketID,CLAIMNS:.spec.bucketClaim.namespace" 2>&1
              kubectl get bucketaccesses.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,GRANTED:.status.accessGranted,CLAIM:.spec.bucketClaimName" 2>&1
            ' || true
            echo "::endgroup::"
            exit 1
          fi
          echo "✅ All selected E2E tests passed"

      - name: Collect chainsaw report
        if: always()
        run: |
          cd /tmp/$SANDBOX_NAME
          mkdir -p _out
          docker cp "$SANDBOX_NAME":/workspace/hack/e2e-chainsaw/chainsaw-report.xml \
            _out/chainsaw-report.xml || true

      - name: Upload chainsaw report
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: chainsaw-report
          path: /tmp/${{ env.SANDBOX_NAME }}/_out/chainsaw-report.xml
          if-no-files-found: ignore

      # ▸ Collect debug information (always runs)
      - name: Collect report
        if: always()
        run: |
          cd /tmp/$SANDBOX_NAME
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true

      - name: Upload cozyreport.tgz
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: cozyreport
          path: /tmp/${{ env.SANDBOX_NAME }}/_out/cozyreport.tgz

      - name: Collect images list
        if: always()
        run: |
          cd /tmp/$SANDBOX_NAME
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-images || true

      - name: Upload image list
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: image-list
          path: /tmp/${{ env.SANDBOX_NAME }}/_out/images.txt

      # ▸ Open an SSH breakpoint to the failing sandbox so maintainers can attach,
      #   inspect Talos/Cozystack state and resume with `breakpoint resume`.
      #
      #   Gated by the `debug` or `release` label (release PRs are bot-created,
      #   so their presence implies maintainer authorship). authorized-users is
      #   kept as a second defense layer at the breakpoint level.
      #
      #   Uses cozystack/breakpoint-action (fork of namespacelabs/breakpoint-action)
      #   pinned by SHA. The fork adds: pause-idle mode (initial grace period for
      #   the first SSH connection, idle-aware exit afterwards), endpoint output
      #   and ::notice:: annotation, and a dedicated Check Run "Breakpoint Open"
      #   that carries the SSH endpoint in output.summary while the breakpoint is
      #   paused (conclusion=failure → standard ✗ in `gh pr checks`; updated to
      #   conclusion=success when the breakpoint exits).
      #
      #   Configured via the BREAKPOINT_ENDPOINT repository variable — in fork
      #   PRs repository variables are not exposed, so the step is skipped and
      #   forks cannot reach the self-hosted rendezvous server.
      - name: Breakpoint on E2E failure
        if: |
          failure() &&
          vars.BREAKPOINT_ENDPOINT != '' &&
          (contains(github.event.pull_request.labels.*.name, 'debug')
            || contains(github.event.pull_request.labels.*.name, 'release'))
        # cozystack/breakpoint-action v2-cozy.1
        # mode: pause-idle defaults: grace-period=20m, idle-timeout=10m
        uses: cozystack/breakpoint-action@a6f3a6f87be398ad63b6577351e3398e53f578e4
        with:
          mode: pause-idle
          endpoint: ${{ vars.BREAKPOINT_ENDPOINT }}
          authorized-users: androndo, Arsolitt, IvanHunters, kvaps, lexfrei, lllamnyp, mattia-eleuteri, matthieu-robin, myasnikovdaniil, sircthulhu, tym83
          check-run-name: "Breakpoint Open"
          github-token: ${{ github.token }}
          check-run-summary-template: |
            ## 🔴 SSH breakpoint open — paused for debug

            ```
            {endpoint}
            ```

            Enter the e2e sandbox after SSH:
            ```
            docker exec -ti $(docker ps --filter name=cozy-e2e-sandbox -q | head -1) bash
            export KUBECONFIG=/workspace/kubeconfig
            ```

            Resume from inside: `breakpoint resume`. Otherwise the breakpoint
            exits 10 minutes after the last SSH session disconnects.

      # ▸ Tear down environment (always runs)
      #
      #   Both halves, and in this order: the compose project owns the nodes and
      #   the zpools outlive it. A leftover pool makes the next run on a reused
      #   host adopt storage that already carries a dead cluster's volumes.
      - name: Tear down sandbox
        if: always()
        run: |
          # Loud, but never the job's verdict. A failed teardown means a dirty
          # host, and this runner is discarded after the job, so there is no next
          # run to protect -- while failing the job here turns the required
          # "E2E Tests" check red in a way no reader can tell apart from a real
          # failure. The annotation is what a reused host (a developer machine)
          # needs; `delete-cluster-container` returning non-zero is not evidence
          # about the product.
          cleanup_rc=0
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete-cluster-container || cleanup_rc=$?
          make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true
          if [ "$cleanup_rc" -ne 0 ]; then
            echo "::error title=sandbox teardown::teardown exited $cleanup_rc; the host may hold a stale compose project or zpool"
          fi

      - name: Remove workspace
        if: always()
        run: rm -rf /tmp/$SANDBOX_NAME