name: Pull Request
env:
# TODO: unhardcode this
REGISTRY: iad.ocir.io/idyksih5sir9/cozystack
on:
pull_request:
# `labeled` makes the promote PR's release-candidate verification and manual
# full-e2e opt-in actionable after the bot opens it. The plan/resolve guards
# below discard every other label event before it can launch work.
types: [opened, synchronize, reopened, labeled]
# Cancel in‑flight runs for the same PR when a new push arrives.
#
# A `labeled` event that `plan` discards gets its OWN group, so labelling an
# open PR cannot cancel a live opened/synchronize run. Any label other than
# full-e2e/release makes `plan` skip, and `e2e-report` skips with it, so without the
# split the pair "cancel the real run, then skip" left the head SHA with no
# "E2E Tests" status at all — and an unposted required status stays "Expected",
# blocking the merge until someone pushes again or re-runs the cancelled run.
# Observed on live CI: run 30614258694 (cancelled) followed by 30614280603
# (skipped) on the same SHA, 24s apart, from a maintainer adding kind/* labels
# just after a push. (`pr-labeler` cannot cause this — it labels with the
# default GITHUB_TOKEN, which by design triggers no further workflow runs.)
#
# The condition is the exact complement of `plan`'s guard, so only a run that
# will publish NOTHING is moved aside. A `labeled` run carrying full-e2e or
# release does publish, so it stays in the main group and supersedes the head
# run the way a push does. That keeps exactly one publisher of the `E2E Tests`
# context alive per head SHA, which the status depends on: two live publishers are latest-wins
# per context, and the full suite is the superset, so a full-e2e run that fails
# early — a build flake, a suite failing at minute twenty — would finish before
# the narrower head run it was meant to widen, and that run's later green would
# erase the failure. Cancelling also means opting into the full suite costs one
# suite rather than two, and a second label landing afterwards cannot cancel the
# full-e2e run, because it is no longer in the same group as it.
#
# `release` publishes on the SAME-REPO lane only, hence the fork term. A promote
# PR is opened same-repo by the release bot, so the label on a fork PR is always
# a mislabel — and on that lane it would be a destructive one. e2e-fork.yaml's
# `resolve` decides "this run built nothing, so it is not a verdict" by asking
# whether `Plan build` skipped; a fork run that gets past `plan` then fails
# closed on the pr-patch artifact its own `release` label gated off, posting red
# on a suite that was never going to run, with no `unlabeled` trigger to undo
# it. Excluding forks keeps the label as inert on that lane as it was before
# promotion started using it. The term appears in both this key and `plan`'s
# guard because the two must stay exact complements: NOT(A OR B OR (C AND NOT
# F)) is (NOT A) AND (NOT B) AND (NOT C OR F), and for a same-repo PR F is false
# and both sides reduce to exactly what they were.
concurrency:
group: pr-${{ github.workflow }}-${{ github.event.pull_request.number }}${{ (github.event.action == 'labeled' && github.event.label.name != 'full-e2e' && (github.event.label.name != 'release' || github.event.pull_request.head.repo.fork)) && '-label' || '' }}
cancel-in-progress: true
# Top-level token defaults to read-only. A job-level block replaces this one
# rather than adding to it, so each job restates every scope it needs.
permissions:
contents: read
jobs:
# Compute the docs-only gate and the per-package build matrix from the PR
# diff. hack/build-matrix.sh parses the root Makefile `build:` target (the
# single source of truth) and emits only the units whose package dir changed,
# escalating to the full set on a shared-dependency change. talos + installer
# are handled by dedicated jobs (see build-matrix.sh for why).
plan:
name: Plan build
runs-on: ubuntu-latest
# Exact complement of the concurrency key above; see the reasoning there,
# including why `release` is admitted for same-repo PRs only.
if: github.event.action != 'labeled' || github.event.label.name == 'full-e2e' || (github.event.label.name == 'release' && !github.event.pull_request.head.repo.fork)
permissions:
contents: read
# For the opening `E2E Tests` status below; e2e-report writes the terminal
# one with the same scope.
statuses: write
outputs:
code: ${{ steps.p.outputs.code }}
matrix: ${{ steps.p.outputs.matrix }}
any: ${{ steps.p.outputs.any }}
touched: ${{ steps.p.outputs.touched }}
talos: ${{ steps.p.outputs.talos }}
steps:
# Claim the required context for THIS run before anything can be merged on
# the strength of the last one. Commit statuses are latest-wins per (sha,
# context) and nothing expires them, so a head SHA that already went green
# keeps reading green while a newly started run works — re-run all jobs, a
# full-e2e label, a reopen — and the PR stays mergeable through a suite
# that may be about to fail. While the gate was a job, GitHub created a
# fresh non-success check run each time and that window did not exist.
# Overwriting with `pending` here restores it: the run that owns the
# status says so at the start and concludes it in e2e-report.
#
# Same-repo only. A fork's `pull_request` token is read-only, so this call
# would 403 and fail `plan` on every fork PR, and that read-only token is
# exactly what makes the whole commit-status design safe (§10). Fork PRs
# get their pending from e2e-fork.yaml's `resolve` instead, which is
# privileged.
#
# Note what that costs a fork. `resolve` cannot start until this entire
# run has finished, so the window this step shuts in seconds stays open
# for the length of the build on the fork lane. Structural rather than an
# oversight, since a read-only token cannot post a status at any point in
# the run, and bounded by the build. §10 records it.
- name: Open the "E2E Tests" status for this run
if: ${{ !github.event.pull_request.head.repo.fork }}
# A transient status-API failure here must not take the run with it.
# This step is the first in `plan`, and `plan` is what `checks`,
# `build`, `build-talos`, `finalize` and `e2e` all hang off, so a
# failure costs a full rebuild for nothing. Skipping the opener only
# degrades to the behaviour that preceded it — the terminal status
# still gets posted by `e2e-report`, and a run that dies before that
# leaves the merge blocked either way.
continue-on-error: true
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
env:
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
script: |
const { HEAD_SHA, RUN_URL } = process.env;
await github.rest.repos.createCommitStatus({
owner: context.repo.owner,
repo: context.repo.repo,
sha: HEAD_SHA,
state: 'pending',
context: 'E2E Tests',
description: 'e2e in progress — this run has not reported yet',
target_url: RUN_URL,
});
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
fetch-depth: 0
# e2e-fork.yaml pushes a fork's images only under names the base tree
# already exports, so an image the PR adds is refused there after the
# whole build. Refuse it here instead, before anything builds. The push
# check stays the boundary; a fork can edit this step away.
- name: Refuse images the base branch does not build (fork PR)
if: ${{ github.event.pull_request.head.repo.fork }}
run: |
bt="$(mktemp -d)/basetree"
git worktree add --quiet --detach "$bt" HEAD^1
hack/fork-new-images.sh "$bt" .
- name: Compute changed files and build matrix
id: p
env:
BASE_REF: ${{ github.base_ref }}
run: |
git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt
# The docs-only decision reads a RENAME-BLIND list. `git diff
# --name-only` applies rename detection and reports a rename as its
# new path alone, so moving a code file into docs/ reads as a
# docs-only change: nothing builds, no e2e runs, and the removal of
# that code path ships untested. It also deadlocks the fork lane,
# which answers the same question from `pulls.listFiles` and does see
# the old path through `previous_filename`. That lane concludes e2e is
# required, then fails closed on a pr-patch artifact this lane never
# produced — an unfixable red on a legitimate contribution, with no
# label override and no push that clears it. Both lanes have to answer
# "is this docs-only" the same way, and the only answer that is also
# correct keeps both sides of a rename.
#
# The build matrix still reads the rename-detected list, which is not
# a claim that doing so is right. Moving a file between packages
# leaves the source package changed and unselected, so the matrix
# under-builds on a cross-package rename. That behaviour predates this
# line and is not what the docs-only decision is for, so it is left
# alone here rather than widened by a side effect.
git diff --name-only --no-renames "origin/${BASE_REF}...HEAD" > /tmp/changed_norenames.txt
echo "Changed files:"; sed 's/^/ /' /tmp/changed_norenames.txt
if grep -qvE '^docs/' /tmp/changed_norenames.txt; then
echo "code=true" >> "$GITHUB_OUTPUT"
else
echo "code=false" >> "$GITHUB_OUTPUT"
fi
matrix="$(hack/build-matrix.sh /tmp/changed.txt)"
echo "Build matrix: $matrix"
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
talos="$(hack/build-talos-needed.sh /tmp/changed_norenames.txt)"
echo "Talos build needed: $talos"
echo "talos=$talos" >> "$GITHUB_OUTPUT"
# Package dirs the PR edited (packages//). Their committed
# refs are the PR's intent and must survive the current-main overlay —
# otherwise an upstream image bump in a non-build-unit package would be
# silently reverted in e2e/the installer artifact (see
# hack/overlay-main-images.sh). Whitespace-separated, no trailing slash.
touched="$(sed -nE 's#^(packages/[^/]+/[^/]+)/.*#\1#p' /tmp/changed.txt | sort -u | tr '\n' ' ')"
echo "Touched packages: $touched"
echo "touched=$touched" >> "$GITHUB_OUTPUT"
if [ "$matrix" = "[]" ]; then
echo "any=false" >> "$GITHUB_OUTPUT"
else
echo "any=true" >> "$GITHUB_OUTPUT"
fi
# Promotion publishes the rewritten packages artifact before opening its PR.
# Verify GitHub's prospective merge tree here, so a `packages/` edit pushed to
# this PR fails the required E2E Tests status before it can merge.
#
# Checking the MERGE tree rather than the head is what makes base drift
# visible when this job runs — but nothing re-runs it when the base moves:
# GitHub does not fire `pull_request` on the base branch advancing, it only
# recomputes refs/pull/N/merge. So a `packages/` change landing on the base
# after this went green is never re-verified here, and the PR stays mergeable
# on the stale result. Finalize repeats the check against the real merge
# commit before creating the write-once stable tag, which is where that case
# is actually caught — loudly, and before any stable name exists.
verify-release-candidate:
name: Verify release packages candidate
runs-on: ubuntu-latest
# Matching `checks` and `finalize`. Without a ceiling the job inherits the
# 6-hour default, and a stalled `flux pull artifact` would hold the required
# `E2E Tests` status pending for all of it, since e2e-report needs this job.
timeout-minutes: 30
if: |
contains(github.event.pull_request.labels.*.name, 'release')
&& github.event.pull_request.user.login == 'cozystack-ci[bot]'
&& (github.event.action != 'labeled' || github.event.label.name == 'full-e2e' || github.event.label.name == 'release')
permissions:
contents: read
packages: read
steps:
# The default pull_request checkout is GitHub's prospective merge commit,
# not just the head branch. That makes base-branch package drift visible.
- name: Checkout prospective merge
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
fetch-depth: 0
# Execute the verifier from the trusted base SHA. An older rc-derived head
# may not contain it, and a PR must not be able to weaken its own guard.
- name: Checkout release tooling from base
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
ref: ${{ github.event.pull_request.base.sha }}
path: .release-tooling
sparse-checkout: hack
persist-credentials: false
# Both tools are version-pinned. This job decides whether a stable release
# may be created, so what it runs has to be a fixed input: `latest/download`
# resolves to whatever mikefarah shipped this morning, and a yq that changed
# how it emits or compares a value turns a release gate into a moving one.
# The version tracks packages/core/testing/images/e2e-sandbox/Dockerfile.
# yq is additionally pinned by content; flux comes from the commit-pinned
# upstream setup action, which checksums the release tarball.
# promote-rc.yaml's copy of this step carries the full reasoning for both.
- name: Set up flux
uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6
with:
version: "2.8.6"
- name: Set up promotion toolchain (yq)
env:
YQ_VERSION: "4.53.3"
YQ_SHA256: "fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4"
run: |
if ! yq --version 2>/dev/null | grep -q "mikefarah.* version v${YQ_VERSION}\$"; then
yq_bin="$(mktemp)"
curl -fsSL -o "$yq_bin" \
"https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64"
printf '%s %s\n' "$YQ_SHA256" "$yq_bin" | sha256sum -c -
sudo install -m 0755 "$yq_bin" /usr/local/bin/yq
rm -f "$yq_bin"
fi
- name: Login to registry (GHCR)
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
registry: ghcr.io
env:
DOCKER_CONFIG: ${{ runner.temp }}/.docker
- name: Verify stable packages candidate
env:
DOCKER_CONFIG: ${{ runner.temp }}/.docker
HEAD_BRANCH: ${{ github.head_ref }}
run: |
STABLE_VERSION="${HEAD_BRANCH#release-}"
.release-tooling/hack/verify-promoted-packages.sh "$STABLE_VERSION"
# Helm-unittest + Go controller tests. Decoupled from the image build and from
# e2e so they give fast, independent signal; they gate the PR as their own
# required check rather than blocking e2e from starting.
checks:
name: Unit & controller tests
runs-on: oracle-vm-4cpu-16gb-x86-64
timeout-minutes: 30
env:
# The pipe falls back to direct on any proxy error (hack/common-envs.mk).
GOPROXY: "https://proxy.golang.org|direct"
permissions:
contents: read
needs: ["plan"]
# Unit contracts read both code and documentation. Run this job for every
# non-release PR: code changes take the complete unit path, while docs-only
# changes run the Bats lane that owns those cross-tree contracts. Builds and
# E2E remain gated on `code` below.
if: |
!contains(github.event.pull_request.labels.*.name, 'release')
steps:
- name: Checkout code
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
fetch-depth: 0
# The module cache is restored from the key build-main.yaml writes on
# main, which every PR run can read, so a PR's first run starts warm. The
# download fetches whatever the cache lacks, the whole set on a miss. The
# Go steps are for code changes only: a docs-only run is the Bats lane alone.
- name: Locate Go module cache
if: needs.plan.outputs.code == 'true'
id: gomodcache
run: echo "dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
- name: Cache Go modules
if: needs.plan.outputs.code == 'true'
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ steps.gomodcache.outputs.dir }}
key: go-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
restore-keys: go-mod-${{ runner.os }}-
- name: Download Go modules
if: needs.plan.outputs.code == 'true'
run: |
for mod in $(git ls-files go.mod '*/go.mod' | xargs -n1 dirname); do
(cd "$mod" && "$GITHUB_WORKSPACE/hack/go-mod-download.sh")
done
# Ephemeral runners don't carry the helm-unittest plugin the persistent
# self-hosted runner accreted. The plugin embeds its own Helm, so its
# version is pinned: a plugin release must not change test results. Bats is
# installed on every run, docs-only ones included.
- name: Set up test toolchain
env:
CODE_CHANGED: ${{ needs.plan.outputs.code }}
HELM_UNITTEST_VERSION: "1.1.2"
BATS_VERSION: "1.14.0"
BATS_COMMIT: eb7f42f8d608ac693d7a4b67474f6714ea68cfc5
run: |
set -euo pipefail
if [ "$CODE_CHANGED" = true ]; then
installed="$(helm plugin list 2>/dev/null | awk '$1 == "unittest" { print $2 }' || true)"
if [ "$installed" != "$HELM_UNITTEST_VERSION" ]; then
[ -z "$installed" ] || helm plugin uninstall unittest
helm plugin install https://github.com/helm-unittest/helm-unittest --version "v${HELM_UNITTEST_VERSION}"
fi
fi
# Source archive bytes can change; verify the tag's commit before install.
if ! bats --version 2>/dev/null | grep -qx "Bats $BATS_VERSION"; then
bats_parent=$(mktemp -d "$RUNNER_TEMP/bats-core.XXXXXX")
bats_source="$bats_parent/source"
git clone --quiet --depth=1 --branch "v$BATS_VERSION" \
https://github.com/bats-core/bats-core.git "$bats_source"
test "$(git -C "$bats_source" rev-parse HEAD)" = "$BATS_COMMIT"
sudo "$bats_source/install.sh" /usr/local
fi
bats --version | grep -qx "Bats $BATS_VERSION"
# setup-envtest and the kube-apiserver and etcd it fetches, so a pull
# request downloads them only when the versions pinned in the Makefile
# change, and an outage at their sources cannot fail unrelated changes.
# The envtest target runs the controller suites a fake client cannot
# stand in for.
- name: Cache envtest binaries
if: needs.plan.outputs.code == 'true'
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: _out/envtest
key: envtest-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}
# -k is required, not cosmetic: unit-tests and test-controllers are two
# independent goals on the same command line, and without -k make stops
# handing out new work the moment any unit-tests prerequisite fails —
# test-controllers then never starts and reports nothing, so a red BATS
# file hides the entire Go controller suite. -k lets already-schedulable
# goals keep running; the job still fails overall on any red target.
- name: Run unit and controller tests
if: needs.plan.outputs.code == 'true'
run: make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target
- name: Run Bats unit tests for docs-only changes
if: needs.plan.outputs.code == 'false'
run: make bats-unit-tests
- name: Upload unit test report
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: bats-unit-report
path: _out/test-reports/
if-no-files-found: ignore
# One image (package) per job, fanned out across the ephemeral pool. Each job
# gets its own buildx (docker-container driver) so the shared-buildkit
# contention that wedged the old single Build job (#2937) is severed by
# construction. Each job patches only its own package's values.yaml/.tag files
# and uploads that diff as a patch fragment; `finalize` reassembles them.
build:
name: Build ${{ matrix.pkg }}
needs: ["plan"]
if: |
needs.plan.outputs.code == 'true'
&& needs.plan.outputs.any == 'true'
&& !contains(github.event.pull_request.labels.*.name, 'release')
runs-on: oracle-vm-4cpu-16gb-x86-64
timeout-minutes: 30
permissions:
contents: read
strategy:
fail-fast: false
matrix:
pkg: ${{ fromJSON(needs.plan.outputs.matrix) }}
steps:
- name: Checkout code
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
fetch-depth: 0
fetch-tags: true
# installer is built in finalize, but some package image targets shell out
# to flux.
- name: Set up flux
uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6
with:
version: "2.8.6"
- name: Set up Docker config
run: |
if [ -d ~/.docker ]; then
cp -r ~/.docker "${{ runner.temp }}/.docker"
fi
- name: Login to GitHub Container Registry
if: ${{ !github.event.pull_request.head.repo.fork }}
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.OCIR_USER }}
password: ${{ secrets.OCIR_TOKEN }}
registry: iad.ocir.io
env:
DOCKER_CONFIG: ${{ runner.temp }}/.docker
# docker-container driver: isolated buildkit per job (no shared-buildkit
# contention) and a prerequisite for the mode=max cache reads (#2938).
# Its buildkitd also routes docker.io base-image pulls through
# mirror.gcr.io (hack/buildkitd.toml): the Build step authenticates only
# to the push registry (OCIR), so base images (golang/ubuntu/nginx/...)
# would otherwise be pulled from Docker Hub anonymously and intermittently
# trip its anonymous-pull rate limit; the mirror sidesteps that and falls
# back to Docker Hub on a miss/outage. Create the builder under the SAME
# DOCKER_CONFIG the Build step reads (runner.temp/.docker, holding the
# OCIR auth) so buildx forwards push/cache auth to the container builder.
- name: Set up Buildx (docker-container driver, docker.io mirror)
id: buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
with:
driver: docker-container
buildkitd-config: hack/buildkitd.toml
env:
DOCKER_CONFIG: ${{ runner.temp }}/.docker
- name: Build ${{ matrix.pkg }}
run: |
mkdir -p _out/oci
make -C "$PKG" image
env:
PKG: ${{ matrix.pkg }}
DOCKER_CONFIG: ${{ runner.temp }}/.docker
# Build on the docker-container builder created above so base-image
# pulls go through the docker.io mirror (BUILDX_ARGS adds
# --builder=$(BUILDER)).
BUILDER: ${{ steps.buildx.outputs.name }}
# common-envs.mk defaults to amd64+arm64. Nothing here sets up QEMU,
# and on this x86-64 runner the arm64 half would be emulated anyway;
# CI builds amd64 until arm64 has a native leg of its own.
PLATFORM: linux/amd64
# Unique per (PR, commit) so concurrent PR builds never push the same
# tag. PR images are consumed by digest; :latest belongs to releases.
IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
PUBLISH_VERSIONED: '0'
PUBLISH_FLOATING: '0'
# Fork PRs get no registry credentials, so instead of pushing, build
# each image to /_out/oci/.oci.tar and upload it below;
# the privileged e2e-fork.yaml workflow_run pushes them by digest. Empty
# on same-repo PRs → normal push path. Setting it also forces PUSH/LOAD
# off in common-envs.mk (#3257). Must be ABSOLUTE (workspace-rooted):
# the build runs `make -C `, so buildx resolves a relative dest
# against the package dir — the archive would land in /_out/oci and
# miss the repo-root _out/oci the upload reads.
OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}
- name: Capture digest patch fragment
id: frag
env:
PKG: ${{ matrix.pkg }}
run: |
safe="$(echo "$PKG" | tr '/' '-')"
mkdir -p _out/fragments
# --binary so vendored binary provider bundles (components*.gz),
# which some package image targets regenerate, survive reassembly
# in finalize (a plain "Binary files differ" stub is unappliable).
git diff --binary HEAD > "_out/fragments/${safe}.patch"
echo "safe=${safe}" >> "$GITHUB_OUTPUT"
echo "Fragment for ${PKG}:"; git --no-pager diff --stat HEAD
- name: Upload patch fragment
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: pr-patch-fragment-${{ steps.frag.outputs.safe }}
path: _out/fragments/${{ steps.frag.outputs.safe }}.patch
if-no-files-found: error
# Fork PR only: the images were exported to OCI archives instead of pushed
# (OCI_EXPORT_DIR above). Upload them for e2e-fork.yaml to push by digest.
- name: Upload OCI image archives (fork PR)
if: ${{ github.event.pull_request.head.repo.fork }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: pr-oci-${{ steps.frag.outputs.safe }}
path: _out/oci/*.oci.tar
if-no-files-found: error
# Talos is its own leg: the installer tarball is heavy and the Talos image is
# built from it, so both live in one job. No nocloud disk is built here any
# more, and e2e no longer downloads one -- the container lane boots the
# upstream Talos container image. `images/talos/profiles/nocloud.yaml` is
# therefore compiled by nightly.yaml and by the tag build's `make assets`.
#
# For same-repo PRs, the plan job classifies the rename-blind changed-path
# list with build-talos-needed.sh. That list includes direct Talos sources and
# the shared Makefile/Buildx/workflow inputs that determine this job's output;
# line-based classification avoids depending on the formatting of `touched`.
# A successful run stamps packages/extra/bootbox/images/matchbox.tag, and
# finalize preserves that generated ref when it overlays base-branch images.
#
# Fork PRs keep the previous unconditional behavior as a conservative scope
# boundary. Their build/export and privileged publication path spans a second
# workflow and is not optimized here. The installer job also exports an OCI
# archive, so archive availability is not a reason to require Talos itself.
build-talos:
name: Build Talos
needs: ["plan"]
if: |
needs.plan.outputs.code == 'true'
&& !contains(github.event.pull_request.labels.*.name, 'release')
&& (
github.event.pull_request.head.repo.fork
|| needs.plan.outputs.talos == 'true'
)
runs-on: oracle-vm-4cpu-16gb-x86-64
timeout-minutes: 40
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
fetch-depth: 0
fetch-tags: true
- name: Set up Docker config
run: |
if [ -d ~/.docker ]; then
cp -r ~/.docker "${{ runner.temp }}/.docker"
fi
- name: Login to GitHub Container Registry
if: ${{ !github.event.pull_request.head.repo.fork }}
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.OCIR_USER }}
password: ${{ secrets.OCIR_TOKEN }}
registry: iad.ocir.io
env:
DOCKER_CONFIG: ${{ runner.temp }}/.docker
# buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
# (hack/buildkitd.toml) so the talos image build dodges Docker Hub's
# anonymous-pull rate limit; see the build job for the full rationale.
- name: Set up Buildx (docker-container driver, docker.io mirror)
id: buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
with:
driver: docker-container
buildkitd-config: hack/buildkitd.toml
env:
DOCKER_CONFIG: ${{ runner.temp }}/.docker
- name: Build Talos image (matchbox + installer tarball)
run: |
mkdir -p _out/oci
make -C packages/core/talos image
env:
DOCKER_CONFIG: ${{ runner.temp }}/.docker
BUILDER: ${{ steps.buildx.outputs.name }}
# amd64 only, as in the build job. The arm64 installer needs no QEMU,
# but nothing a PR runs boots it. matchbox still gets the arm64 kernel
# and initramfs.
PLATFORM: linux/amd64
IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
PUBLISH_VERSIONED: '0'
PUBLISH_FLOATING: '0'
# Fork PR: export to OCI archive instead of pushing (see build job).
OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}
- name: Capture Talos patch fragment
run: |
mkdir -p _out/fragments
git diff --binary HEAD > _out/fragments/talos.patch
echo "Talos fragment:"; git --no-pager diff --stat HEAD
- name: Upload Talos patch fragment
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: pr-patch-fragment-talos
path: _out/fragments/talos.patch
if-no-files-found: error
# Fork PR only: upload the Talos image OCI archive for e2e-fork.yaml.
- name: Upload Talos OCI image archive (fork PR)
if: ${{ github.event.pull_request.head.repo.fork }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: pr-oci-talos
path: _out/oci/*.oci.tar
if-no-files-found: error
# Reassemble the unified pr.patch from every fragment, then build the installer
# LAST: `make -C packages/core/installer image` runs `flux push artifact
# --path=packages`, which bundles the WHOLE digest-patched tree into the OCI
# artifact the operator pulls — so every other package's digest edit must
# already be applied here. e2e consumes the resulting pr.patch unchanged.
finalize:
name: Finalize (merge patches + installer)
# Depends on `checks` too: gating finalize on unit/controller tests cascades
# to e2e (which needs finalize), so a red unit test skips the expensive
# install+e2e path instead of wasting ~1h on it. Restores the gating the old
# in-build-job tests gave, without re-coupling the cheap parallel builds.
needs: ["plan", "checks", "build", "build-talos"]
if: |
!cancelled()
&& needs.plan.outputs.code == 'true'
&& !contains(github.event.pull_request.labels.*.name, 'release')
&& needs.checks.result == 'success'
&& (needs.build.result == 'success' || needs.build.result == 'skipped')
&& (needs.build-talos.result == 'success' || needs.build-talos.result == 'skipped')
runs-on: oracle-vm-4cpu-16gb-x86-64
timeout-minutes: 30
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
fetch-depth: 0
fetch-tags: true
- name: Set up flux
uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6
with:
version: "2.8.6"
- name: Download patch fragments
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
pattern: pr-patch-fragment-*
merge-multiple: true
path: _out/fragments
- name: Apply digest fragments
run: |
applied=0
# Each fragment patches only its own package's files; --3way lets two
# fragments that touch adjacent lines of a shared file (e.g. a package
# Makefile that writes a sibling package's values.yaml) merge cleanly
# instead of the second apply aborting the whole finalize.
for f in _out/fragments/*.patch; do
[ -s "$f" ] || continue
echo "Applying $f"
git apply --3way "$f"
applied=$((applied + 1))
done
echo "Applied $applied fragment(s)"
git --no-pager diff --stat HEAD
- name: Set up Docker config
run: |
if [ -d ~/.docker ]; then
cp -r ~/.docker "${{ runner.temp }}/.docker"
fi
- name: Login to GitHub Container Registry
if: ${{ !github.event.pull_request.head.repo.fork }}
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.OCIR_USER }}
password: ${{ secrets.OCIR_TOKEN }}
registry: iad.ocir.io
env:
DOCKER_CONFIG: ${{ runner.temp }}/.docker
# Overlay current-main image refs onto every package this PR did NOT
# rebuild. Without it, unbuilt packages fall back to the last-release refs
# committed in the tree (e.g. v1.5.0) even when main is hundreds of commits
# ahead — so e2e and the installer's packages artifact would ship stale
# images for everything outside this PR's build matrix. Source of truth is
# the cozystack-packages:main OCI artifact build-main.yaml already
# publishes (the whole current-main packages tree, refs digest-pinned to
# main's images). Runs after login (creds available) and before the
# installer build below so the flux-pushed packages tree — and the
# pr.patch e2e applies — already reflect the overlay. A missing artifact
# (build-main not yet green) is a no-op: packages keep their committed
# refs (prior behaviour). See hack/overlay-main-images.sh.
#
# Read the artifact for the PR's OWN base branch, not always main. Overlaying
# main's images onto a release-line PR mixes generations: main's binaries
# against that line's charts. #3437 (a one-line change on release-1.6) failed
# install deterministically that way, because main's cozystack-controller
# served an aggregated OpenAPI release-1.6's charts could not validate
# against (`unknown model in reference: …v1alpha1.OptionSpec`) — a broken
# lane reading as a broken PR. build-release.yaml publishes
# cozystack-packages: for every maintained release-.
# branch, so each base branch has its own artifact to read.
#
# A base that is neither main nor a line (a stacked PR's feature branch) has
# no artifact, and a `/` in it is not even a valid OCI tag. It reads the
# artifact of whichever of main and the lines it has the fewest commits on
# top of. A tie goes to main. A line with no commit of its own yet is, in
# the graph, main at the cut commit, so a stack on it ties and reads the
# artifact built from main's tip. That artifact carries newer images than
# the line once main moves past the cut. The stack keeps reading it until
# it contains a commit of the line's own. Breaking the tie towards the
# line instead would misroute the far more common branch forked from main
# before the cut, which ties the same way.
#
# A missing artifact stays a no-op (packages keep their committed refs), which
# also covers a line whose first build-release run has not landed yet.
- name: Pull base-branch packages tree
run: |
ARTIFACT_REF="$BASE_REF"
if [ "$BASE_REF" != "main" ] \
&& ! printf '%s' "$BASE_REF" | grep -qE '^release-[0-9]+\.[0-9]+$'; then
ARTIFACT_REF=main
# A re-run can outlive its base: the payload keeps a branch deleted since.
if git rev-parse --verify --quiet "origin/${BASE_REF}" >/dev/null; then
best="$(git rev-list --count "origin/main..origin/${BASE_REF}")"
for line in $(git for-each-ref --format='%(refname:lstrip=3)' 'refs/remotes/origin/release-*' \
| grep -E '^release-[0-9]+\.[0-9]+$'); do
n="$(git rev-list --count "origin/${line}..origin/${BASE_REF}")"
if [ "$n" -lt "$best" ]; then best="$n"; ARTIFACT_REF="$line"; fi
done
fi
echo "::notice::Base ${BASE_REF} publishes no packages artifact; overlaying cozystack-packages:${ARTIFACT_REF}."
fi
rm -rf _out/mainpkgs
mkdir -p _out/mainpkgs
if ! flux pull artifact "oci://${REGISTRY}/cozystack-packages:${ARTIFACT_REF}" \
--output _out/mainpkgs; then
rm -rf _out/mainpkgs
# Degrading to committed refs is safe, but SAY so: on a release line a
# missing artifact means build-release.yaml has not published this line
# yet (first build pending, or its branch filter does not match this
# branch), and the PR is then testing the line's last release instead of
# its tip. Silent would look identical to a working overlay.
if [ "${ARTIFACT_REF}" = "main" ]; then
echo "cozystack-packages:${ARTIFACT_REF} pull failed — keeping committed refs"
else
echo "::warning title=No packages artifact for ${ARTIFACT_REF}::Keeping committed refs, so unbuilt packages use this line's last release rather than its tip. Check that build-release.yaml has run for ${ARTIFACT_REF}."
fi
fi
env:
DOCKER_CONFIG: ${{ runner.temp }}/.docker
# Env-passed, not interpolated into the script: base_ref is repo-controlled
# here (a branch name in this repo), but the workflow-injection hygiene
# applies uniformly.
BASE_REF: ${{ github.base_ref }}
- name: Overlay base-branch refs for unbuilt packages
# Best-effort: this only improves WHICH images e2e and the installer use,
# so an unexpected failure must degrade to the committed refs (prior
# behaviour), never block the PR. overlay-main-images.sh already handles
# the expected paths (missing artifact, drift, absent unit) internally —
# including the empty directory the step above leaves when a line has no
# artifact yet.
run: |
rebuilt="$BUILD_MATRIX"
if [ "$TALOS_RESULT" = "success" ]; then
case "$rebuilt" in
'[]') rebuilt='["packages/extra/bootbox"]' ;;
*) rebuilt="${rebuilt%]},\"packages/extra/bootbox\"]" ;;
esac
fi
hack/overlay-main-images.sh _out/mainpkgs "$rebuilt" "$PR_TOUCHED" \
|| echo "::warning::overlay-main-images failed; unbuilt packages keep committed refs"
env:
# The set of packages THIS PR rebuilt; their pr-- refs are
# authoritative, so overlay-main-images.sh skips them. Computed by the
# plan job from the Makefile (a JSON array of package dirs), not from
# untrusted PR input — passed via env per workflow-injection hygiene.
BUILD_MATRIX: ${{ needs.plan.outputs.matrix }}
# build-talos writes matchbox.tag under extra/bootbox rather than its
# own package. Preserve that generated ref only when the job actually
# succeeded; when it was skipped, the base-branch ref should overlay.
TALOS_RESULT: ${{ needs.build-talos.result }}
# Packages THIS PR edited (changed-file dirs from the plan job's git
# diff). Their committed refs are the PR's intent — an upstream image
# bump in a non-build-unit package would otherwise be silently reverted
# by the overlay — so overlay-main-images.sh preserves them. Also
# env-passed, derived from the diff, not from untrusted PR input.
PR_TOUCHED: ${{ needs.plan.outputs.touched }}
# buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
# (hack/buildkitd.toml) so the installer build dodges Docker Hub's
# anonymous-pull rate limit; see the build job for the full rationale.
- name: Set up Buildx (docker-container driver, docker.io mirror)
id: buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
with:
driver: docker-container
buildkitd-config: hack/buildkitd.toml
env:
DOCKER_CONFIG: ${{ runner.temp }}/.docker
# Same-repo PR: build the operator image AND flux-push the now-fully-patched
# packages tree, then patch the installer's platformSourceRef to that
# artifact. Fork PR: no registry creds — build ONLY the operator image to an
# OCI archive (image-packages/chart, which push, are skipped) and upload it;
# e2e-fork.yaml pushes the operator + flux-pushes the packages tree and
# bakes platformSourceRef there. The overlay above already ran, so the
# pr.patch assembled below carries the fully digest-pinned tree either way.
- name: Build installer (operator + packages artifact)
run: |
mkdir -p _out/oci
if [ "$IS_FORK" = "true" ]; then
make -C packages/core/installer pre-checks image-operator
else
make -C packages/core/installer image
fi
env:
DOCKER_CONFIG: ${{ runner.temp }}/.docker
BUILDER: ${{ steps.buildx.outputs.name }}
# amd64 only, as in the build job.
PLATFORM: linux/amd64
IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
PUBLISH_VERSIONED: '0'
PUBLISH_FLOATING: '0'
IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}
# Fork PR only: upload the operator image OCI archive for e2e-fork.yaml.
- name: Upload installer OCI image archive (fork PR)
if: ${{ github.event.pull_request.head.repo.fork }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: pr-oci-installer
path: _out/oci/*.oci.tar
if-no-files-found: error
- name: Assemble pr.patch
run: |
mkdir -p _out/assets
# --binary so regenerated provider bundles (components*.gz) stay in
# the patch e2e applies; mirrors main's pr.patch binary fix (#e566992).
git diff --binary HEAD > _out/assets/pr.patch
echo "Final pr.patch:"; git --no-pager diff --stat HEAD
- name: Upload git diff patch
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: pr-patch
path: _out/assets/pr.patch
if-no-files-found: error
resolve_assets:
name: "Resolve assets"
runs-on: ubuntu-latest
# In lockstep with `plan`, fork term included: this job is on `e2e`'s
# release arm, so a label event `plan` runs for and this job skips for
# leaves `e2e` with both arms false while `e2e-report` still runs. On a
# promote PR already carrying full-e2e that reaches the final `else` — the
# "e2e not repeated" branch is bypassed by full-e2e — and posts `E2E Tests`
# = failure over a green one, unclearable without a push because
# `unlabeled` is not a trigger. The fork term keeps a `release` label on a
# fork PR as inert as it was before promotion used the label: without it
# this job runs there and fails resolving a draft release for a branch that
# is not `release-X.Y.Z`. It short-circuits on `action != 'labeled'`, so the
# pre-existing behaviour on a push to such a PR is unchanged.
if: |
contains(github.event.pull_request.labels.*.name, 'release')
&& (github.event.action != 'labeled' || github.event.label.name == 'full-e2e' || (github.event.label.name == 'release' && !github.event.pull_request.head.repo.fork))
permissions:
contents: read
steps:
- name: Generate GitHub App token
id: app-token
uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1
with:
app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}
private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}
owner: cozystack
repositories: ${{ github.event.repository.name }}
permission-contents: read
- name: Checkout code
if: contains(github.event.pull_request.labels.*.name, 'release')
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
fetch-depth: 0
fetch-tags: true
- name: Extract tag from PR branch (release PR)
if: contains(github.event.pull_request.labels.*.name, 'release')
id: get_tag
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
with:
script: |
const branch = context.payload.pull_request.head.ref;
const m = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/);
if (!m) {
core.setFailed(`❌ Branch '${branch}' does not match 'release-X.Y.Z[-suffix]'`);
return;
}
core.setOutput('tag', `v${m[1]}`);
# The draft release has to exist for a promote PR's e2e arm to mean
# anything, and this job is what proves it. It used to also resolve the
# nocloud disk asset and export its id; the container lane boots no disk,
# so nothing in this workflow reads that id any more and resolving it only
# added a way to fail a release PR over an asset it does not use. The
# asset's presence is still checked in e2e-tag.yaml and promote-rc.yaml.
- name: Verify draft release exists (release PR)
if: contains(github.event.pull_request.labels.*.name, 'release')
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
with:
github-token: ${{ steps.app-token.outputs.token }}
script: |
const tag = '${{ steps.get_tag.outputs.tag }}';
const releases = await github.rest.repos.listReleases({
owner: context.repo.owner,
repo: context.repo.repo,
per_page: 100
});
const draft = releases.data.find(r => r.tag_name === tag && r.draft);
if (!draft) {
core.setFailed(`Draft release '${tag}' not found`);
return;
}
# In-tree e2e for SAME-REPO PRs, on Talos nodes running as CONTAINERS, like
# every other e2e lane. What containers cannot carry has no lane at all:
# DRBD, and with it the `replicated` StorageClass, its Immediate binding mode
# and the tenant StorageClass propagation that only applies to
# remotely-accessible classes.
#
# Deliberately NOT named "E2E Tests": the
# required "E2E Tests" context is a commit status posted by the `e2e-report`
# job (non-fork) or by e2e-fork.yaml (fork). A required *job* that skips
# counts as PASSED by branch protection, which is exactly how fork PRs used
# to merge with e2e never run (#3257); routing the required context through
# a status removes that skip-to-green path. Fork PRs skip this job
# (they have no pushed images to pull) and run e2e in the privileged
# workflow_run instead.
e2e:
name: "E2E (in-tree)"
# hack/e2e-compose.yaml gives each of the three nodes `cpus: 8` and
# `mem_limit: 24576m`, matching what the QEMU guests were given, so the nodes
# still demand 24 vCPU and 72 GiB between them. The same headroom argument
# therefore still decides the class: on a 24-vCPU runner that is a 1:1 commit
# with nothing left for containerd, the kernel, the host iptables MASQUERADE
# every image pull traverses and the runner agent, and during "Install
# Cozystack" the nodes contend with those and lose cycles. Readiness and
# liveness probes then miss their deadlines, a HelmRelease flips InProgress ->
# Failed and its dependents cascade to DependencyNotReady -- on a different
# component each run, the signature of resource thrash rather than a product
# bug. The 32-vCPU / 128 GiB class leaves 8 vCPU (25%) and 56 GiB of headroom
# and removes that oversubscription.
#
# One thing this class does NOT fix, and it is worth knowing before reading a
# red run: a container node reports the HOST's CPU and memory as its capacity,
# and hack/e2e-container-up.sh counterbalances that with kubelet
# systemReserved so the scheduler sees 8 CPU / 24 GiB. That corrects the
# scheduler and not what a pod reads from /proc, so a workload that sizes
# itself from /proc/cpuinfo or /proc/meminfo is configured for the whole host.
runs-on: oracle-vm-32cpu-128gb-x86-64
# Covers a green full suite. It does not cover a tenant-Kubernetes suite
# running into its node-join deadline, so a red there can end as a
# cancelled job. Held to one figure across every lane that runs the suite.
timeout-minutes: 215
permissions:
contents: read
packages: read
checks: write
needs: ["finalize", "resolve_assets", "verify-release-candidate"]
# Regular PRs still enter through finalize. A release-labelled promote PR
# enters through resolve_assets only when a maintainer explicitly adds
# full-e2e; without that label the immutable rc's already-green suite is not
# repeated. The labeled trigger above makes that post-open opt-in start a run.
#
# Fork PRs never enter here: their build exports OCI archives instead of
# pushing images, so there is no closure for this job to test. e2e-fork.yaml
# runs the suite for them from the privileged workflow_run and concludes the
# required "E2E Tests" status; for same-repo PRs e2e-report concludes it.
if: |
!cancelled()
&& !github.event.pull_request.head.repo.fork
&& (
needs.finalize.result == 'success'
|| (
needs.resolve_assets.result == 'success'
&& needs.verify-release-candidate.result == 'success'
&& contains(github.event.pull_request.labels.*.name, 'release')
&& contains(github.event.pull_request.labels.*.name, 'full-e2e')
)
)
steps:
# ▸ Checkout and prepare the codebase
# fetch-depth: 0 is required by the "Select E2E tests" step so that
# `git diff origin/${BASE_REF}...HEAD` can resolve the base ref.
- name: Checkout code
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
fetch-depth: 0
# ▸ Regular PR path – pr.patch from finalize. No Talos disk: the nodes boot
# from ghcr.io/siderolabs/talos directly (hack/e2e-compose.yaml), so this
# lane consumes no nocloud asset and waits on no Talos build.
- name: Download PR patch
if: "!contains(github.event.pull_request.labels.*.name, 'release')"
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: pr-patch
path: _out/assets
- name: Apply patch
if: "!contains(github.event.pull_request.labels.*.name, 'release')"
run: |
git apply _out/assets/pr.patch
- name: Set sandbox ID
run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> $GITHUB_ENV
# ▸ Host kernel modules. MANDATORY and up front, because nothing later can
# supply them: the nodes take their kernel from this host, and Talos
# cannot load modules in container mode -- `machine.kernel.modules` is a
# silent no-op there, `kernel_module_spec.go` returns early on
# ModeContainer with no error and no event.
#
# A module missing here is missing for every node for the whole run, and
# it surfaces in disguise: no `openvswitch` and kube-ovn comes up with no
# dataplane, no `zfs` and every LINSTOR pool creation fails. Both read as
# a CNI or a storage regression an hour later. So this loads both and
# then asserts both, and a failure here names the host rather than the
# product.
- name: Prepare container-lane host
run: |
set -eu
echo "::group::Loading kernel modules required by the container lane"
for m in openvswitch zfs; do
if grep -q "^${m} " /proc/modules; then
echo "${m}: already loaded"
continue
fi
if [ "${m}" = zfs ]; then
sudo apt-get update -qq
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq zfsutils-linux
fi
echo "${m}: loading"
sudo modprobe "${m}" || echo "${m}: modprobe failed"
done
echo "::endgroup::"
missing=''
for m in openvswitch zfs; do
grep -q "^${m} " /proc/modules || missing="${missing} ${m}"
done
if [ -n "${missing}" ]; then
echo "::error title=container-lane host::kernel module(s) could not be loaded:${missing}"
echo "This is a host problem, not a product failure. The e2e nodes run as" >&2
echo "containers on this runner's kernel and Talos cannot load modules in" >&2
echo "container mode, so what is not here now is unavailable for the whole" >&2
echo "run: openvswitch backs the kube-ovn dataplane, zfs backs every" >&2
echo "LINSTOR pool. Failing here rather than an hour later is the point." >&2
exit 1
fi
grep -E '^(openvswitch|zfs) ' /proc/modules
# ▸ Prepare environment. Two make targets rather than one: the nodes are
# siblings on the runner's docker daemon (the /dev and /run/udev binds in
# hack/e2e-compose.yaml must resolve against the real host devtmpfs, which
# is what lets this lane carry LINSTOR at all), so the host-side half
# needs a docker socket the sandbox does not have.
- name: Prepare environment
run: |
attempt=0
until make SANDBOX_NAME=$SANDBOX_NAME prepare-env-container; do
attempt=$((attempt + 1))
if [ $attempt -ge 3 ]; then
echo "Attempt $attempt failed, exiting..."
exit 1
fi
echo "Attempt $attempt failed, retrying..."
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete-cluster-container || true
done
echo "Prepare environment completed after $((attempt + 1)) attempts"
# Test Impact Analysis (TIA): walk the PackageSource graph for the PR
# diff and select only the Chainsaw suites affected. Runs by default; opt
# out via the `full-e2e` label to run the full suite.
#
# `skip=true` means every changed path was matched by one of select-e2e.sh's
# rules that select nothing — docs and dashboards, a disabled Chainsaw
# suite, a backups example with no suite, or the inert repo-meta pattern
# — so skipping Chainsaw and reporting green is the correct verdict. A
# path none of the rules recognise escalates to the full suite instead of
# returning empty, so `skip=true` can no longer mean "TIA had no idea"
# (#3392). What it still cannot rule out is a workflow nobody classified:
# .github/ is inert as a directory and the e2e workflows are exempted
# from it by name, so a workflow added under .github/workflows/ reads as
# inert until someone puts it on that list, and the fall-through never
# sees it.
- name: Select E2E tests
id: select
if: ${{ !contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
env:
BASE_REF: ${{ github.base_ref }}
run: |
git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt
apps=$(./hack/select-e2e.sh /tmp/changed.txt)
echo "Selected apps: ${apps:-}"
echo "apps=${apps}" >> $GITHUB_OUTPUT
if [ -z "${apps}" ]; then
echo "skip=true" >> $GITHUB_OUTPUT
else
echo "skip=false" >> $GITHUB_OUTPUT
fi
# The complement of what the selected suites need, for a reduced
# install. An empty selection installs everything as before.
disabled=""
if [ -n "${apps}" ]; then
disabled="$(./hack/select-install.sh --disabled "${apps}")"
fi
echo "Disabled packages: ${disabled:-}"
echo "disabled=${disabled}" >> $GITHUB_OUTPUT
# ▸ Install Cozystack
- name: Install Cozystack into sandbox
env:
# This lane has no ARP VIP: .11 is a plain control-plane node standing
# in for the QEMU lane's 192.168.123.10.
COZY_APISERVER_ENDPOINT: https://192.168.123.11:6443
# Talos containers share the runner kernel and cannot load DRBD, so the
# harness suppresses DRBD-dependent chart content, registers the host's
# pre-created zpools and keeps LINSTOR's local ZFS path.
COZY_LINSTOR_DRBD_ENABLED: "false"
# Root monitoring is the one root-tenant workload whose chart selects
# `replicated` explicitly instead of inheriting the default class. The
# install leaves it off but still seeds its local-storage override, so
# a run that turns it on does not create a replicated PVC.
COZY_E2E_STORAGE_CLASS: local
# Empty on the full-e2e label and on an empty selection, which keeps
# the full install.
COZY_DISABLED_PACKAGES: ${{ steps.select.outputs.disabled }}
run: |
if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then
echo "❌ Install failed (no retry — see diagnostics below)"
echo "::group::Diagnostics: HelmRelease status"
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true
echo "::endgroup::"
echo "::group::Diagnostics: not-Ready HelmReleases (describe)"
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \
| while read -r ns name; do
echo "--- $ns/$name ---"
docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true
done
echo "::endgroup::"
echo "::group::Diagnostics: recent events"
docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -50' || true
echo "::endgroup::"
exit 1
fi
- name: Run OpenAPI tests
run: |
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi
# ▸ Run E2E tests (Chainsaw, scoped by Test Impact Analysis)
#
# The "Select E2E tests" step above picks the suites impacted by the PR
# diff. With the `full-e2e` label that step is skipped, so FULL_E2E is
# the only way to reach this step with no selection and we run the whole
# suite; an all-inert selection skips this step entirely (the
# skip != 'true' guard).
#
# Chainsaw discovers each named suite under hack/e2e-chainsaw/ and polls
# every assertion to its own timeout, so the old per-app 3-retry loop is
# gone — a failure is reported as a structured assertion diff, with
# cluster-state diagnostics dumped below on a non-zero exit.
- name: Run E2E tests
id: e2e_tests
if: ${{ steps.select.outputs.skip != 'true' }}
env:
# No DRBD-backed class exists here. Chainsaw templates the storage
# fixtures with `local`, and the Kubernetes helper omits only the
# remote-propagation and RWX assertions while running the same suites.
COZY_E2E_STORAGE_CLASS: local
SELECTED_APPS: ${{ steps.select.outputs.apps }}
FULL_E2E: ${{ contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
run: |
# Only the full-e2e label can get here with an empty selection: when
# `select` runs, an empty result sets skip=true and this step never
# starts. Testing FULL_E2E alone keeps that invariant visible.
if [ "$FULL_E2E" = "true" ]; then
suites="" # empty => test-chainsaw runs the whole suite
echo "Running full chainsaw suite"
else
suites="$SELECTED_APPS"
echo "Running selected chainsaw suites: $suites"
fi
if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES="$suites"; then
echo "❌ Chainsaw E2E failed (see the assertion diffs or LEAKED-PODS lines above and diagnostics below)"
echo "::group::Diagnostics: HelmRelease status"
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true
echo "::endgroup::"
echo "::group::Diagnostics: recent events"
docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true
echo "::endgroup::"
# COSI state at the moment of failure — the CRDs ship no printer
# columns, so pull the readiness fields explicitly (cozyreport
# captures the same at end-of-job, but bucket states move between
# a test failing and the report being taken).
echo "::group::Diagnostics: COSI bucket state"
docker exec "$SANDBOX_NAME" sh -c '
kubectl get bucketclaims.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.bucketReady,BUCKET:.status.bucketName" 2>&1
kubectl get buckets.objectstorage.k8s.io -o custom-columns="NAME:.metadata.name,READY:.status.bucketReady,ID:.status.bucketID,CLAIMNS:.spec.bucketClaim.namespace" 2>&1
kubectl get bucketaccesses.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,GRANTED:.status.accessGranted,CLAIM:.spec.bucketClaimName" 2>&1
' || true
echo "::endgroup::"
exit 1
fi
echo "✅ All selected E2E tests passed"
- name: Collect chainsaw report
if: always()
run: |
mkdir -p _out
docker cp "$SANDBOX_NAME":/workspace/hack/e2e-chainsaw/chainsaw-report.xml \
_out/chainsaw-report.xml || true
- name: Upload chainsaw report
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: chainsaw-report
path: _out/chainsaw-report.xml
if-no-files-found: ignore
# ▸ Collect debug information (always runs)
#
# A ceiling of its own, not the job's 215 minutes. `Upload
# cozyreport.tgz` is the next step and the collector writes its tarball
# at the very end of its run, so a read that never returns spends what
# is left of the job budget, the job is cancelled, and the upload never
# runs -- the artifact is lost rather than truncated, on exactly the run
# where it is the only evidence left. `|| true` guards against a
# non-zero exit, not against a read that does not return.
#
# 30 minutes is measured against the step: it takes 3-42s across sampled
# runs, on failed e2e jobs as well as green ones. It RESERVES nothing --
# `timeout-minutes` caps a step, it does not carve budget out of the job,
# so a suite that has already burned 210 of the 215 leaves this step five
# minutes and the upload is cancelled with everything else. What the cap
# buys is that a hung read cannot be the reason.
# `continue-on-error` because the ceiling must not change what the job
# REPORTS. `|| true` on the make call is this step's standing statement
# that collecting diagnostics is never fatal, and a step killed at its
# ceiling never reaches that `|| true` -- the runner ends the process. So
# without this, a passing E2E run whose collection ran long is published
# as a failed one, and every later `failure()` in this job starts meaning
# "the tests failed OR the report was slow".
- name: Collect report
id: collect_report
if: always()
timeout-minutes: 30
continue-on-error: true
run: |
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true
# `continue-on-error` keeps the overrun out of the job's result, which also
# keeps it out of everyone's view: a step that ended at its ceiling renders
# green and the only trace is in the raw log. This branch's own standard is
# that a bound which fires is recorded, and that has to hold for the outer
# bound too.
#
# The wording says what `outcome` establishes and no more: a step can end
# non-zero without any ceiling firing, so naming the timeout as the cause
# would be the false-mechanism claim the collector refuses to make for a 137.
- name: Report collection overran its ceiling
if: always() && steps.collect_report.outcome == 'failure'
run: |
echo "::warning title=cozyreport::Collect report ended non-zero -- its timeout-minutes fired, or the step failed before the collector ran; cozyreport.tgz is missing or partial for this run"
- name: Upload cozyreport.tgz
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: cozyreport
path: _out/cozyreport.tgz
- name: Collect images list
if: always()
run: |
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-images || true
- name: Upload image list
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: image-list
path: _out/images.txt
# ▸ Open an SSH breakpoint to the failing sandbox so maintainers can attach,
# inspect Talos/Cozystack state and resume with `breakpoint resume`.
#
# Opens when the E2E job fails AND the PR carries the `debug` label or
# the `release` label (release PRs are bot-created, so the label
# implies maintainer authorship). authorized-users is kept as a second
# defense layer controlling who may actually SSH in.
#
# `failure()` is the whole condition now. It used to be joined by a
# soft-red output, because a node-join red the lane decided not to block
# on arrived here as a SUCCESSFUL job and would have been skipped. That
# carve-out is gone with the QEMU substrate: the node-join deadline
# fails its suite like any other deadline, so a sandbox worth attaching
# to always belongs to a failed job. Label-gating is what keeps this
# affordable, and it is unchanged.
#
# Uses cozystack/breakpoint-action (fork of namespacelabs/breakpoint-action)
# pinned by SHA. The fork adds: pause-idle mode (initial grace period for
# the first SSH connection, idle-aware exit afterwards), endpoint output
# and ::notice:: annotation, and a dedicated Check Run "Breakpoint Open"
# that carries the SSH endpoint in output.summary while the breakpoint is
# paused (conclusion=failure → standard ✗ in `gh pr checks`; updated to
# conclusion=success when the breakpoint exits).
#
# Configured via the BREAKPOINT_ENDPOINT repository variable — in fork
# PRs repository variables are not exposed, so the step is skipped and
# forks cannot reach the self-hosted rendezvous server.
- name: Breakpoint on E2E failure
if: |
failure() &&
vars.BREAKPOINT_ENDPOINT != '' &&
(contains(github.event.pull_request.labels.*.name, 'debug')
|| contains(github.event.pull_request.labels.*.name, 'release'))
# A step that fails here would fail the job on its own account. Same
# reason `Collect report` above carries it: what the job REPORTS must not
# depend on what a helper did.
continue-on-error: true
# cozystack/breakpoint-action v2-cozy.1
# mode: pause-idle defaults: grace-period=20m, idle-timeout=10m
uses: cozystack/breakpoint-action@a6f3a6f87be398ad63b6577351e3398e53f578e4
with:
mode: pause-idle
endpoint: ${{ vars.BREAKPOINT_ENDPOINT }}
authorized-users: androndo, Arsolitt, IvanHunters, kvaps, lexfrei, lllamnyp, mattia-eleuteri, matthieu-robin, myasnikovdaniil, sircthulhu, tym83
check-run-name: "Breakpoint Open"
github-token: ${{ github.token }}
check-run-summary-template: |
## 🔴 SSH breakpoint open — paused for debug
```
{endpoint}
```
Enter the e2e sandbox after SSH:
```
docker exec -ti $(docker ps --filter name=cozy-e2e-sandbox -q | head -1) bash
export KUBECONFIG=/workspace/kubeconfig
```
Resume from inside: `breakpoint resume`. Otherwise the breakpoint
exits 10 minutes after the last SSH session disconnects.
# ▸ Tear down environment (always runs)
#
# Both halves, and in this order: the compose project owns the nodes and
# the zpools outlive it. A leftover pool makes the next run on a reused
# host adopt storage that already carries a dead cluster's volumes.
- name: Tear down sandbox
if: always()
run: |
# Loud, but never the job's verdict. A failed teardown means a dirty
# host, and this runner is discarded after the job, so there is no next
# run to protect -- while failing the job here posts the required
# "E2E Tests" status as a failure that no reader can tell apart from a
# real one. The annotation is what a reused host (a developer machine)
# needs; `delete-cluster-container` returning non-zero is not evidence
# about the product.
cleanup_rc=0
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete-cluster-container || cleanup_rc=$?
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true
if [ "$cleanup_rc" -ne 0 ]; then
echo "::error title=sandbox teardown::teardown exited $cleanup_rc; the host may hold a stale compose project or zpool"
fi
# Report the required "E2E Tests" gate as a commit status rather than relying
# on a job named "E2E Tests". A required *job* that is skipped counts
# as PASSED by branch protection — desired for docs-only PRs, but the exact
# hole that let fork PRs merge with e2e never run (#3257). This job always runs
# on same-repo PRs and concludes "E2E Tests" deterministically:
# • plan failed → failure
# • docs-only PR (code!=true)→ success (e2e not required)
# • in-tree e2e succeeded → success
# • otherwise (skipped/failed/cancelled with code changes) → failure
# Fork PRs skip this job; e2e-fork.yaml concludes "E2E Tests" for them, so a
# fork PR stays "Expected"/not-mergeable until the privileged run reports.
#
# The required "E2E Tests" merge gate is published as a COMMIT STATUS with the
# default GITHUB_TOKEN (+ statuses:write) — NOT as a job, and NOT as a
# check-run.
# - Not a job: a *skipped* required job counts as passed by branch
# protection, which is how fork PRs used to merge with e2e never run
# (#3257). A status is "success" only when this job posts it; a
# never-posted required status stays "Expected" and blocks the merge.
# - Not a check-run: a check-run made with the default token floats into an
# arbitrary GitHub Actions check-suite (it rendered under "PR size label"),
# and an app-token check-run needs the app granted checks:write. A commit
# status has neither problem — branch protection matches it by context name
# and it carries no check-suite grouping. Safe with the default token: a
# fork's pull_request token is read-only and cannot post a status, so only
# same-repo runs (this job, guarded !fork) and the privileged workflow_run
# in e2e-fork.yaml can set it.
# REQUIRES main branch protection to require the "E2E Tests" status context —
# see docs/agents/e2e-testing.md §10.
#
# The `labeled` guard must stay in lockstep with `plan`'s: this job runs if and
# only if `plan` runs. `labeled` events other than full-e2e/release are
# discarded there, so without the same guard this job would run with plan *skipped*, read
# PLAN_RESULT != 'success', and post "E2E Tests" = failure — clobbering an
# already-green required status every time anyone labels an open PR.
e2e-report:
name: "Report E2E status"
needs: ["plan", "e2e", "verify-release-candidate"]
# !cancelled() rather than always(). A cancelled run no longer owns the
# status for its head SHA, because something superseded it and that
# something is now responsible. Posting from here would stamp a red on a
# suite that never finished, and the run that replaced it may be one that
# deliberately posts nothing, leaving nothing to flip it back. A cancelled
# run must report nothing.
#
# Cancellation still reaches this job from a push, from a `full-e2e` label
# run superseding the head run, and from a manual cancel. It no longer
# reaches it from a label the workflow discards: those run in their own
# concurrency group (see the key at the top of this file) precisely so they
# cannot cancel a live run.
if: ${{ !cancelled() && !github.event.pull_request.head.repo.fork && (github.event.action != 'labeled' || github.event.label.name == 'full-e2e' || github.event.label.name == 'release') }}
runs-on: ubuntu-latest
permissions:
contents: read
statuses: write
steps:
- name: Report "E2E Tests" commit status
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
env:
PLAN_RESULT: ${{ needs.plan.result }}
CODE: ${{ needs.plan.outputs.code }}
E2E_RESULT: ${{ needs.e2e.result }}
CANDIDATE_RESULT: ${{ needs.verify-release-candidate.result }}
IS_RELEASE: ${{ contains(github.event.pull_request.labels.*.name, 'release') }}
IS_FULL_E2E: ${{ contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
script: |
const { PLAN_RESULT, CODE, E2E_RESULT, CANDIDATE_RESULT, IS_RELEASE, IS_FULL_E2E, HEAD_SHA, RUN_URL } = process.env;
let state, description;
if (PLAN_RESULT !== 'success') {
state = 'failure';
description = `plan job did not succeed (result: ${PLAN_RESULT})`;
} else if (IS_RELEASE === 'true' && CANDIDATE_RESULT !== 'success') {
state = 'failure';
description = `release packages candidate did not verify (result: ${CANDIDATE_RESULT})`;
} else if (CODE !== 'true') {
state = 'success';
description = 'docs-only PR — e2e not required';
} else if (IS_RELEASE === 'true' && IS_FULL_E2E !== 'true') {
// Release promotion PRs deliberately do not repeat e2e: the rc was
// already verified against the immutable candidate whose digests
// this tree pins, which is the whole point of the rc gate. The `e2e`
// job skips for exactly this case, and while the gate was a job that
// skip satisfied branch protection. Now that the gate is a status,
// the skip has to be reported explicitly or every promotion needs a
// full suite re-run or an admin override.
state = 'success';
description = 'release promotion — verified at rc, e2e not repeated (add full-e2e to force a run)';
} else if (E2E_RESULT === 'success') {
state = 'success';
description = 'in-tree e2e passed';
} else {
state = 'failure';
description = `in-tree e2e did not pass (result: ${E2E_RESULT})`;
}
try {
await github.rest.repos.createCommitStatus({
owner: context.repo.owner,
repo: context.repo.repo,
sha: HEAD_SHA,
state,
context: 'E2E Tests',
description,
target_url: RUN_URL,
});
} catch (err) {
// The 'E2E Tests' status is the required merge gate. If we cannot
// publish it, fail loudly and fail-closed: setFailed keeps the job
// red and the required status unreported ("Expected"), so the PR
// stays non-mergeable.
core.setFailed(`Failed to report E2E Tests status: ${err}`);
return;
}
core.info(`E2E Tests → ${state}: ${description}`);