mirror of
https://github.com/cozystack/cozystack.git
synced 2026-10-23 03:27:50 +09:00
Myasnikov Daniil
cc942fed68
The shell translator rewrites test bodies and loses Bats reporting. The unit lane needs complete TAP results and JUnit reports without changing how live-cluster suites run. Sourced production helpers still need a dash pass, and unit test bodies need nounset without exporting shell options into subprocesses. Assisted-by: LLM Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
1471 lines
77 KiB
YAML
1471 lines
77 KiB
YAML
name: Pull Request
|
||
|
||
env:
|
||
# TODO: unhardcode this
|
||
REGISTRY: iad.ocir.io/idyksih5sir9/cozystack
|
||
on:
|
||
pull_request:
|
||
# `labeled` makes the promote PR's release-candidate verification and manual
|
||
# full-e2e opt-in actionable after the bot opens it. The plan/resolve guards
|
||
# below discard every other label event before it can launch work.
|
||
types: [opened, synchronize, reopened, labeled]
|
||
|
||
# Cancel in‑flight runs for the same PR when a new push arrives.
|
||
#
|
||
# A `labeled` event that `plan` discards gets its OWN group, so labelling an
|
||
# open PR cannot cancel a live opened/synchronize run. Any label other than
|
||
# full-e2e/release makes `plan` skip, and `e2e-report` skips with it, so without the
|
||
# split the pair "cancel the real run, then skip" left the head SHA with no
|
||
# "E2E Tests" status at all — and an unposted required status stays "Expected",
|
||
# blocking the merge until someone pushes again or re-runs the cancelled run.
|
||
# Observed on live CI: run 30614258694 (cancelled) followed by 30614280603
|
||
# (skipped) on the same SHA, 24s apart, from a maintainer adding kind/* labels
|
||
# just after a push. (`pr-labeler` cannot cause this — it labels with the
|
||
# default GITHUB_TOKEN, which by design triggers no further workflow runs.)
|
||
#
|
||
# The condition is the exact complement of `plan`'s guard, so only a run that
|
||
# will publish NOTHING is moved aside. A `labeled` run carrying full-e2e or
|
||
# release does publish, so it stays in the main group and supersedes the head
|
||
# run the way a push does. That keeps exactly one publisher of the `E2E Tests`
|
||
# context alive per head SHA, which the status depends on: two live publishers are latest-wins
|
||
# per context, and the full suite is the superset, so a full-e2e run that fails
|
||
# early — a build flake, a suite failing at minute twenty — would finish before
|
||
# the narrower head run it was meant to widen, and that run's later green would
|
||
# erase the failure. Cancelling also means opting into the full suite costs one
|
||
# suite rather than two, and a second label landing afterwards cannot cancel the
|
||
# full-e2e run, because it is no longer in the same group as it.
|
||
#
|
||
# `release` publishes on the SAME-REPO lane only, hence the fork term. A promote
|
||
# PR is opened same-repo by the release bot, so the label on a fork PR is always
|
||
# a mislabel — and on that lane it would be a destructive one. e2e-fork.yaml's
|
||
# `resolve` decides "this run built nothing, so it is not a verdict" by asking
|
||
# whether `Plan build` skipped; a fork run that gets past `plan` then fails
|
||
# closed on the pr-patch artifact its own `release` label gated off, posting red
|
||
# on a suite that was never going to run, with no `unlabeled` trigger to undo
|
||
# it. Excluding forks keeps the label as inert on that lane as it was before
|
||
# promotion started using it. The term appears in both this key and `plan`'s
|
||
# guard because the two must stay exact complements: NOT(A OR B OR (C AND NOT
|
||
# F)) is (NOT A) AND (NOT B) AND (NOT C OR F), and for a same-repo PR F is false
|
||
# and both sides reduce to exactly what they were.
|
||
concurrency:
|
||
group: pr-${{ github.workflow }}-${{ github.event.pull_request.number }}${{ (github.event.action == 'labeled' && github.event.label.name != 'full-e2e' && (github.event.label.name != 'release' || github.event.pull_request.head.repo.fork)) && '-label' || '' }}
|
||
cancel-in-progress: true
|
||
|
||
# Top-level token defaults to read-only. A job-level block replaces this one
|
||
# rather than adding to it, so each job restates every scope it needs.
|
||
permissions:
|
||
contents: read
|
||
|
||
jobs:
|
||
# Compute the docs-only gate and the per-package build matrix from the PR
|
||
# diff. hack/build-matrix.sh parses the root Makefile `build:` target (the
|
||
# single source of truth) and emits only the units whose package dir changed,
|
||
# escalating to the full set on a shared-dependency change. talos + installer
|
||
# are handled by dedicated jobs (see build-matrix.sh for why).
|
||
plan:
|
||
name: Plan build
|
||
runs-on: ubuntu-latest
|
||
# Exact complement of the concurrency key above; see the reasoning there,
|
||
# including why `release` is admitted for same-repo PRs only.
|
||
if: github.event.action != 'labeled' || github.event.label.name == 'full-e2e' || (github.event.label.name == 'release' && !github.event.pull_request.head.repo.fork)
|
||
permissions:
|
||
contents: read
|
||
# For the opening `E2E Tests` status below; e2e-report writes the terminal
|
||
# one with the same scope.
|
||
statuses: write
|
||
outputs:
|
||
code: ${{ steps.p.outputs.code }}
|
||
matrix: ${{ steps.p.outputs.matrix }}
|
||
any: ${{ steps.p.outputs.any }}
|
||
touched: ${{ steps.p.outputs.touched }}
|
||
talos: ${{ steps.p.outputs.talos }}
|
||
steps:
|
||
# Claim the required context for THIS run before anything can be merged on
|
||
# the strength of the last one. Commit statuses are latest-wins per (sha,
|
||
# context) and nothing expires them, so a head SHA that already went green
|
||
# keeps reading green while a newly started run works — re-run all jobs, a
|
||
# full-e2e label, a reopen — and the PR stays mergeable through a suite
|
||
# that may be about to fail. While the gate was a job, GitHub created a
|
||
# fresh non-success check run each time and that window did not exist.
|
||
# Overwriting with `pending` here restores it: the run that owns the
|
||
# status says so at the start and concludes it in e2e-report.
|
||
#
|
||
# Same-repo only. A fork's `pull_request` token is read-only, so this call
|
||
# would 403 and fail `plan` on every fork PR, and that read-only token is
|
||
# exactly what makes the whole commit-status design safe (§10). Fork PRs
|
||
# get their pending from e2e-fork.yaml's `resolve` instead, which is
|
||
# privileged.
|
||
#
|
||
# Note what that costs a fork. `resolve` cannot start until this entire
|
||
# run has finished, so the window this step shuts in seconds stays open
|
||
# for the length of the build on the fork lane. Structural rather than an
|
||
# oversight, since a read-only token cannot post a status at any point in
|
||
# the run, and bounded by the build. §10 records it.
|
||
- name: Open the "E2E Tests" status for this run
|
||
if: ${{ !github.event.pull_request.head.repo.fork }}
|
||
# A transient status-API failure here must not take the run with it.
|
||
# This step is the first in `plan`, and `plan` is what `checks`,
|
||
# `build`, `build-talos`, `finalize` and `e2e` all hang off, so a
|
||
# failure costs a full rebuild for nothing. Skipping the opener only
|
||
# degrades to the behaviour that preceded it — the terminal status
|
||
# still gets posted by `e2e-report`, and a run that dies before that
|
||
# leaves the merge blocked either way.
|
||
continue-on-error: true
|
||
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
|
||
env:
|
||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||
with:
|
||
script: |
|
||
const { HEAD_SHA, RUN_URL } = process.env;
|
||
await github.rest.repos.createCommitStatus({
|
||
owner: context.repo.owner,
|
||
repo: context.repo.repo,
|
||
sha: HEAD_SHA,
|
||
state: 'pending',
|
||
context: 'E2E Tests',
|
||
description: 'e2e in progress — this run has not reported yet',
|
||
target_url: RUN_URL,
|
||
});
|
||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||
with:
|
||
fetch-depth: 0
|
||
- name: Compute changed files and build matrix
|
||
id: p
|
||
env:
|
||
BASE_REF: ${{ github.base_ref }}
|
||
run: |
|
||
git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt
|
||
# The docs-only decision reads a RENAME-BLIND list. `git diff
|
||
# --name-only` applies rename detection and reports a rename as its
|
||
# new path alone, so moving a code file into docs/ reads as a
|
||
# docs-only change: nothing builds, no e2e runs, and the removal of
|
||
# that code path ships untested. It also deadlocks the fork lane,
|
||
# which answers the same question from `pulls.listFiles` and does see
|
||
# the old path through `previous_filename`. That lane concludes e2e is
|
||
# required, then fails closed on a pr-patch artifact this lane never
|
||
# produced — an unfixable red on a legitimate contribution, with no
|
||
# label override and no push that clears it. Both lanes have to answer
|
||
# "is this docs-only" the same way, and the only answer that is also
|
||
# correct keeps both sides of a rename.
|
||
#
|
||
# The build matrix still reads the rename-detected list, which is not
|
||
# a claim that doing so is right. Moving a file between packages
|
||
# leaves the source package changed and unselected, so the matrix
|
||
# under-builds on a cross-package rename. That behaviour predates this
|
||
# line and is not what the docs-only decision is for, so it is left
|
||
# alone here rather than widened by a side effect.
|
||
git diff --name-only --no-renames "origin/${BASE_REF}...HEAD" > /tmp/changed_norenames.txt
|
||
echo "Changed files:"; sed 's/^/ /' /tmp/changed_norenames.txt
|
||
if grep -qvE '^docs/' /tmp/changed_norenames.txt; then
|
||
echo "code=true" >> "$GITHUB_OUTPUT"
|
||
else
|
||
echo "code=false" >> "$GITHUB_OUTPUT"
|
||
fi
|
||
matrix="$(hack/build-matrix.sh /tmp/changed.txt)"
|
||
echo "Build matrix: $matrix"
|
||
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
|
||
talos="$(hack/build-talos-needed.sh /tmp/changed_norenames.txt)"
|
||
echo "Talos build needed: $talos"
|
||
echo "talos=$talos" >> "$GITHUB_OUTPUT"
|
||
# Package dirs the PR edited (packages/<area>/<name>). Their committed
|
||
# refs are the PR's intent and must survive the current-main overlay —
|
||
# otherwise an upstream image bump in a non-build-unit package would be
|
||
# silently reverted in e2e/the installer artifact (see
|
||
# hack/overlay-main-images.sh). Whitespace-separated, no trailing slash.
|
||
touched="$(sed -nE 's#^(packages/[^/]+/[^/]+)/.*#\1#p' /tmp/changed.txt | sort -u | tr '\n' ' ')"
|
||
echo "Touched packages: $touched"
|
||
echo "touched=$touched" >> "$GITHUB_OUTPUT"
|
||
if [ "$matrix" = "[]" ]; then
|
||
echo "any=false" >> "$GITHUB_OUTPUT"
|
||
else
|
||
echo "any=true" >> "$GITHUB_OUTPUT"
|
||
fi
|
||
|
||
# Promotion publishes the rewritten packages artifact before opening its PR.
|
||
# Verify GitHub's prospective merge tree here, so a `packages/` edit pushed to
|
||
# this PR fails the required E2E Tests status before it can merge.
|
||
#
|
||
# Checking the MERGE tree rather than the head is what makes base drift
|
||
# visible when this job runs — but nothing re-runs it when the base moves:
|
||
# GitHub does not fire `pull_request` on the base branch advancing, it only
|
||
# recomputes refs/pull/N/merge. So a `packages/` change landing on the base
|
||
# after this went green is never re-verified here, and the PR stays mergeable
|
||
# on the stale result. Finalize repeats the check against the real merge
|
||
# commit before creating the write-once stable tag, which is where that case
|
||
# is actually caught — loudly, and before any stable name exists.
|
||
verify-release-candidate:
|
||
name: Verify release packages candidate
|
||
runs-on: ubuntu-latest
|
||
# Matching `checks` and `finalize`. Without a ceiling the job inherits the
|
||
# 6-hour default, and a stalled `flux pull artifact` would hold the required
|
||
# `E2E Tests` status pending for all of it, since e2e-report needs this job.
|
||
timeout-minutes: 30
|
||
if: |
|
||
contains(github.event.pull_request.labels.*.name, 'release')
|
||
&& github.event.pull_request.user.login == 'cozystack-ci[bot]'
|
||
&& (github.event.action != 'labeled' || github.event.label.name == 'full-e2e' || github.event.label.name == 'release')
|
||
permissions:
|
||
contents: read
|
||
packages: read
|
||
steps:
|
||
# The default pull_request checkout is GitHub's prospective merge commit,
|
||
# not just the head branch. That makes base-branch package drift visible.
|
||
- name: Checkout prospective merge
|
||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||
with:
|
||
fetch-depth: 0
|
||
|
||
# Execute the verifier from the trusted base SHA. An older rc-derived head
|
||
# may not contain it, and a PR must not be able to weaken its own guard.
|
||
- name: Checkout release tooling from base
|
||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||
with:
|
||
ref: ${{ github.event.pull_request.base.sha }}
|
||
path: .release-tooling
|
||
sparse-checkout: hack
|
||
persist-credentials: false
|
||
|
||
# Both tools are version-pinned. This job decides whether a stable release
|
||
# may be created, so what it runs has to be a fixed input: `latest/download`
|
||
# resolves to whatever mikefarah shipped this morning, and a yq that changed
|
||
# how it emits or compares a value turns a release gate into a moving one.
|
||
# The version tracks packages/core/testing/images/e2e-sandbox/Dockerfile.
|
||
# yq is additionally pinned by content; flux comes from the commit-pinned
|
||
# upstream setup action, which checksums the release tarball.
|
||
# promote-rc.yaml's copy of this step carries the full reasoning for both.
|
||
- name: Set up flux
|
||
uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6
|
||
with:
|
||
version: "2.8.6"
|
||
- name: Set up promotion toolchain (yq)
|
||
env:
|
||
YQ_VERSION: "4.53.3"
|
||
YQ_SHA256: "fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4"
|
||
run: |
|
||
if ! yq --version 2>/dev/null | grep -q "mikefarah.* version v${YQ_VERSION}\$"; then
|
||
yq_bin="$(mktemp)"
|
||
curl -fsSL -o "$yq_bin" \
|
||
"https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64"
|
||
printf '%s %s\n' "$YQ_SHA256" "$yq_bin" | sha256sum -c -
|
||
sudo install -m 0755 "$yq_bin" /usr/local/bin/yq
|
||
rm -f "$yq_bin"
|
||
fi
|
||
|
||
- name: Login to registry (GHCR)
|
||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||
with:
|
||
username: ${{ github.repository_owner }}
|
||
password: ${{ secrets.GITHUB_TOKEN }}
|
||
registry: ghcr.io
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
|
||
- name: Verify stable packages candidate
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
HEAD_BRANCH: ${{ github.head_ref }}
|
||
run: |
|
||
STABLE_VERSION="${HEAD_BRANCH#release-}"
|
||
.release-tooling/hack/verify-promoted-packages.sh "$STABLE_VERSION"
|
||
|
||
# Helm-unittest + Go controller tests. Decoupled from the image build and from
|
||
# e2e so they give fast, independent signal; they gate the PR as their own
|
||
# required check rather than blocking e2e from starting.
|
||
checks:
|
||
name: Unit & controller tests
|
||
runs-on: oracle-vm-4cpu-16gb-x86-64
|
||
timeout-minutes: 30
|
||
env:
|
||
# The pipe falls back to direct on any proxy error (hack/common-envs.mk).
|
||
GOPROXY: "https://proxy.golang.org|direct"
|
||
permissions:
|
||
contents: read
|
||
needs: ["plan"]
|
||
# Unit contracts read both code and documentation. Run this job for every
|
||
# non-release PR: code changes take the complete unit path, while docs-only
|
||
# changes run the Bats lane that owns those cross-tree contracts. Builds and
|
||
# E2E remain gated on `code` below.
|
||
if: |
|
||
!contains(github.event.pull_request.labels.*.name, 'release')
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||
with:
|
||
fetch-depth: 0
|
||
# The module cache is restored from the key build-main.yaml writes on
|
||
# main, which every PR run can read, so a PR's first run starts warm. The
|
||
# download fetches whatever the cache lacks, the whole set on a miss. The
|
||
# Go steps are for code changes only: a docs-only run is the Bats lane alone.
|
||
- name: Locate Go module cache
|
||
if: needs.plan.outputs.code == 'true'
|
||
id: gomodcache
|
||
run: echo "dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
|
||
- name: Cache Go modules
|
||
if: needs.plan.outputs.code == 'true'
|
||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||
with:
|
||
path: ${{ steps.gomodcache.outputs.dir }}
|
||
key: go-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
|
||
restore-keys: go-mod-${{ runner.os }}-
|
||
- name: Download Go modules
|
||
if: needs.plan.outputs.code == 'true'
|
||
run: |
|
||
for mod in $(git ls-files go.mod '*/go.mod' | xargs -n1 dirname); do
|
||
(cd "$mod" && "$GITHUB_WORKSPACE/hack/go-mod-download.sh")
|
||
done
|
||
# Ephemeral runners don't carry the helm-unittest plugin the persistent
|
||
# self-hosted runner accreted. The plugin embeds its own Helm, so its
|
||
# version is pinned: a plugin release must not change test results. Bats is
|
||
# installed on every run, docs-only ones included.
|
||
- name: Set up test toolchain
|
||
env:
|
||
CODE_CHANGED: ${{ needs.plan.outputs.code }}
|
||
HELM_UNITTEST_VERSION: "1.1.2"
|
||
BATS_VERSION: "1.14.0"
|
||
BATS_COMMIT: eb7f42f8d608ac693d7a4b67474f6714ea68cfc5
|
||
run: |
|
||
set -euo pipefail
|
||
if [ "$CODE_CHANGED" = true ]; then
|
||
installed="$(helm plugin list 2>/dev/null | awk '$1 == "unittest" { print $2 }' || true)"
|
||
if [ "$installed" != "$HELM_UNITTEST_VERSION" ]; then
|
||
[ -z "$installed" ] || helm plugin uninstall unittest
|
||
helm plugin install https://github.com/helm-unittest/helm-unittest --version "v${HELM_UNITTEST_VERSION}"
|
||
fi
|
||
fi
|
||
# Source archive bytes can change; verify the tag's commit before install.
|
||
if ! bats --version 2>/dev/null | grep -qx "Bats $BATS_VERSION"; then
|
||
bats_parent=$(mktemp -d "$RUNNER_TEMP/bats-core.XXXXXX")
|
||
bats_source="$bats_parent/source"
|
||
git clone --quiet --depth=1 --branch "v$BATS_VERSION" \
|
||
https://github.com/bats-core/bats-core.git "$bats_source"
|
||
test "$(git -C "$bats_source" rev-parse HEAD)" = "$BATS_COMMIT"
|
||
sudo "$bats_source/install.sh" /usr/local
|
||
fi
|
||
bats --version | grep -qx "Bats $BATS_VERSION"
|
||
# -k is required, not cosmetic: unit-tests and test-controllers are two
|
||
# independent goals on the same command line, and without -k make stops
|
||
# handing out new work the moment any unit-tests prerequisite fails —
|
||
# test-controllers then never starts and reports nothing, so a red BATS
|
||
# file hides the entire Go controller suite. -k lets already-schedulable
|
||
# goals keep running; the job still fails overall on any red target.
|
||
- name: Run unit and controller tests
|
||
if: needs.plan.outputs.code == 'true'
|
||
run: make unit-tests test-controllers -j4 -k --output-sync=target
|
||
- name: Run Bats unit tests for docs-only changes
|
||
if: needs.plan.outputs.code == 'false'
|
||
run: make bats-unit-tests
|
||
- name: Upload unit test report
|
||
if: always()
|
||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||
with:
|
||
name: bats-unit-report
|
||
path: _out/test-reports/
|
||
if-no-files-found: ignore
|
||
|
||
# One image (package) per job, fanned out across the ephemeral pool. Each job
|
||
# gets its own buildx (docker-container driver) so the shared-buildkit
|
||
# contention that wedged the old single Build job (#2937) is severed by
|
||
# construction. Each job patches only its own package's values.yaml/.tag files
|
||
# and uploads that diff as a patch fragment; `finalize` reassembles them.
|
||
build:
|
||
name: Build ${{ matrix.pkg }}
|
||
needs: ["plan"]
|
||
if: |
|
||
needs.plan.outputs.code == 'true'
|
||
&& needs.plan.outputs.any == 'true'
|
||
&& !contains(github.event.pull_request.labels.*.name, 'release')
|
||
runs-on: oracle-vm-4cpu-16gb-x86-64
|
||
timeout-minutes: 30
|
||
permissions:
|
||
contents: read
|
||
strategy:
|
||
fail-fast: false
|
||
matrix:
|
||
pkg: ${{ fromJSON(needs.plan.outputs.matrix) }}
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||
with:
|
||
fetch-depth: 0
|
||
fetch-tags: true
|
||
# installer is built in finalize, but some package image targets shell out
|
||
# to flux.
|
||
- name: Set up flux
|
||
uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6
|
||
with:
|
||
version: "2.8.6"
|
||
- name: Set up Docker config
|
||
run: |
|
||
if [ -d ~/.docker ]; then
|
||
cp -r ~/.docker "${{ runner.temp }}/.docker"
|
||
fi
|
||
- name: Login to GitHub Container Registry
|
||
if: ${{ !github.event.pull_request.head.repo.fork }}
|
||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||
with:
|
||
username: ${{ secrets.OCIR_USER }}
|
||
password: ${{ secrets.OCIR_TOKEN }}
|
||
registry: iad.ocir.io
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
# docker-container driver: isolated buildkit per job (no shared-buildkit
|
||
# contention) and a prerequisite for the mode=max cache reads (#2938).
|
||
# Its buildkitd also routes docker.io base-image pulls through
|
||
# mirror.gcr.io (hack/buildkitd.toml): the Build step authenticates only
|
||
# to the push registry (OCIR), so base images (golang/ubuntu/nginx/...)
|
||
# would otherwise be pulled from Docker Hub anonymously and intermittently
|
||
# trip its anonymous-pull rate limit; the mirror sidesteps that and falls
|
||
# back to Docker Hub on a miss/outage. Create the builder under the SAME
|
||
# DOCKER_CONFIG the Build step reads (runner.temp/.docker, holding the
|
||
# OCIR auth) so buildx forwards push/cache auth to the container builder.
|
||
- name: Set up Buildx (docker-container driver, docker.io mirror)
|
||
id: buildx
|
||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||
with:
|
||
driver: docker-container
|
||
buildkitd-config: hack/buildkitd.toml
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
- name: Build ${{ matrix.pkg }}
|
||
run: |
|
||
mkdir -p _out/oci
|
||
make -C "$PKG" image
|
||
env:
|
||
PKG: ${{ matrix.pkg }}
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
# Build on the docker-container builder created above so base-image
|
||
# pulls go through the docker.io mirror (BUILDX_ARGS adds
|
||
# --builder=$(BUILDER)).
|
||
BUILDER: ${{ steps.buildx.outputs.name }}
|
||
# common-envs.mk defaults to amd64+arm64. Nothing here sets up QEMU,
|
||
# and on this x86-64 runner the arm64 half would be emulated anyway;
|
||
# CI builds amd64 until arm64 has a native leg of its own.
|
||
PLATFORM: linux/amd64
|
||
# Unique per (PR, commit) so concurrent PR builds never push the same
|
||
# tag. PR images are consumed by digest; :latest belongs to releases.
|
||
IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
|
||
PUBLISH_VERSIONED: '0'
|
||
PUBLISH_FLOATING: '0'
|
||
# Fork PRs get no registry credentials, so instead of pushing, build
|
||
# each image to <workspace>/_out/oci/<name>.oci.tar and upload it below;
|
||
# the privileged e2e-fork.yaml workflow_run pushes them by digest. Empty
|
||
# on same-repo PRs → normal push path. Setting it also forces PUSH/LOAD
|
||
# off in common-envs.mk (#3257). Must be ABSOLUTE (workspace-rooted):
|
||
# the build runs `make -C <pkg>`, so buildx resolves a relative dest
|
||
# against the package dir — the archive would land in <pkg>/_out/oci and
|
||
# miss the repo-root _out/oci the upload reads.
|
||
OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}
|
||
- name: Capture digest patch fragment
|
||
id: frag
|
||
env:
|
||
PKG: ${{ matrix.pkg }}
|
||
run: |
|
||
safe="$(echo "$PKG" | tr '/' '-')"
|
||
mkdir -p _out/fragments
|
||
# --binary so vendored binary provider bundles (components*.gz),
|
||
# which some package image targets regenerate, survive reassembly
|
||
# in finalize (a plain "Binary files differ" stub is unappliable).
|
||
git diff --binary HEAD > "_out/fragments/${safe}.patch"
|
||
echo "safe=${safe}" >> "$GITHUB_OUTPUT"
|
||
echo "Fragment for ${PKG}:"; git --no-pager diff --stat HEAD
|
||
- name: Upload patch fragment
|
||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||
with:
|
||
name: pr-patch-fragment-${{ steps.frag.outputs.safe }}
|
||
path: _out/fragments/${{ steps.frag.outputs.safe }}.patch
|
||
if-no-files-found: error
|
||
# Fork PR only: the images were exported to OCI archives instead of pushed
|
||
# (OCI_EXPORT_DIR above). Upload them for e2e-fork.yaml to push by digest.
|
||
- name: Upload OCI image archives (fork PR)
|
||
if: ${{ github.event.pull_request.head.repo.fork }}
|
||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||
with:
|
||
name: pr-oci-${{ steps.frag.outputs.safe }}
|
||
path: _out/oci/*.oci.tar
|
||
if-no-files-found: error
|
||
|
||
# Talos is its own leg: the installer tarball is heavy and the Talos image is
|
||
# built from it, so both live in one job. No nocloud disk is built here any
|
||
# more, and e2e no longer downloads one -- the container lane boots the
|
||
# upstream Talos container image. `images/talos/profiles/nocloud.yaml` is
|
||
# therefore compiled by nightly.yaml and by the tag build's `make assets`.
|
||
#
|
||
# For same-repo PRs, the plan job classifies the rename-blind changed-path
|
||
# list with build-talos-needed.sh. That list includes direct Talos sources and
|
||
# the shared Makefile/Buildx/workflow inputs that determine this job's output;
|
||
# line-based classification avoids depending on the formatting of `touched`.
|
||
# A successful run stamps packages/extra/bootbox/images/matchbox.tag, and
|
||
# finalize preserves that generated ref when it overlays base-branch images.
|
||
#
|
||
# Fork PRs keep the previous unconditional behavior as a conservative scope
|
||
# boundary. Their build/export and privileged publication path spans a second
|
||
# workflow and is not optimized here. The installer job also exports an OCI
|
||
# archive, so archive availability is not a reason to require Talos itself.
|
||
build-talos:
|
||
name: Build Talos
|
||
needs: ["plan"]
|
||
if: |
|
||
needs.plan.outputs.code == 'true'
|
||
&& !contains(github.event.pull_request.labels.*.name, 'release')
|
||
&& (
|
||
github.event.pull_request.head.repo.fork
|
||
|| needs.plan.outputs.talos == 'true'
|
||
)
|
||
runs-on: oracle-vm-4cpu-16gb-x86-64
|
||
timeout-minutes: 40
|
||
permissions:
|
||
contents: read
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||
with:
|
||
fetch-depth: 0
|
||
fetch-tags: true
|
||
- name: Set up Docker config
|
||
run: |
|
||
if [ -d ~/.docker ]; then
|
||
cp -r ~/.docker "${{ runner.temp }}/.docker"
|
||
fi
|
||
- name: Login to GitHub Container Registry
|
||
if: ${{ !github.event.pull_request.head.repo.fork }}
|
||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||
with:
|
||
username: ${{ secrets.OCIR_USER }}
|
||
password: ${{ secrets.OCIR_TOKEN }}
|
||
registry: iad.ocir.io
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
# buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
|
||
# (hack/buildkitd.toml) so the talos image build dodges Docker Hub's
|
||
# anonymous-pull rate limit; see the build job for the full rationale.
|
||
- name: Set up Buildx (docker-container driver, docker.io mirror)
|
||
id: buildx
|
||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||
with:
|
||
driver: docker-container
|
||
buildkitd-config: hack/buildkitd.toml
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
- name: Build Talos image (matchbox + installer tarball)
|
||
run: |
|
||
mkdir -p _out/oci
|
||
make -C packages/core/talos image
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
BUILDER: ${{ steps.buildx.outputs.name }}
|
||
# amd64 only, as in the build job. The arm64 installer needs no QEMU,
|
||
# but nothing a PR runs boots it. matchbox still gets the arm64 kernel
|
||
# and initramfs.
|
||
PLATFORM: linux/amd64
|
||
IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
|
||
PUBLISH_VERSIONED: '0'
|
||
PUBLISH_FLOATING: '0'
|
||
# Fork PR: export to OCI archive instead of pushing (see build job).
|
||
OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}
|
||
- name: Capture Talos patch fragment
|
||
run: |
|
||
mkdir -p _out/fragments
|
||
git diff --binary HEAD > _out/fragments/talos.patch
|
||
echo "Talos fragment:"; git --no-pager diff --stat HEAD
|
||
- name: Upload Talos patch fragment
|
||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||
with:
|
||
name: pr-patch-fragment-talos
|
||
path: _out/fragments/talos.patch
|
||
if-no-files-found: error
|
||
# Fork PR only: upload the Talos image OCI archive for e2e-fork.yaml.
|
||
- name: Upload Talos OCI image archive (fork PR)
|
||
if: ${{ github.event.pull_request.head.repo.fork }}
|
||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||
with:
|
||
name: pr-oci-talos
|
||
path: _out/oci/*.oci.tar
|
||
if-no-files-found: error
|
||
|
||
# Reassemble the unified pr.patch from every fragment, then build the installer
|
||
# LAST: `make -C packages/core/installer image` runs `flux push artifact
|
||
# --path=packages`, which bundles the WHOLE digest-patched tree into the OCI
|
||
# artifact the operator pulls — so every other package's digest edit must
|
||
# already be applied here. e2e consumes the resulting pr.patch unchanged.
|
||
finalize:
|
||
name: Finalize (merge patches + installer)
|
||
# Depends on `checks` too: gating finalize on unit/controller tests cascades
|
||
# to e2e (which needs finalize), so a red unit test skips the expensive
|
||
# install+e2e path instead of wasting ~1h on it. Restores the gating the old
|
||
# in-build-job tests gave, without re-coupling the cheap parallel builds.
|
||
needs: ["plan", "checks", "build", "build-talos"]
|
||
if: |
|
||
!cancelled()
|
||
&& needs.plan.outputs.code == 'true'
|
||
&& !contains(github.event.pull_request.labels.*.name, 'release')
|
||
&& needs.checks.result == 'success'
|
||
&& (needs.build.result == 'success' || needs.build.result == 'skipped')
|
||
&& (needs.build-talos.result == 'success' || needs.build-talos.result == 'skipped')
|
||
runs-on: oracle-vm-4cpu-16gb-x86-64
|
||
timeout-minutes: 30
|
||
permissions:
|
||
contents: read
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||
with:
|
||
fetch-depth: 0
|
||
fetch-tags: true
|
||
- name: Set up flux
|
||
uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6
|
||
with:
|
||
version: "2.8.6"
|
||
- name: Download patch fragments
|
||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||
with:
|
||
pattern: pr-patch-fragment-*
|
||
merge-multiple: true
|
||
path: _out/fragments
|
||
- name: Apply digest fragments
|
||
run: |
|
||
applied=0
|
||
# Each fragment patches only its own package's files; --3way lets two
|
||
# fragments that touch adjacent lines of a shared file (e.g. a package
|
||
# Makefile that writes a sibling package's values.yaml) merge cleanly
|
||
# instead of the second apply aborting the whole finalize.
|
||
for f in _out/fragments/*.patch; do
|
||
[ -s "$f" ] || continue
|
||
echo "Applying $f"
|
||
git apply --3way "$f"
|
||
applied=$((applied + 1))
|
||
done
|
||
echo "Applied $applied fragment(s)"
|
||
git --no-pager diff --stat HEAD
|
||
- name: Set up Docker config
|
||
run: |
|
||
if [ -d ~/.docker ]; then
|
||
cp -r ~/.docker "${{ runner.temp }}/.docker"
|
||
fi
|
||
- name: Login to GitHub Container Registry
|
||
if: ${{ !github.event.pull_request.head.repo.fork }}
|
||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||
with:
|
||
username: ${{ secrets.OCIR_USER }}
|
||
password: ${{ secrets.OCIR_TOKEN }}
|
||
registry: iad.ocir.io
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
# Overlay current-main image refs onto every package this PR did NOT
|
||
# rebuild. Without it, unbuilt packages fall back to the last-release refs
|
||
# committed in the tree (e.g. v1.5.0) even when main is hundreds of commits
|
||
# ahead — so e2e and the installer's packages artifact would ship stale
|
||
# images for everything outside this PR's build matrix. Source of truth is
|
||
# the cozystack-packages:main OCI artifact build-main.yaml already
|
||
# publishes (the whole current-main packages tree, refs digest-pinned to
|
||
# main's images). Runs after login (creds available) and before the
|
||
# installer build below so the flux-pushed packages tree — and the
|
||
# pr.patch e2e applies — already reflect the overlay. A missing artifact
|
||
# (build-main not yet green) is a no-op: packages keep their committed
|
||
# refs (prior behaviour). See hack/overlay-main-images.sh.
|
||
#
|
||
# Read the artifact for the PR's OWN base branch, not always main. Overlaying
|
||
# main's images onto a release-line PR mixes generations: main's binaries
|
||
# against that line's charts. #3437 (a one-line change on release-1.6) failed
|
||
# install deterministically that way, because main's cozystack-controller
|
||
# served an aggregated OpenAPI release-1.6's charts could not validate
|
||
# against (`unknown model in reference: …v1alpha1.OptionSpec`) — a broken
|
||
# lane reading as a broken PR. build-release.yaml publishes
|
||
# cozystack-packages:<line> for every maintained release-<major>.<minor>
|
||
# branch, so each base branch has its own artifact to read.
|
||
#
|
||
# A base that is neither main nor a line (a stacked PR's feature branch) has
|
||
# no artifact, and a `/` in it is not even a valid OCI tag. It reads the
|
||
# artifact of whichever of main and the lines it has the fewest commits on
|
||
# top of. A tie goes to main. A line with no commit of its own yet is, in
|
||
# the graph, main at the cut commit, so a stack on it ties and reads the
|
||
# artifact built from main's tip. That artifact carries newer images than
|
||
# the line once main moves past the cut. The stack keeps reading it until
|
||
# it contains a commit of the line's own. Breaking the tie towards the
|
||
# line instead would misroute the far more common branch forked from main
|
||
# before the cut, which ties the same way.
|
||
#
|
||
# A missing artifact stays a no-op (packages keep their committed refs), which
|
||
# also covers a line whose first build-release run has not landed yet.
|
||
- name: Pull base-branch packages tree
|
||
run: |
|
||
ARTIFACT_REF="$BASE_REF"
|
||
if [ "$BASE_REF" != "main" ] \
|
||
&& ! printf '%s' "$BASE_REF" | grep -qE '^release-[0-9]+\.[0-9]+$'; then
|
||
ARTIFACT_REF=main
|
||
# A re-run can outlive its base: the payload keeps a branch deleted since.
|
||
if git rev-parse --verify --quiet "origin/${BASE_REF}" >/dev/null; then
|
||
best="$(git rev-list --count "origin/main..origin/${BASE_REF}")"
|
||
for line in $(git for-each-ref --format='%(refname:lstrip=3)' 'refs/remotes/origin/release-*' \
|
||
| grep -E '^release-[0-9]+\.[0-9]+$'); do
|
||
n="$(git rev-list --count "origin/${line}..origin/${BASE_REF}")"
|
||
if [ "$n" -lt "$best" ]; then best="$n"; ARTIFACT_REF="$line"; fi
|
||
done
|
||
fi
|
||
echo "::notice::Base ${BASE_REF} publishes no packages artifact; overlaying cozystack-packages:${ARTIFACT_REF}."
|
||
fi
|
||
rm -rf _out/mainpkgs
|
||
mkdir -p _out/mainpkgs
|
||
if ! flux pull artifact "oci://${REGISTRY}/cozystack-packages:${ARTIFACT_REF}" \
|
||
--output _out/mainpkgs; then
|
||
rm -rf _out/mainpkgs
|
||
# Degrading to committed refs is safe, but SAY so: on a release line a
|
||
# missing artifact means build-release.yaml has not published this line
|
||
# yet (first build pending, or its branch filter does not match this
|
||
# branch), and the PR is then testing the line's last release instead of
|
||
# its tip. Silent would look identical to a working overlay.
|
||
if [ "${ARTIFACT_REF}" = "main" ]; then
|
||
echo "cozystack-packages:${ARTIFACT_REF} pull failed — keeping committed refs"
|
||
else
|
||
echo "::warning title=No packages artifact for ${ARTIFACT_REF}::Keeping committed refs, so unbuilt packages use this line's last release rather than its tip. Check that build-release.yaml has run for ${ARTIFACT_REF}."
|
||
fi
|
||
fi
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
# Env-passed, not interpolated into the script: base_ref is repo-controlled
|
||
# here (a branch name in this repo), but the workflow-injection hygiene
|
||
# applies uniformly.
|
||
BASE_REF: ${{ github.base_ref }}
|
||
- name: Overlay base-branch refs for unbuilt packages
|
||
# Best-effort: this only improves WHICH images e2e and the installer use,
|
||
# so an unexpected failure must degrade to the committed refs (prior
|
||
# behaviour), never block the PR. overlay-main-images.sh already handles
|
||
# the expected paths (missing artifact, drift, absent unit) internally —
|
||
# including the empty directory the step above leaves when a line has no
|
||
# artifact yet.
|
||
run: |
|
||
rebuilt="$BUILD_MATRIX"
|
||
if [ "$TALOS_RESULT" = "success" ]; then
|
||
case "$rebuilt" in
|
||
'[]') rebuilt='["packages/extra/bootbox"]' ;;
|
||
*) rebuilt="${rebuilt%]},\"packages/extra/bootbox\"]" ;;
|
||
esac
|
||
fi
|
||
hack/overlay-main-images.sh _out/mainpkgs "$rebuilt" "$PR_TOUCHED" \
|
||
|| echo "::warning::overlay-main-images failed; unbuilt packages keep committed refs"
|
||
env:
|
||
# The set of packages THIS PR rebuilt; their pr-<N>-<sha> refs are
|
||
# authoritative, so overlay-main-images.sh skips them. Computed by the
|
||
# plan job from the Makefile (a JSON array of package dirs), not from
|
||
# untrusted PR input — passed via env per workflow-injection hygiene.
|
||
BUILD_MATRIX: ${{ needs.plan.outputs.matrix }}
|
||
# build-talos writes matchbox.tag under extra/bootbox rather than its
|
||
# own package. Preserve that generated ref only when the job actually
|
||
# succeeded; when it was skipped, the base-branch ref should overlay.
|
||
TALOS_RESULT: ${{ needs.build-talos.result }}
|
||
# Packages THIS PR edited (changed-file dirs from the plan job's git
|
||
# diff). Their committed refs are the PR's intent — an upstream image
|
||
# bump in a non-build-unit package would otherwise be silently reverted
|
||
# by the overlay — so overlay-main-images.sh preserves them. Also
|
||
# env-passed, derived from the diff, not from untrusted PR input.
|
||
PR_TOUCHED: ${{ needs.plan.outputs.touched }}
|
||
# buildkitd-config routes docker.io base-image pulls through mirror.gcr.io
|
||
# (hack/buildkitd.toml) so the installer build dodges Docker Hub's
|
||
# anonymous-pull rate limit; see the build job for the full rationale.
|
||
- name: Set up Buildx (docker-container driver, docker.io mirror)
|
||
id: buildx
|
||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
|
||
with:
|
||
driver: docker-container
|
||
buildkitd-config: hack/buildkitd.toml
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
# Same-repo PR: build the operator image AND flux-push the now-fully-patched
|
||
# packages tree, then patch the installer's platformSourceRef to that
|
||
# artifact. Fork PR: no registry creds — build ONLY the operator image to an
|
||
# OCI archive (image-packages/chart, which push, are skipped) and upload it;
|
||
# e2e-fork.yaml pushes the operator + flux-pushes the packages tree and
|
||
# bakes platformSourceRef there. The overlay above already ran, so the
|
||
# pr.patch assembled below carries the fully digest-pinned tree either way.
|
||
- name: Build installer (operator + packages artifact)
|
||
run: |
|
||
mkdir -p _out/oci
|
||
if [ "$IS_FORK" = "true" ]; then
|
||
make -C packages/core/installer pre-checks image-operator
|
||
else
|
||
make -C packages/core/installer image
|
||
fi
|
||
env:
|
||
DOCKER_CONFIG: ${{ runner.temp }}/.docker
|
||
BUILDER: ${{ steps.buildx.outputs.name }}
|
||
# amd64 only, as in the build job.
|
||
PLATFORM: linux/amd64
|
||
IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}
|
||
PUBLISH_VERSIONED: '0'
|
||
PUBLISH_FLOATING: '0'
|
||
IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
|
||
OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}
|
||
# Fork PR only: upload the operator image OCI archive for e2e-fork.yaml.
|
||
- name: Upload installer OCI image archive (fork PR)
|
||
if: ${{ github.event.pull_request.head.repo.fork }}
|
||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||
with:
|
||
name: pr-oci-installer
|
||
path: _out/oci/*.oci.tar
|
||
if-no-files-found: error
|
||
- name: Assemble pr.patch
|
||
run: |
|
||
mkdir -p _out/assets
|
||
# --binary so regenerated provider bundles (components*.gz) stay in
|
||
# the patch e2e applies; mirrors main's pr.patch binary fix (#e566992).
|
||
git diff --binary HEAD > _out/assets/pr.patch
|
||
echo "Final pr.patch:"; git --no-pager diff --stat HEAD
|
||
- name: Upload git diff patch
|
||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||
with:
|
||
name: pr-patch
|
||
path: _out/assets/pr.patch
|
||
if-no-files-found: error
|
||
|
||
resolve_assets:
|
||
name: "Resolve assets"
|
||
runs-on: ubuntu-latest
|
||
# In lockstep with `plan`, fork term included: this job is on `e2e`'s
|
||
# release arm, so a label event `plan` runs for and this job skips for
|
||
# leaves `e2e` with both arms false while `e2e-report` still runs. On a
|
||
# promote PR already carrying full-e2e that reaches the final `else` — the
|
||
# "e2e not repeated" branch is bypassed by full-e2e — and posts `E2E Tests`
|
||
# = failure over a green one, unclearable without a push because
|
||
# `unlabeled` is not a trigger. The fork term keeps a `release` label on a
|
||
# fork PR as inert as it was before promotion used the label: without it
|
||
# this job runs there and fails resolving a draft release for a branch that
|
||
# is not `release-X.Y.Z`. It short-circuits on `action != 'labeled'`, so the
|
||
# pre-existing behaviour on a push to such a PR is unchanged.
|
||
if: |
|
||
contains(github.event.pull_request.labels.*.name, 'release')
|
||
&& (github.event.action != 'labeled' || github.event.label.name == 'full-e2e' || (github.event.label.name == 'release' && !github.event.pull_request.head.repo.fork))
|
||
permissions:
|
||
contents: read
|
||
steps:
|
||
- name: Generate GitHub App token
|
||
id: app-token
|
||
uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1
|
||
with:
|
||
app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}
|
||
private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}
|
||
owner: cozystack
|
||
repositories: ${{ github.event.repository.name }}
|
||
permission-contents: read
|
||
|
||
- name: Checkout code
|
||
if: contains(github.event.pull_request.labels.*.name, 'release')
|
||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||
with:
|
||
fetch-depth: 0
|
||
fetch-tags: true
|
||
|
||
- name: Extract tag from PR branch (release PR)
|
||
if: contains(github.event.pull_request.labels.*.name, 'release')
|
||
id: get_tag
|
||
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
|
||
with:
|
||
script: |
|
||
const branch = context.payload.pull_request.head.ref;
|
||
const m = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/);
|
||
if (!m) {
|
||
core.setFailed(`❌ Branch '${branch}' does not match 'release-X.Y.Z[-suffix]'`);
|
||
return;
|
||
}
|
||
core.setOutput('tag', `v${m[1]}`);
|
||
|
||
# The draft release has to exist for a promote PR's e2e arm to mean
|
||
# anything, and this job is what proves it. It used to also resolve the
|
||
# nocloud disk asset and export its id; the container lane boots no disk,
|
||
# so nothing in this workflow reads that id any more and resolving it only
|
||
# added a way to fail a release PR over an asset it does not use. The
|
||
# asset is still resolved where it is still consumed, in e2e-tag.yaml.
|
||
- name: Verify draft release exists (release PR)
|
||
if: contains(github.event.pull_request.labels.*.name, 'release')
|
||
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
|
||
with:
|
||
github-token: ${{ steps.app-token.outputs.token }}
|
||
script: |
|
||
const tag = '${{ steps.get_tag.outputs.tag }}';
|
||
const releases = await github.rest.repos.listReleases({
|
||
owner: context.repo.owner,
|
||
repo: context.repo.repo,
|
||
per_page: 100
|
||
});
|
||
const draft = releases.data.find(r => r.tag_name === tag && r.draft);
|
||
if (!draft) {
|
||
core.setFailed(`Draft release '${tag}' not found`);
|
||
return;
|
||
}
|
||
|
||
|
||
# In-tree e2e for SAME-REPO PRs, on Talos nodes running as CONTAINERS. The
|
||
# QEMU substrate this lane used to boot is off the per-PR path only: it is
|
||
# still wired into nightly.yaml and e2e-tag.yaml, both of which run `make
|
||
# prepare-env` -> `make -C packages/core/testing prepare-cluster`. That target
|
||
# and hack/e2e-prepare-cluster.bats are therefore live code, not leftovers,
|
||
# and deleting them takes nightly and every release-candidate run with them.
|
||
# They carry the coverage containers cannot: DRBD, and with it the
|
||
# `replicated` StorageClass, its Immediate binding mode and the tenant
|
||
# StorageClass propagation that only applies to remotely-accessible classes.
|
||
#
|
||
# Deliberately NOT named "E2E Tests": the
|
||
# required "E2E Tests" context is a commit status posted by the `e2e-report`
|
||
# job (non-fork) or by e2e-fork.yaml (fork). A required *job* that skips
|
||
# counts as PASSED by branch protection, which is exactly how fork PRs used
|
||
# to merge with e2e never run (#3257); routing the required context through
|
||
# a status removes that skip-to-green path. Fork PRs skip this job
|
||
# (they have no pushed images to pull) and run e2e in the privileged
|
||
# workflow_run instead.
|
||
e2e:
|
||
name: "E2E (in-tree)"
|
||
# hack/e2e-compose.yaml gives each of the three nodes `cpus: 8` and
|
||
# `mem_limit: 24576m`, matching what the QEMU guests were given, so the nodes
|
||
# still demand 24 vCPU and 72 GiB between them. The same headroom argument
|
||
# therefore still decides the class: on a 24-vCPU runner that is a 1:1 commit
|
||
# with nothing left for containerd, the kernel, the host iptables MASQUERADE
|
||
# every image pull traverses and the runner agent, and during "Install
|
||
# Cozystack" the nodes contend with those and lose cycles. Readiness and
|
||
# liveness probes then miss their deadlines, a HelmRelease flips InProgress ->
|
||
# Failed and its dependents cascade to DependencyNotReady -- on a different
|
||
# component each run, the signature of resource thrash rather than a product
|
||
# bug. The 32-vCPU / 128 GiB class leaves 8 vCPU (25%) and 56 GiB of headroom
|
||
# and removes that oversubscription.
|
||
#
|
||
# One thing this class does NOT fix, and it is worth knowing before reading a
|
||
# red run: a container node reports the HOST's CPU and memory as its capacity,
|
||
# and hack/e2e-container-up.sh counterbalances that with kubelet
|
||
# systemReserved so the scheduler sees 8 CPU / 24 GiB. That corrects the
|
||
# scheduler and not what a pod reads from /proc, so a workload that sizes
|
||
# itself from /proc/cpuinfo or /proc/meminfo is configured for the whole host.
|
||
runs-on: oracle-vm-32cpu-128gb-x86-64
|
||
# Sized above what the tenant-Kubernetes pair can spend on a failing run, and
|
||
# held to one figure across every lane that runs the suite.
|
||
timeout-minutes: 215
|
||
permissions:
|
||
contents: read
|
||
packages: read
|
||
checks: write
|
||
needs: ["finalize", "resolve_assets", "verify-release-candidate"]
|
||
# Regular PRs still enter through finalize. A release-labelled promote PR
|
||
# enters through resolve_assets only when a maintainer explicitly adds
|
||
# full-e2e; without that label the immutable rc's already-green suite is not
|
||
# repeated. The labeled trigger above makes that post-open opt-in start a run.
|
||
#
|
||
# Fork PRs never enter here: their build exports OCI archives instead of
|
||
# pushing images, so there is no closure for this job to test. e2e-fork.yaml
|
||
# runs the suite for them from the privileged workflow_run and concludes the
|
||
# required "E2E Tests" status; for same-repo PRs e2e-report concludes it.
|
||
if: |
|
||
!cancelled()
|
||
&& !github.event.pull_request.head.repo.fork
|
||
&& (
|
||
needs.finalize.result == 'success'
|
||
|| (
|
||
needs.resolve_assets.result == 'success'
|
||
&& needs.verify-release-candidate.result == 'success'
|
||
&& contains(github.event.pull_request.labels.*.name, 'release')
|
||
&& contains(github.event.pull_request.labels.*.name, 'full-e2e')
|
||
)
|
||
)
|
||
|
||
steps:
|
||
# ▸ Checkout and prepare the codebase
|
||
# fetch-depth: 0 is required by the "Select E2E tests" step so that
|
||
# `git diff origin/${BASE_REF}...HEAD` can resolve the base ref.
|
||
- name: Checkout code
|
||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||
with:
|
||
fetch-depth: 0
|
||
|
||
# ▸ Regular PR path – pr.patch from finalize. No Talos disk: the nodes boot
|
||
# from ghcr.io/siderolabs/talos directly (hack/e2e-compose.yaml), so this
|
||
# lane consumes no nocloud asset and waits on no Talos build.
|
||
- name: Download PR patch
|
||
if: "!contains(github.event.pull_request.labels.*.name, 'release')"
|
||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||
with:
|
||
name: pr-patch
|
||
path: _out/assets
|
||
|
||
- name: Apply patch
|
||
if: "!contains(github.event.pull_request.labels.*.name, 'release')"
|
||
run: |
|
||
git apply _out/assets/pr.patch
|
||
|
||
- name: Set sandbox ID
|
||
run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> $GITHUB_ENV
|
||
|
||
# ▸ Host kernel modules. MANDATORY and up front, because nothing later can
|
||
# supply them: the nodes take their kernel from this host, and Talos
|
||
# cannot load modules in container mode -- `machine.kernel.modules` is a
|
||
# silent no-op there, `kernel_module_spec.go` returns early on
|
||
# ModeContainer with no error and no event.
|
||
#
|
||
# A module missing here is missing for every node for the whole run, and
|
||
# it surfaces in disguise: no `openvswitch` and kube-ovn comes up with no
|
||
# dataplane, no `zfs` and every LINSTOR pool creation fails. Both read as
|
||
# a CNI or a storage regression an hour later. So this loads both and
|
||
# then asserts both, and a failure here names the host rather than the
|
||
# product.
|
||
- name: Prepare container-lane host
|
||
run: |
|
||
set -eu
|
||
echo "::group::Loading kernel modules required by the container lane"
|
||
for m in openvswitch zfs; do
|
||
if grep -q "^${m} " /proc/modules; then
|
||
echo "${m}: already loaded"
|
||
continue
|
||
fi
|
||
if [ "${m}" = zfs ]; then
|
||
sudo apt-get update -qq
|
||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq zfsutils-linux
|
||
fi
|
||
echo "${m}: loading"
|
||
sudo modprobe "${m}" || echo "${m}: modprobe failed"
|
||
done
|
||
echo "::endgroup::"
|
||
missing=''
|
||
for m in openvswitch zfs; do
|
||
grep -q "^${m} " /proc/modules || missing="${missing} ${m}"
|
||
done
|
||
if [ -n "${missing}" ]; then
|
||
echo "::error title=container-lane host::kernel module(s) could not be loaded:${missing}"
|
||
echo "This is a host problem, not a product failure. The e2e nodes run as" >&2
|
||
echo "containers on this runner's kernel and Talos cannot load modules in" >&2
|
||
echo "container mode, so what is not here now is unavailable for the whole" >&2
|
||
echo "run: openvswitch backs the kube-ovn dataplane, zfs backs every" >&2
|
||
echo "LINSTOR pool. Failing here rather than an hour later is the point." >&2
|
||
exit 1
|
||
fi
|
||
grep -E '^(openvswitch|zfs) ' /proc/modules
|
||
|
||
# ▸ Prepare environment. Two make targets rather than one: the nodes are
|
||
# siblings on the runner's docker daemon (the /dev and /run/udev binds in
|
||
# hack/e2e-compose.yaml must resolve against the real host devtmpfs, which
|
||
# is what lets this lane carry LINSTOR at all), so the host-side half
|
||
# needs a docker socket the sandbox does not have.
|
||
- name: Prepare environment
|
||
run: |
|
||
attempt=0
|
||
until make SANDBOX_NAME=$SANDBOX_NAME prepare-env-container; do
|
||
attempt=$((attempt + 1))
|
||
if [ $attempt -ge 3 ]; then
|
||
echo "Attempt $attempt failed, exiting..."
|
||
exit 1
|
||
fi
|
||
echo "Attempt $attempt failed, retrying..."
|
||
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete-cluster-container || true
|
||
done
|
||
echo "Prepare environment completed after $((attempt + 1)) attempts"
|
||
|
||
# Test Impact Analysis (TIA): walk the PackageSource graph for the PR
|
||
# diff and select only the Chainsaw suites affected. Runs by default; opt
|
||
# out via the `full-e2e` label to run the full suite.
|
||
#
|
||
# `skip=true` means every changed path was matched by one of select-e2e.sh's
|
||
# rules that select nothing — docs and dashboards, a disabled Chainsaw
|
||
# suite, a backups example with no suite, or the inert repo-meta pattern
|
||
# — so skipping Chainsaw and reporting green is the correct verdict. A
|
||
# path none of the rules recognise escalates to the full suite instead of
|
||
# returning empty, so `skip=true` can no longer mean "TIA had no idea"
|
||
# (#3392). What it still cannot rule out is a workflow nobody classified:
|
||
# .github/ is inert as a directory and the e2e workflows are exempted
|
||
# from it by name, so a workflow added under .github/workflows/ reads as
|
||
# inert until someone puts it on that list, and the fall-through never
|
||
# sees it.
|
||
- name: Select E2E tests
|
||
id: select
|
||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
|
||
env:
|
||
BASE_REF: ${{ github.base_ref }}
|
||
run: |
|
||
git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt
|
||
apps=$(./hack/select-e2e.sh /tmp/changed.txt)
|
||
echo "Selected apps: ${apps:-<none>}"
|
||
echo "apps=${apps}" >> $GITHUB_OUTPUT
|
||
if [ -z "${apps}" ]; then
|
||
echo "skip=true" >> $GITHUB_OUTPUT
|
||
else
|
||
echo "skip=false" >> $GITHUB_OUTPUT
|
||
fi
|
||
# The complement of what the selected suites need, for a reduced
|
||
# install. An empty selection installs everything as before.
|
||
disabled=""
|
||
if [ -n "${apps}" ]; then
|
||
disabled="$(./hack/select-install.sh --disabled "${apps}")"
|
||
fi
|
||
echo "Disabled packages: ${disabled:-<none>}"
|
||
echo "disabled=${disabled}" >> $GITHUB_OUTPUT
|
||
|
||
# ▸ Install Cozystack
|
||
- name: Install Cozystack into sandbox
|
||
env:
|
||
# This lane has no ARP VIP: .11 is a plain control-plane node standing
|
||
# in for the QEMU lane's 192.168.123.10.
|
||
COZY_APISERVER_ENDPOINT: https://192.168.123.11:6443
|
||
# Talos containers share the runner kernel and cannot load DRBD, so the
|
||
# harness suppresses DRBD-dependent chart content, registers the host's
|
||
# pre-created zpools and keeps LINSTOR's local ZFS path.
|
||
COZY_LINSTOR_DRBD_ENABLED: "false"
|
||
# Root monitoring is the one root-tenant workload whose chart selects
|
||
# `replicated` explicitly instead of inheriting the default class. The
|
||
# install leaves it off but still seeds its local-storage override, so
|
||
# a run that turns it on does not create a replicated PVC.
|
||
COZY_E2E_STORAGE_CLASS: local
|
||
# Empty on the full-e2e label and on an empty selection, which keeps
|
||
# the full install.
|
||
COZY_DISABLED_PACKAGES: ${{ steps.select.outputs.disabled }}
|
||
run: |
|
||
if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then
|
||
echo "❌ Install failed (no retry — see diagnostics below)"
|
||
echo "::group::Diagnostics: HelmRelease status"
|
||
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true
|
||
echo "::endgroup::"
|
||
echo "::group::Diagnostics: not-Ready HelmReleases (describe)"
|
||
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \
|
||
| while read -r ns name; do
|
||
echo "--- $ns/$name ---"
|
||
docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true
|
||
done
|
||
echo "::endgroup::"
|
||
echo "::group::Diagnostics: recent events"
|
||
docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -50' || true
|
||
echo "::endgroup::"
|
||
exit 1
|
||
fi
|
||
|
||
- name: Run OpenAPI tests
|
||
run: |
|
||
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi
|
||
|
||
# ▸ Run E2E tests (Chainsaw, scoped by Test Impact Analysis)
|
||
#
|
||
# The "Select E2E tests" step above picks the suites impacted by the PR
|
||
# diff. With the `full-e2e` label that step is skipped, so FULL_E2E is
|
||
# the only way to reach this step with no selection and we run the whole
|
||
# suite; an all-inert selection skips this step entirely (the
|
||
# skip != 'true' guard).
|
||
#
|
||
# Chainsaw discovers each named suite under hack/e2e-chainsaw/ and polls
|
||
# every assertion to its own timeout, so the old per-app 3-retry loop is
|
||
# gone — a failure is reported as a structured assertion diff, with
|
||
# cluster-state diagnostics dumped below on a non-zero exit.
|
||
- name: Run E2E tests
|
||
id: e2e_tests
|
||
if: ${{ steps.select.outputs.skip != 'true' }}
|
||
env:
|
||
# No DRBD-backed class exists here. Chainsaw templates the storage
|
||
# fixtures with `local`, and the Kubernetes helper omits only the
|
||
# remote-propagation and RWX assertions while running the same suites.
|
||
COZY_E2E_STORAGE_CLASS: local
|
||
SELECTED_APPS: ${{ steps.select.outputs.apps }}
|
||
FULL_E2E: ${{ contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
|
||
run: |
|
||
# Only the full-e2e label can get here with an empty selection: when
|
||
# `select` runs, an empty result sets skip=true and this step never
|
||
# starts. Testing FULL_E2E alone keeps that invariant visible.
|
||
if [ "$FULL_E2E" = "true" ]; then
|
||
suites="" # empty => test-chainsaw runs the whole suite
|
||
echo "Running full chainsaw suite"
|
||
else
|
||
suites="$SELECTED_APPS"
|
||
echo "Running selected chainsaw suites: $suites"
|
||
fi
|
||
if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES="$suites"; then
|
||
echo "❌ Chainsaw E2E failed (see the assertion diffs or LEAKED-PODS lines above and diagnostics below)"
|
||
echo "::group::Diagnostics: HelmRelease status"
|
||
docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true
|
||
echo "::endgroup::"
|
||
echo "::group::Diagnostics: recent events"
|
||
docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true
|
||
echo "::endgroup::"
|
||
# COSI state at the moment of failure — the CRDs ship no printer
|
||
# columns, so pull the readiness fields explicitly (cozyreport
|
||
# captures the same at end-of-job, but bucket states move between
|
||
# a test failing and the report being taken).
|
||
echo "::group::Diagnostics: COSI bucket state"
|
||
docker exec "$SANDBOX_NAME" sh -c '
|
||
kubectl get bucketclaims.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.bucketReady,BUCKET:.status.bucketName" 2>&1
|
||
kubectl get buckets.objectstorage.k8s.io -o custom-columns="NAME:.metadata.name,READY:.status.bucketReady,ID:.status.bucketID,CLAIMNS:.spec.bucketClaim.namespace" 2>&1
|
||
kubectl get bucketaccesses.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,GRANTED:.status.accessGranted,CLAIM:.spec.bucketClaimName" 2>&1
|
||
' || true
|
||
echo "::endgroup::"
|
||
exit 1
|
||
fi
|
||
echo "✅ All selected E2E tests passed"
|
||
|
||
- name: Collect chainsaw report
|
||
if: always()
|
||
run: |
|
||
mkdir -p _out
|
||
docker cp "$SANDBOX_NAME":/workspace/hack/e2e-chainsaw/chainsaw-report.xml \
|
||
_out/chainsaw-report.xml || true
|
||
|
||
- name: Upload chainsaw report
|
||
if: always()
|
||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||
with:
|
||
name: chainsaw-report
|
||
path: _out/chainsaw-report.xml
|
||
if-no-files-found: ignore
|
||
|
||
# ▸ Collect debug information (always runs)
|
||
#
|
||
# A ceiling of its own, not the job's 215 minutes. `Upload
|
||
# cozyreport.tgz` is the next step and the collector writes its tarball
|
||
# at the very end of its run, so a read that never returns spends what
|
||
# is left of the job budget, the job is cancelled, and the upload never
|
||
# runs -- the artifact is lost rather than truncated, on exactly the run
|
||
# where it is the only evidence left. `|| true` guards against a
|
||
# non-zero exit, not against a read that does not return.
|
||
#
|
||
# 30 minutes is measured against the step: it takes 3-42s across sampled
|
||
# runs, on failed e2e jobs as well as green ones. It RESERVES nothing --
|
||
# `timeout-minutes` caps a step, it does not carve budget out of the job,
|
||
# so a suite that has already burned 210 of the 215 leaves this step five
|
||
# minutes and the upload is cancelled with everything else. What the cap
|
||
# buys is that a hung read cannot be the reason.
|
||
# `continue-on-error` because the ceiling must not change what the job
|
||
# REPORTS. `|| true` on the make call is this step's standing statement
|
||
# that collecting diagnostics is never fatal, and a step killed at its
|
||
# ceiling never reaches that `|| true` -- the runner ends the process. So
|
||
# without this, a passing E2E run whose collection ran long is published
|
||
# as a failed one, and every later `failure()` in this job starts meaning
|
||
# "the tests failed OR the report was slow".
|
||
- name: Collect report
|
||
id: collect_report
|
||
if: always()
|
||
timeout-minutes: 30
|
||
continue-on-error: true
|
||
run: |
|
||
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true
|
||
|
||
# `continue-on-error` keeps the overrun out of the job's result, which also
|
||
# keeps it out of everyone's view: a step that ended at its ceiling renders
|
||
# green and the only trace is in the raw log. This branch's own standard is
|
||
# that a bound which fires is recorded, and that has to hold for the outer
|
||
# bound too.
|
||
#
|
||
# The wording says what `outcome` establishes and no more: a step can end
|
||
# non-zero without any ceiling firing, so naming the timeout as the cause
|
||
# would be the false-mechanism claim the collector refuses to make for a 137.
|
||
- name: Report collection overran its ceiling
|
||
if: always() && steps.collect_report.outcome == 'failure'
|
||
run: |
|
||
echo "::warning title=cozyreport::Collect report ended non-zero -- its timeout-minutes fired, or the step failed before the collector ran; cozyreport.tgz is missing or partial for this run"
|
||
|
||
- name: Upload cozyreport.tgz
|
||
if: always()
|
||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||
with:
|
||
name: cozyreport
|
||
path: _out/cozyreport.tgz
|
||
|
||
- name: Collect images list
|
||
if: always()
|
||
run: |
|
||
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-images || true
|
||
|
||
- name: Upload image list
|
||
if: always()
|
||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||
with:
|
||
name: image-list
|
||
path: _out/images.txt
|
||
|
||
# ▸ Open an SSH breakpoint to the failing sandbox so maintainers can attach,
|
||
# inspect Talos/Cozystack state and resume with `breakpoint resume`.
|
||
#
|
||
# Opens when the E2E job fails AND the PR carries the `debug` label or
|
||
# the `release` label (release PRs are bot-created, so the label
|
||
# implies maintainer authorship). authorized-users is kept as a second
|
||
# defense layer controlling who may actually SSH in.
|
||
#
|
||
# `failure()` is the whole condition now. It used to be joined by a
|
||
# soft-red output, because a node-join red the lane decided not to block
|
||
# on arrived here as a SUCCESSFUL job and would have been skipped. That
|
||
# carve-out is gone with the QEMU substrate: the node-join deadline
|
||
# fails its suite like any other deadline, so a sandbox worth attaching
|
||
# to always belongs to a failed job. Label-gating is what keeps this
|
||
# affordable, and it is unchanged.
|
||
#
|
||
# Uses cozystack/breakpoint-action (fork of namespacelabs/breakpoint-action)
|
||
# pinned by SHA. The fork adds: pause-idle mode (initial grace period for
|
||
# the first SSH connection, idle-aware exit afterwards), endpoint output
|
||
# and ::notice:: annotation, and a dedicated Check Run "Breakpoint Open"
|
||
# that carries the SSH endpoint in output.summary while the breakpoint is
|
||
# paused (conclusion=failure → standard ✗ in `gh pr checks`; updated to
|
||
# conclusion=success when the breakpoint exits).
|
||
#
|
||
# Configured via the BREAKPOINT_ENDPOINT repository variable — in fork
|
||
# PRs repository variables are not exposed, so the step is skipped and
|
||
# forks cannot reach the self-hosted rendezvous server.
|
||
- name: Breakpoint on E2E failure
|
||
if: |
|
||
failure() &&
|
||
vars.BREAKPOINT_ENDPOINT != '' &&
|
||
(contains(github.event.pull_request.labels.*.name, 'debug')
|
||
|| contains(github.event.pull_request.labels.*.name, 'release'))
|
||
# A step that fails here would fail the job on its own account. Same
|
||
# reason `Collect report` above carries it: what the job REPORTS must not
|
||
# depend on what a helper did.
|
||
continue-on-error: true
|
||
# cozystack/breakpoint-action v2-cozy.1
|
||
# mode: pause-idle defaults: grace-period=20m, idle-timeout=10m
|
||
uses: cozystack/breakpoint-action@a6f3a6f87be398ad63b6577351e3398e53f578e4
|
||
with:
|
||
mode: pause-idle
|
||
endpoint: ${{ vars.BREAKPOINT_ENDPOINT }}
|
||
authorized-users: androndo, Arsolitt, IvanHunters, kvaps, lexfrei, lllamnyp, mattia-eleuteri, matthieu-robin, myasnikovdaniil, sircthulhu, tym83
|
||
check-run-name: "Breakpoint Open"
|
||
github-token: ${{ github.token }}
|
||
check-run-summary-template: |
|
||
## 🔴 SSH breakpoint open — paused for debug
|
||
|
||
```
|
||
{endpoint}
|
||
```
|
||
|
||
Enter the e2e sandbox after SSH:
|
||
```
|
||
docker exec -ti $(docker ps --filter name=cozy-e2e-sandbox -q | head -1) bash
|
||
export KUBECONFIG=/workspace/kubeconfig
|
||
```
|
||
|
||
Resume from inside: `breakpoint resume`. Otherwise the breakpoint
|
||
exits 10 minutes after the last SSH session disconnects.
|
||
|
||
# ▸ Tear down environment (always runs)
|
||
#
|
||
# Both halves, and in this order: the compose project owns the nodes and
|
||
# the zpools outlive it. A leftover pool makes the next run on a reused
|
||
# host adopt storage that already carries a dead cluster's volumes.
|
||
- name: Tear down sandbox
|
||
if: always()
|
||
run: |
|
||
# Loud, but never the job's verdict. A failed teardown means a dirty
|
||
# host, and this runner is discarded after the job, so there is no next
|
||
# run to protect -- while failing the job here posts the required
|
||
# "E2E Tests" status as a failure that no reader can tell apart from a
|
||
# real one. The annotation is what a reused host (a developer machine)
|
||
# needs; `delete-cluster-container` returning non-zero is not evidence
|
||
# about the product.
|
||
cleanup_rc=0
|
||
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete-cluster-container || cleanup_rc=$?
|
||
make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true
|
||
if [ "$cleanup_rc" -ne 0 ]; then
|
||
echo "::error title=sandbox teardown::teardown exited $cleanup_rc; the host may hold a stale compose project or zpool"
|
||
fi
|
||
|
||
# Report the required "E2E Tests" gate as a commit status rather than relying
|
||
# on a job named "E2E Tests". A required *job* that is skipped counts
|
||
# as PASSED by branch protection — desired for docs-only PRs, but the exact
|
||
# hole that let fork PRs merge with e2e never run (#3257). This job always runs
|
||
# on same-repo PRs and concludes "E2E Tests" deterministically:
|
||
# • plan failed → failure
|
||
# • docs-only PR (code!=true)→ success (e2e not required)
|
||
# • in-tree e2e succeeded → success
|
||
# • otherwise (skipped/failed/cancelled with code changes) → failure
|
||
# Fork PRs skip this job; e2e-fork.yaml concludes "E2E Tests" for them, so a
|
||
# fork PR stays "Expected"/not-mergeable until the privileged run reports.
|
||
#
|
||
# The required "E2E Tests" merge gate is published as a COMMIT STATUS with the
|
||
# default GITHUB_TOKEN (+ statuses:write) — NOT as a job, and NOT as a
|
||
# check-run.
|
||
# - Not a job: a *skipped* required job counts as passed by branch
|
||
# protection, which is how fork PRs used to merge with e2e never run
|
||
# (#3257). A status is "success" only when this job posts it; a
|
||
# never-posted required status stays "Expected" and blocks the merge.
|
||
# - Not a check-run: a check-run made with the default token floats into an
|
||
# arbitrary GitHub Actions check-suite (it rendered under "PR size label"),
|
||
# and an app-token check-run needs the app granted checks:write. A commit
|
||
# status has neither problem — branch protection matches it by context name
|
||
# and it carries no check-suite grouping. Safe with the default token: a
|
||
# fork's pull_request token is read-only and cannot post a status, so only
|
||
# same-repo runs (this job, guarded !fork) and the privileged workflow_run
|
||
# in e2e-fork.yaml can set it.
|
||
# REQUIRES main branch protection to require the "E2E Tests" status context —
|
||
# see docs/agents/e2e-testing.md §10.
|
||
#
|
||
# The `labeled` guard must stay in lockstep with `plan`'s: this job runs if and
|
||
# only if `plan` runs. `labeled` events other than full-e2e/release are
|
||
# discarded there, so without the same guard this job would run with plan *skipped*, read
|
||
# PLAN_RESULT != 'success', and post "E2E Tests" = failure — clobbering an
|
||
# already-green required status every time anyone labels an open PR.
|
||
e2e-report:
|
||
name: "Report E2E status"
|
||
needs: ["plan", "e2e", "verify-release-candidate"]
|
||
# !cancelled() rather than always(). A cancelled run no longer owns the
|
||
# status for its head SHA, because something superseded it and that
|
||
# something is now responsible. Posting from here would stamp a red on a
|
||
# suite that never finished, and the run that replaced it may be one that
|
||
# deliberately posts nothing, leaving nothing to flip it back. A cancelled
|
||
# run must report nothing.
|
||
#
|
||
# Cancellation still reaches this job from a push, from a `full-e2e` label
|
||
# run superseding the head run, and from a manual cancel. It no longer
|
||
# reaches it from a label the workflow discards: those run in their own
|
||
# concurrency group (see the key at the top of this file) precisely so they
|
||
# cannot cancel a live run.
|
||
if: ${{ !cancelled() && !github.event.pull_request.head.repo.fork && (github.event.action != 'labeled' || github.event.label.name == 'full-e2e' || github.event.label.name == 'release') }}
|
||
runs-on: ubuntu-latest
|
||
permissions:
|
||
contents: read
|
||
statuses: write
|
||
steps:
|
||
- name: Report "E2E Tests" commit status
|
||
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
|
||
env:
|
||
PLAN_RESULT: ${{ needs.plan.result }}
|
||
CODE: ${{ needs.plan.outputs.code }}
|
||
E2E_RESULT: ${{ needs.e2e.result }}
|
||
CANDIDATE_RESULT: ${{ needs.verify-release-candidate.result }}
|
||
IS_RELEASE: ${{ contains(github.event.pull_request.labels.*.name, 'release') }}
|
||
IS_FULL_E2E: ${{ contains(github.event.pull_request.labels.*.name, 'full-e2e') }}
|
||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||
with:
|
||
script: |
|
||
const { PLAN_RESULT, CODE, E2E_RESULT, CANDIDATE_RESULT, IS_RELEASE, IS_FULL_E2E, HEAD_SHA, RUN_URL } = process.env;
|
||
let state, description;
|
||
if (PLAN_RESULT !== 'success') {
|
||
state = 'failure';
|
||
description = `plan job did not succeed (result: ${PLAN_RESULT})`;
|
||
} else if (IS_RELEASE === 'true' && CANDIDATE_RESULT !== 'success') {
|
||
state = 'failure';
|
||
description = `release packages candidate did not verify (result: ${CANDIDATE_RESULT})`;
|
||
} else if (CODE !== 'true') {
|
||
state = 'success';
|
||
description = 'docs-only PR — e2e not required';
|
||
} else if (IS_RELEASE === 'true' && IS_FULL_E2E !== 'true') {
|
||
// Release promotion PRs deliberately do not repeat e2e: the rc was
|
||
// already verified against the immutable candidate whose digests
|
||
// this tree pins, which is the whole point of the rc gate. The `e2e`
|
||
// job skips for exactly this case, and while the gate was a job that
|
||
// skip satisfied branch protection. Now that the gate is a status,
|
||
// the skip has to be reported explicitly or every promotion needs a
|
||
// full suite re-run or an admin override.
|
||
state = 'success';
|
||
description = 'release promotion — verified at rc, e2e not repeated (add full-e2e to force a run)';
|
||
} else if (E2E_RESULT === 'success') {
|
||
state = 'success';
|
||
description = 'in-tree e2e passed';
|
||
} else {
|
||
state = 'failure';
|
||
description = `in-tree e2e did not pass (result: ${E2E_RESULT})`;
|
||
}
|
||
try {
|
||
await github.rest.repos.createCommitStatus({
|
||
owner: context.repo.owner,
|
||
repo: context.repo.repo,
|
||
sha: HEAD_SHA,
|
||
state,
|
||
context: 'E2E Tests',
|
||
description,
|
||
target_url: RUN_URL,
|
||
});
|
||
} catch (err) {
|
||
// The 'E2E Tests' status is the required merge gate. If we cannot
|
||
// publish it, fail loudly and fail-closed: setFailed keeps the job
|
||
// red and the required status unreported ("Expected"), so the PR
|
||
// stays non-mergeable.
|
||
core.setFailed(`Failed to report E2E Tests status: ${err}`);
|
||
return;
|
||
}
|
||
core.info(`E2E Tests → ${state}: ${description}`);
|