From 1619cb4eb94d2d1f82b0a44e59a73c591766cf5c Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 12 Nov 2026 15:42:32 +0500 Subject: [PATCH 1/2] ci: keep the CI-only Go tools from escalating builds and e2e cmd/backport-batch and cmd/api-gate are built into no image and run nowhere in the e2e sandbox, yet a change to either rebuilt every image and ran the full Chainsaw suite, and widened a package change made in the same pull request to everything. Every other directory under cmd/ is a shipped binary and keeps escalating. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- hack/build-matrix.sh | 8 +++++++- hack/build-matrix_test.bats | 17 +++++++++++++++++ hack/select-e2e.sh | 13 +++++++++++-- hack/select-e2e_test.bats | 17 +++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/hack/build-matrix.sh b/hack/build-matrix.sh index 95e894a7f..d15267631 100755 --- a/hack/build-matrix.sh +++ b/hack/build-matrix.sh @@ -67,7 +67,13 @@ if [ -z "$CHANGED" ] || [ "$CHANGED" = "FULL" ]; then exit 0 fi -if grep -qE "$full_rebuild_pattern" "$CHANGED"; then +# CI-only Go tools that no image builds. Dropped before the full-rebuild check, +# whose cmd/ would otherwise rebuild everything for an edit no image contains. +# hack/select-e2e.sh skips the same two, and a diff made only of them and docs/ +# never reaches this script's build jobs (the plan job's `code` is false). +tooling_pattern='^cmd/(backport-batch|api-gate)/' + +if grep -vE "$tooling_pattern" "$CHANGED" | grep -qE "$full_rebuild_pattern"; then emit_json "$units" exit 0 fi diff --git a/hack/build-matrix_test.bats b/hack/build-matrix_test.bats index 44a9f2c04..c11f2707f 100644 --- a/hack/build-matrix_test.bats +++ b/hack/build-matrix_test.bats @@ -97,6 +97,23 @@ load test_helper rm -f "$tmp" } +@test "the CI-only tools do not force the full matrix" { + tmp=$(mktemp) + # No image builds cmd/backport-batch or cmd/api-gate, so they select nothing + # and leave the rest of the diff to its own rules. + printf '%s\n' cmd/backport-batch/report_test.go cmd/api-gate/main.go > "$tmp" + out=$(hack/build-matrix.sh "$tmp") + [ "$out" = '[]' ] + printf '%s\n' cmd/backport-batch/main.go packages/apps/mariadb/Makefile > "$tmp" + out=$(hack/build-matrix.sh "$tmp") + [ "$out" = '["packages/apps/mariadb"]' ] + # Every other cmd/ directory is a shipped binary and still rebuilds all. + printf '%s\n' cmd/backport-batch/main.go cmd/cozystack-api/main.go > "$tmp" + out=$(hack/build-matrix.sh "$tmp") + [ "$out" = "$(hack/build-matrix.sh)" ] + rm -f "$tmp" +} + @test "a package with no image target selects nothing" { tmp=$(mktemp) # postgres ships no in-repo image build, so it is not a build unit. diff --git a/hack/select-e2e.sh b/hack/select-e2e.sh index fcb7651e3..59db4fe25 100755 --- a/hack/select-e2e.sh +++ b/hack/select-e2e.sh @@ -8,8 +8,9 @@ # Output: # - empty no E2E impact, decided by one of the rules that select # nothing: step 1 for docs/, dashboards/ and *.md, step 2 for -# a *.disabled Chainsaw suite and for an -# examples/backups// with no suite, and +# a *.disabled Chainsaw suite, for an +# examples/backups// with no suite and for the CI-only +# tools cmd/backport-batch/ and cmd/api-gate/, and # inert_config_pattern for repo meta and agent config # - selected per the PackageSource dependency graph # - full list any path that affects all tests, OR an unrecognised @@ -497,6 +498,14 @@ while IFS= read -r file || [ -n "$file" ]; do echo "select-e2e: '$file' belongs to no single backup walkthrough — escalating to the full suite" >&2 trigger_full=1 continue ;; + cmd/backport-batch/*|cmd/api-gate/*) + # CI-only Go tools. No image builds them and nothing in the e2e sandbox + # runs them, so an edit cannot regress a suite. Matched here because + # full_suite_pattern takes all of cmd/ and is checked before the inert + # list. The plan job's `code` output and e2e-fork.yaml skip e2e for a + # diff made only of these and docs/; this arm keeps them from escalating + # a diff that also touches something else. + continue ;; esac # 3. Full-suite trigger diff --git a/hack/select-e2e_test.bats b/hack/select-e2e_test.bats index 6ce198830..f79d6b05c 100644 --- a/hack/select-e2e_test.bats +++ b/hack/select-e2e_test.bats @@ -202,6 +202,23 @@ assert_full_suite() { rm -rf "$tmp" } +@test "the CI-only tools select nothing and do not escalate the rest" { + tmp=$(mktemp -d) + cp -r packages/core/platform/sources "$tmp/sources" + printf '%s\n' cmd/backport-batch/report_test.go cmd/api-gate/main.go > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + [ -z "$output" ] + # Beside a package, the package alone decides the selection. + printf '%s\n' cmd/backport-batch/main.go packages/apps/postgres/values.yaml > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + assert_selection "a CI-only tool must not widen a package selection" "$output" "postgres" + # Any other cmd/ directory is a shipped binary and still escalates. + echo "cmd/cozystack-api/main.go" > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources" 2>/dev/null) + assert_full_suite "$output" + rm -rf "$tmp" +} + @test "kubernetes-application maps to the two kubernetes suites" { tmp=$(mktemp -d) cp -r packages/core/platform/sources "$tmp/sources" From 48eae9ad361206355ddb8224470ab5053cedd0e3 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 12 Nov 2026 15:43:05 +0500 Subject: [PATCH 2/2] ci: run only the backport-batch tests for a backport-batch change A pull request that changed only cmd/backport-batch built every image, installed the platform for e2e and ran the whole unit lane, so a one-line test fix waited hours on suites that cannot exercise it. The backport-batch tests themselves ran on every code pull request, although only a change to the tool, to what it imports or to the pages its tests read can break them. Their inputs are listed beside the tool and checked against its import graph, because a hand-kept list that fell behind would switch the tests off without anyone noticing. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- .github/workflows/e2e-fork.yaml | 21 +++-- .github/workflows/pull-requests.yaml | 51 +++++++++--- Makefile | 5 +- cmd/backport-batch/inputs_test.go | 112 ++++++++++++++++++++++++++ cmd/backport-batch/test-inputs.txt | 26 ++++++ docs/agents/e2e-testing.md | 2 +- docs/agents/overview.md | 2 +- hack/backport-batch-tests-needed.bats | 87 ++++++++++++++++++++ hack/backport-batch-tests-needed.sh | 44 ++++++++++ hack/downstream-trigger-map.bats | 16 +++- hack/gate-concurrency-contract.bats | 40 ++++++++- 11 files changed, 378 insertions(+), 28 deletions(-) create mode 100644 cmd/backport-batch/inputs_test.go create mode 100644 cmd/backport-batch/test-inputs.txt create mode 100644 hack/backport-batch-tests-needed.bats create mode 100755 hack/backport-batch-tests-needed.sh diff --git a/.github/workflows/e2e-fork.yaml b/.github/workflows/e2e-fork.yaml index 65b78230d..03c568395 100644 --- a/.github/workflows/e2e-fork.yaml +++ b/.github/workflows/e2e-fork.yaml @@ -61,7 +61,8 @@ jobs: # base-repo PR file list — NOT from the presence/absence of an artifact the # fork's own build controls. Fail-closed if the fork build failed or the # pr-patch artifact is missing; green-and-stop only for a PR whose file list is - # entirely under docs/ (verified against the base-repo API). + # entirely under docs/ or the CI-only tools cmd/backport-batch/ and + # cmd/api-gate/ (verified against the base-repo API). resolve: name: Resolve fork PR # Only fork PRs: head_repository differs from the base repository. Same-repo @@ -266,15 +267,21 @@ jobs: await decide('failure', 'PR file list hit the 3000-file API cap — cannot verify docs-only'); return; } + // Paths with nothing to build and nothing for e2e to install: docs, + // and the CI-only Go tools no image builds. pull-requests.yaml's + // plan job answers the same question with the same prefixes, and + // hack/gate-concurrency-contract.bats holds the two lists equal. + // // A rename reports the NEW path in `filename` and the old one in // `previous_filename`. Checking only `filename` would read a // packages/… -> docs/… rename as docs-only, when it in fact removes - // a code path. BOTH sides must be under docs/ to count as docs-only. - const isDocs = p => typeof p === 'string' && p.startsWith('docs/'); - const hasNonDocs = files.some(f => - !isDocs(f.filename) || (f.previous_filename && !isDocs(f.previous_filename))); - if (!hasNonDocs) { - await decide('success', 'docs-only PR (verified via PR file list) — e2e not required'); + // a code path. BOTH sides must be under these prefixes to count. + const noRuntimePrefixes = ['docs/', 'cmd/backport-batch/', 'cmd/api-gate/']; + const isNoRuntime = p => typeof p === 'string' && noRuntimePrefixes.some(x => p.startsWith(x)); + const hasRuntime = files.some(f => + !isNoRuntime(f.filename) || (f.previous_filename && !isNoRuntime(f.previous_filename))); + if (!hasRuntime) { + await decide('success', 'docs or CI-only tools (verified via PR file list) — e2e not required'); return; } diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 24ee473ef..62cbbb1ad 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -75,6 +75,8 @@ jobs: statuses: write outputs: code: ${{ steps.p.outputs.code }} + docs: ${{ steps.p.outputs.docs }} + backport_batch: ${{ steps.p.outputs.backport_batch }} matrix: ${{ steps.p.outputs.matrix }} any: ${{ steps.p.outputs.any }} touched: ${{ steps.p.outputs.touched }} @@ -167,11 +169,27 @@ jobs: # alone here rather than widened by a side effect. git diff --name-only --no-renames "origin/${BASE_REF}...HEAD" > /tmp/changed_norenames.txt echo "Changed files:"; sed 's/^/ /' /tmp/changed_norenames.txt - if grep -qvE '^docs/' /tmp/changed_norenames.txt; then + # `code` is false when every path is under docs/ or in a CI-only Go + # tool that no image builds (cmd/backport-batch, cmd/api-gate): there + # is nothing to build and nothing for e2e to install. e2e-fork.yaml + # answers the same question from the PR file list with the same + # prefixes, and hack/gate-concurrency-contract.bats holds the two lists + # equal. + if grep -qvE '^(docs/|cmd/backport-batch/|cmd/api-gate/)' /tmp/changed_norenames.txt; then echo "code=true" >> "$GITHUB_OUTPUT" else echo "code=false" >> "$GITHUB_OUTPUT" fi + # The Bats contracts read docs, so a docs change runs them even when + # `code` is false; a change confined to the CI-only tools does not. + if grep -qE '^docs/' /tmp/changed_norenames.txt; then + echo "docs=true" >> "$GITHUB_OUTPUT" + else + echo "docs=false" >> "$GITHUB_OUTPUT" + fi + backport_batch="$(hack/backport-batch-tests-needed.sh /tmp/changed_norenames.txt)" + echo "backport-batch tests needed: $backport_batch" + echo "backport_batch=$backport_batch" >> "$GITHUB_OUTPUT" matrix="$(hack/build-matrix.sh /tmp/changed.txt)" echo "Build matrix: $matrix" echo "matrix=$matrix" >> "$GITHUB_OUTPUT" @@ -293,9 +311,11 @@ jobs: contents: read needs: ["plan"] # Unit contracts read both code and documentation. Run this job for every - # non-release PR: code changes take the complete unit path, while docs-only - # changes run the Bats lane that owns those cross-tree contracts. Builds and - # E2E remain gated on `code` below. + # non-release PR: code changes take the complete unit path, docs changes run + # the Bats lane that owns those cross-tree contracts, and the backport-batch + # tests run whenever one of their inputs changed and at no other time. A + # change confined to the CI-only tools runs no other lane. Builds and E2E + # remain gated on `code` below. if: | !contains(github.event.pull_request.labels.*.name, 'release') steps: @@ -306,20 +326,21 @@ jobs: # The module cache is restored from the key build-main.yaml writes on # main, which every PR run can read, so a PR's first run starts warm. The # download fetches whatever the cache lacks, the whole set on a miss. The - # Go steps are for code changes only: a docs-only run is the Bats lane alone. + # Go steps are for code changes and the backport-batch tests only: a + # docs-only run is the Bats lane alone. - name: Locate Go module cache - if: needs.plan.outputs.code == 'true' + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true' id: gomodcache run: echo "dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" - name: Cache Go modules - if: needs.plan.outputs.code == 'true' + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.gomodcache.outputs.dir }} key: go-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }} restore-keys: go-mod-${{ runner.os }}- - name: Download Go modules - if: needs.plan.outputs.code == 'true' + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true' run: | for mod in $(git ls-files go.mod '*/go.mod' | xargs -n1 dirname); do (cd "$mod" && "$GITHUB_WORKSPACE/hack/go-mod-download.sh") @@ -327,8 +348,9 @@ jobs: # Ephemeral runners don't carry the helm-unittest plugin the persistent # self-hosted runner accreted. The plugin embeds its own Helm, so its # version is pinned: a plugin release must not change test results. Bats is - # installed on every run, docs-only ones included. + # installed whenever a Bats lane runs, docs-only ones included. - name: Set up test toolchain + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.docs == 'true' env: CODE_CHANGED: ${{ needs.plan.outputs.code }} HELM_UNITTEST_VERSION: "1.1.2" @@ -374,8 +396,15 @@ jobs: if: needs.plan.outputs.code == 'true' run: make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target - name: Run Bats unit tests for docs-only changes - if: needs.plan.outputs.code == 'false' + if: needs.plan.outputs.code == 'false' && needs.plan.outputs.docs == 'true' run: make bats-unit-tests + # Not part of `make unit-tests`: hack/backport-batch-tests-needed.sh + # decides from cmd/backport-batch/test-inputs.txt, whatever `code` is. + # `!cancelled()` for the reason the unit lane runs with -k: a red step + # above must not hide this suite's own result. + - name: Run backport-batch tests + if: ${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }} + run: make test-backport-batch - name: Upload unit test report if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -1427,7 +1456,7 @@ jobs: description = `release packages candidate did not verify (result: ${CANDIDATE_RESULT})`; } else if (CODE !== 'true') { state = 'success'; - description = 'docs-only PR — e2e not required'; + description = 'docs or CI-only tools — e2e not required'; } else if (IS_RELEASE === 'true' && IS_FULL_E2E !== 'true') { // Release promotion PRs deliberately do not repeat e2e: the rc was // already verified against the immutable candidate whose digests diff --git a/Makefile b/Makefile index 2feea00e9..a32e33af4 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ test: make -C packages/core/testing apply make -C packages/core/testing e2e -unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness test-backport-batch migrations-target-check +unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness migrations-target-check helm-unit-tests: hack/helm-unit-tests.sh @@ -196,6 +196,9 @@ test-check-readiness: go test ./test/check-readiness/ -count=1 # ./cmd/... is excluded from go-unit-tests, so the batch tool needs its own target. +# It is not a prerequisite of unit-tests: the checks job in pull-requests.yaml +# runs it only when the diff touches a path listed in +# cmd/backport-batch/test-inputs.txt. test-backport-batch: go test ./cmd/backport-batch/... -count=1 diff --git a/cmd/backport-batch/inputs_test.go b/cmd/backport-batch/inputs_test.go new file mode 100644 index 000000000..0632f80ef --- /dev/null +++ b/cmd/backport-batch/inputs_test.go @@ -0,0 +1,112 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "os/exec" + "strings" + "testing" +) + +const modulePath = "github.com/cozystack/cozystack/" + +// readTestInputs returns the entries of test-inputs.txt, the list CI reads to +// decide whether a pull request runs these tests. +func readTestInputs(t *testing.T) []string { + t.Helper() + data, err := os.ReadFile("test-inputs.txt") + if err != nil { + t.Fatal(err) + } + var entries []string + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + entries = append(entries, line) + } + if len(entries) == 0 { + t.Fatal("test-inputs.txt lists no path") + } + return entries +} + +// covered reports whether a change to path matches an entry the way the plan +// job matches it: a trailing slash is a directory prefix, anything else is +// one file. +func covered(entries []string, path string) bool { + for _, e := range entries { + if strings.HasSuffix(e, "/") && strings.HasPrefix(path, e) || path == e { + return true + } + } + return false +} + +// A pull request that changes an input of these tests has to run them, and CI +// knows the inputs only from test-inputs.txt. An import added here, or added to +// a package this tool imports, lands in a pull request that touches a listed +// path, so it runs this test, which fails until the new package is listed. +func TestInputsListed(t *testing.T) { + entries := readTestInputs(t) + + out, err := exec.Command("go", "list", "-deps", "-test", "-f", "{{.ImportPath}}", ".").Output() + if err != nil { + t.Fatalf("go list: %v", err) + } + var deps int + for _, pkg := range strings.Split(strings.TrimSpace(string(out)), "\n") { + // A test variant prints as "pkg [pkg.test]"; the synthesized main + // package as "pkg.test". Both are this directory. + pkg, _, _ = strings.Cut(pkg, " ") + pkg = strings.TrimSuffix(pkg, ".test") + rel, ok := strings.CutPrefix(pkg, modulePath) + if !ok { + continue + } + deps++ + if !covered(entries, rel+"/x.go") { + t.Errorf("these tests import %s, which test-inputs.txt does not list: add %s/", pkg, rel) + } + } + if deps == 0 { + t.Fatalf("go list named no package of %s, so nothing was checked:\n%s", modulePath, out) + } + + for _, page := range []string{policyPage, "docs/release/backport-tasks/x.md"} { + if !covered(entries, page) { + t.Errorf("the tests read %s, which test-inputs.txt does not cover", page) + } + } + for _, f := range []string{"go.mod", "go.sum"} { + if !covered(entries, f) { + t.Errorf("test-inputs.txt does not list %s", f) + } + } +} + +// A listed path that no longer exists makes the list read as broader than it +// is, and usually means a dependency moved and its new home is missing. +func TestInputsExist(t *testing.T) { + for _, e := range readTestInputs(t) { + if _, err := os.Stat("../../" + e); err != nil { + t.Errorf("test-inputs.txt lists %s: %v", e, err) + } + } +} diff --git a/cmd/backport-batch/test-inputs.txt b/cmd/backport-batch/test-inputs.txt new file mode 100644 index 000000000..4262b0e1f --- /dev/null +++ b/cmd/backport-batch/test-inputs.txt @@ -0,0 +1,26 @@ +# Every path whose change can change the result of +# `go test ./cmd/backport-batch/...`. The plan job in +# .github/workflows/pull-requests.yaml runs those tests when, and only when, the +# pull request touches one of these paths. +# +# One repository-relative path per line. A trailing slash names a directory and +# matches everything below it; anything else matches that file alone. Blank lines +# and lines starting with # are ignored. +# +# TestInputsListed in inputs_test.go keeps this list in step with the code: it +# fails when the tests' in-repo package dependencies (go list -deps -test) or the +# docs pages they read are missing from it. + +cmd/backport-batch/ +go.mod +go.sum + +# In-repo packages the tool and its tests import. +internal/apigate/ +pkg/apis/apps/v1alpha1/ +pkg/apis/apps/validation/ +pkg/config/ + +# Pages the tests read from the tree. +docs/release/backports.md +docs/release/backport-tasks/ diff --git a/docs/agents/e2e-testing.md b/docs/agents/e2e-testing.md index 9ad31d967..2d37ab274 100644 --- a/docs/agents/e2e-testing.md +++ b/docs/agents/e2e-testing.md @@ -121,7 +121,7 @@ Be precise about TIA's scope — it is narrower than "skip E2E for unrelated PRs - **It trims only which suites `chainsaw test` runs, not the expensive stages.** `Prepare environment` and `Install Cozystack` (the full platform install) run regardless of the selection. TIA only decides the `CHAINSAW_SUITES` passed to the final E2E step. A perfect narrow saves the suite tail, not the bulk of wall-time. (Package image builds are scoped separately by `hack/build-matrix.sh`; the Talos leg runs when its direct or shared inputs changed and remains unconditional for forks, while installer/finalize runs for every non-docs PR. That is a distinct mechanism from TIA's suite trimming.) - **`Select E2E tests` runs *after* `Install Cozystack` and has no `always()`.** GitHub implicitly ANDs a step's `if:` with `success()`, so if install fails the selector (and `Run E2E tests`) are skipped entirely. On a red install you will not see TIA narrow anything — that is the ordering, not a selector bug. - **Most PRs legitimately escalate to the full suite.** Platform manifests live under `packages/core/`, Go under `api/`/`internal/`, helpers under `hack/`, plus the `Makefile` — touching any one triggers the full suite. So "all suites selected" is usually correct, not a failure to narrow. -- **Two skip layers, with a gap.** The `plan` job computes a coarse docs-only gate (`code`, true for any change outside `docs/**`) that skips the whole build+E2E pipeline for docs-only PRs, plus a per-package build matrix (`hack/build-matrix.sh`) that scopes which images build. TIA's finer skip (`select-e2e.sh` ignoring `docs/`, `dashboards/`, `*.md` and the rest of `inert_config_pattern`) only trims the suite selection. A PR touching only `dashboards/*.json` therefore still runs install, then skips just the Chainsaw suites — near-zero savings. If you need such PRs to skip the pipeline, widen the `plan` docs-gate, not `select-e2e.sh`. +- **Two skip layers, with a gap.** The `plan` job computes a coarse gate (`code`, false only when every changed path is under `docs/` or one of the CI-only Go tools no image builds, `cmd/backport-batch/` and `cmd/api-gate/`) that skips the whole build+E2E pipeline for such PRs, and `e2e-fork.yaml` skips on the same prefixes (`hack/gate-concurrency-contract.bats` holds the two lists equal), plus a per-package build matrix (`hack/build-matrix.sh`) that scopes which images build. TIA's finer skip (`select-e2e.sh` ignoring `docs/`, `dashboards/`, `*.md` and the rest of `inert_config_pattern`) only trims the suite selection. A PR touching only `dashboards/*.json` therefore still runs install, then skips just the Chainsaw suites — near-zero savings. If you need such PRs to skip the pipeline, widen the `plan` docs-gate, not `select-e2e.sh`. ### 8. Failure must be self-explanatory — diagnostics are first-class diff --git a/docs/agents/overview.md b/docs/agents/overview.md index 9ab67c41d..3c0b50d14 100644 --- a/docs/agents/overview.md +++ b/docs/agents/overview.md @@ -106,7 +106,7 @@ A chart also declares the limits on its own resource name there, with `## @name ## Testing -- **Helm and BATS unit tests:** `make helm-unit-tests` runs `hack/helm-unit-tests.sh` over every package that defines a `test` target. A `test` target is expected to run helm-unittest: the sweep fails a package whose run reports no suite, because exiting 0 having asserted nothing is indistinguishable from passing. A target that drives something else belongs under another name, as `packages/core/testing` already does with its sandbox flow. `make bats-unit-tests` requires bats-core 1.5 or newer; GNU parallel is optional and the target runs serially without it. Run the complete local BATS and POSIX-compatibility pass with `pre-commit run bats-unit-tests --hook-stage manual --all-files`; pull-request CI runs both lanes for code changes and the BATS contracts for docs-only changes. On a host whose `/bin/sh` differs from CI, `make BATS_POSIX_SHELL=dash bats-posix-compat-tests` selects the same interpreter explicitly. `make unit-tests` runs the full unit suite — Helm, BATS, the targeted POSIX-shell compatibility pass, Go, and the preset/readiness checks. It does not include the controller tests under `./internal/...`; run `make test-controllers` as well. CI invokes both targets together. +- **Helm and BATS unit tests:** `make helm-unit-tests` runs `hack/helm-unit-tests.sh` over every package that defines a `test` target. A `test` target is expected to run helm-unittest: the sweep fails a package whose run reports no suite, because exiting 0 having asserted nothing is indistinguishable from passing. A target that drives something else belongs under another name, as `packages/core/testing` already does with its sandbox flow. `make bats-unit-tests` requires bats-core 1.5 or newer; GNU parallel is optional and the target runs serially without it. Run the complete local BATS and POSIX-compatibility pass with `pre-commit run bats-unit-tests --hook-stage manual --all-files`; pull-request CI runs both lanes for code changes and the BATS contracts for docs-only changes. On a host whose `/bin/sh` differs from CI, `make BATS_POSIX_SHELL=dash bats-posix-compat-tests` selects the same interpreter explicitly. `make unit-tests` runs the full unit suite — Helm, BATS, the targeted POSIX-shell compatibility pass, Go, and the preset/readiness checks. It does not include the controller tests under `./internal/...`; run `make test-controllers` as well. CI invokes both targets together. The backport-batch tests are not part of it: run `make test-backport-batch` after changing anything listed in `cmd/backport-batch/test-inputs.txt`, which is also what CI keys them on. - **E2E tests:** Kyverno Chainsaw suites in `hack/e2e-chainsaw/` (one directory per app), run with `chainsaw test`. Cluster bootstrap (`hack/e2e-install-cozystack.bats`) and the OpenAPI checks (`hack/e2e-test-openapi.bats`) remain BATS. Conventions for writing and stabilising them — and the CI that runs them — live in [`e2e-testing.md`](./e2e-testing.md). - **Go tests:** standard `go test`, with Ginkgo/Gomega for controllers. diff --git a/hack/backport-batch-tests-needed.bats b/hack/backport-batch-tests-needed.bats new file mode 100644 index 000000000..9f37e768f --- /dev/null +++ b/hack/backport-batch-tests-needed.bats @@ -0,0 +1,87 @@ +#!/usr/bin/env bats +# Unit tests for hack/backport-batch-tests-needed.sh, which decides whether a +# pull request runs `make test-backport-batch`, and for its wiring into the +# checks job of pull-requests.yaml. +# +# Test-level EXIT traps replace Bats' own handler and hide failing TAP results. +# Cleanup follows aborting assertions; see docs/agents/e2e-testing.md. + +load test_helper + +CLASSIFIER="hack/backport-batch-tests-needed.sh" +WF=".github/workflows/pull-requests.yaml" + +classify() { + list="$1" + shift + printf '%s\n' "$@" > "$list" + "$CLASSIFIER" "$list" +} + +@test "a change to the tool, an imported package, a page its tests read or the module runs the tests" { + tmp=$(mktemp) + [ "$(classify "$tmp" cmd/backport-batch/report_test.go)" = true ] + [ "$(classify "$tmp" internal/apigate/surface.go)" = true ] + [ "$(classify "$tmp" pkg/config/config.go)" = true ] + [ "$(classify "$tmp" docs/release/backports.md)" = true ] + [ "$(classify "$tmp" docs/release/backport-tasks/conflict.md)" = true ] + [ "$(classify "$tmp" go.sum)" = true ] + # One input among unrelated paths is enough. + [ "$(classify "$tmp" packages/apps/mariadb/values.yaml pkg/config/config.go)" = true ] + rm -f "$tmp" +} + +@test "a change that reaches none of its inputs does not run the tests" { + tmp=$(mktemp) + [ "$(classify "$tmp" packages/apps/mariadb/values.yaml)" = false ] + [ "$(classify "$tmp" cmd/api-gate/main.go)" = false ] + [ "$(classify "$tmp" hack/backport-batch-scheduled.sh)" = false ] + [ "$(classify "$tmp" docs/release.md docs/release/other.md)" = false ] + # A file entry is not a prefix: backports.md does not cover backports.md.bak, + # and a directory entry does not cover a sibling sharing its name as prefix. + [ "$(classify "$tmp" docs/release/backports.md.bak)" = false ] + [ "$(classify "$tmp" pkg/configx/x.go)" = false ] + [ "$(classify "$tmp" internal/apigate-old/x.go)" = false ] + rm -f "$tmp" +} + +@test "entries are trimmed as the Go test trims them" { + tmp=$(mktemp -d) + printf '%s\n' cmd/backport-batch/main.go > "$tmp/diff" + printf ' cmd/backport-batch/ \r\n' > "$tmp/inputs" + [ "$("$CLASSIFIER" "$tmp/diff" "$tmp/inputs")" = true ] + rm -rf "$tmp" +} + +@test "an inputs file with no entry is refused rather than read as never" { + tmp=$(mktemp -d) + printf '%s\n' cmd/backport-batch/main.go > "$tmp/diff" + printf '# only a comment\n\n' > "$tmp/inputs" + if "$CLASSIFIER" "$tmp/diff" "$tmp/inputs" >"$tmp/out" 2>"$tmp/err"; then + echo "an empty inputs file must fail, it printed: $(cat "$tmp/out")" >&2 + exit 1 + fi + grep -q 'lists no path' "$tmp/err" + rm -rf "$tmp" +} + +@test "the checks job runs the target on the classifier's answer alone" { + plan_run=$(yq -r '.jobs.plan.steps[] | select(.id == "p") | .run' "$WF") + printf '%s\n' "$plan_run" | grep -qF 'backport_batch="$(hack/backport-batch-tests-needed.sh /tmp/changed_norenames.txt)"' + [ "$(yq -r '.jobs.plan.outputs.backport_batch' "$WF")" = '${{ steps.p.outputs.backport_batch }}' ] + + step_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Run backport-batch tests") | .if' "$WF") + step_run=$(yq -r '.jobs.checks.steps[] | select(.name == "Run backport-batch tests") | .run' "$WF") + # !cancelled(): a failed unit lane above must not skip this suite too. + [ "$step_if" = "\${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }}" ] + [ "$step_run" = 'make test-backport-batch' ] + + # And nowhere else: the unit lane every code PR runs must not carry it. + prereqs=$(sed -n 's/^unit-tests:[[:space:]]*//p' Makefile) + [ -n "$prereqs" ] + case " $prereqs " in + *" test-backport-batch "*) + echo "make unit-tests still runs test-backport-batch on every code PR: $prereqs" >&2 + exit 1 ;; + esac +} diff --git a/hack/backport-batch-tests-needed.sh b/hack/backport-batch-tests-needed.sh new file mode 100755 index 000000000..71803e730 --- /dev/null +++ b/hack/backport-batch-tests-needed.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Print whether a changed-path list touches an input of +# `go test ./cmd/backport-batch/...`, as listed in +# cmd/backport-batch/test-inputs.txt. An entry ending in / matches every path +# below it, any other entry matches that file alone. The caller supplies the +# rename-blind list so both sides of a rename are classified. +# +# Usage: hack/backport-batch-tests-needed.sh [] +set -eu + +inputs="${2:-cmd/backport-batch/test-inputs.txt}" +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || [ ! -r "$1" ] || [ ! -r "$inputs" ]; then + echo "usage: backport-batch-tests-needed.sh []" >&2 + exit 2 +fi + +# An inputs file with no entry would answer false for every diff and switch the +# tests off without a word; refuse instead. +if ! grep -qvE '^[[:space:]]*(#|$)' "$inputs"; then + echo "backport-batch-tests-needed: $inputs lists no path" >&2 + exit 2 +fi + +# Entries are trimmed the way TestInputsListed trims them, so a trailing space +# or a CR cannot make the Go test accept an entry this never matches. +awk ' + NR == FNR { + e = $0 + sub(/^[[:space:]]+/, "", e) + sub(/[[:space:]]+$/, "", e) + if (e != "" && e !~ /^#/) entries[++n] = e + next + } + { + for (i = 1; i <= n; i++) { + e = entries[i] + if ((substr(e, length(e)) == "/" && index($0, e) == 1) || $0 == e) { + found = 1 + exit + } + } + } + END { print (found ? "true" : "false") } +' "$inputs" "$1" diff --git a/hack/downstream-trigger-map.bats b/hack/downstream-trigger-map.bats index c5ebf6af3..39e1d5433 100644 --- a/hack/downstream-trigger-map.bats +++ b/hack/downstream-trigger-map.bats @@ -35,8 +35,9 @@ # a human's job, and reviewers should not assume CI did it for them. # # A docs-only PR skips builds and E2E, but the `checks` job still runs the Bats -# lane. Otherwise editing this map, or another document audited by a unit -# contract, would exempt the changed input from its own guard. +# lane whenever a docs/ path changed. Otherwise editing this map, or another +# document audited by a unit contract, would exempt the changed input from its +# own guard. load test_helper @@ -117,9 +118,18 @@ map_section() { [ "$job_if" = "!contains(github.event.pull_request.labels.*.name, 'release')" ] [ "$code_if" = "needs.plan.outputs.code == 'true'" ] - [ "$docs_if" = "needs.plan.outputs.code == 'false'" ] + [ "$docs_if" = "needs.plan.outputs.code == 'false' && needs.plan.outputs.docs == 'true'" ] [ "$docs_run" = 'make bats-unit-tests' ] [ "$code_run" = 'make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target' ] + + # `code` is false for the CI-only tools too, so the docs lane is keyed on a + # docs/ path being present, not on `code` alone. Pin that the output means + # exactly that and that Bats gets installed for it. + plan_run=$(yq -r '.jobs.plan.steps[] | select(.id == "p") | .run' "$WORKFLOW") + printf '%s\n' "$plan_run" | grep -qF "if grep -qE '^docs/' /tmp/changed_norenames.txt; then" + [ "$(yq -r '.jobs.plan.outputs.docs' "$WORKFLOW")" = '${{ steps.p.outputs.docs }}' ] + toolchain_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Set up test toolchain") | .if' "$WORKFLOW") + [ "$toolchain_if" = "needs.plan.outputs.code == 'true' || needs.plan.outputs.docs == 'true'" ] } @test "the trigger map and the PR-template checklist list the same repositories" { diff --git a/hack/gate-concurrency-contract.bats b/hack/gate-concurrency-contract.bats index 5133a4cd8..0c4c90f15 100644 --- a/hack/gate-concurrency-contract.bats +++ b/hack/gate-concurrency-contract.bats @@ -266,24 +266,56 @@ resolve_silent_conclusions() { plan_block="$(job_block plan "$PULL_REQUESTS" | code_lines)" [ -n "$plan_block" ] - count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^docs/' /tmp/changed_norenames.txt" || true)" + count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^(.*)' /tmp/changed_norenames.txt" || true)" [ "${count:-0}" -eq 1 ] # The rename-detected list may still feed the build matrix, but never this. - count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^docs/' /tmp/changed.txt" || true)" + count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^(.*)' /tmp/changed.txt" || true)" [ "${count:-0}" -eq 0 ] # The fork half of the same agreement: both sides of a rename are inspected. - # Pinned on the docs-only expression itself — `previous_filename` also appears + # Pinned on the skip expression itself — `previous_filename` also appears # in the TIA file list a few lines down, so counting the identifier would stay # satisfied by that one with this check deleted. block="$(job_block resolve "$FORK" | code_lines)" [ -n "$block" ] count="$(printf '%s\n' "$block" \ - | grep -c 'f\.previous_filename && !isDocs(f\.previous_filename)' || true)" + | grep -c 'f\.previous_filename && !isNoRuntime(f\.previous_filename)' || true)" [ "${count:-0}" -eq 1 ] } +@test "both lanes skip e2e for the same path prefixes" { + # The rename test above pins HOW each lane reads its list; this one pins + # WHAT it skips. A prefix added to one lane only is a fork PR that one lane + # reads as nothing-to-build and the other as needing the pr-patch artifact, + # which nothing will ever produce. + plan_prefixes="$(job_block plan "$PULL_REQUESTS" | code_lines \ + | sed -nE "s,.*grep -qvE '\^\(([^)]*)\)' /tmp/changed_norenames\.txt.*,\1,p" \ + | tr '|' '\n' | sort)" + fork_prefixes="$(job_block resolve "$FORK" | code_lines \ + | sed -nE "s,.*const noRuntimePrefixes = \[([^]]*)\];.*,\1,p" \ + | tr ',' '\n' | tr -d " '" | sort)" + [ -n "$plan_prefixes" ] + printf '%s\n' "$plan_prefixes" | grep -qx 'docs/' + if [ "$plan_prefixes" != "$fork_prefixes" ]; then + echo "plan skips:" >&2; printf ' %s\n' $plan_prefixes >&2 + echo "fork skips:" >&2; printf ' %s\n' $fork_prefixes >&2 + exit 1 + fi + + # The tools in that list must also be the ones the build matrix and the suite + # selector leave alone in a mixed diff; otherwise a tool skipped on its own + # still rebuilds everything beside a one-line package change. + tools="$(printf '%s\n' "$plan_prefixes" | grep -vx 'docs/')" + [ -n "$tools" ] + matrix_tools="$(sed -nE "s,^tooling_pattern='\^cmd/\(([^)]*)\)/'$,\1,p" "$REPO_ROOT/hack/build-matrix.sh" \ + | tr '|' '\n' | sed 's,^,cmd/,; s,$,/,' | sort)" + select_tools="$(sed -nE 's,^ (cmd/[^)]*)\)$,\1,p' "$REPO_ROOT/hack/select-e2e.sh" \ + | tr '|' '\n' | sed 's,\*$,,' | sort)" + [ "$tools" = "$matrix_tools" ] + [ "$tools" = "$select_tools" ] +} + @test "same-repo gate: every publishing guard names the same label set" { # `plan`, `verify-release-candidate`, `e2e-report` and the concurrency key # each decide from the label name whether this run owns the status. Renaming