diff --git a/.github/workflows/e2e-fork.yaml b/.github/workflows/e2e-fork.yaml index 65b78230d..03c568395 100644 --- a/.github/workflows/e2e-fork.yaml +++ b/.github/workflows/e2e-fork.yaml @@ -61,7 +61,8 @@ jobs: # base-repo PR file list — NOT from the presence/absence of an artifact the # fork's own build controls. Fail-closed if the fork build failed or the # pr-patch artifact is missing; green-and-stop only for a PR whose file list is - # entirely under docs/ (verified against the base-repo API). + # entirely under docs/ or the CI-only tools cmd/backport-batch/ and + # cmd/api-gate/ (verified against the base-repo API). resolve: name: Resolve fork PR # Only fork PRs: head_repository differs from the base repository. Same-repo @@ -266,15 +267,21 @@ jobs: await decide('failure', 'PR file list hit the 3000-file API cap — cannot verify docs-only'); return; } + // Paths with nothing to build and nothing for e2e to install: docs, + // and the CI-only Go tools no image builds. pull-requests.yaml's + // plan job answers the same question with the same prefixes, and + // hack/gate-concurrency-contract.bats holds the two lists equal. + // // A rename reports the NEW path in `filename` and the old one in // `previous_filename`. Checking only `filename` would read a // packages/… -> docs/… rename as docs-only, when it in fact removes - // a code path. BOTH sides must be under docs/ to count as docs-only. - const isDocs = p => typeof p === 'string' && p.startsWith('docs/'); - const hasNonDocs = files.some(f => - !isDocs(f.filename) || (f.previous_filename && !isDocs(f.previous_filename))); - if (!hasNonDocs) { - await decide('success', 'docs-only PR (verified via PR file list) — e2e not required'); + // a code path. BOTH sides must be under these prefixes to count. + const noRuntimePrefixes = ['docs/', 'cmd/backport-batch/', 'cmd/api-gate/']; + const isNoRuntime = p => typeof p === 'string' && noRuntimePrefixes.some(x => p.startsWith(x)); + const hasRuntime = files.some(f => + !isNoRuntime(f.filename) || (f.previous_filename && !isNoRuntime(f.previous_filename))); + if (!hasRuntime) { + await decide('success', 'docs or CI-only tools (verified via PR file list) — e2e not required'); return; } diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 24ee473ef..62cbbb1ad 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -75,6 +75,8 @@ jobs: statuses: write outputs: code: ${{ steps.p.outputs.code }} + docs: ${{ steps.p.outputs.docs }} + backport_batch: ${{ steps.p.outputs.backport_batch }} matrix: ${{ steps.p.outputs.matrix }} any: ${{ steps.p.outputs.any }} touched: ${{ steps.p.outputs.touched }} @@ -167,11 +169,27 @@ jobs: # alone here rather than widened by a side effect. git diff --name-only --no-renames "origin/${BASE_REF}...HEAD" > /tmp/changed_norenames.txt echo "Changed files:"; sed 's/^/ /' /tmp/changed_norenames.txt - if grep -qvE '^docs/' /tmp/changed_norenames.txt; then + # `code` is false when every path is under docs/ or in a CI-only Go + # tool that no image builds (cmd/backport-batch, cmd/api-gate): there + # is nothing to build and nothing for e2e to install. e2e-fork.yaml + # answers the same question from the PR file list with the same + # prefixes, and hack/gate-concurrency-contract.bats holds the two lists + # equal. + if grep -qvE '^(docs/|cmd/backport-batch/|cmd/api-gate/)' /tmp/changed_norenames.txt; then echo "code=true" >> "$GITHUB_OUTPUT" else echo "code=false" >> "$GITHUB_OUTPUT" fi + # The Bats contracts read docs, so a docs change runs them even when + # `code` is false; a change confined to the CI-only tools does not. + if grep -qE '^docs/' /tmp/changed_norenames.txt; then + echo "docs=true" >> "$GITHUB_OUTPUT" + else + echo "docs=false" >> "$GITHUB_OUTPUT" + fi + backport_batch="$(hack/backport-batch-tests-needed.sh /tmp/changed_norenames.txt)" + echo "backport-batch tests needed: $backport_batch" + echo "backport_batch=$backport_batch" >> "$GITHUB_OUTPUT" matrix="$(hack/build-matrix.sh /tmp/changed.txt)" echo "Build matrix: $matrix" echo "matrix=$matrix" >> "$GITHUB_OUTPUT" @@ -293,9 +311,11 @@ jobs: contents: read needs: ["plan"] # Unit contracts read both code and documentation. Run this job for every - # non-release PR: code changes take the complete unit path, while docs-only - # changes run the Bats lane that owns those cross-tree contracts. Builds and - # E2E remain gated on `code` below. + # non-release PR: code changes take the complete unit path, docs changes run + # the Bats lane that owns those cross-tree contracts, and the backport-batch + # tests run whenever one of their inputs changed and at no other time. A + # change confined to the CI-only tools runs no other lane. Builds and E2E + # remain gated on `code` below. if: | !contains(github.event.pull_request.labels.*.name, 'release') steps: @@ -306,20 +326,21 @@ jobs: # The module cache is restored from the key build-main.yaml writes on # main, which every PR run can read, so a PR's first run starts warm. The # download fetches whatever the cache lacks, the whole set on a miss. The - # Go steps are for code changes only: a docs-only run is the Bats lane alone. + # Go steps are for code changes and the backport-batch tests only: a + # docs-only run is the Bats lane alone. - name: Locate Go module cache - if: needs.plan.outputs.code == 'true' + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true' id: gomodcache run: echo "dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" - name: Cache Go modules - if: needs.plan.outputs.code == 'true' + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.gomodcache.outputs.dir }} key: go-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }} restore-keys: go-mod-${{ runner.os }}- - name: Download Go modules - if: needs.plan.outputs.code == 'true' + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true' run: | for mod in $(git ls-files go.mod '*/go.mod' | xargs -n1 dirname); do (cd "$mod" && "$GITHUB_WORKSPACE/hack/go-mod-download.sh") @@ -327,8 +348,9 @@ jobs: # Ephemeral runners don't carry the helm-unittest plugin the persistent # self-hosted runner accreted. The plugin embeds its own Helm, so its # version is pinned: a plugin release must not change test results. Bats is - # installed on every run, docs-only ones included. + # installed whenever a Bats lane runs, docs-only ones included. - name: Set up test toolchain + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.docs == 'true' env: CODE_CHANGED: ${{ needs.plan.outputs.code }} HELM_UNITTEST_VERSION: "1.1.2" @@ -374,8 +396,15 @@ jobs: if: needs.plan.outputs.code == 'true' run: make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target - name: Run Bats unit tests for docs-only changes - if: needs.plan.outputs.code == 'false' + if: needs.plan.outputs.code == 'false' && needs.plan.outputs.docs == 'true' run: make bats-unit-tests + # Not part of `make unit-tests`: hack/backport-batch-tests-needed.sh + # decides from cmd/backport-batch/test-inputs.txt, whatever `code` is. + # `!cancelled()` for the reason the unit lane runs with -k: a red step + # above must not hide this suite's own result. + - name: Run backport-batch tests + if: ${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }} + run: make test-backport-batch - name: Upload unit test report if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -1427,7 +1456,7 @@ jobs: description = `release packages candidate did not verify (result: ${CANDIDATE_RESULT})`; } else if (CODE !== 'true') { state = 'success'; - description = 'docs-only PR — e2e not required'; + description = 'docs or CI-only tools — e2e not required'; } else if (IS_RELEASE === 'true' && IS_FULL_E2E !== 'true') { // Release promotion PRs deliberately do not repeat e2e: the rc was // already verified against the immutable candidate whose digests diff --git a/Makefile b/Makefile index adc652dd7..aac1d6617 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ test: make -C packages/core/testing apply make -C packages/core/testing e2e -unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness test-backport-batch migrations-target-check +unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness migrations-target-check helm-unit-tests: hack/helm-unit-tests.sh @@ -195,7 +195,9 @@ test-check-readiness: go test ./test/check-readiness/ -count=1 # The batch tool's suite drives real git repositories for minutes, so it is -# kept out of go-unit-tests in a target of its own. +# kept out of go-unit-tests in a target of its own. It is not a prerequisite +# of unit-tests: the checks job in pull-requests.yaml runs it only when the +# diff touches a path listed in cmd/backport-batch/test-inputs.txt. test-backport-batch: go test ./cmd/backport-batch/... -count=1 diff --git a/cmd/backport-batch/inputs_test.go b/cmd/backport-batch/inputs_test.go new file mode 100644 index 000000000..0632f80ef --- /dev/null +++ b/cmd/backport-batch/inputs_test.go @@ -0,0 +1,112 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "os/exec" + "strings" + "testing" +) + +const modulePath = "github.com/cozystack/cozystack/" + +// readTestInputs returns the entries of test-inputs.txt, the list CI reads to +// decide whether a pull request runs these tests. +func readTestInputs(t *testing.T) []string { + t.Helper() + data, err := os.ReadFile("test-inputs.txt") + if err != nil { + t.Fatal(err) + } + var entries []string + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + entries = append(entries, line) + } + if len(entries) == 0 { + t.Fatal("test-inputs.txt lists no path") + } + return entries +} + +// covered reports whether a change to path matches an entry the way the plan +// job matches it: a trailing slash is a directory prefix, anything else is +// one file. +func covered(entries []string, path string) bool { + for _, e := range entries { + if strings.HasSuffix(e, "/") && strings.HasPrefix(path, e) || path == e { + return true + } + } + return false +} + +// A pull request that changes an input of these tests has to run them, and CI +// knows the inputs only from test-inputs.txt. An import added here, or added to +// a package this tool imports, lands in a pull request that touches a listed +// path, so it runs this test, which fails until the new package is listed. +func TestInputsListed(t *testing.T) { + entries := readTestInputs(t) + + out, err := exec.Command("go", "list", "-deps", "-test", "-f", "{{.ImportPath}}", ".").Output() + if err != nil { + t.Fatalf("go list: %v", err) + } + var deps int + for _, pkg := range strings.Split(strings.TrimSpace(string(out)), "\n") { + // A test variant prints as "pkg [pkg.test]"; the synthesized main + // package as "pkg.test". Both are this directory. + pkg, _, _ = strings.Cut(pkg, " ") + pkg = strings.TrimSuffix(pkg, ".test") + rel, ok := strings.CutPrefix(pkg, modulePath) + if !ok { + continue + } + deps++ + if !covered(entries, rel+"/x.go") { + t.Errorf("these tests import %s, which test-inputs.txt does not list: add %s/", pkg, rel) + } + } + if deps == 0 { + t.Fatalf("go list named no package of %s, so nothing was checked:\n%s", modulePath, out) + } + + for _, page := range []string{policyPage, "docs/release/backport-tasks/x.md"} { + if !covered(entries, page) { + t.Errorf("the tests read %s, which test-inputs.txt does not cover", page) + } + } + for _, f := range []string{"go.mod", "go.sum"} { + if !covered(entries, f) { + t.Errorf("test-inputs.txt does not list %s", f) + } + } +} + +// A listed path that no longer exists makes the list read as broader than it +// is, and usually means a dependency moved and its new home is missing. +func TestInputsExist(t *testing.T) { + for _, e := range readTestInputs(t) { + if _, err := os.Stat("../../" + e); err != nil { + t.Errorf("test-inputs.txt lists %s: %v", e, err) + } + } +} diff --git a/cmd/backport-batch/test-inputs.txt b/cmd/backport-batch/test-inputs.txt new file mode 100644 index 000000000..4262b0e1f --- /dev/null +++ b/cmd/backport-batch/test-inputs.txt @@ -0,0 +1,26 @@ +# Every path whose change can change the result of +# `go test ./cmd/backport-batch/...`. The plan job in +# .github/workflows/pull-requests.yaml runs those tests when, and only when, the +# pull request touches one of these paths. +# +# One repository-relative path per line. A trailing slash names a directory and +# matches everything below it; anything else matches that file alone. Blank lines +# and lines starting with # are ignored. +# +# TestInputsListed in inputs_test.go keeps this list in step with the code: it +# fails when the tests' in-repo package dependencies (go list -deps -test) or the +# docs pages they read are missing from it. + +cmd/backport-batch/ +go.mod +go.sum + +# In-repo packages the tool and its tests import. +internal/apigate/ +pkg/apis/apps/v1alpha1/ +pkg/apis/apps/validation/ +pkg/config/ + +# Pages the tests read from the tree. +docs/release/backports.md +docs/release/backport-tasks/ diff --git a/docs/agents/e2e-testing.md b/docs/agents/e2e-testing.md index 9ad31d967..2d37ab274 100644 --- a/docs/agents/e2e-testing.md +++ b/docs/agents/e2e-testing.md @@ -121,7 +121,7 @@ Be precise about TIA's scope — it is narrower than "skip E2E for unrelated PRs - **It trims only which suites `chainsaw test` runs, not the expensive stages.** `Prepare environment` and `Install Cozystack` (the full platform install) run regardless of the selection. TIA only decides the `CHAINSAW_SUITES` passed to the final E2E step. A perfect narrow saves the suite tail, not the bulk of wall-time. (Package image builds are scoped separately by `hack/build-matrix.sh`; the Talos leg runs when its direct or shared inputs changed and remains unconditional for forks, while installer/finalize runs for every non-docs PR. That is a distinct mechanism from TIA's suite trimming.) - **`Select E2E tests` runs *after* `Install Cozystack` and has no `always()`.** GitHub implicitly ANDs a step's `if:` with `success()`, so if install fails the selector (and `Run E2E tests`) are skipped entirely. On a red install you will not see TIA narrow anything — that is the ordering, not a selector bug. - **Most PRs legitimately escalate to the full suite.** Platform manifests live under `packages/core/`, Go under `api/`/`internal/`, helpers under `hack/`, plus the `Makefile` — touching any one triggers the full suite. So "all suites selected" is usually correct, not a failure to narrow. -- **Two skip layers, with a gap.** The `plan` job computes a coarse docs-only gate (`code`, true for any change outside `docs/**`) that skips the whole build+E2E pipeline for docs-only PRs, plus a per-package build matrix (`hack/build-matrix.sh`) that scopes which images build. TIA's finer skip (`select-e2e.sh` ignoring `docs/`, `dashboards/`, `*.md` and the rest of `inert_config_pattern`) only trims the suite selection. A PR touching only `dashboards/*.json` therefore still runs install, then skips just the Chainsaw suites — near-zero savings. If you need such PRs to skip the pipeline, widen the `plan` docs-gate, not `select-e2e.sh`. +- **Two skip layers, with a gap.** The `plan` job computes a coarse gate (`code`, false only when every changed path is under `docs/` or one of the CI-only Go tools no image builds, `cmd/backport-batch/` and `cmd/api-gate/`) that skips the whole build+E2E pipeline for such PRs, and `e2e-fork.yaml` skips on the same prefixes (`hack/gate-concurrency-contract.bats` holds the two lists equal), plus a per-package build matrix (`hack/build-matrix.sh`) that scopes which images build. TIA's finer skip (`select-e2e.sh` ignoring `docs/`, `dashboards/`, `*.md` and the rest of `inert_config_pattern`) only trims the suite selection. A PR touching only `dashboards/*.json` therefore still runs install, then skips just the Chainsaw suites — near-zero savings. If you need such PRs to skip the pipeline, widen the `plan` docs-gate, not `select-e2e.sh`. ### 8. Failure must be self-explanatory — diagnostics are first-class diff --git a/docs/agents/overview.md b/docs/agents/overview.md index 4ebf430c8..a8aebbab1 100644 --- a/docs/agents/overview.md +++ b/docs/agents/overview.md @@ -106,7 +106,7 @@ A chart also declares the limits on its own resource name there, with `## @name ## Testing -- **Helm and BATS unit tests:** `make helm-unit-tests` runs `hack/helm-unit-tests.sh` over every package that defines a `test` target. A `test` target is expected to run helm-unittest: the sweep fails a package whose run reports no suite, because exiting 0 having asserted nothing is indistinguishable from passing. A target that drives something else belongs under another name, as `packages/core/testing` already does with its sandbox flow. `make bats-unit-tests` requires bats-core 1.5 or newer; GNU parallel is optional and the target runs serially without it. Run the complete local BATS and POSIX-compatibility pass with `pre-commit run bats-unit-tests --hook-stage manual --all-files`; pull-request CI runs both lanes for code changes and the BATS contracts for docs-only changes. On a host whose `/bin/sh` differs from CI, `make BATS_POSIX_SHELL=dash bats-posix-compat-tests` selects the same interpreter explicitly. `make unit-tests` runs the full unit suite — Helm, BATS, the targeted POSIX-shell compatibility pass, Go, and the preset/readiness checks. It does not include the controller tests under `./internal/...`; run `make test-controllers` as well. CI invokes both targets together. +- **Helm and BATS unit tests:** `make helm-unit-tests` runs `hack/helm-unit-tests.sh` over every package that defines a `test` target. A `test` target is expected to run helm-unittest: the sweep fails a package whose run reports no suite, because exiting 0 having asserted nothing is indistinguishable from passing. A target that drives something else belongs under another name, as `packages/core/testing` already does with its sandbox flow. `make bats-unit-tests` requires bats-core 1.5 or newer; GNU parallel is optional and the target runs serially without it. Run the complete local BATS and POSIX-compatibility pass with `pre-commit run bats-unit-tests --hook-stage manual --all-files`; pull-request CI runs both lanes for code changes and the BATS contracts for docs-only changes. On a host whose `/bin/sh` differs from CI, `make BATS_POSIX_SHELL=dash bats-posix-compat-tests` selects the same interpreter explicitly. `make unit-tests` runs the full unit suite — Helm, BATS, the targeted POSIX-shell compatibility pass, Go, and the preset/readiness checks. It does not include the controller tests under `./internal/...`; run `make test-controllers` as well. CI invokes both targets together. The backport-batch tests are not part of it: run `make test-backport-batch` after changing anything listed in `cmd/backport-batch/test-inputs.txt`, which is also what CI keys them on. - **E2E tests:** Kyverno Chainsaw suites in `hack/e2e-chainsaw/` (one directory per app), run with `chainsaw test`. Cluster bootstrap (`hack/e2e-install-cozystack.bats`) and the OpenAPI checks (`hack/e2e-test-openapi.bats`) remain BATS. Conventions for writing and stabilising them — and the CI that runs them — live in [`e2e-testing.md`](./e2e-testing.md). - **Go tests:** standard `go test`, with Ginkgo/Gomega for controllers. diff --git a/hack/backport-batch-tests-needed.bats b/hack/backport-batch-tests-needed.bats new file mode 100644 index 000000000..9f37e768f --- /dev/null +++ b/hack/backport-batch-tests-needed.bats @@ -0,0 +1,87 @@ +#!/usr/bin/env bats +# Unit tests for hack/backport-batch-tests-needed.sh, which decides whether a +# pull request runs `make test-backport-batch`, and for its wiring into the +# checks job of pull-requests.yaml. +# +# Test-level EXIT traps replace Bats' own handler and hide failing TAP results. +# Cleanup follows aborting assertions; see docs/agents/e2e-testing.md. + +load test_helper + +CLASSIFIER="hack/backport-batch-tests-needed.sh" +WF=".github/workflows/pull-requests.yaml" + +classify() { + list="$1" + shift + printf '%s\n' "$@" > "$list" + "$CLASSIFIER" "$list" +} + +@test "a change to the tool, an imported package, a page its tests read or the module runs the tests" { + tmp=$(mktemp) + [ "$(classify "$tmp" cmd/backport-batch/report_test.go)" = true ] + [ "$(classify "$tmp" internal/apigate/surface.go)" = true ] + [ "$(classify "$tmp" pkg/config/config.go)" = true ] + [ "$(classify "$tmp" docs/release/backports.md)" = true ] + [ "$(classify "$tmp" docs/release/backport-tasks/conflict.md)" = true ] + [ "$(classify "$tmp" go.sum)" = true ] + # One input among unrelated paths is enough. + [ "$(classify "$tmp" packages/apps/mariadb/values.yaml pkg/config/config.go)" = true ] + rm -f "$tmp" +} + +@test "a change that reaches none of its inputs does not run the tests" { + tmp=$(mktemp) + [ "$(classify "$tmp" packages/apps/mariadb/values.yaml)" = false ] + [ "$(classify "$tmp" cmd/api-gate/main.go)" = false ] + [ "$(classify "$tmp" hack/backport-batch-scheduled.sh)" = false ] + [ "$(classify "$tmp" docs/release.md docs/release/other.md)" = false ] + # A file entry is not a prefix: backports.md does not cover backports.md.bak, + # and a directory entry does not cover a sibling sharing its name as prefix. + [ "$(classify "$tmp" docs/release/backports.md.bak)" = false ] + [ "$(classify "$tmp" pkg/configx/x.go)" = false ] + [ "$(classify "$tmp" internal/apigate-old/x.go)" = false ] + rm -f "$tmp" +} + +@test "entries are trimmed as the Go test trims them" { + tmp=$(mktemp -d) + printf '%s\n' cmd/backport-batch/main.go > "$tmp/diff" + printf ' cmd/backport-batch/ \r\n' > "$tmp/inputs" + [ "$("$CLASSIFIER" "$tmp/diff" "$tmp/inputs")" = true ] + rm -rf "$tmp" +} + +@test "an inputs file with no entry is refused rather than read as never" { + tmp=$(mktemp -d) + printf '%s\n' cmd/backport-batch/main.go > "$tmp/diff" + printf '# only a comment\n\n' > "$tmp/inputs" + if "$CLASSIFIER" "$tmp/diff" "$tmp/inputs" >"$tmp/out" 2>"$tmp/err"; then + echo "an empty inputs file must fail, it printed: $(cat "$tmp/out")" >&2 + exit 1 + fi + grep -q 'lists no path' "$tmp/err" + rm -rf "$tmp" +} + +@test "the checks job runs the target on the classifier's answer alone" { + plan_run=$(yq -r '.jobs.plan.steps[] | select(.id == "p") | .run' "$WF") + printf '%s\n' "$plan_run" | grep -qF 'backport_batch="$(hack/backport-batch-tests-needed.sh /tmp/changed_norenames.txt)"' + [ "$(yq -r '.jobs.plan.outputs.backport_batch' "$WF")" = '${{ steps.p.outputs.backport_batch }}' ] + + step_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Run backport-batch tests") | .if' "$WF") + step_run=$(yq -r '.jobs.checks.steps[] | select(.name == "Run backport-batch tests") | .run' "$WF") + # !cancelled(): a failed unit lane above must not skip this suite too. + [ "$step_if" = "\${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }}" ] + [ "$step_run" = 'make test-backport-batch' ] + + # And nowhere else: the unit lane every code PR runs must not carry it. + prereqs=$(sed -n 's/^unit-tests:[[:space:]]*//p' Makefile) + [ -n "$prereqs" ] + case " $prereqs " in + *" test-backport-batch "*) + echo "make unit-tests still runs test-backport-batch on every code PR: $prereqs" >&2 + exit 1 ;; + esac +} diff --git a/hack/backport-batch-tests-needed.sh b/hack/backport-batch-tests-needed.sh new file mode 100755 index 000000000..71803e730 --- /dev/null +++ b/hack/backport-batch-tests-needed.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Print whether a changed-path list touches an input of +# `go test ./cmd/backport-batch/...`, as listed in +# cmd/backport-batch/test-inputs.txt. An entry ending in / matches every path +# below it, any other entry matches that file alone. The caller supplies the +# rename-blind list so both sides of a rename are classified. +# +# Usage: hack/backport-batch-tests-needed.sh [] +set -eu + +inputs="${2:-cmd/backport-batch/test-inputs.txt}" +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || [ ! -r "$1" ] || [ ! -r "$inputs" ]; then + echo "usage: backport-batch-tests-needed.sh []" >&2 + exit 2 +fi + +# An inputs file with no entry would answer false for every diff and switch the +# tests off without a word; refuse instead. +if ! grep -qvE '^[[:space:]]*(#|$)' "$inputs"; then + echo "backport-batch-tests-needed: $inputs lists no path" >&2 + exit 2 +fi + +# Entries are trimmed the way TestInputsListed trims them, so a trailing space +# or a CR cannot make the Go test accept an entry this never matches. +awk ' + NR == FNR { + e = $0 + sub(/^[[:space:]]+/, "", e) + sub(/[[:space:]]+$/, "", e) + if (e != "" && e !~ /^#/) entries[++n] = e + next + } + { + for (i = 1; i <= n; i++) { + e = entries[i] + if ((substr(e, length(e)) == "/" && index($0, e) == 1) || $0 == e) { + found = 1 + exit + } + } + } + END { print (found ? "true" : "false") } +' "$inputs" "$1" diff --git a/hack/build-matrix.sh b/hack/build-matrix.sh index 95e894a7f..d15267631 100755 --- a/hack/build-matrix.sh +++ b/hack/build-matrix.sh @@ -67,7 +67,13 @@ if [ -z "$CHANGED" ] || [ "$CHANGED" = "FULL" ]; then exit 0 fi -if grep -qE "$full_rebuild_pattern" "$CHANGED"; then +# CI-only Go tools that no image builds. Dropped before the full-rebuild check, +# whose cmd/ would otherwise rebuild everything for an edit no image contains. +# hack/select-e2e.sh skips the same two, and a diff made only of them and docs/ +# never reaches this script's build jobs (the plan job's `code` is false). +tooling_pattern='^cmd/(backport-batch|api-gate)/' + +if grep -vE "$tooling_pattern" "$CHANGED" | grep -qE "$full_rebuild_pattern"; then emit_json "$units" exit 0 fi diff --git a/hack/build-matrix_test.bats b/hack/build-matrix_test.bats index 44a9f2c04..c11f2707f 100644 --- a/hack/build-matrix_test.bats +++ b/hack/build-matrix_test.bats @@ -97,6 +97,23 @@ load test_helper rm -f "$tmp" } +@test "the CI-only tools do not force the full matrix" { + tmp=$(mktemp) + # No image builds cmd/backport-batch or cmd/api-gate, so they select nothing + # and leave the rest of the diff to its own rules. + printf '%s\n' cmd/backport-batch/report_test.go cmd/api-gate/main.go > "$tmp" + out=$(hack/build-matrix.sh "$tmp") + [ "$out" = '[]' ] + printf '%s\n' cmd/backport-batch/main.go packages/apps/mariadb/Makefile > "$tmp" + out=$(hack/build-matrix.sh "$tmp") + [ "$out" = '["packages/apps/mariadb"]' ] + # Every other cmd/ directory is a shipped binary and still rebuilds all. + printf '%s\n' cmd/backport-batch/main.go cmd/cozystack-api/main.go > "$tmp" + out=$(hack/build-matrix.sh "$tmp") + [ "$out" = "$(hack/build-matrix.sh)" ] + rm -f "$tmp" +} + @test "a package with no image target selects nothing" { tmp=$(mktemp) # postgres ships no in-repo image build, so it is not a build unit. diff --git a/hack/downstream-trigger-map.bats b/hack/downstream-trigger-map.bats index c5ebf6af3..39e1d5433 100644 --- a/hack/downstream-trigger-map.bats +++ b/hack/downstream-trigger-map.bats @@ -35,8 +35,9 @@ # a human's job, and reviewers should not assume CI did it for them. # # A docs-only PR skips builds and E2E, but the `checks` job still runs the Bats -# lane. Otherwise editing this map, or another document audited by a unit -# contract, would exempt the changed input from its own guard. +# lane whenever a docs/ path changed. Otherwise editing this map, or another +# document audited by a unit contract, would exempt the changed input from its +# own guard. load test_helper @@ -117,9 +118,18 @@ map_section() { [ "$job_if" = "!contains(github.event.pull_request.labels.*.name, 'release')" ] [ "$code_if" = "needs.plan.outputs.code == 'true'" ] - [ "$docs_if" = "needs.plan.outputs.code == 'false'" ] + [ "$docs_if" = "needs.plan.outputs.code == 'false' && needs.plan.outputs.docs == 'true'" ] [ "$docs_run" = 'make bats-unit-tests' ] [ "$code_run" = 'make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target' ] + + # `code` is false for the CI-only tools too, so the docs lane is keyed on a + # docs/ path being present, not on `code` alone. Pin that the output means + # exactly that and that Bats gets installed for it. + plan_run=$(yq -r '.jobs.plan.steps[] | select(.id == "p") | .run' "$WORKFLOW") + printf '%s\n' "$plan_run" | grep -qF "if grep -qE '^docs/' /tmp/changed_norenames.txt; then" + [ "$(yq -r '.jobs.plan.outputs.docs' "$WORKFLOW")" = '${{ steps.p.outputs.docs }}' ] + toolchain_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Set up test toolchain") | .if' "$WORKFLOW") + [ "$toolchain_if" = "needs.plan.outputs.code == 'true' || needs.plan.outputs.docs == 'true'" ] } @test "the trigger map and the PR-template checklist list the same repositories" { diff --git a/hack/gate-concurrency-contract.bats b/hack/gate-concurrency-contract.bats index 5133a4cd8..0c4c90f15 100644 --- a/hack/gate-concurrency-contract.bats +++ b/hack/gate-concurrency-contract.bats @@ -266,24 +266,56 @@ resolve_silent_conclusions() { plan_block="$(job_block plan "$PULL_REQUESTS" | code_lines)" [ -n "$plan_block" ] - count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^docs/' /tmp/changed_norenames.txt" || true)" + count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^(.*)' /tmp/changed_norenames.txt" || true)" [ "${count:-0}" -eq 1 ] # The rename-detected list may still feed the build matrix, but never this. - count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^docs/' /tmp/changed.txt" || true)" + count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^(.*)' /tmp/changed.txt" || true)" [ "${count:-0}" -eq 0 ] # The fork half of the same agreement: both sides of a rename are inspected. - # Pinned on the docs-only expression itself — `previous_filename` also appears + # Pinned on the skip expression itself — `previous_filename` also appears # in the TIA file list a few lines down, so counting the identifier would stay # satisfied by that one with this check deleted. block="$(job_block resolve "$FORK" | code_lines)" [ -n "$block" ] count="$(printf '%s\n' "$block" \ - | grep -c 'f\.previous_filename && !isDocs(f\.previous_filename)' || true)" + | grep -c 'f\.previous_filename && !isNoRuntime(f\.previous_filename)' || true)" [ "${count:-0}" -eq 1 ] } +@test "both lanes skip e2e for the same path prefixes" { + # The rename test above pins HOW each lane reads its list; this one pins + # WHAT it skips. A prefix added to one lane only is a fork PR that one lane + # reads as nothing-to-build and the other as needing the pr-patch artifact, + # which nothing will ever produce. + plan_prefixes="$(job_block plan "$PULL_REQUESTS" | code_lines \ + | sed -nE "s,.*grep -qvE '\^\(([^)]*)\)' /tmp/changed_norenames\.txt.*,\1,p" \ + | tr '|' '\n' | sort)" + fork_prefixes="$(job_block resolve "$FORK" | code_lines \ + | sed -nE "s,.*const noRuntimePrefixes = \[([^]]*)\];.*,\1,p" \ + | tr ',' '\n' | tr -d " '" | sort)" + [ -n "$plan_prefixes" ] + printf '%s\n' "$plan_prefixes" | grep -qx 'docs/' + if [ "$plan_prefixes" != "$fork_prefixes" ]; then + echo "plan skips:" >&2; printf ' %s\n' $plan_prefixes >&2 + echo "fork skips:" >&2; printf ' %s\n' $fork_prefixes >&2 + exit 1 + fi + + # The tools in that list must also be the ones the build matrix and the suite + # selector leave alone in a mixed diff; otherwise a tool skipped on its own + # still rebuilds everything beside a one-line package change. + tools="$(printf '%s\n' "$plan_prefixes" | grep -vx 'docs/')" + [ -n "$tools" ] + matrix_tools="$(sed -nE "s,^tooling_pattern='\^cmd/\(([^)]*)\)/'$,\1,p" "$REPO_ROOT/hack/build-matrix.sh" \ + | tr '|' '\n' | sed 's,^,cmd/,; s,$,/,' | sort)" + select_tools="$(sed -nE 's,^ (cmd/[^)]*)\)$,\1,p' "$REPO_ROOT/hack/select-e2e.sh" \ + | tr '|' '\n' | sed 's,\*$,,' | sort)" + [ "$tools" = "$matrix_tools" ] + [ "$tools" = "$select_tools" ] +} + @test "same-repo gate: every publishing guard names the same label set" { # `plan`, `verify-release-candidate`, `e2e-report` and the concurrency key # each decide from the label name whether this run owns the status. Renaming diff --git a/hack/select-e2e.sh b/hack/select-e2e.sh index fcb7651e3..59db4fe25 100755 --- a/hack/select-e2e.sh +++ b/hack/select-e2e.sh @@ -8,8 +8,9 @@ # Output: # - empty no E2E impact, decided by one of the rules that select # nothing: step 1 for docs/, dashboards/ and *.md, step 2 for -# a *.disabled Chainsaw suite and for an -# examples/backups// with no suite, and +# a *.disabled Chainsaw suite, for an +# examples/backups// with no suite and for the CI-only +# tools cmd/backport-batch/ and cmd/api-gate/, and # inert_config_pattern for repo meta and agent config # - selected per the PackageSource dependency graph # - full list any path that affects all tests, OR an unrecognised @@ -497,6 +498,14 @@ while IFS= read -r file || [ -n "$file" ]; do echo "select-e2e: '$file' belongs to no single backup walkthrough — escalating to the full suite" >&2 trigger_full=1 continue ;; + cmd/backport-batch/*|cmd/api-gate/*) + # CI-only Go tools. No image builds them and nothing in the e2e sandbox + # runs them, so an edit cannot regress a suite. Matched here because + # full_suite_pattern takes all of cmd/ and is checked before the inert + # list. The plan job's `code` output and e2e-fork.yaml skip e2e for a + # diff made only of these and docs/; this arm keeps them from escalating + # a diff that also touches something else. + continue ;; esac # 3. Full-suite trigger diff --git a/hack/select-e2e_test.bats b/hack/select-e2e_test.bats index 6ce198830..f79d6b05c 100644 --- a/hack/select-e2e_test.bats +++ b/hack/select-e2e_test.bats @@ -202,6 +202,23 @@ assert_full_suite() { rm -rf "$tmp" } +@test "the CI-only tools select nothing and do not escalate the rest" { + tmp=$(mktemp -d) + cp -r packages/core/platform/sources "$tmp/sources" + printf '%s\n' cmd/backport-batch/report_test.go cmd/api-gate/main.go > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + [ -z "$output" ] + # Beside a package, the package alone decides the selection. + printf '%s\n' cmd/backport-batch/main.go packages/apps/postgres/values.yaml > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + assert_selection "a CI-only tool must not widen a package selection" "$output" "postgres" + # Any other cmd/ directory is a shipped binary and still escalates. + echo "cmd/cozystack-api/main.go" > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources" 2>/dev/null) + assert_full_suite "$output" + rm -rf "$tmp" +} + @test "kubernetes-application maps to the two kubernetes suites" { tmp=$(mktemp -d) cp -r packages/core/platform/sources "$tmp/sources"