ci: run only the backport-batch tests for a backport-batch change

A pull request that changed only cmd/backport-batch built every image,
installed the platform for e2e and ran the whole unit lane, so a
one-line test fix waited hours on suites that cannot exercise it. The
backport-batch tests themselves ran on every code pull request,
although only a change to the tool, to what it imports or to the pages
its tests read can break them.

Their inputs are listed beside the tool and checked against its import
graph, because a hand-kept list that fell behind would switch the
tests off without anyone noticing.

Assisted-by: LLM
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
This commit is contained in:
Myasnikov Daniil committed 2026-11-13 13:11:46 +05:00
1 parent 1619cb4eb9
commit 48eae9ad36
11 files changed
+378 -28

No files matched your search

+14 -7
View File
@@ -61,7 +61,8 @@ jobs:
# base-repo PR file list — NOT from the presence/absence of an artifact the
# fork's own build controls. Fail-closed if the fork build failed or the
# pr-patch artifact is missing; green-and-stop only for a PR whose file list is
# entirely under docs/ (verified against the base-repo API).
# entirely under docs/ or the CI-only tools cmd/backport-batch/ and
# cmd/api-gate/ (verified against the base-repo API).
resolve:
name: Resolve fork PR
# Only fork PRs: head_repository differs from the base repository. Same-repo
@@ -266,15 +267,21 @@ jobs:
await decide('failure', 'PR file list hit the 3000-file API cap — cannot verify docs-only');
return;
}
// Paths with nothing to build and nothing for e2e to install: docs,
// and the CI-only Go tools no image builds. pull-requests.yaml's
// plan job answers the same question with the same prefixes, and
// hack/gate-concurrency-contract.bats holds the two lists equal.
//
// A rename reports the NEW path in `filename` and the old one in
// `previous_filename`. Checking only `filename` would read a
// packages/… -> docs/… rename as docs-only, when it in fact removes
// a code path. BOTH sides must be under docs/ to count as docs-only.
const isDocs = p => typeof p === 'string' && p.startsWith('docs/');
const hasNonDocs = files.some(f =>
!isDocs(f.filename) || (f.previous_filename && !isDocs(f.previous_filename)));
if (!hasNonDocs) {
await decide('success', 'docs-only PR (verified via PR file list) — e2e not required');
// a code path. BOTH sides must be under these prefixes to count.
const noRuntimePrefixes = ['docs/', 'cmd/backport-batch/', 'cmd/api-gate/'];
const isNoRuntime = p => typeof p === 'string' && noRuntimePrefixes.some(x => p.startsWith(x));
const hasRuntime = files.some(f =>
!isNoRuntime(f.filename) || (f.previous_filename && !isNoRuntime(f.previous_filename)));
if (!hasRuntime) {
await decide('success', 'docs or CI-only tools (verified via PR file list) — e2e not required');
return;
}
+40 -11
View File
@@ -75,6 +75,8 @@ jobs:
statuses: write
outputs:
code: ${{ steps.p.outputs.code }}
docs: ${{ steps.p.outputs.docs }}
backport_batch: ${{ steps.p.outputs.backport_batch }}
matrix: ${{ steps.p.outputs.matrix }}
any: ${{ steps.p.outputs.any }}
touched: ${{ steps.p.outputs.touched }}
@@ -167,11 +169,27 @@ jobs:
# alone here rather than widened by a side effect.
git diff --name-only --no-renames "origin/${BASE_REF}...HEAD" > /tmp/changed_norenames.txt
echo "Changed files:"; sed 's/^/ /' /tmp/changed_norenames.txt
if grep -qvE '^docs/' /tmp/changed_norenames.txt; then
# `code` is false when every path is under docs/ or in a CI-only Go
# tool that no image builds (cmd/backport-batch, cmd/api-gate): there
# is nothing to build and nothing for e2e to install. e2e-fork.yaml
# answers the same question from the PR file list with the same
# prefixes, and hack/gate-concurrency-contract.bats holds the two lists
# equal.
if grep -qvE '^(docs/|cmd/backport-batch/|cmd/api-gate/)' /tmp/changed_norenames.txt; then
echo "code=true" >> "$GITHUB_OUTPUT"
else
echo "code=false" >> "$GITHUB_OUTPUT"
fi
# The Bats contracts read docs, so a docs change runs them even when
# `code` is false; a change confined to the CI-only tools does not.
if grep -qE '^docs/' /tmp/changed_norenames.txt; then
echo "docs=true" >> "$GITHUB_OUTPUT"
else
echo "docs=false" >> "$GITHUB_OUTPUT"
fi
backport_batch="$(hack/backport-batch-tests-needed.sh /tmp/changed_norenames.txt)"
echo "backport-batch tests needed: $backport_batch"
echo "backport_batch=$backport_batch" >> "$GITHUB_OUTPUT"
matrix="$(hack/build-matrix.sh /tmp/changed.txt)"
echo "Build matrix: $matrix"
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
@@ -293,9 +311,11 @@ jobs:
contents: read
needs: ["plan"]
# Unit contracts read both code and documentation. Run this job for every
# non-release PR: code changes take the complete unit path, while docs-only
# changes run the Bats lane that owns those cross-tree contracts. Builds and
# E2E remain gated on `code` below.
# non-release PR: code changes take the complete unit path, docs changes run
# the Bats lane that owns those cross-tree contracts, and the backport-batch
# tests run whenever one of their inputs changed and at no other time. A
# change confined to the CI-only tools runs no other lane. Builds and E2E
# remain gated on `code` below.
if: |
!contains(github.event.pull_request.labels.*.name, 'release')
steps:
@@ -306,20 +326,21 @@ jobs:
# The module cache is restored from the key build-main.yaml writes on
# main, which every PR run can read, so a PR's first run starts warm. The
# download fetches whatever the cache lacks, the whole set on a miss. The
# Go steps are for code changes only: a docs-only run is the Bats lane alone.
# Go steps are for code changes and the backport-batch tests only: a
# docs-only run is the Bats lane alone.
- name: Locate Go module cache
if: needs.plan.outputs.code == 'true'
if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true'
id: gomodcache
run: echo "dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
- name: Cache Go modules
if: needs.plan.outputs.code == 'true'
if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true'
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ steps.gomodcache.outputs.dir }}
key: go-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
restore-keys: go-mod-${{ runner.os }}-
- name: Download Go modules
if: needs.plan.outputs.code == 'true'
if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true'
run: |
for mod in $(git ls-files go.mod '*/go.mod' | xargs -n1 dirname); do
(cd "$mod" && "$GITHUB_WORKSPACE/hack/go-mod-download.sh")
@@ -327,8 +348,9 @@ jobs:
# Ephemeral runners don't carry the helm-unittest plugin the persistent
# self-hosted runner accreted. The plugin embeds its own Helm, so its
# version is pinned: a plugin release must not change test results. Bats is
# installed on every run, docs-only ones included.
# installed whenever a Bats lane runs, docs-only ones included.
- name: Set up test toolchain
if: needs.plan.outputs.code == 'true' || needs.plan.outputs.docs == 'true'
env:
CODE_CHANGED: ${{ needs.plan.outputs.code }}
HELM_UNITTEST_VERSION: "1.1.2"
@@ -374,8 +396,15 @@ jobs:
if: needs.plan.outputs.code == 'true'
run: make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target
- name: Run Bats unit tests for docs-only changes
if: needs.plan.outputs.code == 'false'
if: needs.plan.outputs.code == 'false' && needs.plan.outputs.docs == 'true'
run: make bats-unit-tests
# Not part of `make unit-tests`: hack/backport-batch-tests-needed.sh
# decides from cmd/backport-batch/test-inputs.txt, whatever `code` is.
# `!cancelled()` for the reason the unit lane runs with -k: a red step
# above must not hide this suite's own result.
- name: Run backport-batch tests
if: ${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }}
run: make test-backport-batch
- name: Upload unit test report
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
@@ -1427,7 +1456,7 @@ jobs:
description = `release packages candidate did not verify (result: ${CANDIDATE_RESULT})`;
} else if (CODE !== 'true') {
state = 'success';
description = 'docs-only PR — e2e not required';
description = 'docs or CI-only tools — e2e not required';
} else if (IS_RELEASE === 'true' && IS_FULL_E2E !== 'true') {
// Release promotion PRs deliberately do not repeat e2e: the rc was
// already verified against the immutable candidate whose digests
+4 -1
View File
@@ -120,7 +120,7 @@ test:
make -C packages/core/testing apply
make -C packages/core/testing e2e
unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness test-backport-batch migrations-target-check
unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness migrations-target-check
helm-unit-tests:
hack/helm-unit-tests.sh
@@ -196,6 +196,9 @@ test-check-readiness:
go test ./test/check-readiness/ -count=1
# ./cmd/... is excluded from go-unit-tests, so the batch tool needs its own target.
# It is not a prerequisite of unit-tests: the checks job in pull-requests.yaml
# runs it only when the diff touches a path listed in
# cmd/backport-batch/test-inputs.txt.
test-backport-batch:
go test ./cmd/backport-batch/... -count=1
+112
View File
@@ -0,0 +1,112 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"os"
"os/exec"
"strings"
"testing"
)
const modulePath = "github.com/cozystack/cozystack/"
// readTestInputs returns the entries of test-inputs.txt, the list CI reads to
// decide whether a pull request runs these tests.
func readTestInputs(t *testing.T) []string {
t.Helper()
data, err := os.ReadFile("test-inputs.txt")
if err != nil {
t.Fatal(err)
}
var entries []string
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
entries = append(entries, line)
}
if len(entries) == 0 {
t.Fatal("test-inputs.txt lists no path")
}
return entries
}
// covered reports whether a change to path matches an entry the way the plan
// job matches it: a trailing slash is a directory prefix, anything else is
// one file.
func covered(entries []string, path string) bool {
for _, e := range entries {
if strings.HasSuffix(e, "/") && strings.HasPrefix(path, e) || path == e {
return true
}
}
return false
}
// A pull request that changes an input of these tests has to run them, and CI
// knows the inputs only from test-inputs.txt. An import added here, or added to
// a package this tool imports, lands in a pull request that touches a listed
// path, so it runs this test, which fails until the new package is listed.
func TestInputsListed(t *testing.T) {
entries := readTestInputs(t)
out, err := exec.Command("go", "list", "-deps", "-test", "-f", "{{.ImportPath}}", ".").Output()
if err != nil {
t.Fatalf("go list: %v", err)
}
var deps int
for _, pkg := range strings.Split(strings.TrimSpace(string(out)), "\n") {
// A test variant prints as "pkg [pkg.test]"; the synthesized main
// package as "pkg.test". Both are this directory.
pkg, _, _ = strings.Cut(pkg, " ")
pkg = strings.TrimSuffix(pkg, ".test")
rel, ok := strings.CutPrefix(pkg, modulePath)
if !ok {
continue
}
deps++
if !covered(entries, rel+"/x.go") {
t.Errorf("these tests import %s, which test-inputs.txt does not list: add %s/", pkg, rel)
}
}
if deps == 0 {
t.Fatalf("go list named no package of %s, so nothing was checked:\n%s", modulePath, out)
}
for _, page := range []string{policyPage, "docs/release/backport-tasks/x.md"} {
if !covered(entries, page) {
t.Errorf("the tests read %s, which test-inputs.txt does not cover", page)
}
}
for _, f := range []string{"go.mod", "go.sum"} {
if !covered(entries, f) {
t.Errorf("test-inputs.txt does not list %s", f)
}
}
}
// A listed path that no longer exists makes the list read as broader than it
// is, and usually means a dependency moved and its new home is missing.
func TestInputsExist(t *testing.T) {
for _, e := range readTestInputs(t) {
if _, err := os.Stat("../../" + e); err != nil {
t.Errorf("test-inputs.txt lists %s: %v", e, err)
}
}
}
+26
View File
@@ -0,0 +1,26 @@
# Every path whose change can change the result of
# `go test ./cmd/backport-batch/...`. The plan job in
# .github/workflows/pull-requests.yaml runs those tests when, and only when, the
# pull request touches one of these paths.
#
# One repository-relative path per line. A trailing slash names a directory and
# matches everything below it; anything else matches that file alone. Blank lines
# and lines starting with # are ignored.
#
# TestInputsListed in inputs_test.go keeps this list in step with the code: it
# fails when the tests' in-repo package dependencies (go list -deps -test) or the
# docs pages they read are missing from it.
cmd/backport-batch/
go.mod
go.sum
# In-repo packages the tool and its tests import.
internal/apigate/
pkg/apis/apps/v1alpha1/
pkg/apis/apps/validation/
pkg/config/
# Pages the tests read from the tree.
docs/release/backports.md
docs/release/backport-tasks/
+1 -1
View File
@@ -121,7 +121,7 @@ Be precise about TIA's scope — it is narrower than "skip E2E for unrelated PRs
- **It trims only which suites `chainsaw test` runs, not the expensive stages.** `Prepare environment` and `Install Cozystack` (the full platform install) run regardless of the selection. TIA only decides the `CHAINSAW_SUITES` passed to the final E2E step. A perfect narrow saves the suite tail, not the bulk of wall-time. (Package image builds are scoped separately by `hack/build-matrix.sh`; the Talos leg runs when its direct or shared inputs changed and remains unconditional for forks, while installer/finalize runs for every non-docs PR. That is a distinct mechanism from TIA's suite trimming.)
- **`Select E2E tests` runs *after* `Install Cozystack` and has no `always()`.** GitHub implicitly ANDs a step's `if:` with `success()`, so if install fails the selector (and `Run E2E tests`) are skipped entirely. On a red install you will not see TIA narrow anything — that is the ordering, not a selector bug.
- **Most PRs legitimately escalate to the full suite.** Platform manifests live under `packages/core/`, Go under `api/`/`internal/`, helpers under `hack/`, plus the `Makefile` — touching any one triggers the full suite. So "all suites selected" is usually correct, not a failure to narrow.
- **Two skip layers, with a gap.** The `plan` job computes a coarse docs-only gate (`code`, true for any change outside `docs/**`) that skips the whole build+E2E pipeline for docs-only PRs, plus a per-package build matrix (`hack/build-matrix.sh`) that scopes which images build. TIA's finer skip (`select-e2e.sh` ignoring `docs/`, `dashboards/`, `*.md` and the rest of `inert_config_pattern`) only trims the suite selection. A PR touching only `dashboards/*.json` therefore still runs install, then skips just the Chainsaw suites — near-zero savings. If you need such PRs to skip the pipeline, widen the `plan` docs-gate, not `select-e2e.sh`.
- **Two skip layers, with a gap.** The `plan` job computes a coarse gate (`code`, false only when every changed path is under `docs/` or one of the CI-only Go tools no image builds, `cmd/backport-batch/` and `cmd/api-gate/`) that skips the whole build+E2E pipeline for such PRs, and `e2e-fork.yaml` skips on the same prefixes (`hack/gate-concurrency-contract.bats` holds the two lists equal), plus a per-package build matrix (`hack/build-matrix.sh`) that scopes which images build. TIA's finer skip (`select-e2e.sh` ignoring `docs/`, `dashboards/`, `*.md` and the rest of `inert_config_pattern`) only trims the suite selection. A PR touching only `dashboards/*.json` therefore still runs install, then skips just the Chainsaw suites — near-zero savings. If you need such PRs to skip the pipeline, widen the `plan` docs-gate, not `select-e2e.sh`.
### 8. Failure must be self-explanatory — diagnostics are first-class
+1 -1
View File
@@ -106,7 +106,7 @@ A chart also declares the limits on its own resource name there, with `## @name
## Testing
- **Helm and BATS unit tests:** `make helm-unit-tests` runs `hack/helm-unit-tests.sh` over every package that defines a `test` target. A `test` target is expected to run helm-unittest: the sweep fails a package whose run reports no suite, because exiting 0 having asserted nothing is indistinguishable from passing. A target that drives something else belongs under another name, as `packages/core/testing` already does with its sandbox flow. `make bats-unit-tests` requires bats-core 1.5 or newer; GNU parallel is optional and the target runs serially without it. Run the complete local BATS and POSIX-compatibility pass with `pre-commit run bats-unit-tests --hook-stage manual --all-files`; pull-request CI runs both lanes for code changes and the BATS contracts for docs-only changes. On a host whose `/bin/sh` differs from CI, `make BATS_POSIX_SHELL=dash bats-posix-compat-tests` selects the same interpreter explicitly. `make unit-tests` runs the full unit suite — Helm, BATS, the targeted POSIX-shell compatibility pass, Go, and the preset/readiness checks. It does not include the controller tests under `./internal/...`; run `make test-controllers` as well. CI invokes both targets together.
- **Helm and BATS unit tests:** `make helm-unit-tests` runs `hack/helm-unit-tests.sh` over every package that defines a `test` target. A `test` target is expected to run helm-unittest: the sweep fails a package whose run reports no suite, because exiting 0 having asserted nothing is indistinguishable from passing. A target that drives something else belongs under another name, as `packages/core/testing` already does with its sandbox flow. `make bats-unit-tests` requires bats-core 1.5 or newer; GNU parallel is optional and the target runs serially without it. Run the complete local BATS and POSIX-compatibility pass with `pre-commit run bats-unit-tests --hook-stage manual --all-files`; pull-request CI runs both lanes for code changes and the BATS contracts for docs-only changes. On a host whose `/bin/sh` differs from CI, `make BATS_POSIX_SHELL=dash bats-posix-compat-tests` selects the same interpreter explicitly. `make unit-tests` runs the full unit suite — Helm, BATS, the targeted POSIX-shell compatibility pass, Go, and the preset/readiness checks. It does not include the controller tests under `./internal/...`; run `make test-controllers` as well. CI invokes both targets together. The backport-batch tests are not part of it: run `make test-backport-batch` after changing anything listed in `cmd/backport-batch/test-inputs.txt`, which is also what CI keys them on.
- **E2E tests:** Kyverno Chainsaw suites in `hack/e2e-chainsaw/` (one directory per app), run with `chainsaw test`. Cluster bootstrap (`hack/e2e-install-cozystack.bats`) and the OpenAPI checks (`hack/e2e-test-openapi.bats`) remain BATS. Conventions for writing and stabilising them — and the CI that runs them — live in [`e2e-testing.md`](./e2e-testing.md).
- **Go tests:** standard `go test`, with Ginkgo/Gomega for controllers.
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env bats
# Unit tests for hack/backport-batch-tests-needed.sh, which decides whether a
# pull request runs `make test-backport-batch`, and for its wiring into the
# checks job of pull-requests.yaml.
#
# Test-level EXIT traps replace Bats' own handler and hide failing TAP results.
# Cleanup follows aborting assertions; see docs/agents/e2e-testing.md.
load test_helper
CLASSIFIER="hack/backport-batch-tests-needed.sh"
WF=".github/workflows/pull-requests.yaml"
classify() {
list="$1"
shift
printf '%s\n' "$@" > "$list"
"$CLASSIFIER" "$list"
}
@test "a change to the tool, an imported package, a page its tests read or the module runs the tests" {
tmp=$(mktemp)
[ "$(classify "$tmp" cmd/backport-batch/report_test.go)" = true ]
[ "$(classify "$tmp" internal/apigate/surface.go)" = true ]
[ "$(classify "$tmp" pkg/config/config.go)" = true ]
[ "$(classify "$tmp" docs/release/backports.md)" = true ]
[ "$(classify "$tmp" docs/release/backport-tasks/conflict.md)" = true ]
[ "$(classify "$tmp" go.sum)" = true ]
# One input among unrelated paths is enough.
[ "$(classify "$tmp" packages/apps/mariadb/values.yaml pkg/config/config.go)" = true ]
rm -f "$tmp"
}
@test "a change that reaches none of its inputs does not run the tests" {
tmp=$(mktemp)
[ "$(classify "$tmp" packages/apps/mariadb/values.yaml)" = false ]
[ "$(classify "$tmp" cmd/api-gate/main.go)" = false ]
[ "$(classify "$tmp" hack/backport-batch-scheduled.sh)" = false ]
[ "$(classify "$tmp" docs/release.md docs/release/other.md)" = false ]
# A file entry is not a prefix: backports.md does not cover backports.md.bak,
# and a directory entry does not cover a sibling sharing its name as prefix.
[ "$(classify "$tmp" docs/release/backports.md.bak)" = false ]
[ "$(classify "$tmp" pkg/configx/x.go)" = false ]
[ "$(classify "$tmp" internal/apigate-old/x.go)" = false ]
rm -f "$tmp"
}
@test "entries are trimmed as the Go test trims them" {
tmp=$(mktemp -d)
printf '%s\n' cmd/backport-batch/main.go > "$tmp/diff"
printf ' cmd/backport-batch/ \r\n' > "$tmp/inputs"
[ "$("$CLASSIFIER" "$tmp/diff" "$tmp/inputs")" = true ]
rm -rf "$tmp"
}
@test "an inputs file with no entry is refused rather than read as never" {
tmp=$(mktemp -d)
printf '%s\n' cmd/backport-batch/main.go > "$tmp/diff"
printf '# only a comment\n\n' > "$tmp/inputs"
if "$CLASSIFIER" "$tmp/diff" "$tmp/inputs" >"$tmp/out" 2>"$tmp/err"; then
echo "an empty inputs file must fail, it printed: $(cat "$tmp/out")" >&2
exit 1
fi
grep -q 'lists no path' "$tmp/err"
rm -rf "$tmp"
}
@test "the checks job runs the target on the classifier's answer alone" {
plan_run=$(yq -r '.jobs.plan.steps[] | select(.id == "p") | .run' "$WF")
printf '%s\n' "$plan_run" | grep -qF 'backport_batch="$(hack/backport-batch-tests-needed.sh /tmp/changed_norenames.txt)"'
[ "$(yq -r '.jobs.plan.outputs.backport_batch' "$WF")" = '${{ steps.p.outputs.backport_batch }}' ]
step_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Run backport-batch tests") | .if' "$WF")
step_run=$(yq -r '.jobs.checks.steps[] | select(.name == "Run backport-batch tests") | .run' "$WF")
# !cancelled(): a failed unit lane above must not skip this suite too.
[ "$step_if" = "\${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }}" ]
[ "$step_run" = 'make test-backport-batch' ]
# And nowhere else: the unit lane every code PR runs must not carry it.
prereqs=$(sed -n 's/^unit-tests:[[:space:]]*//p' Makefile)
[ -n "$prereqs" ]
case " $prereqs " in
*" test-backport-batch "*)
echo "make unit-tests still runs test-backport-batch on every code PR: $prereqs" >&2
exit 1 ;;
esac
}
+44
View File
@@ -0,0 +1,44 @@
#!/bin/sh
# Print whether a changed-path list touches an input of
# `go test ./cmd/backport-batch/...`, as listed in
# cmd/backport-batch/test-inputs.txt. An entry ending in / matches every path
# below it, any other entry matches that file alone. The caller supplies the
# rename-blind list so both sides of a rename are classified.
#
# Usage: hack/backport-batch-tests-needed.sh <changed-files> [<inputs-file>]
set -eu
inputs="${2:-cmd/backport-batch/test-inputs.txt}"
if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || [ ! -r "$1" ] || [ ! -r "$inputs" ]; then
echo "usage: backport-batch-tests-needed.sh <readable-changed-files> [<readable-inputs-file>]" >&2
exit 2
fi
# An inputs file with no entry would answer false for every diff and switch the
# tests off without a word; refuse instead.
if ! grep -qvE '^[[:space:]]*(#|$)' "$inputs"; then
echo "backport-batch-tests-needed: $inputs lists no path" >&2
exit 2
fi
# Entries are trimmed the way TestInputsListed trims them, so a trailing space
# or a CR cannot make the Go test accept an entry this never matches.
awk '
NR == FNR {
e = $0
sub(/^[[:space:]]+/, "", e)
sub(/[[:space:]]+$/, "", e)
if (e != "" && e !~ /^#/) entries[++n] = e
next
}
{
for (i = 1; i <= n; i++) {
e = entries[i]
if ((substr(e, length(e)) == "/" && index($0, e) == 1) || $0 == e) {
found = 1
exit
}
}
}
END { print (found ? "true" : "false") }
' "$inputs" "$1"
+13 -3
View File
@@ -35,8 +35,9 @@
# a human's job, and reviewers should not assume CI did it for them.
#
# A docs-only PR skips builds and E2E, but the `checks` job still runs the Bats
# lane. Otherwise editing this map, or another document audited by a unit
# contract, would exempt the changed input from its own guard.
# lane whenever a docs/ path changed. Otherwise editing this map, or another
# document audited by a unit contract, would exempt the changed input from its
# own guard.
load test_helper
@@ -117,9 +118,18 @@ map_section() {
[ "$job_if" = "!contains(github.event.pull_request.labels.*.name, 'release')" ]
[ "$code_if" = "needs.plan.outputs.code == 'true'" ]
[ "$docs_if" = "needs.plan.outputs.code == 'false'" ]
[ "$docs_if" = "needs.plan.outputs.code == 'false' && needs.plan.outputs.docs == 'true'" ]
[ "$docs_run" = 'make bats-unit-tests' ]
[ "$code_run" = 'make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target' ]
# `code` is false for the CI-only tools too, so the docs lane is keyed on a
# docs/ path being present, not on `code` alone. Pin that the output means
# exactly that and that Bats gets installed for it.
plan_run=$(yq -r '.jobs.plan.steps[] | select(.id == "p") | .run' "$WORKFLOW")
printf '%s\n' "$plan_run" | grep -qF "if grep -qE '^docs/' /tmp/changed_norenames.txt; then"
[ "$(yq -r '.jobs.plan.outputs.docs' "$WORKFLOW")" = '${{ steps.p.outputs.docs }}' ]
toolchain_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Set up test toolchain") | .if' "$WORKFLOW")
[ "$toolchain_if" = "needs.plan.outputs.code == 'true' || needs.plan.outputs.docs == 'true'" ]
}
@test "the trigger map and the PR-template checklist list the same repositories" {
+36 -4
View File
@@ -266,24 +266,56 @@ resolve_silent_conclusions() {
plan_block="$(job_block plan "$PULL_REQUESTS" | code_lines)"
[ -n "$plan_block" ]
count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^docs/' /tmp/changed_norenames.txt" || true)"
count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^(.*)' /tmp/changed_norenames.txt" || true)"
[ "${count:-0}" -eq 1 ]
# The rename-detected list may still feed the build matrix, but never this.
count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^docs/' /tmp/changed.txt" || true)"
count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^(.*)' /tmp/changed.txt" || true)"
[ "${count:-0}" -eq 0 ]
# The fork half of the same agreement: both sides of a rename are inspected.
# Pinned on the docs-only expression itself — `previous_filename` also appears
# Pinned on the skip expression itself — `previous_filename` also appears
# in the TIA file list a few lines down, so counting the identifier would stay
# satisfied by that one with this check deleted.
block="$(job_block resolve "$FORK" | code_lines)"
[ -n "$block" ]
count="$(printf '%s\n' "$block" \
| grep -c 'f\.previous_filename && !isDocs(f\.previous_filename)' || true)"
| grep -c 'f\.previous_filename && !isNoRuntime(f\.previous_filename)' || true)"
[ "${count:-0}" -eq 1 ]
}
@test "both lanes skip e2e for the same path prefixes" {
# The rename test above pins HOW each lane reads its list; this one pins
# WHAT it skips. A prefix added to one lane only is a fork PR that one lane
# reads as nothing-to-build and the other as needing the pr-patch artifact,
# which nothing will ever produce.
plan_prefixes="$(job_block plan "$PULL_REQUESTS" | code_lines \
| sed -nE "s,.*grep -qvE '\^\(([^)]*)\)' /tmp/changed_norenames\.txt.*,\1,p" \
| tr '|' '\n' | sort)"
fork_prefixes="$(job_block resolve "$FORK" | code_lines \
| sed -nE "s,.*const noRuntimePrefixes = \[([^]]*)\];.*,\1,p" \
| tr ',' '\n' | tr -d " '" | sort)"
[ -n "$plan_prefixes" ]
printf '%s\n' "$plan_prefixes" | grep -qx 'docs/'
if [ "$plan_prefixes" != "$fork_prefixes" ]; then
echo "plan skips:" >&2; printf ' %s\n' $plan_prefixes >&2
echo "fork skips:" >&2; printf ' %s\n' $fork_prefixes >&2
exit 1
fi
# The tools in that list must also be the ones the build matrix and the suite
# selector leave alone in a mixed diff; otherwise a tool skipped on its own
# still rebuilds everything beside a one-line package change.
tools="$(printf '%s\n' "$plan_prefixes" | grep -vx 'docs/')"
[ -n "$tools" ]
matrix_tools="$(sed -nE "s,^tooling_pattern='\^cmd/\(([^)]*)\)/'$,\1,p" "$REPO_ROOT/hack/build-matrix.sh" \
| tr '|' '\n' | sed 's,^,cmd/,; s,$,/,' | sort)"
select_tools="$(sed -nE 's,^ (cmd/[^)]*)\)$,\1,p' "$REPO_ROOT/hack/select-e2e.sh" \
| tr '|' '\n' | sed 's,\*$,,' | sort)"
[ "$tools" = "$matrix_tools" ]
[ "$tools" = "$select_tools" ]
}
@test "same-repo gate: every publishing guard names the same label set" {
# `plan`, `verify-release-candidate`, `e2e-report` and the concurrency key
# each decide from the label name whether this run owns the status. Renaming