From a8c4640bea007c19d012bbc332b3803051f31a2a Mon Sep 17 00:00:00 2001 From: Ahmed Kamal Date: Sat, 31 Oct 2026 23:07:29 +0200 Subject: [PATCH 1/4] test(tests): select resources in absence-only Helm checks Signed-off-by: Ahmed Kamal Assisted-by: LLM --- .github/workflows/pull-requests.yaml | 4 + hack/check-helm-document-indices.bats | 141 ++++++++++++++++++ hack/check-helm-document-indices.py | 79 ++++++++++ .../tests/dashboard-resourcemap_test.yaml | 4 +- .../tests/kamaji_managed_args_test.yaml | 8 +- packages/apps/mariadb/tests/tls_test.yaml | 27 +++- packages/apps/mongodb/tests/tls_test.yaml | 32 +++- packages/apps/redis/tests/tls_test.yaml | 8 +- .../tests/solver_test.yaml | 8 +- .../tests/volume_resize_hook_test.yaml | 6 +- 10 files changed, 294 insertions(+), 23 deletions(-) create mode 100644 hack/check-helm-document-indices.bats create mode 100644 hack/check-helm-document-indices.py diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index b8b06fedf..96dce2056 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -364,6 +364,10 @@ jobs: with: path: _out/envtest key: envtest-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }} + - name: Check new Helm document indices + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: python3 hack/check-helm-document-indices.py "$(git merge-base HEAD "origin/$BASE_REF")" # -k is required, not cosmetic: unit-tests and test-controllers are two # independent goals on the same command line, and without -k make stops # handing out new work the moment any unit-tests prerequisite fails — diff --git a/hack/check-helm-document-indices.bats b/hack/check-helm-document-indices.bats new file mode 100644 index 000000000..f797d0b7f --- /dev/null +++ b/hack/check-helm-document-indices.bats @@ -0,0 +1,141 @@ +#!/usr/bin/env bats + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" + +setup() { + strict_setup + tmp=$(mktemp -d) + cd "$tmp" + git init -q + mkdir -p packages/apps/example/templates packages/apps/example/tests + cat > packages/apps/example/templates/multi.yaml <<'EOF' +apiVersion: v1 +kind: ConfigMap +metadata: + name: first +--- +apiVersion: v1 +kind: Secret +metadata: + name: second +EOF + head -n 4 packages/apps/example/templates/multi.yaml > packages/apps/example/templates/single.yaml + git add . + git -c user.name=Test -c user.email=test@example.org commit -qm fixture + base=$(git rev-parse HEAD) +} + +teardown() { + cd "$REPO_ROOT" + rm -rf "$tmp" +} + +write_suite() { + cat > packages/apps/example/tests/resource_test.yaml < output 2>&1; then + echo "accepted a positional selector on a multi-document template" >&2 + false + fi + grep -q 'resource_test.yaml:6: use documentSelector' output +} + +@test "allows an index on a single-document template and ignores comments" { + write_suite single + printf '\n# documentIndex: 1\n' >> packages/apps/example/tests/resource_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "leaves existing indices alone when another assertion changes" { + write_suite multi + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + sed -i 's/spec.missing/spec.other/' packages/apps/example/tests/resource_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "follows assertion template overrides in later YAML suites" { + write_suite single + cat >> packages/apps/example/tests/resource_test.yaml <<'EOF' +--- +suite: overridden template +templates: + - templates/single.yaml +tests: + - it: checks another resource + asserts: + - notExists: + path: spec.missing + template: templates/multi.yaml + documentIndex: 1 +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted an assertion-level positional selector" >&2 + false + fi + grep -q 'use documentSelector.*templates/multi.yaml' output +} + +@test "rejects an invalid base rather than passing an empty diff" { + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" missing-base > output 2>&1; then + echo "accepted an invalid base" >&2 + false + fi +} + +@test "test-level indices follow each assertion's template override" { + write_suite multi + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: assertion override +templates: [templates/single.yaml, templates/multi.yaml] +tests: + - it: checks only the single-document template + documentIndex: 0 + asserts: + - template: templates/single.yaml + notExists: + path: spec.missing +EOF + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "conditional kinds inside one document are not multiple documents" { + write_suite single + cat > packages/apps/example/templates/single.yaml <<'EOF' +apiVersion: v1 +{{ if .Values.secret }} +kind: Secret +{{ else }} +kind: ConfigMap +{{ end }} +metadata: + name: single +EOF + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "rejects a positional selector whose value is on a separate line" { + write_suite multi + sed -i 's/documentIndex: 0/documentIndex:\n 0/' packages/apps/example/tests/resource_test.yaml + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a multiline positional selector" >&2 + false + fi + grep -q 'resource_test.yaml:6: use documentSelector' output +} diff --git a/hack/check-helm-document-indices.py b/hack/check-helm-document-indices.py new file mode 100644 index 000000000..bf9c56f47 --- /dev/null +++ b/hack/check-helm-document-indices.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Reject added positional selectors on multi-resource Helm templates. + +Usage: hack/check-helm-document-indices.py +Requires mikefarah/yq, like the chart test tooling. Existing indices are left +alone. This static check counts --- separated blocks with a top-level kind; +it does not evaluate conditional documents or resources emitted by helpers/ranges. +""" + +import json +from pathlib import Path +import re +import subprocess +import sys + + +def git(*args): + return subprocess.check_output(["git", *args], text=True) + + +def check(base): + root = Path(git("rev-parse", "--show-toplevel").strip()) + paths = git("diff", "--name-only", "--diff-filter=ACMR", base, "--", + "packages/*/*/tests/*_test.yaml").splitlines() + errors = [] + for path in paths: + added = set() + for line in git("diff", "--no-ext-diff", "--unified=0", base, "--", path).splitlines(): + if line.startswith("@@"): + number = int(re.search(r"\+(\d+)", line).group(1)) + elif line.startswith(("+++", "---", "diff ", "index ", "\\")): + continue + elif line.startswith("+"): + if "documentIndex" in line: + added.add(number) + number += 1 + elif line.startswith(" "): + number += 1 + if not added: + continue + + # Preserve the key's source line while yq resolves multi-document YAML, + # flow mappings and anchors; a text-only suite parser misses those. + expression = ('[. | (.. | select(tag == "!!map" and has("documentIndex")) ' + '| .documentIndex) |= {"line": key | line}]') + suites = json.loads(subprocess.check_output( + ["yq", "eval-all", "-o=json", expression, str(root / path)], text=True)) + chart = (root / path).parent.parent + for suite in suites: + if not suite: + continue + for test in suite.get("tests", []): + templates = test.get("templates", suite.get("templates", ["templates/*"])) + for assertion in test.get("asserts", []): + index = assertion.get("documentIndex", test.get("documentIndex", {})) + number = index.get("line") + if number not in added: + continue + selected = [assertion["template"]] if "template" in assertion else templates + for pattern in selected: + matches = list(chart.glob(pattern)) + if not matches: + matches = list((chart / "templates").glob(pattern)) + for template in matches: + if not template.is_file(): + continue + documents = re.split(r"(?m)^---[ \t]*(?:#.*)?$", template.read_text()) + if sum(bool(re.search(r"(?m)^kind\s*:", doc)) for doc in documents) > 1: + errors.append(f"{path}:{number}: use documentSelector, not " + f"documentIndex, for {template.relative_to(root)}") + for error in sorted(set(errors)): + print(error, file=sys.stderr) + return bool(errors) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit("usage: check-helm-document-indices.py ") + sys.exit(check(sys.argv[1])) diff --git a/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml b/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml index ace053c58..88da0d69a 100644 --- a/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml +++ b/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml @@ -137,11 +137,13 @@ tests: release: name: test-kafka namespace: tenant-test + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[2].resourceNames content: test-kafka-zookeeper - documentIndex: 0 # External bootstrap service absent when external=false, regardless of tls.enabled - it: does not grant access to external bootstrap service when tls.enabled=true and external=false diff --git a/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml b/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml index b09343cf5..85045c9a1 100644 --- a/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml +++ b/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml @@ -241,18 +241,22 @@ tests: errorMessage: 'controlPlane.apiServer.extraArgs contains "--secure-port=6444" — the tenant control plane sets --secure-port itself and drops every entry naming it before the kube-apiserver sees the list, so this entry has no effect, and written without =value it leaves its value behind as an argument the apiserver refuses. Remove the entry; the description of controlPlane.apiServer.extraArgs lists every flag the control plane owns.' - it: leaves spec.admissionControllers out by default, so the control plane keeps its own default set + documentSelector: + path: kind + value: KamajiControlPlane asserts: - notExists: path: spec.admissionControllers - documentIndex: 1 - it: leaves spec.admissionControllers out for an explicitly empty list set: controlPlane.apiServer.admissionControllers: [] + documentSelector: + path: kind + value: KamajiControlPlane asserts: - notExists: path: spec.admissionControllers - documentIndex: 1 - it: renders controlPlane.apiServer.admissionControllers into spec.admissionControllers verbatim set: diff --git a/packages/apps/mariadb/tests/tls_test.yaml b/packages/apps/mariadb/tests/tls_test.yaml index 2f5f4548a..ac7b8ae79 100644 --- a/packages/apps/mariadb/tests/tls_test.yaml +++ b/packages/apps/mariadb/tests/tls_test.yaml @@ -530,7 +530,9 @@ tests: - templates/dashboard-resourcemap.yaml set: external: false - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -544,7 +546,9 @@ tests: - templates/dashboard-resourcemap.yaml set: external: true - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -560,7 +564,9 @@ tests: external: true tls: issuer: operator - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -576,7 +582,9 @@ tests: external: false tls: issuer: cert-manager - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -695,7 +703,9 @@ tests: set: tls: issuer: cert-manager - documentIndex: 3 + documentSelector: + path: metadata.name + value: my-mariadb-tls asserts: - notContains: path: spec.dnsNames @@ -1187,7 +1197,9 @@ tests: - it: "dashboard Role never grants the CA bundle even when TLS is unmanaged" templates: - templates/dashboard-resourcemap.yaml - documentIndex: 0 + documentSelector: + path: kind + value: Role set: tls: issuer: operator @@ -1436,6 +1448,9 @@ tests: enabled: true s3Bucket: s3.example.org/test cleanupStrategy: "--keep-last=3" + documentSelector: + path: kind + value: Secret asserts: - notMatchRegex: path: stringData["backup.sh"] diff --git a/packages/apps/mongodb/tests/tls_test.yaml b/packages/apps/mongodb/tests/tls_test.yaml index 0ddb38e46..8cf64b8cc 100644 --- a/packages/apps/mongodb/tests/tls_test.yaml +++ b/packages/apps/mongodb/tests/tls_test.yaml @@ -53,7 +53,9 @@ tests: value: mg-test-ssl - it: spec.tls has no issuerConf (the operator-managed secret is passed directly) - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: - notExists: path: spec.tls.issuerConf @@ -127,13 +129,17 @@ tests: # test would pass even with the external check removed. _namespace: host: example.com - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: - notExists: path: spec.replsets[0].splitHorizons - it: chart does not set allowInvalidCertificates, leaving the operator default - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # The README states the permissive default is inherited rather than # chosen here. Pin that, so the claim stays falsifiable. @@ -141,7 +147,9 @@ tests: path: spec.tls.allowInvalidCertificates - it: chart does not set certValidityDuration, leaving the operator default - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # Same class as allowInvalidCertificates above: the leaf lifetime is the # operator's to choose, not this chart's. Deliberately left unpinned. @@ -177,7 +185,9 @@ tests: # the CRD and notExists on it can never fail. - it: no TLS-disabling escape hatches with a default replica count - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: - notExists: path: spec.allowUnsafeConfigurations @@ -235,7 +245,9 @@ tests: - name: rs0 replicas: 3 size: 10Gi - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # splitHorizons is replica-set scoped and rendered only for the # non-sharded topology. In sharded mode external access goes through the @@ -255,7 +267,9 @@ tests: replicas: 0 _namespace: host: example.com - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # With replicas: 0 the range body renders nothing. Emitting the key with # a null value would be rejected by the API server: the CRD types @@ -269,7 +283,9 @@ tests: external: true _namespace: host: "" - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # The controller may create _namespace before populating host, so the # bootstrap window presents host as an empty string rather than as an diff --git a/packages/apps/redis/tests/tls_test.yaml b/packages/apps/redis/tests/tls_test.yaml index 9b1b3ac0b..c8f55ee91 100644 --- a/packages/apps/redis/tests/tls_test.yaml +++ b/packages/apps/redis/tests/tls_test.yaml @@ -557,7 +557,9 @@ tests: external: false tls: enabled: true - documentIndex: 3 + documentSelector: + path: metadata.name + value: my-redis-tls asserts: - notContains: path: spec.dnsNames @@ -840,7 +842,9 @@ tests: external: false tls: enabled: true - documentIndex: 3 + documentSelector: + path: metadata.name + value: my-redis-tls asserts: - notContains: path: spec.dnsNames diff --git a/packages/system/cert-manager-issuers/tests/solver_test.yaml b/packages/system/cert-manager-issuers/tests/solver_test.yaml index 953d978bc..2a9faebbf 100644 --- a/packages/system/cert-manager-issuers/tests/solver_test.yaml +++ b/packages/system/cert-manager-issuers/tests/solver_test.yaml @@ -228,10 +228,14 @@ tests: gateway-class-name: " cloudflare-tunnel " gateway-edge-classes: cloudflare-tunnel asserts: - - documentIndex: 0 + - documentSelector: + path: metadata.name + value: letsencrypt-prod notExists: path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].sectionName - - documentIndex: 1 + - documentSelector: + path: metadata.name + value: letsencrypt-stage notExists: path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].sectionName diff --git a/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml b/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml index 402748312..011a3de94 100644 --- a/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml +++ b/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml @@ -156,9 +156,11 @@ tests: resources: requests: storage: 10Gi + documentSelector: + path: kind + value: Job asserts: - - documentIndex: 0 - notMatchRegex: + - notMatchRegex: path: spec.template.spec.containers[0].args[0] pattern: kubectl patch pvc From 8d40fc8e00ef099fd827d643d2485cfa7d316b18 Mon Sep 17 00:00:00 2001 From: Ahmed Kamal Date: Sat, 31 Oct 2026 23:36:26 +0200 Subject: [PATCH 2/4] fix(tests): honor singular Helm test template overrides Signed-off-by: Ahmed Kamal Assisted-by: LLM --- hack/check-helm-document-indices.bats | 33 +++++++++++++++++++++++++++ hack/check-helm-document-indices.py | 4 +++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/hack/check-helm-document-indices.bats b/hack/check-helm-document-indices.bats index f797d0b7f..ca84af677 100644 --- a/hack/check-helm-document-indices.bats +++ b/hack/check-helm-document-indices.bats @@ -106,6 +106,7 @@ suite: assertion override templates: [templates/single.yaml, templates/multi.yaml] tests: - it: checks only the single-document template + template: templates/multi.yaml documentIndex: 0 asserts: - template: templates/single.yaml @@ -139,3 +140,35 @@ EOF fi grep -q 'resource_test.yaml:6: use documentSelector' output } + +@test "rejects a singular test template overriding a single-document suite" { + write_suite single + sed -i '/ - it:/a\ template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a positional selector on the test template override" >&2 + false + fi + grep -q 'use documentSelector.*templates/multi.yaml' output +} + +@test "allows a singular test template selecting one document" { + write_suite multi + sed -i '/ - it:/a\ template: templates/single.yaml' packages/apps/example/tests/resource_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "plural test templates take precedence over the singular test template" { + write_suite multi + sed -i '/ - it:/a\ templates: [templates/single.yaml]\n template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "an empty plural test override falls back to the singular template" { + write_suite single + sed -i '/ - it:/a\ templates: []\n template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a positional selector after an empty plural override" >&2 + false + fi + grep -q 'use documentSelector.*templates/multi.yaml' output +} diff --git a/hack/check-helm-document-indices.py b/hack/check-helm-document-indices.py index bf9c56f47..212a50964 100644 --- a/hack/check-helm-document-indices.py +++ b/hack/check-helm-document-indices.py @@ -50,7 +50,9 @@ def check(base): if not suite: continue for test in suite.get("tests", []): - templates = test.get("templates", suite.get("templates", ["templates/*"])) + templates = test.get("templates") or ( + [test["template"]] if test.get("template") + else suite.get("templates", ["templates/*"])) for assertion in test.get("asserts", []): index = assertion.get("documentIndex", test.get("documentIndex", {})) number = index.get("line") From 2757e2c60ad35e40d6e378a6049b8a20f6242ea5 Mon Sep 17 00:00:00 2001 From: Ahmed Kamal Date: Sun, 1 Nov 2026 01:23:55 +0200 Subject: [PATCH 3/4] test(tests): tighten Helm selector guard scope and coverage Signed-off-by: Ahmed Kamal Assisted-by: LLM --- .github/workflows/pull-requests.yaml | 9 +- hack/check-helm-document-indices.bats | 174 +++++++++++++++++- hack/check-helm-document-indices.py | 41 +++-- .../apps/kafka/tests/jvm_sizing_test.yaml | 4 +- 4 files changed, 197 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 96dce2056..1f44fc287 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -364,10 +364,6 @@ jobs: with: path: _out/envtest key: envtest-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }} - - name: Check new Helm document indices - env: - BASE_REF: ${{ github.event.pull_request.base.ref }} - run: python3 hack/check-helm-document-indices.py "$(git merge-base HEAD "origin/$BASE_REF")" # -k is required, not cosmetic: unit-tests and test-controllers are two # independent goals on the same command line, and without -k make stops # handing out new work the moment any unit-tests prerequisite fails — @@ -380,6 +376,11 @@ jobs: - name: Run Bats unit tests for docs-only changes if: needs.plan.outputs.code == 'false' run: make bats-unit-tests + - name: Check new Helm document indices + if: ${{ !cancelled() }} + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: python3 hack/check-helm-document-indices.py "$(git merge-base HEAD "origin/$BASE_REF")" - name: Upload unit test report if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/hack/check-helm-document-indices.bats b/hack/check-helm-document-indices.bats index ca84af677..5188f5b26 100644 --- a/hack/check-helm-document-indices.bats +++ b/hack/check-helm-document-indices.bats @@ -66,7 +66,8 @@ EOF write_suite multi git -c user.name=Test -c user.email=test@example.org commit -qm existing base=$(git rev-parse HEAD) - sed -i 's/spec.missing/spec.other/' packages/apps/example/tests/resource_test.yaml + sed 's/spec.missing/spec.other/' packages/apps/example/tests/resource_test.yaml > changed.yaml + mv changed.yaml packages/apps/example/tests/resource_test.yaml python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" } @@ -133,17 +134,37 @@ EOF @test "rejects a positional selector whose value is on a separate line" { write_suite multi - sed -i 's/documentIndex: 0/documentIndex:\n 0/' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: multiline index +templates: [templates/multi.yaml] +tests: + - it: checks a resource + documentIndex: + 0 + asserts: + - notExists: + path: spec.missing +EOF if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then echo "accepted a multiline positional selector" >&2 false fi - grep -q 'resource_test.yaml:6: use documentSelector' output + grep -q 'resource_test.yaml:5: use documentSelector' output } @test "rejects a singular test template overriding a single-document suite" { write_suite single - sed -i '/ - it:/a\ template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/single.yaml] +tests: + - it: checks a resource + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then echo "accepted a positional selector on the test template override" >&2 false @@ -153,22 +174,161 @@ EOF @test "allows a singular test template selecting one document" { write_suite multi - sed -i '/ - it:/a\ template: templates/single.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/multi.yaml] +tests: + - it: checks a resource + template: templates/single.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" } @test "plural test templates take precedence over the singular test template" { write_suite multi - sed -i '/ - it:/a\ templates: [templates/single.yaml]\n template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/multi.yaml] +tests: + - it: checks a resource + templates: [templates/single.yaml] + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" } @test "an empty plural test override falls back to the singular template" { write_suite single - sed -i '/ - it:/a\ templates: []\n template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/single.yaml] +tests: + - it: checks a resource + templates: [] + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then echo "accepted a positional selector after an empty plural override" >&2 false fi grep -q 'use documentSelector.*templates/multi.yaml' output } + +@test "does not check vendored chart suites" { + mkdir -p packages/apps/example/charts/sub/templates packages/apps/example/charts/sub/tests + cp packages/apps/example/templates/multi.yaml packages/apps/example/charts/sub/templates/ + cat > packages/apps/example/charts/sub/tests/resource_test.yaml <<'EOF' +suite: upstream resource selection +templates: [templates/multi.yaml] +tests: + - it: checks an upstream resource + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + git add . + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "a new test-level index overrides existing assertion indices" { + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: index inheritance +templates: [templates/multi.yaml] +tests: + - it: checks a resource + asserts: + - notExists: + path: spec.missing + documentIndex: 0 + - notExists: + path: spec.other + documentIndex: 0 +EOF + git add . + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: index inheritance +templates: [templates/multi.yaml] +tests: + - it: checks a resource + documentIndex: 1 + asserts: + - notExists: + path: spec.missing + documentIndex: 0 + - notExists: + path: spec.other + documentIndex: 0 +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a new inherited positional selector" >&2 + false + fi + grep -q 'resource_test.yaml:5: use documentSelector' output +} + +@test "allows a pure suite rename with existing indices" { + write_suite multi + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + git mv packages/apps/example/tests/resource_test.yaml packages/apps/example/tests/renamed_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "checks added index lines in a renamed suite" { + write_suite multi + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + git mv packages/apps/example/tests/resource_test.yaml packages/apps/example/tests/renamed_test.yaml + cat > packages/apps/example/tests/renamed_test.yaml <<'EOF' +suite: resource selection +templates: + - templates/multi.yaml +tests: + - it: checks a resource + documentIndex: 1 + asserts: + - notExists: + path: spec.missing +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a new positional selector in a renamed suite" >&2 + false + fi + grep -q 'renamed_test.yaml:6: use documentSelector' output +} + +@test "an empty suite template list includes nested chart templates" { + mkdir packages/apps/example/templates/nested + mv packages/apps/example/templates/multi.yaml packages/apps/example/templates/nested/ + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: default templates +templates: [] +tests: + - it: checks a resource + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + git add . + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a positional selector with default templates" >&2 + false + fi + grep -q 'use documentSelector.*templates/nested/multi.yaml' output +} diff --git a/hack/check-helm-document-indices.py b/hack/check-helm-document-indices.py index 212a50964..4fd36b06f 100644 --- a/hack/check-helm-document-indices.py +++ b/hack/check-helm-document-indices.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Reject added positional selectors on multi-resource Helm templates. -Usage: hack/check-helm-document-indices.py +Usage: python3 hack/check-helm-document-indices.py Requires mikefarah/yq, like the chart test tooling. Existing indices are left alone. This static check counts --- separated blocks with a top-level kind; it does not evaluate conditional documents or resources emitted by helpers/ranges. @@ -20,22 +20,25 @@ def git(*args): def check(base): root = Path(git("rev-parse", "--show-toplevel").strip()) - paths = git("diff", "--name-only", "--diff-filter=ACMR", base, "--", - "packages/*/*/tests/*_test.yaml").splitlines() + additions = {} + path = None + for line in git("diff", "--no-ext-diff", "-M", "--unified=0", base, "--", + ":(glob)packages/*/*/tests/*_test.yaml").splitlines(): + if line.startswith("diff --git "): + path = None + elif line.startswith("+++ b/"): + path = line[6:] + additions[path] = set() + elif line.startswith("@@"): + number = int(re.search(r"\+(\d+)", line).group(1)) + elif path and line.startswith("+"): + if "documentIndex" in line: + additions[path].add(number) + number += 1 + elif line.startswith(" "): + number += 1 errors = [] - for path in paths: - added = set() - for line in git("diff", "--no-ext-diff", "--unified=0", base, "--", path).splitlines(): - if line.startswith("@@"): - number = int(re.search(r"\+(\d+)", line).group(1)) - elif line.startswith(("+++", "---", "diff ", "index ", "\\")): - continue - elif line.startswith("+"): - if "documentIndex" in line: - added.add(number) - number += 1 - elif line.startswith(" "): - number += 1 + for path, added in additions.items(): if not added: continue @@ -52,9 +55,9 @@ def check(base): for test in suite.get("tests", []): templates = test.get("templates") or ( [test["template"]] if test.get("template") - else suite.get("templates", ["templates/*"])) + else suite.get("templates") or ["templates/**/*"]) for assertion in test.get("asserts", []): - index = assertion.get("documentIndex", test.get("documentIndex", {})) + index = test.get("documentIndex") or assertion.get("documentIndex", {}) number = index.get("line") if number not in added: continue @@ -77,5 +80,5 @@ def check(base): if __name__ == "__main__": if len(sys.argv) != 2: - sys.exit("usage: check-helm-document-indices.py ") + sys.exit("usage: python3 hack/check-helm-document-indices.py ") sys.exit(check(sys.argv[1])) diff --git a/packages/apps/kafka/tests/jvm_sizing_test.yaml b/packages/apps/kafka/tests/jvm_sizing_test.yaml index 26a065d17..f4569a1e2 100644 --- a/packages/apps/kafka/tests/jvm_sizing_test.yaml +++ b/packages/apps/kafka/tests/jvm_sizing_test.yaml @@ -27,7 +27,9 @@ tests: # kafka-run-class.sh then drops its G1 settings. - it: leaves the node pool JVM options to Strimzi and Kafka template: templates/migration-hook.yaml - documentIndex: 3 + documentSelector: + path: kind + value: Job asserts: - notMatchRegex: path: spec.template.spec.containers[0].args[0] From 342283cfc781e343760148fb85775f914d114bcc Mon Sep 17 00:00:00 2001 From: Ahmed Kamal Date: Sun, 1 Nov 2026 02:17:57 +0200 Subject: [PATCH 4/4] docs(tests): document Helm selector guard limits Signed-off-by: Ahmed Kamal Assisted-by: LLM --- hack/check-helm-document-indices.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hack/check-helm-document-indices.py b/hack/check-helm-document-indices.py index 4fd36b06f..dd75e799a 100644 --- a/hack/check-helm-document-indices.py +++ b/hack/check-helm-document-indices.py @@ -2,9 +2,7 @@ """Reject added positional selectors on multi-resource Helm templates. Usage: python3 hack/check-helm-document-indices.py -Requires mikefarah/yq, like the chart test tooling. Existing indices are left -alone. This static check counts --- separated blocks with a top-level kind; -it does not evaluate conditional documents or resources emitted by helpers/ranges. +Requires mikefarah/yq, like the chart test tooling. """ import json @@ -19,6 +17,11 @@ def git(*args): def check(base): + """Check added indices, leaving existing ones alone. + + This static check counts --- separated blocks with a top-level kind; it does + not evaluate conditional documents or resources emitted by helpers/ranges. + """ root = Path(git("rev-parse", "--show-toplevel").strip()) additions = {} path = None