diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 96dce2056..1f44fc287 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -364,10 +364,6 @@ jobs: with: path: _out/envtest key: envtest-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }} - - name: Check new Helm document indices - env: - BASE_REF: ${{ github.event.pull_request.base.ref }} - run: python3 hack/check-helm-document-indices.py "$(git merge-base HEAD "origin/$BASE_REF")" # -k is required, not cosmetic: unit-tests and test-controllers are two # independent goals on the same command line, and without -k make stops # handing out new work the moment any unit-tests prerequisite fails — @@ -380,6 +376,11 @@ jobs: - name: Run Bats unit tests for docs-only changes if: needs.plan.outputs.code == 'false' run: make bats-unit-tests + - name: Check new Helm document indices + if: ${{ !cancelled() }} + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: python3 hack/check-helm-document-indices.py "$(git merge-base HEAD "origin/$BASE_REF")" - name: Upload unit test report if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/hack/check-helm-document-indices.bats b/hack/check-helm-document-indices.bats index ca84af677..5188f5b26 100644 --- a/hack/check-helm-document-indices.bats +++ b/hack/check-helm-document-indices.bats @@ -66,7 +66,8 @@ EOF write_suite multi git -c user.name=Test -c user.email=test@example.org commit -qm existing base=$(git rev-parse HEAD) - sed -i 's/spec.missing/spec.other/' packages/apps/example/tests/resource_test.yaml + sed 's/spec.missing/spec.other/' packages/apps/example/tests/resource_test.yaml > changed.yaml + mv changed.yaml packages/apps/example/tests/resource_test.yaml python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" } @@ -133,17 +134,37 @@ EOF @test "rejects a positional selector whose value is on a separate line" { write_suite multi - sed -i 's/documentIndex: 0/documentIndex:\n 0/' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: multiline index +templates: [templates/multi.yaml] +tests: + - it: checks a resource + documentIndex: + 0 + asserts: + - notExists: + path: spec.missing +EOF if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then echo "accepted a multiline positional selector" >&2 false fi - grep -q 'resource_test.yaml:6: use documentSelector' output + grep -q 'resource_test.yaml:5: use documentSelector' output } @test "rejects a singular test template overriding a single-document suite" { write_suite single - sed -i '/ - it:/a\ template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/single.yaml] +tests: + - it: checks a resource + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then echo "accepted a positional selector on the test template override" >&2 false @@ -153,22 +174,161 @@ EOF @test "allows a singular test template selecting one document" { write_suite multi - sed -i '/ - it:/a\ template: templates/single.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/multi.yaml] +tests: + - it: checks a resource + template: templates/single.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" } @test "plural test templates take precedence over the singular test template" { write_suite multi - sed -i '/ - it:/a\ templates: [templates/single.yaml]\n template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/multi.yaml] +tests: + - it: checks a resource + templates: [templates/single.yaml] + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" } @test "an empty plural test override falls back to the singular template" { write_suite single - sed -i '/ - it:/a\ templates: []\n template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/single.yaml] +tests: + - it: checks a resource + templates: [] + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then echo "accepted a positional selector after an empty plural override" >&2 false fi grep -q 'use documentSelector.*templates/multi.yaml' output } + +@test "does not check vendored chart suites" { + mkdir -p packages/apps/example/charts/sub/templates packages/apps/example/charts/sub/tests + cp packages/apps/example/templates/multi.yaml packages/apps/example/charts/sub/templates/ + cat > packages/apps/example/charts/sub/tests/resource_test.yaml <<'EOF' +suite: upstream resource selection +templates: [templates/multi.yaml] +tests: + - it: checks an upstream resource + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + git add . + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "a new test-level index overrides existing assertion indices" { + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: index inheritance +templates: [templates/multi.yaml] +tests: + - it: checks a resource + asserts: + - notExists: + path: spec.missing + documentIndex: 0 + - notExists: + path: spec.other + documentIndex: 0 +EOF + git add . + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: index inheritance +templates: [templates/multi.yaml] +tests: + - it: checks a resource + documentIndex: 1 + asserts: + - notExists: + path: spec.missing + documentIndex: 0 + - notExists: + path: spec.other + documentIndex: 0 +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a new inherited positional selector" >&2 + false + fi + grep -q 'resource_test.yaml:5: use documentSelector' output +} + +@test "allows a pure suite rename with existing indices" { + write_suite multi + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + git mv packages/apps/example/tests/resource_test.yaml packages/apps/example/tests/renamed_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "checks added index lines in a renamed suite" { + write_suite multi + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + git mv packages/apps/example/tests/resource_test.yaml packages/apps/example/tests/renamed_test.yaml + cat > packages/apps/example/tests/renamed_test.yaml <<'EOF' +suite: resource selection +templates: + - templates/multi.yaml +tests: + - it: checks a resource + documentIndex: 1 + asserts: + - notExists: + path: spec.missing +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a new positional selector in a renamed suite" >&2 + false + fi + grep -q 'renamed_test.yaml:6: use documentSelector' output +} + +@test "an empty suite template list includes nested chart templates" { + mkdir packages/apps/example/templates/nested + mv packages/apps/example/templates/multi.yaml packages/apps/example/templates/nested/ + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: default templates +templates: [] +tests: + - it: checks a resource + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + git add . + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a positional selector with default templates" >&2 + false + fi + grep -q 'use documentSelector.*templates/nested/multi.yaml' output +} diff --git a/hack/check-helm-document-indices.py b/hack/check-helm-document-indices.py index 212a50964..4fd36b06f 100644 --- a/hack/check-helm-document-indices.py +++ b/hack/check-helm-document-indices.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Reject added positional selectors on multi-resource Helm templates. -Usage: hack/check-helm-document-indices.py +Usage: python3 hack/check-helm-document-indices.py Requires mikefarah/yq, like the chart test tooling. Existing indices are left alone. This static check counts --- separated blocks with a top-level kind; it does not evaluate conditional documents or resources emitted by helpers/ranges. @@ -20,22 +20,25 @@ def git(*args): def check(base): root = Path(git("rev-parse", "--show-toplevel").strip()) - paths = git("diff", "--name-only", "--diff-filter=ACMR", base, "--", - "packages/*/*/tests/*_test.yaml").splitlines() + additions = {} + path = None + for line in git("diff", "--no-ext-diff", "-M", "--unified=0", base, "--", + ":(glob)packages/*/*/tests/*_test.yaml").splitlines(): + if line.startswith("diff --git "): + path = None + elif line.startswith("+++ b/"): + path = line[6:] + additions[path] = set() + elif line.startswith("@@"): + number = int(re.search(r"\+(\d+)", line).group(1)) + elif path and line.startswith("+"): + if "documentIndex" in line: + additions[path].add(number) + number += 1 + elif line.startswith(" "): + number += 1 errors = [] - for path in paths: - added = set() - for line in git("diff", "--no-ext-diff", "--unified=0", base, "--", path).splitlines(): - if line.startswith("@@"): - number = int(re.search(r"\+(\d+)", line).group(1)) - elif line.startswith(("+++", "---", "diff ", "index ", "\\")): - continue - elif line.startswith("+"): - if "documentIndex" in line: - added.add(number) - number += 1 - elif line.startswith(" "): - number += 1 + for path, added in additions.items(): if not added: continue @@ -52,9 +55,9 @@ def check(base): for test in suite.get("tests", []): templates = test.get("templates") or ( [test["template"]] if test.get("template") - else suite.get("templates", ["templates/*"])) + else suite.get("templates") or ["templates/**/*"]) for assertion in test.get("asserts", []): - index = assertion.get("documentIndex", test.get("documentIndex", {})) + index = test.get("documentIndex") or assertion.get("documentIndex", {}) number = index.get("line") if number not in added: continue @@ -77,5 +80,5 @@ def check(base): if __name__ == "__main__": if len(sys.argv) != 2: - sys.exit("usage: check-helm-document-indices.py ") + sys.exit("usage: python3 hack/check-helm-document-indices.py ") sys.exit(check(sys.argv[1])) diff --git a/packages/apps/kafka/tests/jvm_sizing_test.yaml b/packages/apps/kafka/tests/jvm_sizing_test.yaml index 26a065d17..f4569a1e2 100644 --- a/packages/apps/kafka/tests/jvm_sizing_test.yaml +++ b/packages/apps/kafka/tests/jvm_sizing_test.yaml @@ -27,7 +27,9 @@ tests: # kafka-run-class.sh then drops its G1 settings. - it: leaves the node pool JVM options to Strimzi and Kafka template: templates/migration-hook.yaml - documentIndex: 3 + documentSelector: + path: kind + value: Job asserts: - notMatchRegex: path: spec.template.spec.containers[0].args[0]