From cc942fed6865086d681f1707ee61bc67a2b1cb09 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 28 Sep 2026 15:33:30 +0500 Subject: [PATCH] test(tests): run unit suites with Bats and retain POSIX coverage The shell translator rewrites test bodies and loses Bats reporting. The unit lane needs complete TAP results and JUnit reports without changing how live-cluster suites run. Sourced production helpers still need a dash pass, and unit test bodies need nounset without exporting shell options into subprocesses. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- .github/workflows/pre-commit.yml | 3 + .github/workflows/pull-requests.yaml | 77 +-- .pre-commit-config.yaml | 11 + Makefile | 108 ++-- docs/agents/e2e-testing.md | 4 +- docs/agents/overview.md | 5 +- hack/admin-kubeconfig-invariant.bats | 2 + hack/alert-severity-contract.bats | 2 + hack/app-cleanup-hooks_test.bats | 9 +- hack/backup-access-preflight.bats | 2 + hack/backup-examples-shared-helpers.bats | 2 + hack/bats-no-exit-trap.bats | 2 + hack/bats-no-negated-assert.bats | 2 + hack/bats-runner-contract.bats | 141 +++++ hack/bats-runner-coverage.bats | 17 +- hack/bats-strict-setup.bats | 483 ++++++++++++++++++ hack/bats-unit-keep-going.bats | 62 ++- hack/build-matrix_test.bats | 10 +- hack/build-talos-gating_test.bats | 2 + hack/capi-core-components_test.bats | 2 + hack/capi-provider-components_test.bats | 2 + hack/capk-provider_test.bats | 2 + hack/capture-dataplane.bats | 8 +- hack/capture-previous-logs.bats | 8 +- hack/chainsaw-leak-guard_test.bats | 20 +- hack/chainsaw-op-budgets.bats | 7 +- ...ck-applicationdefinition-crd-ordering.bats | 2 + hack/check-commit-trailers.bats | 2 + hack/check-gpu-operator-variants.bats | 2 + hack/check-gpu-recording-rules.bats | 4 +- hack/check-hook-request-timeout.bats | 2 + hack/check-host-runtime.bats | 14 +- hack/check-kafka-rd-schema.bats | 2 + hack/check-kafka-rd-secrets.bats | 2 + hack/check-kubeovn-image-copies.bats | 2 + hack/check-mariadb-rd-secrets.bats | 2 + hack/check-mongodb-rd-secrets.bats | 2 + hack/check-nats-rd-secrets.bats | 2 + hack/check-opensearch-rd-secrets.bats | 2 + hack/check-opensearch-readme.bats | 2 + hack/check-packages-artifact-ignores.bats | 2 + hack/check-postgres-postgis-versions.bats | 2 + hack/check-postgres-preload-allowlist.bats | 2 + hack/check-qdrant-rd-secrets.bats | 2 + hack/check-rabbitmq-rd-secrets.bats | 2 + hack/check-rabbitmq-topology-kinds.bats | 2 + hack/check-rd-presets_test.bats | 2 + hack/check-rd-release-prefix.bats | 2 + hack/check-redis-rd-secrets.bats | 2 + hack/check-render-matrix.bats | 2 + ...ck-talosconfigtemplate-name-agreement.bats | 2 + hack/check-values-schema-string-defaults.bats | 2 + hack/check-worker-image-catalog-defaults.bats | 2 + hack/ci-tool-installs-contract.bats | 2 + hack/cilium-leak-healer_test.bats | 4 +- hack/clickhouse-update-versions_test.bats | 10 +- hack/cni-plugins-staging-contract.bats | 4 +- hack/codeowners-invariant.bats | 2 + hack/collect-images_test.bats | 9 +- hack/common-envs_test.bats | 16 +- hack/computeplane-kcp-hold_test.bats | 2 + hack/console-nginx-header-buffers.bats | 11 +- hack/container-lane-capacity_test.bats | 2 + hack/cozyreport-talos.bats | 2 + hack/cozyreport.bats | 8 +- hack/cozystack-controller-secret-rbac.bats | 4 +- hack/cozystack-version-stamp.bats | 13 +- hack/cozytest-capture-gate.bats | 4 +- hack/cozytest-nonascii-title.bats | 8 +- hack/cozytest-trace-switch.bats | 6 +- hack/cozytest.sh | 12 +- hack/downstream-trigger-map.bats | 81 +-- hack/etcd-probe_test.bats | 16 +- hack/fork-pr-checkout_test.bats | 2 + hack/gate-concurrency-contract.bats | 2 + hack/go-module-fetch-contract.bats | 2 + hack/helm-unit-tests.bats | 13 +- hack/helmrelease-readiness.bats | 2 + hack/image-builders-cross-compile.bats | 4 +- hack/image-pin-consistency.bats | 14 +- hack/image-targets-pass-buildx-args.bats | 4 +- hack/installer-image-packages-push_test.bats | 9 +- hack/isp-hosted-lane_test.bats | 2 + hack/issue-triage-contract.bats | 2 + hack/kafka-backup-script.bats | 2 + hack/kafka-metadata-backup-demo-helpers.bats | 2 + hack/kafka-migration-hook.bats | 11 +- hack/kafka-pre-delete-hook.bats | 14 +- hack/kafka-suite-pvc-cleanup_test.bats | 2 + hack/kafka-topic-backup-demo-helpers.bats | 2 + hack/kafka-topic-backup-strategy.bats | 2 + hack/kamaji-crds-schema.bats | 2 + hack/kubectl-wait-retry_test.bats | 10 +- hack/kubernetes-md0-migration.bats | 13 +- hack/kubernetes-nodes-unpin-hook.bats | 9 +- hack/kubernetes-pre-delete-hook.bats | 9 +- hack/linstor-storage-mode_test.bats | 2 + hack/mariadb-first-boot-waits.bats | 2 + hack/md-no-hardwrap.bats | 2 + hack/migration-50-etcd-adopt.bats | 11 +- hack/migration-54-redis-adopt.bats | 15 +- hack/migration-55-fluxcd-orphan.bats | 15 +- hack/migration-56-adopt-path.bats | 9 +- hack/migration-56-adopt-values.bats | 11 +- hack/migration-57-seaweedfs-size-limit.bats | 2 + hack/migration-58-http-cache-freeze.bats | 2 + hack/migration-59-vmi-hotplug-volumes.bats | 2 + hack/migration-seaweedfs-db-adopt.bats | 11 +- hack/monitoring-pvc-backfill-migration.bats | 13 +- hack/multus-install-cni-plugins.bats | 10 +- hack/nats-backup-demo-helpers.bats | 2 + hack/nightly-mirror_test.bats | 16 +- hack/overlay-main-images_test.bats | 7 +- hack/package-mk-phony.bats | 4 +- hack/package-test-target-coverage.bats | 2 + hack/package-update-templates.bats | 2 + hack/platform-packages_test.bats | 2 + hack/pod-label-census_test.bats | 16 +- hack/post-install-prep.bats | 4 +- hack/postgres-backup-app-login_test.bats | 2 + hack/pr-labeler-contract.bats | 2 + hack/pre-commit-generate-packages_test.bats | 4 +- hack/promote-gate-contract.bats | 2 + hack/promote-packages-artifact_test.bats | 4 +- hack/promote-packages-contract.bats | 2 + hack/promote-retag_test.bats | 21 +- hack/promote-rewrite-tags_test.bats | 18 +- hack/promotion-retention-contract.bats | 2 + hack/proxmox-e2e-fixture_test.bats | 5 +- hack/proxmox-step1b-pod-match_test.bats | 7 +- hack/redis-backup-demo-helpers.bats | 2 + hack/redis-backup-strategy.bats | 9 +- hack/release-changelog-behaviour.bats | 2 + hack/release-changelog-contract.bats | 2 + hack/release-freeze-contract.bats | 2 + hack/remediation-guard.bats | 11 +- ...oute-hostname-policy-version-coverage.bats | 11 +- hack/run-kubernetes-block-io_test.bats | 2 + hack/run-kubernetes-cpu-throttle_test.bats | 2 + hack/run-kubernetes-drain_test.bats | 11 +- hack/run-kubernetes-install-wait_test.bats | 2 + hack/run-kubernetes-join-timing_test.bats | 2 + hack/run-kubernetes-kvm-exits_test.bats | 2 + .../run-kubernetes-network-counters_test.bats | 2 + hack/run-kubernetes-node-join_test.bats | 11 +- hack/run-kubernetes-oidc_test.bats | 2 + hack/run-kubernetes-remediation_test.bats | 9 +- hack/run-kubernetes-runner-canary_test.bats | 32 +- hack/run-kubernetes-runner-cpu_test.bats | 8 +- .../run-kubernetes-sandbox-cpu-time_test.bats | 2 + hack/run-kubernetes-schedulable_test.bats | 11 +- hack/run-kubernetes-serial-console_test.bats | 2 + ...run-kubernetes-talos-diagnostics_test.bats | 2 + hack/run-kubernetes-talos-spec_test.bats | 12 +- hack/run-kubernetes-thread-cpu_test.bats | 2 + hack/run-migrations.bats | 4 +- hack/runner-identity.bats | 8 +- hack/sandbox-cidr-disjoint.bats | 18 +- hack/sandbox-runner-headroom.bats | 24 +- hack/seaweedfs-chart-seam.bats | 4 +- hack/seaweedfs-guard-parity.bats | 4 +- hack/seaweedfs-naming-audit.bats | 10 +- hack/seaweedfs-naming-audit.sh | 16 +- hack/select-e2e.sh | 6 +- hack/select-e2e_test.bats | 32 +- hack/select-install_test.bats | 17 +- hack/stitch-multiarch_test.bats | 9 +- hack/talos-image-cache_test.bats | 14 +- hack/talos-installer-index_test.bats | 4 +- hack/talos-reconcile-heredoc_test.bats | 6 +- hack/tenant-pre-delete-hook.bats | 9 +- hack/test_helper.bash | 11 + hack/tracing-central-credentials.bats | 4 +- hack/unit-test-parallelism.bats | 48 +- hack/verify-multiarch_test.bats | 9 +- hack/verify-promoted-packages_test.bats | 4 +- 176 files changed, 1524 insertions(+), 534 deletions(-) create mode 100644 hack/bats-runner-contract.bats create mode 100644 hack/bats-strict-setup.bats create mode 100644 hack/test_helper.bash diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 9a4336023..dbec873fb 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -60,6 +60,9 @@ jobs: curl -sSL https://github.com/cozystack/cozyvalues-gen/releases/download/v1.7.0/cozyvalues-gen-linux-amd64.tar.gz | tar -xzvf- -C /usr/local/bin/ cozyvalues-gen - name: Run pre-commit hooks + # The complete Bats pass is a manual local hook and is therefore outside + # this changed-file lint run. pull-requests.yaml's checks job runs the + # full unit path for code and the Bats contracts for docs-only changes. run: | git fetch origin main || git fetch origin master base_commit=$(git rev-parse --verify origin/main || git rev-parse --verify origin/master || echo "") diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 2a5ac1022..c1df524fe 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -75,7 +75,6 @@ jobs: statuses: write outputs: code: ${{ steps.p.outputs.code }} - trigger_map: ${{ steps.p.outputs.trigger_map }} matrix: ${{ steps.p.outputs.matrix }} any: ${{ steps.p.outputs.any }} touched: ${{ steps.p.outputs.touched }} @@ -163,23 +162,6 @@ jobs: else echo "code=false" >> "$GITHUB_OUTPUT" fi - # docs/agents/contributing.md carries the downstream trigger map, which - # hack/downstream-trigger-map.bats asserts against: every path the map - # cites still exists, and its repository list still matches the PR - # template. Editing the map alone is a docs-only PR, which would skip the - # unit tests and exempt the map from its own guard. This flag lets the - # unit-test job opt back in, and only that job: it deliberately does not - # feed `code`, so a docs PR still builds nothing. - # - # Detect the map by its old path too, which is why this reads the - # rename-blind list above: a PR that moves the map would otherwise skip - # the very test that pins the literal below to it, and break for - # whoever pushes next. - if grep -qxF 'docs/agents/contributing.md' /tmp/changed_norenames.txt; then - echo "trigger_map=true" >> "$GITHUB_OUTPUT" - else - echo "trigger_map=false" >> "$GITHUB_OUTPUT" - fi matrix="$(hack/build-matrix.sh /tmp/changed.txt)" echo "Build matrix: $matrix" echo "matrix=$matrix" >> "$GITHUB_OUTPUT" @@ -300,12 +282,12 @@ jobs: permissions: contents: read needs: ["plan"] - # trigger_map opts a docs-only PR back in when it edits the downstream trigger - # map, so hack/downstream-trigger-map.bats still guards it. No other job reads - # that flag: a docs PR runs the unit tests and nothing heavier. + # Unit contracts read both code and documentation. Run this job for every + # non-release PR: code changes take the complete unit path, while docs-only + # changes run the Bats lane that owns those cross-tree contracts. Builds and + # E2E remain gated on `code` below. if: | - (needs.plan.outputs.code == 'true' || needs.plan.outputs.trigger_map == 'true') - && !contains(github.event.pull_request.labels.*.name, 'release') + !contains(github.event.pull_request.labels.*.name, 'release') steps: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 @@ -313,38 +295,54 @@ jobs: fetch-depth: 0 # The module cache is restored from the key build-main.yaml writes on # main, which every PR run can read, so a PR's first run starts warm. The - # download fetches whatever the cache lacks, the whole set on a miss. + # download fetches whatever the cache lacks, the whole set on a miss. The + # Go steps are for code changes only: a docs-only run is the Bats lane alone. - name: Locate Go module cache + if: needs.plan.outputs.code == 'true' id: gomodcache run: echo "dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" - name: Cache Go modules + if: needs.plan.outputs.code == 'true' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.gomodcache.outputs.dir }} key: go-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }} restore-keys: go-mod-${{ runner.os }}- - name: Download Go modules + if: needs.plan.outputs.code == 'true' run: | for mod in $(git ls-files go.mod '*/go.mod' | xargs -n1 dirname); do (cd "$mod" && "$GITHUB_WORKSPACE/hack/go-mod-download.sh") done # Ephemeral runners don't carry the helm-unittest plugin the persistent # self-hosted runner accreted. The plugin embeds its own Helm, so its - # version is pinned: a plugin release must not change test results. + # version is pinned: a plugin release must not change test results. Bats is + # installed on every run, docs-only ones included. - name: Set up test toolchain env: + CODE_CHANGED: ${{ needs.plan.outputs.code }} HELM_UNITTEST_VERSION: "1.1.2" + BATS_VERSION: "1.14.0" + BATS_COMMIT: eb7f42f8d608ac693d7a4b67474f6714ea68cfc5 run: | - installed="$(helm plugin list 2>/dev/null | awk '$1 == "unittest" { print $2 }')" - if [ "$installed" != "$HELM_UNITTEST_VERSION" ]; then - [ -z "$installed" ] || helm plugin uninstall unittest - helm plugin install https://github.com/helm-unittest/helm-unittest --version "v${HELM_UNITTEST_VERSION}" + set -euo pipefail + if [ "$CODE_CHANGED" = true ]; then + installed="$(helm plugin list 2>/dev/null | awk '$1 == "unittest" { print $2 }' || true)" + if [ "$installed" != "$HELM_UNITTEST_VERSION" ]; then + [ -z "$installed" ] || helm plugin uninstall unittest + helm plugin install https://github.com/helm-unittest/helm-unittest --version "v${HELM_UNITTEST_VERSION}" + fi fi - # The make processes share one four-slot jobserver. Each BATS file is a - # separate target, so the long tail no longer runs as one nine-minute shell - # loop; Helm and Go targets share the same bounded pool. Output stays - # grouped by target instead of four test logs interleaving. - # + # Source archive bytes can change; verify the tag's commit before install. + if ! bats --version 2>/dev/null | grep -qx "Bats $BATS_VERSION"; then + bats_parent=$(mktemp -d "$RUNNER_TEMP/bats-core.XXXXXX") + bats_source="$bats_parent/source" + git clone --quiet --depth=1 --branch "v$BATS_VERSION" \ + https://github.com/bats-core/bats-core.git "$bats_source" + test "$(git -C "$bats_source" rev-parse HEAD)" = "$BATS_COMMIT" + sudo "$bats_source/install.sh" /usr/local + fi + bats --version | grep -qx "Bats $BATS_VERSION" # -k is required, not cosmetic: unit-tests and test-controllers are two # independent goals on the same command line, and without -k make stops # handing out new work the moment any unit-tests prerequisite fails — @@ -352,7 +350,18 @@ jobs: # file hides the entire Go controller suite. -k lets already-schedulable # goals keep running; the job still fails overall on any red target. - name: Run unit and controller tests + if: needs.plan.outputs.code == 'true' run: make unit-tests test-controllers -j4 -k --output-sync=target + - name: Run Bats unit tests for docs-only changes + if: needs.plan.outputs.code == 'false' + run: make bats-unit-tests + - name: Upload unit test report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: bats-unit-report + path: _out/test-reports/ + if-no-files-found: ignore # One image (package) per job, fanned out across the ephemeral pool. Each job # gets its own buildx (docker-container driver) so the shared-buildkit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa6f6ed28..fa3ec818f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,6 +21,17 @@ repos: ' language: system files: ^.*$ + # The complete hack/*.bats and POSIX-compatibility pass is intentionally a + # manual local hook because it requires bats-core and has substantial wall + # time even with GNU parallel. Run it before pushing with: + # pre-commit run bats-unit-tests --hook-stage manual --all-files + # Pull-request CI remains the unconditional gate. + - id: bats-unit-tests + name: Run hack/*.bats unit tests + entry: make bats-unit-tests bats-posix-compat-tests + language: system + stages: [manual] + pass_filenames: false # Lints changed GitHub Actions workflows for unpinned actions and excessive token # permissions (audit scope is set in .github/zizmor.yml). Runs offline so it needs # no GitHub token locally. The same gate runs tree-wide in .github/workflows/zizmor.yml. diff --git a/Makefile b/Makefile index 3342a86b8..2d6dbd9e2 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,4 @@ -.PHONY: manifests assets prepare-env prepare-env-container unit-tests helm-unit-tests bats-unit-tests bats-unit-files-check rd-presets-check migrations-target-check test test-controllers test-backport-audit preflight - -# Before any include, the last file make has read is this one: MAKEFILES and -# -f files given before it are earlier in the list, -f files after it are not -# read yet. -ROOT_MAKEFILE := $(lastword $(MAKEFILE_LIST)) +.PHONY: manifests assets prepare-env prepare-env-container unit-tests helm-unit-tests bats-unit-tests bats-unit-files-check bats-posix-compat-tests print-bats-unit-files print-bats-posix-compat-files print-bats-jobs rd-presets-check migrations-target-check test test-controllers test-backport-audit preflight include hack/common-envs.mk @@ -123,7 +118,7 @@ test: make -C packages/core/testing apply make -C packages/core/testing e2e -unit-tests: helm-unit-tests bats-unit-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness test-backport-audit migrations-target-check +unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness test-backport-audit migrations-target-check helm-unit-tests: hack/helm-unit-tests.sh @@ -179,11 +174,9 @@ test-check-readiness: test-backport-audit: go test ./cmd/backport-audit/ -count=1 -# Discover every hack/*.bats file that is NOT an e2e test and run it -# through cozytest.sh. This glob is one level deep: live-cluster suites under -# hack/e2e-* or in subdirectories need an invocation in packages/core/testing. -# hack/bats-runner-coverage.bats reports files that neither runner selects. -# Park a suite with the .bats.disabled suffix to exclude it from the audit. +# Unit discovery is one level deep. Live-cluster files need an invocation in +# packages/core/testing; hack/bats-runner-coverage.bats checks that boundary. +# Bats supplies TAP and JUnit output; the live-cluster runner keeps its tracing. # # Caveat: $(wildcard ...) returns space-separated names, so a filename # containing a literal space would split into multiple tokens here. All @@ -191,40 +184,81 @@ test-backport-audit: # introduces whitespace-bearing filenames this recipe must be rewritten # (e.g. to use `find ... -print0 | xargs -0`). BATS_UNIT_FILES := $(filter-out hack/e2e-%.bats,$(wildcard hack/*.bats)) -BATS_UNIT_TARGETS := $(patsubst hack/%.bats,bats-unit-%,$(BATS_UNIT_FILES)) -# Quiet by default: cozytest.sh streams every test's xtrace live, which over -# this many suites buried the one failing assertion under ~240k lines of -# trace in CI. COZYTEST_TRACE=0 keeps the per-test start/end lines and still dumps -# the full trace of whichever test FAILS -- but not of one that is killed -# instead, a hang reaped by the job timeout among them, since the dump runs -# after the test returns. See the COZYTEST_TRACE comment in hack/cozytest.sh. -# Override to watch a run line by -# line: `COZYTEST_TRACE=1 make bats-unit-tests`, or invoke the runner -# directly (`hack/cozytest.sh hack/foo.bats [pattern]`), which stays verbose. -# The e2e call sites in packages/core/testing/Makefile are untouched and keep -# the live stream -- see the COZYTEST_TRACE comment in hack/cozytest.sh. -COZYTEST_TRACE ?= 0 +# The same list, one file per line, for hack/bats-strict-setup.bats. Every unit +# file has to load hack/test_helper.bash to get `set -u` back, and that audit is +# only worth anything if the set it walks is the set that actually runs -- so it +# asks here rather than keeping a second copy of the filter above. +print-bats-unit-files: + @printf '%s\n' $(BATS_UNIT_FILES) -# A sub-make with --keep-going, so one red file does not stop make from -# scheduling the files after it; the sub-make still exits non-zero if any -# file failed (hack/bats-unit-keep-going.bats covers both). It shares the -# caller's -j slots and --output-sync through MAKEFLAGS. -bats-unit-tests: bats-unit-files-check - @$(MAKE) --file=$(ROOT_MAKEFILE) --no-print-directory --keep-going $(BATS_UNIT_TARGETS) +# `bats -j` needs GNU parallel and exits non-zero rather than degrading when it +# is missing. moreutils also ships a command named parallel, so identify the GNU +# implementation from its version output before enabling concurrency. Resolve +# the probe once per make process; `?=` would keep the `$(shell ...)` recursive +# and rerun it at every expansion. +ifndef BATS_JOBS +BATS_JOBS := $(shell parallel --version 2>/dev/null | grep -q '^GNU parallel ' && nproc 2>/dev/null || echo 1) +endif + +print-bats-jobs: + @printf '%s\n' "$(BATS_JOBS)" + +# JUnit XML is preserved as a CI artifact for inspection. _out is gitignored. +BATS_REPORT_DIR ?= _out/test-reports bats-unit-files-check: @if [ -z "$(BATS_UNIT_FILES)" ]; then \ echo "ERROR: no hack/*.bats unit test files found"; \ exit 1; \ fi +bats-unit-tests: bats-unit-files-check + @command -v bats >/dev/null 2>&1 || { \ + echo "ERROR: bats not found. Install bats-core >= 1.5 — https://bats-core.readthedocs.io"; \ + exit 1; \ + } + @mkdir -p "$(BATS_REPORT_DIR)" + bats -j $(BATS_JOBS) --report-formatter junit -o "$(BATS_REPORT_DIR)" $(BATS_UNIT_FILES) -# Each file is its own target of the sub-make so `make -jN` can schedule them, -# and the trace switch rides along per target. -.PHONY: $(BATS_UNIT_TARGETS) -$(BATS_UNIT_TARGETS): bats-unit-%: hack/%.bats - @echo "--- running $< ---" - @COZYTEST_TRACE=$(COZYTEST_TRACE) hack/cozytest.sh "$<" +# Real Bats is authoritative. Unit files that source production code whose +# contract is POSIX sh retain a compatibility pass through cozytest.sh's +# /bin/sh translator. Discover literal .sh dependencies wherever they live, +# including hack/lib and package migration helpers, so a new non-Chainsaw +# helper test cannot run only under Bash. The explicit entries retain reviewed +# shell-facing tests, including files whose production path is held in a +# variable and therefore cannot be identified from the source line alone. +BATS_SOURCED_SH_FILES := $(shell grep -El '^[[:space:]]*(\.|source)[[:space:]]+.*\.sh' $(BATS_UNIT_FILES)) +BATS_POSIX_COMPAT_FILES := $(sort \ + $(BATS_SOURCED_SH_FILES) \ + hack/capture-dataplane.bats \ + hack/capture-previous-logs.bats \ + hack/cilium-leak-healer_test.bats \ + hack/container-lane-capacity_test.bats \ + hack/cozyreport-talos.bats \ + hack/cozyreport.bats \ + hack/cozystack-version-stamp.bats \ + hack/nightly-mirror_test.bats \ + hack/pod-label-census_test.bats \ + hack/promote-rewrite-tags_test.bats \ + hack/runner-identity.bats \ + hack/seaweedfs-naming-audit.bats \ +) + +# CI's /bin/sh is dash. Keep the interpreter overridable so contributors can +# run the same compatibility lane explicitly on hosts whose /bin/sh differs. +BATS_POSIX_SHELL ?= /bin/sh +COZYTEST_TRACE ?= 0 + +print-bats-posix-compat-files: + @printf '%s\n' $(BATS_POSIX_COMPAT_FILES) + +bats-posix-compat-tests: + @status=0; \ + for f in $(BATS_POSIX_COMPAT_FILES); do \ + echo "--- running POSIX compatibility: $$f ---"; \ + COZYTEST_TRACE=$(COZYTEST_TRACE) "$(BATS_POSIX_SHELL)" hack/cozytest.sh "$$f" || status=1; \ + done; \ + exit $$status # Operator-facing host preflight check. Warns about a standalone # containerd.service or docker.service running alongside the embedded diff --git a/docs/agents/e2e-testing.md b/docs/agents/e2e-testing.md index 7eb1f46ee..cc10f89d0 100644 --- a/docs/agents/e2e-testing.md +++ b/docs/agents/e2e-testing.md @@ -66,7 +66,7 @@ Trap-based cleanup was the single biggest source of false failures in the BATS s - Chainsaw deletes the resources it `apply`-ed during its cleanup phase (bounded by the `delete`/`cleanup` timeouts in `hack/e2e-chainsaw/.chainsaw.yaml`). Do not hand-roll teardown for resources Chainsaw created. - A self-contained `trap '… ' EXIT` **inside a single `script` step** — to kill a port-forward or remove a temp dir — is fine, because it runs in a contained subprocess with its variables in scope. See `hack/e2e-chainsaw/bucket/chainsaw-test.yaml`. What is banned is test-level trap-based cleanup of the BATS kind. The same carve-out holds inside a BATS `@test` when the trap sits in an explicit subshell — `( … trap "kill $pid" EXIT … )` — because a subshell trap does not replace the one the `bats` binary installs, so a failure inside it still prints its `not ok`. `hack/e2e-test-openapi.bats` relies on this to kill a backgrounded `kubectl proxy`; moving that cleanup to the end of the body would leak a process holding a fixed port rather than fix anything. -- The ban extends to every BATS file under `hack/`, subdirectories and the `e2e-` prefixed ones included, for a second reason worth knowing before you debug one: an `EXIT` trap inside an `@test` body replaces the one the `bats` binary installs for its own bookkeeping, and a test that then **fails** prints no TAP line at all. It does not appear as `not ok`; it disappears, and the run ends with `# bats warning: Executed N instead of expected M tests` and a non-zero exit. Verified with Bats 1.14.0. Anyone reading the tail of the output, or grepping it for `not ok`, sees a green suite — and the CI runner `hack/cozytest.sh`, which is not the `bats` binary, reports the same failure correctly, so the two disagree exactly when it matters. Clean up at the last reachable point after an assertion whose failure aborts the test body; the scratch directory then survives for inspection on failure, which is what a failed test wants anyway. A failure in an errexit-exempt `if` or AND-OR context can continue into cleanup and let cleanup supply the test status, so preserve the verdict explicitly in that shape. When a suite reports zero failures, confirm it also reports how many tests it ran: an exit code answers "did anything fail", never "did anything run". `hack/bats-no-exit-trap.bats` enforces this across every `hack/**/*.bats`, subdirectories included: a file that carries no `# EXIT-TRAP DEBT: N` comment must contain no EXIT-trap line at all, and a file that carries one must install exactly N — so a trap appearing or disappearing fails until the file's own number is corrected. Note what that does and does not buy: it is a ratchet on the number's *accuracy*, not on the debt itself, because adding a trap and raising `N` in the same change is green. Nothing mechanical stops the count growing — review does, which is the point of the number living in the file being reviewed. A trap inside an explicit subshell is exempt from the ban but still counted, so a declaration is not by itself an admission of debt; `hack/e2e-test-openapi.bats` and `hack/capk-provider_test.bats` document their exempt subshell traps in their headers. No file declares conversion debt after the test-level handlers are removed; the remaining declarations count those exempt traps. The scan reads `.bats` files only, so a handler reaching a test body from a sourced `.sh` is outside it — `hack/e2e-chainsaw/_lib/run-kubernetes.sh` installs two, each benign for its own reason rather than by design: the one in `cozy_capture_tenant_talos` because that function is declared with `(` and runs in a subshell, and the one in `run_kubernetes_test` because no `@test` calls it despite being declared with `{`. Several `hack/*.bats` files source that library. Treat the guard as a ratchet over a common spelling, not as proof that a file installs no handler. Whenever a suite reports zero failures, reconcile its `1..N` plan against its `ok` count before believing the run. +- The ban extends to every BATS file under `hack/`, subdirectories and the `e2e-` prefixed ones included, for a second reason worth knowing before you debug one: an `EXIT` trap inside an `@test` body replaces the one the `bats` binary installs for its own bookkeeping, and a test that then **fails** prints no TAP line at all. It does not appear as `not ok`; it disappears, and the run ends with `# bats warning: Executed N instead of expected M tests` and a non-zero exit. Verified with Bats 1.14.0. Anyone reading the tail of the output, or grepping it for `not ok`, sees a green suite. The former unit CI runner `hack/cozytest.sh` reported that failure correctly; the unit lane now runs Bats, which is why test-level handlers had to be removed before the runner switch. Clean up at the last reachable point after an assertion whose failure aborts the test body; the scratch directory then survives for inspection on failure, which is what a failed test wants anyway. A failure in an errexit-exempt `if` or AND-OR context can continue into cleanup and let cleanup supply the test status, so preserve the verdict explicitly in that shape. When a suite reports zero failures, confirm it also reports how many tests it ran: an exit code answers "did anything fail", never "did anything run". `hack/bats-no-exit-trap.bats` enforces this across every `hack/**/*.bats`, subdirectories included: a file that carries no `# EXIT-TRAP DEBT: N` comment must contain no EXIT-trap line at all, and a file that carries one must install exactly N — so a trap appearing or disappearing fails until the file's own number is corrected. Note what that does and does not buy: it is a ratchet on the number's *accuracy*, not on the debt itself, because adding a trap and raising `N` in the same change is green. Nothing mechanical stops the count growing — review does, which is the point of the number living in the file being reviewed. A trap inside an explicit subshell is exempt from the ban but still counted, so a declaration is not by itself an admission of debt; `hack/e2e-test-openapi.bats` and `hack/capk-provider_test.bats` document their exempt subshell traps in their headers. No file declares conversion debt after the test-level handlers are removed; the remaining declarations count those exempt traps. The scan reads `.bats` files only, so a handler reaching a test body from a sourced `.sh` is outside it — `hack/e2e-chainsaw/_lib/run-kubernetes.sh` installs two, each benign for its own reason rather than by design: the one in `cozy_capture_tenant_talos` because that function is declared with `(` and runs in a subshell, and the one in `run_kubernetes_test` because no `@test` calls it despite being declared with `{`. Several `hack/*.bats` files source that library. Treat the guard as a ratchet over a common spelling, not as proof that a file installs no handler. Whenever a suite reports zero failures, reconcile its `1..N` plan against its `ok` count before believing the run. ### 4. Do not mask cleanup or teardown failures @@ -100,7 +100,7 @@ A HelmRelease that hit its wait timeout and failed an action is a silent race we `hack/select-e2e.sh` walks the `packages/core/platform/sources/*.yaml` dependency graph and runs only the Chainsaw suites affected by a diff. A suite is a directory directly under `hack/e2e-chainsaw/` in or below which Chainsaw would find a Test: a `chainsaw-test.yaml`, or a `chainsaw-test.yml`, which Chainsaw loads when the `.yaml` is absent, or, with neither, numbered step files such as `01-install.yaml`, which Chainsaw turns into a Test on its own. Chainsaw walks every subdirectory of the directory it is handed, so a Test in a top-level directory or anywhere below it makes that directory a suite. Every one of those layouts runs, so the selector has to see it or it is never selected. - Conservative escalation: edits to `packages/library/`, `packages/core/`, the Go trees (`api/`, `cmd/`, `internal/`, `pkg/`), the codegen under `tools/`, `go.mod`/`go.sum`, top-level `hack/*.sh` helpers, the `hack/e2e-*.bats` the sandbox runs and the `hack/lib/` they source, `hack/*.mk`, `hack/buildkitd.toml`, `hack/e2e-*.yaml`, the shared `hack/e2e-chainsaw/_lib/`, `examples/backups/_lib/` (the place for helpers shared by the backup walkthroughs) and any file directly in `examples/backups/`, the chainsaw config `hack/e2e-chainsaw/.chainsaw.yaml`, the `Makefile`, or any workflow that runs the suite escalate to the **full suite**. Only `pull-requests.yaml` is executed from the pull request's own head, so escalating on the other three buys generic regression coverage rather than coverage of the edit; the trade and its measured cost are written next to `full_suite_pattern` rather than repeated here. A per-suite edit (any file under `hack/e2e-chainsaw//`) selects **only** that suite, with one exception: a `*.disabled` file selects nothing, because a parked suite is executed by nothing and editing it cannot regress a test. The directory name is membership-tested against the suite list on the spot, so a directory holding no suite — shared material beside `_lib/` — escalates there and then, naming the file. `hack/e2e-apps/.bats` — the pre-Chainsaw per-app BATS suites — maps the same way, off the basename instead of the directory. The files formerly kept there were wired to nothing after migration, and the path remains deliberately mapped rather than marked inert: inert would bake the orphan status into the rule and go quietly wrong the day a lane runs those files again. The basename is membership-tested the same way. Deferring either test to the final intersection instead would leave the verdict to the rest of the diff — alone the selection empties and the backstop escalates, but beside any path that contributed a suite the escalation disappears and the run narrows to that suite in silence, which is the merge-before-escalate shape #3330 removed from the graph walk. Anything else under `hack/e2e-apps/` — a nested path, a non-`.bats` file — is shared material for those suites the way `hack/e2e-chainsaw/_lib/` is for the Chainsaw ones, and escalates for the same reason. -- **Every path is classified, and an unclassified one escalates.** The selector has three selection outcomes — full suite, a scoped set, or nothing — and each is reached by an explicit rule; the fourth outcome in the bullet below is not a selection but a refusal to produce one. Several rules produce the empty one, which is worth knowing before reading an empty selection as "the inert list matched": documentation, dashboards and `*.md` are dropped first wherever they live, a `*.disabled` Chainsaw suite and a `examples/backups//` edit whose app has no suite are dropped where chainsaw and backup-example paths are matched, and `inert_config_pattern` covers repo meta, agent config, `hack/testdata/`, the helm-unittest fixture charts under `packages/tests/`, the three `.gitattributes` that hold nothing but `linguist-generated` markers, which reach no build, chart or test (enumerated rather than matched by filename, because the justification is what those files contain and the name does not carry it — `.gitattributes` can also set `filter`, `eol`, `working-tree-encoding` and `export-subst`, each of which changes what lands in the working tree and therefore what gets built, so a by-name rule would make such a file inert in silence) and the top-level `hack/*.bats` that are not `hack/e2e-*.bats`. That last one is the unit lane, and the root `Makefile` is the authority on the split: `BATS_UNIT_FILES := $(filter-out hack/e2e-%.bats,$(wildcard hack/*.bats))` keeps every non-E2E file for `make unit-tests`, while `packages/core/testing` runs the E2E-prefixed files, so the sandbox never executes a unit file and no Chainsaw suite can regress from an edit to one. Being inert here does not leave them untested: the `unit-tests` job is gated on the `plan` job's `code` output, which `pull-requests.yaml` computes as "any changed path outside `docs/`" and never from `select-e2e.sh`, so a bats-only pull request still runs the whole unit lane — plus install and the OpenAPI tests, since the `e2e` job also reads `code` rather than the selection. `full_suite_pattern` covers shared build inputs and the e2e harness, and the graph lookup handles `packages/(apps|system|extra)/`. A path matching none of them escalates to the full suite and logs why, because both lanes read an empty selection as "skip Chainsaw" and then post `E2E Tests` green — so a silent default is a green gate with nothing tested (#3392). When that escalation fires for something genuinely inert, add it to `inert_config_pattern`; do not relax the fall-through. The escalation list and the inert list are checked in that order, so a specific escalation beats a broad inert directory. +- **Every path is classified, and an unclassified one escalates.** The selector has three selection outcomes — full suite, a scoped set, or nothing — and each is reached by an explicit rule; the fourth outcome in the bullet below is not a selection but a refusal to produce one. Several rules produce the empty one, which is worth knowing before reading an empty selection as "the inert list matched": documentation, dashboards and `*.md` are dropped first wherever they live, a `*.disabled` Chainsaw suite and a `examples/backups//` edit whose app has no suite are dropped where chainsaw and backup-example paths are matched, and `inert_config_pattern` covers repo meta, agent config, `hack/testdata/`, the helm-unittest fixture charts under `packages/tests/`, the three `.gitattributes` that hold nothing but `linguist-generated` markers, which reach no build, chart or test (enumerated rather than matched by filename, because the justification is what those files contain and the name does not carry it — `.gitattributes` can also set `filter`, `eol`, `working-tree-encoding` and `export-subst`, each of which changes what lands in the working tree and therefore what gets built, so a by-name rule would make such a file inert in silence), the top-level `hack/*.bats` that are not `hack/e2e-*.bats`, and their shared `hack/test_helper.bash`. These are the unit lane, and the root `Makefile` is the authority on the split: `BATS_UNIT_FILES := $(filter-out hack/e2e-%.bats,$(wildcard hack/*.bats))` keeps every non-E2E file for `make unit-tests`, while `packages/core/testing` runs the E2E-prefixed files, so the sandbox never executes a unit file and no Chainsaw suite can regress from an edit to one. Being inert here does not leave them untested: the `checks` job runs Bats for both code and docs-only changes. Its code path also runs the remaining unit and controller tests. A bats-only pull request still runs install and the OpenAPI tests, since the `e2e` job reads `code` rather than the selection. `full_suite_pattern` covers shared build inputs and the e2e harness, and the graph lookup handles `packages/(apps|system|extra)/`. A path matching none of them escalates to the full suite and logs why, because both lanes read an empty selection as "skip Chainsaw" and then post `E2E Tests` green — so a silent default is a green gate with nothing tested (#3392). When that escalation fires for something genuinely inert, add it to `inert_config_pattern`; do not relax the fall-through. The escalation list and the inert list are checked in that order, so a specific escalation beats a broad inert directory. - **Coverage is decided per changed path.** A path that resolves through the graph but reaches no runnable suite is covered by nothing, so it forces the full run on its own account, whatever the rest of the diff selected. Reading that off the merged selection instead let any other changed path contributing a suite name swallow the escalation, with nothing in the output to record that it happened: change a platform component, adjust the Chainsaw test beside it, and the component's escalation disappeared behind that one suite (#3330). The unit is the path rather than the source, because one path can belong to several PackageSources, not all of which need reach a suite, and deciding per source would escalate on one that reaches none and run everything for every change to the path. - **Two sources are excluded from the reverse-dependency walk as install-ordering hubs.** Every `*-application` source depends on `cozystack.cozystack-engine` so its `*-rd` HelmRelease waits for the ApplicationDefinition CRD, and the sources that depend on `cozystack.cozystack-basics` do so to get a namespace or a platform-wide policy in place first. Neither edge says the dependent's behaviour changes when the hub does, so carrying selection across them goes wrong in both directions: through the engine a `cert-manager` change fans out to every app in the tree, and through basics the platform's namespace-and-policy package narrows to whichever single suite happens to land downstream of it — 22 suites down to one on #3426, where the new site-router suite sits under `kubevirt-cdi`, which depends on basics. Dropping the reverse edges leaves both hubs reachable and stops them propagating; a change to either still runs everything, because its own group then reaches no suite and the per-path escalation fires on it. Adding a third one is a real decision, not bookkeeping: it says every existing edge into that source is ordering rather than behaviour, so read them before extending the list. - **There is a fourth outcome, and it is a failed step rather than a selection.** It fires when the suite list itself is unavailable: `find` cannot list `hack/e2e-chainsaw/`, or that directory holds no suite in any of the layouts above. There is no fallback left to take, because escalating would print an empty list and both lanes read an empty selection as "skip Chainsaw" before posting `E2E Tests` green — the outcome meaning "run everything" would deliver a run of nothing. So the selector names what broke on stderr and exits non-zero; both lanes run the step under `bash -e`, so the job fails and the required check goes red. A red `Select E2E tests` step is a real signal, not a flake: read its stderr line before rerunning. A `yq` failure is deliberately not in this class — what is missing there is the dependency graph, not the suite list, so the selector still has a safe answer and escalates to the full suite. That is conservative but quiet: the only sign is one stderr line inside a step that passes, so a selector running blind on every PR looks exactly like a project that escalates a lot. diff --git a/docs/agents/overview.md b/docs/agents/overview.md index 712ba9007..632caad78 100644 --- a/docs/agents/overview.md +++ b/docs/agents/overview.md @@ -69,7 +69,7 @@ Root targets (run from the repo root): ```bash make build # Build all Docker images (needs: docker, skopeo, jq, gh, helm, yq, GNU tar/sed/awk) -make unit-tests # Run the unit tests (Helm, BATS, Go, etc.); does not reach ./internal/... +make unit-tests # Run unit tests (requires bats-core >= 1.5; GNU parallel is optional) make test-controllers # Run the Go tests under ./internal/... (controllers, contract tests) make generate # Code generation (hack/update-codegen.sh) — CRDs, DeepCopy, clients, RBAC make manifests # Generate CRD manifests and operator YAML variants @@ -106,7 +106,7 @@ A chart also declares the limits on its own resource name there, with `## @name ## Testing -- **Helm unit tests:** `make helm-unit-tests` (runs `hack/helm-unit-tests.sh` over every package that defines a `test` target). A `test` target is expected to run helm-unittest: the sweep fails a package whose run reports no suite, because exiting 0 having asserted nothing is indistinguishable from passing. A target that drives something else belongs under another name, as `packages/core/testing` already does with its sandbox flow. `make unit-tests` runs the unit suite — Helm, BATS, Go, and the preset/readiness checks. It does not reach the controller and contract tests under `./internal/...`, which have their own target, `make test-controllers`, so covering both locally means running that target as well; CI runs them as two steps of one job. +- **Helm and BATS unit tests:** `make helm-unit-tests` runs `hack/helm-unit-tests.sh` over every package that defines a `test` target. A `test` target is expected to run helm-unittest: the sweep fails a package whose run reports no suite, because exiting 0 having asserted nothing is indistinguishable from passing. A target that drives something else belongs under another name, as `packages/core/testing` already does with its sandbox flow. `make bats-unit-tests` requires bats-core 1.5 or newer; GNU parallel is optional and the target runs serially without it. Run the complete local BATS and POSIX-compatibility pass with `pre-commit run bats-unit-tests --hook-stage manual --all-files`; pull-request CI runs both lanes for code changes and the BATS contracts for docs-only changes. On a host whose `/bin/sh` differs from CI, `make BATS_POSIX_SHELL=dash bats-posix-compat-tests` selects the same interpreter explicitly. `make unit-tests` runs the full unit suite — Helm, BATS, the targeted POSIX-shell compatibility pass, Go, and the preset/readiness checks. It does not include the controller tests under `./internal/...`; run `make test-controllers` as well. CI invokes both targets together. - **E2E tests:** Kyverno Chainsaw suites in `hack/e2e-chainsaw/` (one directory per app), run with `chainsaw test`. Cluster bootstrap (`hack/e2e-install-cozystack.bats`) and the OpenAPI checks (`hack/e2e-test-openapi.bats`) remain BATS. Conventions for writing and stabilising them — and the CI that runs them — live in [`e2e-testing.md`](./e2e-testing.md). - **Go tests:** standard `go test`, with Ginkgo/Gomega for controllers. @@ -171,4 +171,3 @@ Before working on these areas, read the relevant doc first: ### Core Components - Do not modify `packages/core/platform/` without understanding migration impact - diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats index 310590ae0..fb764f557 100644 --- a/hack/admin-kubeconfig-invariant.bats +++ b/hack/admin-kubeconfig-invariant.bats @@ -18,6 +18,8 @@ # project's CI runners and on the maintainer workstation. # ----------------------------------------------------------------------------- +load test_helper + @test "every Deployment mounting admin-kubeconfig has optional:true and wait-for-kubeconfig init" { values_file="packages/apps/kubernetes/tests/values-ci.yaml" [ -f "$values_file" ] diff --git a/hack/alert-severity-contract.bats b/hack/alert-severity-contract.bats index 5180a0d41..45fe0a879 100644 --- a/hack/alert-severity-contract.bats +++ b/hack/alert-severity-contract.bats @@ -1,5 +1,7 @@ #!/usr/bin/env bats +load test_helper + # Alerta's default alarm model, the set the webhook accepts. `none` is left out # on purpose: the Alertmanager config routes severity="none" to a blackhole # receiver, so such an alert is never delivered, which is the outcome this diff --git a/hack/app-cleanup-hooks_test.bats b/hack/app-cleanup-hooks_test.bats index 006d7474c..672dcc541 100644 --- a/hack/app-cleanup-hooks_test.bats +++ b/hack/app-cleanup-hooks_test.bats @@ -19,12 +19,15 @@ # already-clean namespace exits 0; # * a listing that FAILED is distinguishable from one that returned nothing. # -# Run via hack/cozytest.sh from the repo root (make bats-unit-tests); relative -# paths resolve against that cwd. The runner has no setup/teardown, so each -# @test builds its own fixture and removes it at the end of the body. No EXIT +# Run from the repo root through `make bats-unit-tests`; relative paths resolve +# against that cwd. The file also remains compatible with the legacy +# `hack/cozytest.sh` translator, so each @test builds and removes its fixture +# inline instead of relying on setup()/teardown(). No EXIT # trap: docs/agents/e2e-testing.md bans them in hack/*.bats, and under the bats # binary a test that installs one and then fails prints no TAP line at all. +load test_helper + QDRANT_CHART=packages/apps/qdrant HARBOR_CHART=packages/apps/harbor diff --git a/hack/backup-access-preflight.bats b/hack/backup-access-preflight.bats index 600497656..3892c8bcf 100644 --- a/hack/backup-access-preflight.bats +++ b/hack/backup-access-preflight.bats @@ -2,6 +2,8 @@ # Unit tests for the E2E-only S3 access gate that runs before expensive backup # workloads. All cluster and S3 calls are shell stubs. +load test_helper + @test "preflight helper remains sourceable by the POSIX test runner" { helper=hack/e2e-chainsaw/_lib/backup-access-preflight.sh [ "$(sed -n '1p' "$helper")" = '#!/bin/sh' ] diff --git a/hack/backup-examples-shared-helpers.bats b/hack/backup-examples-shared-helpers.bats index 4fc994f13..1821c0192 100644 --- a/hack/backup-examples-shared-helpers.bats +++ b/hack/backup-examples-shared-helpers.bats @@ -5,6 +5,8 @@ # wait_for_field reports why kubectl could not read a field while staying quiet # about a resource that does not exist yet. +load test_helper + SHARED=examples/backups/_lib/wait-helpers.sh SHARED_HELPERS="wait_for_field wait_hr_ready wait_deleted" diff --git a/hack/bats-no-exit-trap.bats b/hack/bats-no-exit-trap.bats index 5235eb0cc..3dc303a5c 100644 --- a/hack/bats-no-exit-trap.bats +++ b/hack/bats-no-exit-trap.bats @@ -85,6 +85,8 @@ # check that catches whatever the pattern missed. # ----------------------------------------------------------------------------- +load test_helper + # The directory to audit. Under the bats binary that is this file's own # location; under cozytest.sh, which does not set BATS_TEST_FILENAME, `$0` is # the runner itself -- and the answer comes out the same only because the runner diff --git a/hack/bats-no-negated-assert.bats b/hack/bats-no-negated-assert.bats index 4eee5ed20..3b6862ad5 100644 --- a/hack/bats-no-negated-assert.bats +++ b/hack/bats-no-negated-assert.bats @@ -77,6 +77,8 @@ # it exists to ban, one level up. # ----------------------------------------------------------------------------- +load test_helper + # The directory to audit. Under the bats binary this file's own location; under # cozytest.sh, which sets no BATS_TEST_FILENAME, `$0` is the runner, and the # answer agrees only because the runner lives beside the files it runs. The diff --git a/hack/bats-runner-contract.bats b/hack/bats-runner-contract.bats new file mode 100644 index 000000000..22426edb2 --- /dev/null +++ b/hack/bats-runner-contract.bats @@ -0,0 +1,141 @@ +#!/usr/bin/env bats +# Keep local discovery, CI provisioning and contributor instructions aligned: +# losing any one can leave a passing lane that other environments never run. + +load test_helper + +BRC_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" +BRC_REPO_ROOT="$(cd "$BRC_DIR/.." && pwd)" +BRC_WORKFLOW="$BRC_REPO_ROOT/.github/workflows/pull-requests.yaml" +BRC_PRECOMMIT_WORKFLOW="$BRC_REPO_ROOT/.github/workflows/pre-commit.yml" + +brc_toolchain_script() { + yq -r '.jobs.checks.steps[] | select(.name == "Set up test toolchain") | .run' "$BRC_WORKFLOW" + return 0 +} + +brc_write_parallel_stub() { + printf '%s\n' '#!/bin/sh' "printf '%s\\n' '$2'" > "$1/parallel" + chmod +x "$1/parallel" + return 0 +} + +@test "CI installs the exact commit behind the Bats release tag" { + version=$(yq -r '.jobs.checks.steps[] | select(.name == "Set up test toolchain") | .env.BATS_VERSION' "$BRC_WORKFLOW") + commit=$(yq -r '.jobs.checks.steps[] | select(.name == "Set up test toolchain") | .env.BATS_COMMIT' "$BRC_WORKFLOW") + [ "$version" = "1.14.0" ] + [ "$commit" = "eb7f42f8d608ac693d7a4b67474f6714ea68cfc5" ] + + script=$(brc_toolchain_script) + printf '%s\n' "$script" | grep -Fq 'git clone --quiet --depth=1 --branch "v$BATS_VERSION"' + printf '%s\n' "$script" | grep -Fq 'test "$(git -C "$bats_source" rev-parse HEAD)" = "$BATS_COMMIT"' + gates=$(printf '%s\n' "$script" | grep -Fc 'grep -qx "Bats $BATS_VERSION"' || :) + [ "$gates" -eq 2 ] + if printf '%s\n' "$script" | grep -q '/archive/'; then + echo "FAIL: the Bats installer pins a byte-unstable generated archive" + false + fi +} + +@test "the job-count fallback accepts GNU Parallel only" { + tmp=$(mktemp -d) + mkdir -p "$tmp/bin" + printf '%s\n' '#!/bin/sh' 'printf "7\\n"' > "$tmp/bin/nproc" + chmod +x "$tmp/bin/nproc" + + printf '%s\n' '#!/bin/sh' 'exit 127' > "$tmp/bin/parallel" + chmod +x "$tmp/bin/parallel" + # Make exports command-line overrides; the probe must see its own default. + jobs=$(cd "$BRC_REPO_ROOT" && env -u BATS_JOBS MAKEFLAGS= MAKELEVEL= PATH="$tmp/bin:$PATH" make --no-print-directory -s print-bats-jobs) + [ "$jobs" = "1" ] + + brc_write_parallel_stub "$tmp/bin" "parallel from moreutils" + jobs=$(cd "$BRC_REPO_ROOT" && env -u BATS_JOBS MAKEFLAGS= MAKELEVEL= PATH="$tmp/bin:$PATH" make --no-print-directory -s print-bats-jobs) + [ "$jobs" = "1" ] + + brc_write_parallel_stub "$tmp/bin" "GNU parallel 20260722" + jobs=$(cd "$BRC_REPO_ROOT" && env -u BATS_JOBS MAKEFLAGS= MAKELEVEL= PATH="$tmp/bin:$PATH" make --no-print-directory -s print-bats-jobs) + [ "$jobs" = "7" ] + rm -rf "$tmp" +} + +@test "the complete local hook is manual and documented" { + hook_entry=$(yq -r '.repos[].hooks[] | select(.id == "bats-unit-tests") | .entry' "$BRC_REPO_ROOT/.pre-commit-config.yaml") + hook_stages=$(yq -r '.repos[].hooks[] | select(.id == "bats-unit-tests") | .stages | join(",")' "$BRC_REPO_ROOT/.pre-commit-config.yaml") + hook_passes_files=$(yq -r '.repos[].hooks[] | select(.id == "bats-unit-tests") | .pass_filenames' "$BRC_REPO_ROOT/.pre-commit-config.yaml") + [ "$hook_entry" = 'make bats-unit-tests bats-posix-compat-tests' ] + [ "$hook_stages" = 'manual' ] + [ "$hook_passes_files" = 'false' ] + grep -Fq 'bats-core 1.5 or newer' "$BRC_REPO_ROOT/docs/agents/overview.md" + grep -Fq 'pre-commit run bats-unit-tests --hook-stage manual --all-files' "$BRC_REPO_ROOT/docs/agents/overview.md" +} + +@test "pull-request CI keeps the required Bats lane unconditional" { + lint_skip=$(yq -r '.jobs.pre-commit.steps[] | select(.name == "Run pre-commit hooks") | .env.SKIP // ""' "$BRC_PRECOMMIT_WORKFLOW") + code_command=$(yq -r '.jobs.checks.steps[] | select(.name == "Run unit and controller tests") | .run' "$BRC_WORKFLOW") + docs_command=$(yq -r '.jobs.checks.steps[] | select(.name == "Run Bats unit tests for docs-only changes") | .run' "$BRC_WORKFLOW") + [ -z "$lint_skip" ] + [ "$code_command" = 'make unit-tests test-controllers -j4 -k --output-sync=target' ] + [ "$docs_command" = 'make bats-unit-tests' ] +} + +@test "the POSIX compatibility lane retains reviewed and sourced shell-facing files" { + compat_files=$(cd "$BRC_REPO_ROOT" && MAKEFLAGS= MAKELEVEL= make --no-print-directory -s print-bats-posix-compat-files) + unit_files=$(cd "$BRC_REPO_ROOT" && MAKEFLAGS= MAKELEVEL= make --no-print-directory -s print-bats-unit-files) + sourced_shell_tests=$(cd "$BRC_REPO_ROOT" && grep -El '^[[:space:]]*(\.|source)[[:space:]]+.*\.sh' $unit_files) + [ -n "$sourced_shell_tests" ] + for file in \ + $sourced_shell_tests \ + hack/capture-dataplane.bats \ + hack/capture-previous-logs.bats \ + hack/cilium-leak-healer_test.bats \ + hack/container-lane-capacity_test.bats \ + hack/cozyreport-talos.bats \ + hack/cozyreport.bats \ + hack/cozystack-version-stamp.bats \ + hack/nightly-mirror_test.bats \ + hack/pod-label-census_test.bats \ + hack/promote-rewrite-tags_test.bats \ + hack/runner-identity.bats \ + hack/seaweedfs-naming-audit.bats; do + printf '%s\n' "$compat_files" | grep -Fxq "$file" + done +} + +@test "compatibility discovery includes new shell dependencies and excludes unrelated files" { + tmp=$(mktemp -d) + fixture="$tmp/non-chainsaw-helper.bats" + unrelated="$tmp/no-shell-helper.bats" + printf '%s\n' '#!/usr/bin/env bats' '. hack/lib/image-refs.sh' > "$fixture" + printf '%s\n' '#!/usr/bin/env bats' 'load test_helper' > "$unrelated" + make_status=0 + compat_files=$(cd "$BRC_REPO_ROOT" && MAKEFLAGS= MAKELEVEL= make --no-print-directory -s "BATS_UNIT_FILES=$fixture $unrelated" print-bats-posix-compat-files) || make_status=$? + rm -rf "$tmp" + + [ "$make_status" -eq 0 ] + printf '%s\n' "$compat_files" | grep -Fxq "$fixture" + if printf '%s\n' "$compat_files" | grep -Fxq "$unrelated"; then + echo "FAIL: a unit file with no production shell dependency entered the compatibility lane" + false + fi +} + +@test "a failed compatibility file does not suppress later files" { + tmp=$(mktemp -d) + first="$tmp/first.bats" + second="$tmp/second.bats" + marker="$tmp/later-ran" + printf '%s\n' '@test "first fails" {' ' false' '}' > "$first" + printf '%s\n' '@test "later runs" {' " : > '$marker'" '}' > "$second" + + make_status=0 + output=$(cd "$BRC_REPO_ROOT" && MAKEFLAGS= MAKELEVEL= make --no-print-directory BATS_POSIX_SHELL=dash "BATS_POSIX_COMPAT_FILES=$first $second" bats-posix-compat-tests 2>&1) || make_status=$? + marker_present=0 + [ -e "$marker" ] && marker_present=1 + rm -rf "$tmp" + + [ "$make_status" -ne 0 ] + [ "$marker_present" -eq 1 ] + printf '%s\n' "$output" | grep -Fq -- "--- running POSIX compatibility: $first ---" + printf '%s\n' "$output" | grep -Fq -- "--- running POSIX compatibility: $second ---" +} diff --git a/hack/bats-runner-coverage.bats b/hack/bats-runner-coverage.bats index 4fe1a1a24..7f0b76885 100644 --- a/hack/bats-runner-coverage.bats +++ b/hack/bats-runner-coverage.bats @@ -9,6 +9,8 @@ # rules so narrowing the real runner cannot silently leave this audit green. # A new runner needs an entry here. Park unused suites with .bats.disabled. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" E2E_MAKEFILE_REL="packages/core/testing/Makefile" @@ -151,20 +153,11 @@ brc_audit() { printf '%s\n' "$makefile" | grep -qF 'BATS_UNIT_FILES := $(filter-out hack/e2e-%.bats,$(wildcard hack/*.bats))' \ || { echo "FAIL: bats-unit-tests no longer selects files the way brc_unit_reachable models it (one level, hack/e2e-* excluded). Update the model in hack/bats-runner-coverage.bats in the same change."; false; } - printf '%s\n' "$makefile" | grep -qF 'BATS_UNIT_TARGETS := $(patsubst hack/%.bats,bats-unit-%,$(BATS_UNIT_FILES))' \ - || { echo "FAIL: update the unit target model in hack/bats-runner-coverage.bats."; false; } - printf '%s\n' "$makefile" | grep -qF 'bats-unit-tests: bats-unit-files-check' \ - || { echo "FAIL: bats-unit-tests no longer checks that unit files were discovered."; false; } + || { echo "FAIL: bats-unit-tests no longer checks for an empty discovery."; false; } - printf '%s\n' "$makefile" | grep -qF -- '--keep-going $(BATS_UNIT_TARGETS)' \ - || { echo "FAIL: bats-unit-tests no longer runs every discovered unit file past a failing one."; false; } - - printf '%s\n' "$makefile" | grep -qF '$(BATS_UNIT_TARGETS): bats-unit-%: hack/%.bats' \ - || { echo "FAIL: each unit target must depend on its discovered bats file."; false; } - - printf '%s\n' "$makefile" | grep -qF 'hack/cozytest.sh "$<"' \ - || { echo "FAIL: bats-unit-tests no longer runs its files through hack/cozytest.sh; the reachability model assumes it does."; false; } + printf '%s\n' "$makefile" | grep -qF 'bats -j $(BATS_JOBS) --report-formatter junit -o "$(BATS_REPORT_DIR)" $(BATS_UNIT_FILES)' \ + || { echo "FAIL: update the Bats invocation model in hack/bats-runner-coverage.bats."; false; } printf '%s\n' "$makefile" | grep -qE '^unit-tests:.*bats-unit-tests' \ || { echo "FAIL: unit-tests no longer depends on bats-unit-tests, so nothing in CI runs the bats sweep -- this guard included."; false; } diff --git a/hack/bats-strict-setup.bats b/hack/bats-strict-setup.bats new file mode 100644 index 000000000..5d2c7eb4a --- /dev/null +++ b/hack/bats-strict-setup.bats @@ -0,0 +1,483 @@ +#!/usr/bin/env bats +# Every discovered unit file must load the nounset helper once. A file-local +# setup() replaces the helper's setup, so its first command must restore it. +# +# This lexical audit accepts one spelling at column zero, not arbitrary shell: +# conditional loads are rejected, while a literal inside a heredoc could be +# mistaken for a load. Fixtures use printf to avoid that ambiguity. Nested Bats +# canaries separately verify the helper's effect in the actual test process. + +load test_helper + +# The directory to audit. Under the bats binary that is this file's own +# location; under cozytest.sh, which does not set BATS_TEST_FILENAME, `$0` is +# the runner itself -- and the answer comes out the same only because the runner +# lives in the directory it runs files from. The first test below refuses to +# pass on an empty enumeration, so a wrong answer here fails rather than audits +# nothing. +BSS_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" +BSS_REPO_ROOT="$(cd "$BSS_DIR/.." && pwd)" + +# Non-recursive, and the exclusion is on the basename, because that is what +# `$(filter-out hack/e2e-%.bats,$(wildcard hack/*.bats))` in the Makefile means. +# A shell glob keeps this portable to the macOS hook instead of relying on +# GNU find's non-POSIX `-maxdepth` flag. +# Nested files are outside this unit enumeration. Runner reachability is +# checked separately by hack/bats-runner-coverage.bats. +bss_units() { + for _f in "$1"/*.bats; do + [ -e "$_f" ] || continue + case ${_f##*/} in + e2e-*) continue ;; + esac + printf '%s\n' "$_f" + done | sort + return 0 +} + +# Report every unit file whose strict-mode setup is missing or overridden, one +# line each, and print nothing when they all comply. +# +# Through stdout rather than an exit status: cozytest.sh rewrites a function's +# closing brace into `return 0`, so a status set in here would be discarded +# before the caller could read it, and the check would pass unconditionally. +bss_audit() { + bss_units "$1" | while IFS= read -r _f; do + [ -e "$_f" ] || continue + _b=$(basename "$_f") + _load_count=$(grep -c '^load test_helper$' "$_f" || :) + if [ "$_load_count" -eq 0 ]; then + echo "$_b: does not load the strict-mode helper; add a column-zero \`load test_helper\` line under the file's leading comment block" + continue + fi + if [ "$_load_count" -ne 1 ]; then + echo "$_b: loads the strict-mode helper $_load_count times; keep exactly one column-zero \`load test_helper\` line" + continue + fi + # A file-local setup replaces the helper's without a word from either + # runner. Recognize the common Bash forms, including indentation, + # `function setup {`, and a brace on the following line, then require the + # suite's one canonical spelling. That makes an ambiguous nested declaration + # fail closed instead of guessing whether Bats will see it at source time. + _setup_count=$(grep -cE '^[[:space:]]*(setup[[:space:]]*\(|function[[:space:]]+setup([[:space:]]|\(|[{]|$))' "$_f" || :) + _canonical_count=$(grep -c '^setup() {$' "$_f" || :) + if [ "$_setup_count" -gt 0 ] \ + && { [ "$_setup_count" -ne 1 ] || [ "$_canonical_count" -ne 1 ]; }; then + echo "$_b: uses a noncanonical or repeated setup declaration; use exactly one column-zero \`setup() {\` block" + continue + fi + if [ "$_canonical_count" -eq 1 ]; then + _first_effective=$(awk ' + /^setup\(\) \{$/ { in_setup=1; next } + in_setup && /^}$/ { exit } + in_setup && /^[[:space:]]*$/ { next } + in_setup && /^[[:space:]]*#/ { next } + in_setup { print; exit } + ' "$_f") + if ! printf '%s\n' "$_first_effective" \ + | grep -qE '^[[:space:]]*strict_setup[[:space:]]*$'; then + echo "$_b: setup() must call strict_setup directly as its first effective command; blank and comment lines may precede it" + fi + fi + done + return 0 +} + +# The load line is assembled from a printf argument rather than +# written as a literal, because this file is audited by the guard it defines and +# a column-zero literal here would be read as a second declaration of its own. +bss_new_fixture() { + printf '%s\n' '#!/usr/bin/env bats' > "$1/subject.bats" + return 0 +} + +bss_add_load() { + printf '%s\n' 'load test_helper' >> "$1/subject.bats" + return 0 +} + +bss_add_indented_load() { + printf ' %s\n' 'load test_helper' >> "$1/subject.bats" + return 0 +} + +bss_add_commented_load() { + printf '# %s\n' 'load test_helper' >> "$1/subject.bats" + return 0 +} + +bss_add_own_setup() { + printf '%s\n' 'setup() {' ' export FIXTURE=1' '}' >> "$1/subject.bats" + return 0 +} + +bss_add_strict_own_setup() { + printf '%s\n' 'setup() {' '' ' # Restore the unit-suite contract first.' ' strict_setup' ' export FIXTURE=1' '}' >> "$1/subject.bats" + return 0 +} + +bss_add_subshell_strict_own_setup() { + printf '%s\n' 'setup() {' ' (' ' strict_setup' ' )' '}' >> "$1/subject.bats" + return 0 +} + +bss_add_conditional_strict_own_setup() { + printf '%s\n' 'setup() {' ' if false; then' ' strict_setup' ' fi' '}' >> "$1/subject.bats" + return 0 +} + +bss_add_function_own_setup() { + printf '%s\n' 'function setup {' ' export FIXTURE=1' '}' >> "$1/subject.bats" + return 0 +} + +bss_add_indented_own_setup() { + printf '%s\n' ' setup() {' ' export FIXTURE=1' ' }' >> "$1/subject.bats" + return 0 +} + +bss_add_next_line_brace_setup() { + printf '%s\n' 'setup()' '{' ' export FIXTURE=1' '}' >> "$1/subject.bats" + return 0 +} + +bss_add_spaced_parens_setup() { + printf '%s\n' 'setup ( ) {' ' export FIXTURE=1' '}' >> "$1/subject.bats" + return 0 +} + +bss_add_commented_strict_own_setup() { + printf '%s\n' 'setup() {' ' # strict_setup' ' export FIXTURE=1' '}' >> "$1/subject.bats" + return 0 +} + +bss_add_unrelated_strict_call() { + printf '%s\n' 'unrelated() {' ' strict_setup' '}' >> "$1/subject.bats" + return 0 +} + +bss_rename() { + mv "$1/subject.bats" "$1/$2" + return 0 +} + +bss_add_nounset_canary() { + printf '%s\n' \ + '@test "nounset canary" {' \ + ' printf "%s\n" "$BSS_NOUNSET_CANARY_UNSET"' \ + '}' >> "$1/subject.bats" + ln -s "$BSS_DIR/test_helper.bash" "$1/test_helper.bash" + return 0 +} + +@test "the unit enumeration is not empty" { + count=$(bss_units "$BSS_DIR" | wc -l) + if [ "$count" -lt 2 ]; then + echo "FAIL: enumerated $count unit file(s) under $BSS_DIR; the audit below would be vacuous" + false + fi +} + +@test "the unit enumeration is non-recursive without GNU find extensions" { + tmp=$(mktemp -d) + mkdir -p "$tmp/bin" "$tmp/nested" + : > "$tmp/root.bats" + : > "$tmp/e2e-live.bats" + : > "$tmp/nested/child.bats" + printf '%s\n' '#!/bin/sh' 'exit 97' > "$tmp/bin/find" + chmod +x "$tmp/bin/find" + + units=$(PATH="$tmp/bin:$PATH" bss_units "$tmp") + [ "$units" = "$tmp/root.bats" ] + rm -rf "$tmp" +} + +@test "the audited set is exactly the set the Makefile hands the runner" { + # Ask make for its own answer instead of trusting this file's copy of the + # rule. The whole point of the audit is that its input tracks the runner's, + # so a divergence between the two is the failure it exists to prevent -- + # narrow the Makefile's filter without narrowing this one and the audit goes + # on reporting clean about files nobody runs, or vice versa. + tmp=$(mktemp -d) + bss_units "$BSS_DIR" | sed "s|^$BSS_REPO_ROOT/||" | sort > "$tmp/mine" + # MAKEFLAGS cleared: this runs from inside `make bats-unit-tests`, and a + # sub-make that inherits the parent's `-j` without its jobserver descriptors + # warns about it. The warning goes to stderr and would not corrupt the + # comparison, but it would land in the middle of a green suite's output. + (cd "$BSS_REPO_ROOT" && MAKEFLAGS= MAKELEVEL= make --no-print-directory -s print-bats-unit-files) | sort > "$tmp/theirs" + # File comparison keeps this audit compatible with cozytest's POSIX shell; + # process substitution would require Bash. + if ! diff -u "$tmp/theirs" "$tmp/mine" > "$tmp/delta"; then + echo "FAIL: the audited set and \$(BATS_UNIT_FILES) disagree (-Makefile +audit):" + cat "$tmp/delta" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "default and direct canonical setup enable nounset in real nested Bats tests" { + tmp=$(mktemp -d) + unset BSS_NOUNSET_CANARY_UNSET + + for form in default direct; do + mkdir "$tmp/$form" + bss_new_fixture "$tmp/$form" + bss_add_load "$tmp/$form" + if [ "$form" = direct ]; then + bss_add_strict_own_setup "$tmp/$form" + fi + bss_add_nounset_canary "$tmp/$form" + report=$(bss_audit "$tmp/$form") + if [ -n "$report" ]; then + echo "FAIL: the supported $form setup was rejected: $report" + rm -rf "$tmp" + false + fi + + child_status=0 + output=$(bats --formatter tap "$tmp/$form/subject.bats" 2>&1) || child_status=$? + if [ "$child_status" -eq 0 ]; then + echo "FAIL: nested Bats accepted an unset test-body read with $form setup" + echo "$output" + rm -rf "$tmp" + false + fi + printf '%s\n' "$output" | grep -Fq 'not ok 1 nounset canary' + printf '%s\n' "$output" | grep -Fq 'BSS_NOUNSET_CANARY_UNSET: unbound variable' + done + rm -rf "$tmp" +} + +@test "the helper leaves subprocess shell options unchanged" { + tmp=$(mktemp -d) + unset BSS_NOUNSET_CANARY_UNSET + bss_new_fixture "$tmp" + bss_add_load "$tmp" + ln -s "$BSS_DIR/test_helper.bash" "$tmp/test_helper.bash" + printf '%s\n' '#!/bin/bash' 'printf "child:%s\n" "$BSS_NOUNSET_CANARY_UNSET"' > "$tmp/child.sh" + printf '@test "subprocess canary" {\n bash "%s/child.sh"\n}\n' "$tmp" >> "$tmp/subject.bats" + child_status=0 + output=$(bats --formatter tap "$tmp/subject.bats" 2>&1) || child_status=$? + [ "$child_status" -eq 0 ] + printf '%s\n' "$output" | grep -Fq 'ok 1 subprocess canary' + rm -rf "$tmp" +} + +@test "every hack/*.bats unit file restores set -u through the shared helper" { + report=$(bss_audit "$BSS_DIR") + if [ -n "$report" ]; then + echo "FAIL: strict-mode setup missing or overridden:" + echo "$report" + false + fi +} + +@test "a unit file with no load line is reported" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + report=$(bss_audit "$tmp") + if ! echo "$report" | grep -q 'does not load the strict-mode helper'; then + echo "FAIL: an uncovered file was not reported; got: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "a unit file with the load line is clean" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + bss_add_load "$tmp" + report=$(bss_audit "$tmp") + if [ -n "$report" ]; then + echo "FAIL: a covered file was reported: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "duplicate load lines are reported" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + bss_add_load "$tmp" + bss_add_load "$tmp" + report=$(bss_audit "$tmp") + if ! echo "$report" | grep -q 'keep exactly one column-zero'; then + echo "FAIL: duplicate load lines were accepted; got: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "an indented or commented-out load line does not count" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + bss_add_indented_load "$tmp" + bss_add_commented_load "$tmp" + report=$(bss_audit "$tmp") + if ! echo "$report" | grep -q 'does not load the strict-mode helper'; then + echo "FAIL: a load line that never runs was credited; got: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "a file-local setup that skips strict_setup is reported" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + bss_add_load "$tmp" + bss_add_own_setup "$tmp" + report=$(bss_audit "$tmp") + if ! echo "$report" | grep -q 'first effective command'; then + echo "FAIL: a setup() override was not reported; got: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "a file-local setup that calls strict_setup is clean" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + bss_add_load "$tmp" + bss_add_strict_own_setup "$tmp" + report=$(bss_audit "$tmp") + if [ -n "$report" ]; then + echo "FAIL: a compliant setup() override was reported: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "subshell and conditional strict_setup placements are rejected and leave nounset off" { + tmp=$(mktemp -d) + unset BSS_NOUNSET_CANARY_UNSET + + for form in subshell conditional; do + mkdir "$tmp/$form" + bss_new_fixture "$tmp/$form" + bss_add_load "$tmp/$form" + if [ "$form" = subshell ]; then + bss_add_subshell_strict_own_setup "$tmp/$form" + else + bss_add_conditional_strict_own_setup "$tmp/$form" + fi + bss_add_nounset_canary "$tmp/$form" + + report=$(bss_audit "$tmp/$form") + if ! printf '%s\n' "$report" | grep -q 'first effective command'; then + echo "FAIL: the unsafe $form setup was accepted: $report" + rm -rf "$tmp" + false + fi + + child_status=0 + output=$(bats --formatter tap "$tmp/$form/subject.bats" 2>&1) || child_status=$? + if [ "$child_status" -ne 0 ]; then + echo "FAIL: the $form fixture unexpectedly enabled nounset:" + echo "$output" + rm -rf "$tmp" + false + fi + printf '%s\n' "$output" | grep -Fq 'ok 1 nounset canary' + done + rm -rf "$tmp" +} + +@test "function-style or indented setup declarations are rejected" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + bss_add_load "$tmp" + bss_add_function_own_setup "$tmp" + report=$(bss_audit "$tmp") + if ! echo "$report" | grep -q 'noncanonical or repeated setup declaration'; then + echo "FAIL: function-style setup was not rejected; got: $report" + rm -rf "$tmp" + false + fi + bss_new_fixture "$tmp" + bss_add_load "$tmp" + bss_add_indented_own_setup "$tmp" + report=$(bss_audit "$tmp") + if ! echo "$report" | grep -q 'noncanonical or repeated setup declaration'; then + echo "FAIL: indented setup was not rejected; got: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "a setup declaration with its brace on the next line is rejected" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + bss_add_load "$tmp" + bss_add_next_line_brace_setup "$tmp" + report=$(bss_audit "$tmp") + if ! echo "$report" | grep -q 'noncanonical or repeated setup declaration'; then + echo "FAIL: a brace-on-next-line setup was not rejected; got: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "a setup declaration with spaced parentheses is rejected" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + bss_add_load "$tmp" + bss_add_spaced_parens_setup "$tmp" + report=$(bss_audit "$tmp") + if ! echo "$report" | grep -q 'noncanonical or repeated setup declaration'; then + echo "FAIL: a spaced-parentheses setup was not rejected; got: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "a comment-only strict_setup mention does not satisfy setup" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + bss_add_load "$tmp" + bss_add_commented_strict_own_setup "$tmp" + report=$(bss_audit "$tmp") + if ! echo "$report" | grep -q 'first effective command'; then + echo "FAIL: a comment-only strict_setup mention was credited; got: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "a strict_setup call outside setup does not satisfy setup" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + bss_add_load "$tmp" + bss_add_unrelated_strict_call "$tmp" + bss_add_own_setup "$tmp" + report=$(bss_audit "$tmp") + if ! echo "$report" | grep -q 'first effective command'; then + echo "FAIL: a strict_setup call outside setup was credited; got: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} + +@test "an e2e- file is outside the audit" { + tmp=$(mktemp -d) + bss_new_fixture "$tmp" + bss_rename "$tmp" "e2e-subject.bats" + report=$(bss_audit "$tmp") + if [ -n "$report" ]; then + echo "FAIL: a live-cluster suite was audited as a unit file: $report" + rm -rf "$tmp" + false + fi + rm -rf "$tmp" +} diff --git a/hack/bats-unit-keep-going.bats b/hack/bats-unit-keep-going.bats index f4dfc1855..91f987d2b 100644 --- a/hack/bats-unit-keep-going.bats +++ b/hack/bats-unit-keep-going.bats @@ -1,42 +1,47 @@ #!/usr/bin/env bats # `make bats-unit-tests` must run every bats file even when an early one fails, -# and still exit non-zero. Each file is its own make target, and without -# --keep-going make stops scheduling targets at the first failure, so the files -# sorting after it never run and the output reads like a complete suite. +# and still exit non-zero, so a red file never hides the ones after it behind +# output that reads like a complete suite. One Bats invocation owns the whole +# set, which gives that for free; this pins that the recipe keeps it that way. # -# The tests drive the real root Makefile against a miniature tree: two bats -# files and a stub hack/cozytest.sh that reports which file it was given. +# The tests drive the real root Makefile against a miniature tree of two bats +# files, with the real bats binary. # -# Requires: make. +# Requires: make, bats. + +load test_helper REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" -# kg_run [makefiles]: builds the fixture, runs the target with -# MAKEFILES set to the second argument, and leaves the output in $fixture/out -# and the exit status in $rc. +# kg_run : builds the fixture, runs the target in it +# and leaves the output in $fixture/out and the exit status in $rc. a.bats sorts +# first, so a failure there is the one that could hide b.bats. kg_run() { fixture="$(mktemp -d)" mkdir "$fixture/hack" : >"$fixture/hack/common-envs.mk" - : >"$fixture/hack/a.bats" - : >"$fixture/hack/b.bats" - printf '%s\n' '#!/bin/sh' 'echo "stub ran $1"' "$1" >"$fixture/hack/cozytest.sh" - chmod +x "$fixture/hack/cozytest.sh" + printf '%s\n' '@test "first file" {' " $1" '}' >"$fixture/hack/a.bats" + printf '%s\n' '@test "second file" {' ' true' '}' >"$fixture/hack/b.bats" # Under `make unit-tests -k` this suite inherits MAKEFLAGS carrying the - # parent's --keep-going, which would let the old behaviour pass here. + # parent's --keep-going, which would hide a recipe that stops on its own. + # + # Bats puts its own libexec first on PATH, and the `bats` found there is a + # helper that needs bats_readlinkf, a bash function bin/bats exports. The + # recipe runs under /bin/sh, and dash (the CI runner's) drops exported + # functions, so the helper dies with "command not found" and no test runs. + # Dropping libexec makes the recipe find the real bats on either shell. rc=0 - env -u MAKEFLAGS -u MFLAGS -u MAKELEVEL MAKEFILES="${2:-}" \ - make --file="$REPO_ROOT/Makefile" --directory="$fixture" bats-unit-tests >"$fixture/out" 2>&1 || rc=$? + env -u MAKEFLAGS -u MFLAGS -u MAKELEVEL PATH="${PATH#"$BATS_LIBEXEC":}" \ + make --file="$REPO_ROOT/Makefile" --directory="$fixture" --no-print-directory \ + BATS_JOBS=1 "BATS_REPORT_DIR=$fixture/reports" bats-unit-tests >"$fixture/out" 2>&1 || rc=$? cat "$fixture/out" } @test "bats-unit-tests runs every file after a failure and still fails" { - # Fail whichever file runs first, whatever order make picks, so the exit - # status cannot come from the last file alone. - kg_run 'mkdir "$(dirname "$0")/.failed" 2>/dev/null && exit 1; exit 0' - ran_a=$(grep -c '^stub ran hack/a.bats$' "$fixture/out" || true) - ran_b=$(grep -c '^stub ran hack/b.bats$' "$fixture/out" || true) + kg_run 'false' + ran_a=$(grep -c '^not ok 1 first file' "$fixture/out" || true) + ran_b=$(grep -c '^ok 2 second file' "$fixture/out" || true) rm -rf "$fixture" [ "$ran_a" = 1 ] @@ -44,18 +49,11 @@ kg_run() { [ "$rc" -ne 0 ] } -@test "bats-unit-tests still finds the root Makefile when MAKEFILES reads another first" { - extra="$(mktemp)" - printf '%s\n' 'unrelated:' ' @true' >"$extra" - kg_run 'exit 0' "$extra" - ran_b=$(grep -c '^stub ran hack/b.bats$' "$fixture/out" || true) - rm -rf "$fixture" "$extra" - [ "$ran_b" = 1 ] - [ "$rc" -eq 0 ] -} - @test "bats-unit-tests exits zero when every file passes" { - kg_run 'exit 0' + kg_run 'true' + ran=$(grep -c '^ok [12] ' "$fixture/out" || true) rm -rf "$fixture" + + [ "$ran" = 2 ] [ "$rc" -eq 0 ] } diff --git a/hack/build-matrix_test.bats b/hack/build-matrix_test.bats index b490d708d..3cfb16d11 100644 --- a/hack/build-matrix_test.bats +++ b/hack/build-matrix_test.bats @@ -1,14 +1,16 @@ #!/usr/bin/env bats # Unit tests for hack/build-matrix.sh — the CI build-matrix selector. # -# Run via hack/cozytest.sh from the repo root (make bats-unit-tests); the -# relative `hack/build-matrix.sh` calls below resolve against that cwd. A bats -# setup() hook would be dead here — cozytest never invokes it — so the -# repo-root cwd is supplied by the runner rather than a setup() cd. +# Run from the repo root through `make bats-unit-tests`; the relative +# `hack/build-matrix.sh` calls below resolve against that cwd. The file also +# remains compatible with the legacy `hack/cozytest.sh` translator, which does +# not invoke setup(), so neither runner depends on a setup() cd. # # Test-level EXIT traps replace Bats' own handler and hide failing TAP results. # Cleanup follows aborting assertions; see docs/agents/e2e-testing.md. +load test_helper + @test "no argument emits the full matrix" { out=$(hack/build-matrix.sh) # The parallel units; assert known members are present. diff --git a/hack/build-talos-gating_test.bats b/hack/build-talos-gating_test.bats index ef73ad89a..780fb5967 100644 --- a/hack/build-talos-gating_test.bats +++ b/hack/build-talos-gating_test.bats @@ -3,6 +3,8 @@ # matchbox-ref handoff into finalize. Workflow expression checks below pin only # producer/consumer wiring; GitHub job-result propagation is CI-only behavior. +load test_helper + ROOT="$(pwd)" CLASSIFIER="$ROOT/hack/build-talos-needed.sh" WF="$ROOT/.github/workflows/pull-requests.yaml" diff --git a/hack/capi-core-components_test.bats b/hack/capi-core-components_test.bats index 4a9689279..e8c5941a3 100644 --- a/hack/capi-core-components_test.bats +++ b/hack/capi-core-components_test.bats @@ -11,6 +11,8 @@ # carries a timestamp and every regeneration rewrites the committed blob even # when the input has not changed. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" FILES="$REPO_ROOT/packages/system/capi-providers-core/files" diff --git a/hack/capi-provider-components_test.bats b/hack/capi-provider-components_test.bats index f82135b2f..647668a3c 100644 --- a/hack/capi-provider-components_test.bats +++ b/hack/capi-provider-components_test.bats @@ -13,6 +13,8 @@ # the header carries a timestamp and every regeneration rewrites the committed # blob even when the input has not changed. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" check_pair() { diff --git a/hack/capk-provider_test.bats b/hack/capk-provider_test.bats index a9f66b4a5..e310f2c77 100644 --- a/hack/capk-provider_test.bats +++ b/hack/capk-provider_test.bats @@ -6,6 +6,8 @@ # declared because the ratchet is exact in both directions: it fails if a # test-level trap is added here, and it fails if one of these is removed. +load test_helper + _capk_fixture() { mkdir -p "$1/packages/system" "$1/hack" "$1/bin" "$1/oci" cp -R packages/system/capi-providers-infraprovider "$1/packages/system/" diff --git a/hack/capture-dataplane.bats b/hack/capture-dataplane.bats index 341c1dc06..ce9115381 100644 --- a/hack/capture-dataplane.bats +++ b/hack/capture-dataplane.bats @@ -45,11 +45,13 @@ # - Only [A-Za-z0-9] from the title survives into the function name, so keep # titles distinctive in their alphanumeric run. # -# Run with: hack/cozytest.sh hack/capture-dataplane.bats -# (or `bats hack/capture-dataplane.bats` if the bats binary is -# installed; cozytest.sh is the CI path.) +# Run from the repo root with `make bats-unit-tests`, or directly with +# `bats hack/capture-dataplane.bats`. The legacy `hack/cozytest.sh` runner +# remains a supported compatibility path. # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" SCRIPT="$HACK_DIR/e2e-capture-dataplane.sh" diff --git a/hack/capture-previous-logs.bats b/hack/capture-previous-logs.bats index 0a205ffd5..3bafefe83 100644 --- a/hack/capture-previous-logs.bats +++ b/hack/capture-previous-logs.bats @@ -30,11 +30,13 @@ # - Only [A-Za-z0-9] from the title survives into the function name, so keep # titles distinctive in their alphanumeric run. # -# Run with: hack/cozytest.sh hack/capture-previous-logs.bats -# (or `bats hack/capture-previous-logs.bats` if the bats binary is -# installed; cozytest.sh is the CI path.) +# Run from the repo root with `make bats-unit-tests`, or directly with +# `bats hack/capture-previous-logs.bats`. The legacy `hack/cozytest.sh` runner +# remains a supported compatibility path. # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" SCRIPT="$HACK_DIR/e2e-capture-previous-logs.sh" diff --git a/hack/chainsaw-leak-guard_test.bats b/hack/chainsaw-leak-guard_test.bats index 9de4911e3..8bf3a2618 100644 --- a/hack/chainsaw-leak-guard_test.bats +++ b/hack/chainsaw-leak-guard_test.bats @@ -12,22 +12,26 @@ # kubectl and chainsaw are stubs on PATH that answer from fixture files, so the # cluster is a directory: pods.json is what `get pods` returns (pods.after.json # once a `kubectl wait` has succeeded), obj/ holds the owners the walk may look -# up, and `now` is the apiserver clock. The guard is bash; cozytest.sh runs these -# bodies under sh, so each test calls it through `bash`. +# up, and `now` is the apiserver clock. The guard is bash; the legacy +# cozytest.sh translator runs these bodies under sh, so each test calls it +# through `bash`. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run` or `$status`, and setup()/teardown() are not -# honored. Each test removes its own temp dir on its last line. The parser also -# puts `return 0` in front of every bare `}` in the file, top-level helpers -# included, so a helper whose exit status matters closes with `} # name`. +# CI runs this file under Bats. It stays compatible with that translator, whose +# awk parser recognizes only @test blocks and a bare `}` on its own line, so the +# tests avoid bats `run`, `$status`, setup() and teardown(). Each test removes +# its own temp dir on its last line. The parser also puts `return 0` in front of +# every bare `}` in the file, top-level helpers included, so a helper whose exit +# status matters closes with `} # name`. # # The file name carries no `e2e-` prefix on purpose: the root Makefile leaves # hack/e2e-*.bats out of `make unit-tests`, and cozytest.sh arms its cluster # captures for them. # -# Run with: hack/cozytest.sh hack/chainsaw-leak-guard_test.bats +# Run with: bats hack/chainsaw-leak-guard_test.bats # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" LG_LIB="$HACK_DIR/e2e-chainsaw/_lib/leak-guard.sh" LG_RUN="$HACK_DIR/e2e-chainsaw-run.sh" diff --git a/hack/chainsaw-op-budgets.bats b/hack/chainsaw-op-budgets.bats index 9dbf4f2ed..9f95c64a6 100644 --- a/hack/chainsaw-op-budgets.bats +++ b/hack/chainsaw-op-budgets.bats @@ -17,13 +17,16 @@ # op. Its failure messages are checked by running the step's script against a # stubbed kubectl. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its +# CI runs this file under Bats. It stays compatible with the legacy cozytest.sh +# translator, whose awk parser recognizes only @test blocks and a bare `}` on its # own line; a `}` at column zero gets `return 0` inserted ahead of it, so the # helpers below report through stdout only. # -# Run with: hack/cozytest.sh hack/chainsaw-op-budgets.bats +# Run with: bats hack/chainsaw-op-budgets.bats # ----------------------------------------------------------------------------- +load test_helper + GATEWAY_SUITE="hack/e2e-chainsaw/gateway/chainsaw-test.yaml" GATEWAY_STEP="parent-child-and-route-drive-listener-set" REDIS_SUITE="hack/e2e-chainsaw/redis/chainsaw-test.yaml" diff --git a/hack/check-applicationdefinition-crd-ordering.bats b/hack/check-applicationdefinition-crd-ordering.bats index 1ff6041f2..b9ea8aa34 100644 --- a/hack/check-applicationdefinition-crd-ordering.bats +++ b/hack/check-applicationdefinition-crd-ordering.bats @@ -16,6 +16,8 @@ # Requires: yq (mikefarah v4+), jq. Both are available on the project's CI # runners and on the maintainer workstation. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" SOURCES_DIR="$REPO_ROOT/packages/core/platform/sources" diff --git a/hack/check-commit-trailers.bats b/hack/check-commit-trailers.bats index 47b28be18..4818c0981 100644 --- a/hack/check-commit-trailers.bats +++ b/hack/check-commit-trailers.bats @@ -14,6 +14,8 @@ # Each test runs in its own subshell, so the `cd` into the fixture does not leak. # ----------------------------------------------------------------------------- +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CHECK="$REPO_ROOT/hack/check-commit-trailers.sh" diff --git a/hack/check-gpu-operator-variants.bats b/hack/check-gpu-operator-variants.bats index 189c387a6..cd64a3d7b 100644 --- a/hack/check-gpu-operator-variants.bats +++ b/hack/check-gpu-operator-variants.bats @@ -42,6 +42,8 @@ # cozytest.sh runs each @test in a fresh subshell with `set -u` and does # not honor bats setup()/teardown(), so we provision TMP inline per test. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CHART="$REPO_ROOT/packages/system/gpu-operator/charts/gpu-operator" VALUES_DIR="$REPO_ROOT/packages/system/gpu-operator" diff --git a/hack/check-gpu-recording-rules.bats b/hack/check-gpu-recording-rules.bats index a150ec37a..0727d4add 100644 --- a/hack/check-gpu-recording-rules.bats +++ b/hack/check-gpu-recording-rules.bats @@ -33,9 +33,11 @@ # - Only [A-Za-z0-9] from the title survives into the function name; titles # differing only in punctuation collapse to the same function. # -# Run with: hack/cozytest.sh hack/check-gpu-recording-rules.bats +# Run with: bats hack/check-gpu-recording-rules.bats # ----------------------------------------------------------------------------- +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" RULES_FILE="$REPO_ROOT/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml" DASHBOARDS_LIST="$REPO_ROOT/packages/system/monitoring/dashboards-infra.list" diff --git a/hack/check-hook-request-timeout.bats b/hack/check-hook-request-timeout.bats index 5395f5e99..175ae606d 100755 --- a/hack/check-hook-request-timeout.bats +++ b/hack/check-hook-request-timeout.bats @@ -22,6 +22,8 @@ # activeDeadlineSeconds bounds the whole Job, which is what # packages/apps/tenant/templates/cleanup-job.yaml uses and documents. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" @test "no in-cluster kubectl call carries a non-zero --request-timeout" { diff --git a/hack/check-host-runtime.bats b/hack/check-host-runtime.bats index 9f88a1f7b..a744c05e0 100644 --- a/hack/check-host-runtime.bats +++ b/hack/check-host-runtime.bats @@ -18,9 +18,9 @@ # Both runners set -e, so a failed test leaves its stub dir behind, which is what # you want to look at anyway. # -# Tests are otherwise self-contained — no shared setup/teardown helpers, -# because cozytest.sh's awk parser only recognizes @test blocks and treats a -# bare `}` on its own line as the end of a test function. +# Fixture setup is otherwise self-contained rather than placed in file-local +# setup()/teardown() hooks, preserving compatibility with the legacy +# cozytest.sh translator. Strict setup still comes from `load test_helper`. # # Title syntax constraints (inherited from cozytest.sh's awk parser): # - Titles must be delimited by ASCII double quotes; embedded literal @@ -30,11 +30,13 @@ # only in punctuation collapse to the same function name. Keep titles # distinctive in their alphanumeric run. # -# Run with: hack/cozytest.sh hack/check-host-runtime.bats -# (or `bats hack/check-host-runtime.bats` if the bats binary is -# installed; cozytest.sh is the CI path.) +# Run from the repo root with `make bats-unit-tests`, or directly with +# `bats hack/check-host-runtime.bats`. The legacy `hack/cozytest.sh` runner +# remains a supported compatibility path. # ----------------------------------------------------------------------------- +load test_helper + @test "clean host with no runtime services exits silently" { STUB_DIR=$(mktemp -d) diff --git a/hack/check-kafka-rd-schema.bats b/hack/check-kafka-rd-schema.bats index 74510d6e0..acdc3c676 100644 --- a/hack/check-kafka-rd-schema.bats +++ b/hack/check-kafka-rd-schema.bats @@ -6,6 +6,8 @@ # absent (omitempty *bool) rather than an explicit null, and the template # detects that via `kindIs "invalid"`. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" COZYRDS="$REPO_ROOT/packages/system/kafka-rd/cozyrds/kafka.yaml" diff --git a/hack/check-kafka-rd-secrets.bats b/hack/check-kafka-rd-secrets.bats index f00c66c95..a5c74f9ad 100644 --- a/hack/check-kafka-rd-secrets.bats +++ b/hack/check-kafka-rd-secrets.bats @@ -2,6 +2,8 @@ # Unit test: kafka-rd cozyrds must reference the Strimzi-generated # clients-ca-cert secret (not the bare clients-ca name, which does not exist). +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" COZYRDS="$REPO_ROOT/packages/system/kafka-rd/cozyrds/kafka.yaml" diff --git a/hack/check-kubeovn-image-copies.bats b/hack/check-kubeovn-image-copies.bats index 006fefb97..4ba4c080a 100644 --- a/hack/check-kubeovn-image-copies.bats +++ b/hack/check-kubeovn-image-copies.bats @@ -16,6 +16,8 @@ # tests, and an expected failure is written as `if CHECK ...; then ... exit 1`. # ----------------------------------------------------------------------------- +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CHECK="$REPO_ROOT/hack/check-kubeovn-image-copies.sh" OURS="$REPO_ROOT/packages/system/kubeovn/images/kubeovn/Dockerfile" diff --git a/hack/check-mariadb-rd-secrets.bats b/hack/check-mariadb-rd-secrets.bats index e8258bf9c..df239daff 100644 --- a/hack/check-mariadb-rd-secrets.bats +++ b/hack/check-mariadb-rd-secrets.bats @@ -54,6 +54,8 @@ # hack/cozytest.sh inserts `return 0` before any line that is exactly `}`, so a # helper whose own last command decides the outcome returns 0 regardless. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CHART="$REPO_ROOT/packages/system/mariadb-rd" diff --git a/hack/check-mongodb-rd-secrets.bats b/hack/check-mongodb-rd-secrets.bats index 2c513730b..23f2d255e 100644 --- a/hack/check-mongodb-rd-secrets.bats +++ b/hack/check-mongodb-rd-secrets.bats @@ -11,6 +11,8 @@ # The declaration that names the extraction source is a TenantProjection the # chart renders, covered by packages/apps/mongodb/tests/tenant_projection_test.yaml. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" COZYRDS="$REPO_ROOT/packages/system/mongodb-rd/cozyrds/mongodb.yaml" # Bound explicitly rather than read from BATS_TEST_FILENAME inside a test body. diff --git a/hack/check-nats-rd-secrets.bats b/hack/check-nats-rd-secrets.bats index 246b2be65..855cb6fb1 100644 --- a/hack/check-nats-rd-secrets.bats +++ b/hack/check-nats-rd-secrets.bats @@ -27,6 +27,8 @@ # or by a fullname helper -- all of which grant the CA private key while the # guard stays green. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" COZYRDS="$REPO_ROOT/packages/system/nats-rd/cozyrds/nats.yaml" CHART="$REPO_ROOT/packages/apps/nats" diff --git a/hack/check-opensearch-rd-secrets.bats b/hack/check-opensearch-rd-secrets.bats index 40218cc7f..9b456abaf 100644 --- a/hack/check-opensearch-rd-secrets.bats +++ b/hack/check-opensearch-rd-secrets.bats @@ -35,6 +35,8 @@ # creates, all three of these among them, so one such entry grants the CA # private key while every name in the file stays correct. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CHART="$REPO_ROOT/packages/system/opensearch-rd" diff --git a/hack/check-opensearch-readme.bats b/hack/check-opensearch-readme.bats index 5c22f64f7..695b74dd2 100644 --- a/hack/check-opensearch-readme.bats +++ b/hack/check-opensearch-readme.bats @@ -7,6 +7,8 @@ # 2. topologySpreadPolicy appears outside the TLS section # 3. opensearch-rd cozyrds labels the tenant CA and no server certificate +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" README="$REPO_ROOT/packages/apps/opensearch/README.md" COZYRDS="$REPO_ROOT/packages/system/opensearch-rd/cozyrds/opensearch.yaml" diff --git a/hack/check-packages-artifact-ignores.bats b/hack/check-packages-artifact-ignores.bats index 854ef8a2b..dd469d550 100644 --- a/hack/check-packages-artifact-ignores.bats +++ b/hack/check-packages-artifact-ignores.bats @@ -12,6 +12,8 @@ # # Requires: make. flux is optional and only used to re-check the defaults. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" INSTALLER_MAKEFILE="$REPO_ROOT/packages/core/installer/Makefile" diff --git a/hack/check-postgres-postgis-versions.bats b/hack/check-postgres-postgis-versions.bats index 47755557e..27b5fe3ad 100644 --- a/hack/check-postgres-postgis-versions.bats +++ b/hack/check-postgres-postgis-versions.bats @@ -10,6 +10,8 @@ # cozytest.sh sources this file into /bin/sh, which is dash on the CI runners: # POSIX shell only, no ${var//} and no <(...). +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" PG_FILES="$REPO_ROOT/packages/apps/postgres/files" diff --git a/hack/check-postgres-preload-allowlist.bats b/hack/check-postgres-preload-allowlist.bats index 531dd97b3..aa68a42c1 100644 --- a/hack/check-postgres-preload-allowlist.bats +++ b/hack/check-postgres-preload-allowlist.bats @@ -8,6 +8,8 @@ # is the only way to exercise that backstop, and it is the case the backstop # exists for. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CHART="$REPO_ROOT/packages/apps/postgres" diff --git a/hack/check-qdrant-rd-secrets.bats b/hack/check-qdrant-rd-secrets.bats index 4f68efb42..5d92e5039 100644 --- a/hack/check-qdrant-rd-secrets.bats +++ b/hack/check-qdrant-rd-secrets.bats @@ -32,6 +32,8 @@ # Secrets it creates, both of these among them, so one such entry grants # the CA private key while every name in the file stays correct. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CHART="$REPO_ROOT/packages/system/qdrant-rd" diff --git a/hack/check-rabbitmq-rd-secrets.bats b/hack/check-rabbitmq-rd-secrets.bats index ba47a1f41..2fe8a5be9 100644 --- a/hack/check-rabbitmq-rd-secrets.bats +++ b/hack/check-rabbitmq-rd-secrets.bats @@ -26,6 +26,8 @@ # templating of the name, or by a label selector that happens to match it -- all # of which grant a private key while the guard stays green. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" COZYRDS="$REPO_ROOT/packages/system/rabbitmq-rd/cozyrds/rabbitmq.yaml" CHART="$REPO_ROOT/packages/apps/rabbitmq" diff --git a/hack/check-rabbitmq-topology-kinds.bats b/hack/check-rabbitmq-topology-kinds.bats index 63a0e60df..c13589a00 100644 --- a/hack/check-rabbitmq-topology-kinds.bats +++ b/hack/check-rabbitmq-topology-kinds.bats @@ -21,6 +21,8 @@ # own reconciler, which sets no finalizer at all, and the Queue, Exchange and # Binding objects it owns are already covered. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" HELPER="$REPO_ROOT/packages/apps/rabbitmq/templates/_topology.tpl" README="$REPO_ROOT/packages/apps/rabbitmq/README.md" diff --git a/hack/check-rd-presets_test.bats b/hack/check-rd-presets_test.bats index fc5dde996..31b252e0f 100644 --- a/hack/check-rd-presets_test.bats +++ b/hack/check-rd-presets_test.bats @@ -1,5 +1,7 @@ #!/usr/bin/env bats +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CHECK="$REPO_ROOT/hack/check-rd-presets.sh" diff --git a/hack/check-rd-release-prefix.bats b/hack/check-rd-release-prefix.bats index 69777a456..1c6f4ae3e 100755 --- a/hack/check-rd-release-prefix.bats +++ b/hack/check-rd-release-prefix.bats @@ -6,6 +6,8 @@ # An empty spec.release.prefix is skipped: hack/update-crd.sh generates the # definitions for packages/extra charts with PREFIX="". +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" @test "every resourceNames selector carries its release prefix" { diff --git a/hack/check-redis-rd-secrets.bats b/hack/check-redis-rd-secrets.bats index eeb035059..915e870fa 100644 --- a/hack/check-redis-rd-secrets.bats +++ b/hack/check-redis-rd-secrets.bats @@ -39,6 +39,8 @@ # Secrets it creates, both of these among them, so one such entry grants # the CA private key while every name in the file stays correct. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CHART="$REPO_ROOT/packages/system/redis-rd" diff --git a/hack/check-render-matrix.bats b/hack/check-render-matrix.bats index 31b9a3545..3498079fb 100644 --- a/hack/check-render-matrix.bats +++ b/hack/check-render-matrix.bats @@ -3,6 +3,8 @@ # Shared by bats and cozytest.sh: inspect exit codes directly, without bats-only # helpers. Scratch directories stay available if an assertion fails. +load test_helper + make_render_chart() { mkdir -p "$1/templates" printf 'apiVersion: v2\nname: render-test\nversion: 0.0.0\n' > "$1/Chart.yaml" diff --git a/hack/check-talosconfigtemplate-name-agreement.bats b/hack/check-talosconfigtemplate-name-agreement.bats index aef847e4b..5f3a29829 100755 --- a/hack/check-talosconfigtemplate-name-agreement.bats +++ b/hack/check-talosconfigtemplate-name-agreement.bats @@ -36,6 +36,8 @@ # teardown(), so TMP is provisioned inline per test and cleaned without an EXIT # trap (docs/agents/e2e-testing.md bans those here). +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CHART="$REPO_ROOT/packages/apps/kubernetes-nodes" diff --git a/hack/check-values-schema-string-defaults.bats b/hack/check-values-schema-string-defaults.bats index 9e6306635..3838473d8 100644 --- a/hack/check-values-schema-string-defaults.bats +++ b/hack/check-values-schema-string-defaults.bats @@ -24,6 +24,8 @@ # Runs under hack/cozytest.sh, which is POSIX /bin/sh — no process # substitution, no arrays, no bats `run`. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" @test "no values.schema.json declares a non-string default on a string-typed property" { diff --git a/hack/check-worker-image-catalog-defaults.bats b/hack/check-worker-image-catalog-defaults.bats index a06062934..261def920 100644 --- a/hack/check-worker-image-catalog-defaults.bats +++ b/hack/check-worker-image-catalog-defaults.bats @@ -26,6 +26,8 @@ # runs each @test in a fresh subshell with `set -u` and does not honor bats # setup()/teardown(). +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" APP_VALUES="$REPO_ROOT/packages/apps/kubernetes-nodes/values.yaml" CATALOG_VALUES="$REPO_ROOT/packages/system/kubernetes-worker-image/values.yaml" diff --git a/hack/ci-tool-installs-contract.bats b/hack/ci-tool-installs-contract.bats index b9ab81484..ed0cf6535 100755 --- a/hack/ci-tool-installs-contract.bats +++ b/hack/ci-tool-installs-contract.bats @@ -6,6 +6,8 @@ # unpin one, which is why this scans every workflow instead of the steps that # happened to be fixed. Other downloaded tools are not covered here. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" WORKFLOWS="$REPO_ROOT/.github/workflows" diff --git a/hack/cilium-leak-healer_test.bats b/hack/cilium-leak-healer_test.bats index b543d1673..8a47da02e 100644 --- a/hack/cilium-leak-healer_test.bats +++ b/hack/cilium-leak-healer_test.bats @@ -21,9 +21,11 @@ # `run` helper (cozytest.sh has no `run`). Mock IPs use the RFC 5737 # documentation range (TEST-NET-1, 192.0.2.0/24). # -# Run with: hack/cozytest.sh hack/cilium-leak-healer_test.bats +# Run with: bats hack/cilium-leak-healer_test.bats # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" SCRIPT="$HACK_DIR/e2e-cilium-endpoint-leak-healer.sh" diff --git a/hack/clickhouse-update-versions_test.bats b/hack/clickhouse-update-versions_test.bats index 7a494001b..c2a5eeb5b 100644 --- a/hack/clickhouse-update-versions_test.bats +++ b/hack/clickhouse-update-versions_test.bats @@ -25,10 +25,12 @@ # # The generator is exercised offline: CH_SERVER_TAGS_FILE / CH_KEEPER_TAGS_FILE # inject the tag lists (no curl), and VALUES_FILE / VERSIONS_FILE stay in a -# scratch dir. Written in the portable subset hack/cozytest.sh understands (no -# `run`/`$status`/`$lines`, no setup/teardown, no standalone `}` inside a test -# body): each assertion exits non-zero on failure and the body relies on -# `set -e`. +# scratch dir. It remains compatible with the legacy hack/cozytest.sh +# translator, so it avoids `run`/`$status`/`$lines`, file-local setup/teardown, +# and a standalone `}` inside a test body. Each assertion exits non-zero on +# failure and the body relies on `set -e`. + +load test_helper REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" GEN="$REPO_ROOT/packages/apps/clickhouse/hack/update-versions.sh" diff --git a/hack/cni-plugins-staging-contract.bats b/hack/cni-plugins-staging-contract.bats index a0332d14e..594531b12 100644 --- a/hack/cni-plugins-staging-contract.bats +++ b/hack/cni-plugins-staging-contract.bats @@ -2,7 +2,9 @@ # Contract checks for the CNI plugin staging that a helm-unittest case cannot # make: they read files outside the chart, or render it more than one way. # -# Run with: hack/cozytest.sh hack/cni-plugins-staging-contract.bats +# Run with: bats hack/cni-plugins-staging-contract.bats + +load test_helper # Keep the closing brace on the same line: hack/cozytest.sh rewrites any line # that is exactly `}` into `return 0` plus `}`, without caring whether it closes diff --git a/hack/codeowners-invariant.bats b/hack/codeowners-invariant.bats index 451b8f3ab..fd208e0d1 100644 --- a/hack/codeowners-invariant.bats +++ b/hack/codeowners-invariant.bats @@ -20,6 +20,8 @@ # policy discussion linked. # ----------------------------------------------------------------------------- +load test_helper + @test "every CODEOWNERS rule that lists owners repeats the catch-all owners" { [ -f .github/CODEOWNERS ] diff --git a/hack/collect-images_test.bats b/hack/collect-images_test.bats index 5ae31391a..c36deeb62 100644 --- a/hack/collect-images_test.bats +++ b/hack/collect-images_test.bats @@ -7,10 +7,13 @@ # talosctl is a stub: it prints the `images ls` table held in # $MOCK_TALOS/-, or fails for a node listed in $MOCK_TALOS/down. # -# Harness note: run by hack/cozytest.sh, not real bats; each @test is a shell -# function under `set -eu`, sourced into POSIX sh (dash on CI). +# Harness note: the unit lane runs this file under Bats. The bodies stay POSIX +# sh (no `run`, no [[ ]]; each @test a shell function under `set -eu`), so +# hack/cozytest.sh can still run it under dash by hand. # -# Run with: hack/cozytest.sh hack/collect-images_test.bats +# Run with: bats hack/collect-images_test.bats + +load test_helper _stub_talos() { MOCK_TALOS="$1" diff --git a/hack/common-envs_test.bats b/hack/common-envs_test.bats index acb7acd0a..9eab7a47b 100644 --- a/hack/common-envs_test.bats +++ b/hack/common-envs_test.bats @@ -13,13 +13,15 @@ # (a two-`--tag` buildx call that put two manifests in one OCI archive) — plus a # plain image-tags package (cozystack-controller) as the happy path. # -# Run via hack/cozytest.sh from the repo root (make bats-unit-tests): the -# relative `make -C packages/...` calls resolve against that cwd. This is NOT -# real bats — no run/$status/$output/setup(); use plain $(...) capture + grep, -# and the build-matrix_test.bats `if grep -q …; then echo FAIL; false; fi` -# negation idiom (cozytest runs each @test under `set -e`, which suppresses any -# command whose status is inverted with `!`, so a regression would silently pass -# a `! grep`). +# Run from the repo root through `make bats-unit-tests`; the relative +# `make -C packages/...` calls resolve against that cwd. The file also remains +# compatible with the legacy `hack/cozytest.sh` translator, so it avoids +# run/$status/$output/setup() and uses plain $(...) capture plus grep and the +# build-matrix_test.bats `if grep -q …; then echo FAIL; false; fi` +# negation idiom (cozytest runs each @test under `set -e`, which suppresses a +# bare `!`-negated pipeline, so a regression would silently pass a `! grep`). + +load test_helper @test "cozystack-controller exports one OCI archive and never pushes under OCI_EXPORT_DIR" { out=$(make -n -C packages/system/cozystack-controller image OCI_EXPORT_DIR=/tmp/ocitest IMAGE_TAG=pr-1-abc BUILDER=b) diff --git a/hack/computeplane-kcp-hold_test.bats b/hack/computeplane-kcp-hold_test.bats index d3b3d430b..dd126f09b 100644 --- a/hack/computeplane-kcp-hold_test.bats +++ b/hack/computeplane-kcp-hold_test.bats @@ -17,6 +17,8 @@ # Run with: hack/cozytest.sh hack/computeplane-kcp-hold_test.bats # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" KH_LIB="$HACK_DIR/e2e-chainsaw/computeplane/kcp-hold.sh" export KH_LIB diff --git a/hack/console-nginx-header-buffers.bats b/hack/console-nginx-header-buffers.bats index 20fa9a046..f8ce4d949 100644 --- a/hack/console-nginx-header-buffers.bats +++ b/hack/console-nginx-header-buffers.bats @@ -14,12 +14,11 @@ # that moved out of the served config -- into a comment, or into a build stage # that ships nothing -- does not read as present. # -# Harness note: the CI path is hack/cozytest.sh, NOT real bats. No `run`, -# `$status`, `$output`, `skip` or setup()/teardown(); each @test is a shell -# function under `set -eu -x`, and paths resolve from the repo root. The two -# runners disagree on negative assertions, so an assertion written for bats can -# be vacuous under the one that gates a merge. -# Run with: hack/cozytest.sh hack/console-nginx-header-buffers.bats +# Paths resolve from the repo root. Direct assertions also keep this file +# compatible with cozytest, which has no Bats run/setup helpers. +# Run with: bats hack/console-nginx-header-buffers.bats + +load test_helper REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CONSOLE_CONTAINERFILE="packages/system/dashboard/images/console/Containerfile" diff --git a/hack/container-lane-capacity_test.bats b/hack/container-lane-capacity_test.bats index dee997d32..318445cee 100644 --- a/hack/container-lane-capacity_test.bats +++ b/hack/container-lane-capacity_test.bats @@ -3,6 +3,8 @@ # container nodes. The live BATS suite checks the resulting Node status; this # file pins the arithmetic and the cross-process plumbing without a cluster. +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" CONTAINER_UP="$HACK_DIR/e2e-container-up.sh" E2E_CONTAINER_UP_LIB=true . "$CONTAINER_UP" diff --git a/hack/cozyreport-talos.bats b/hack/cozyreport-talos.bats index 104fef58b..dc60a1cfa 100644 --- a/hack/cozyreport-talos.bats +++ b/hack/cozyreport-talos.bats @@ -3,6 +3,8 @@ # The full report needs a cluster; this test sources only the focused helper and # replaces talosctl with a shell function that records the exact argument vector. +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" SCRIPT="$HACK_DIR/cozyreport.sh" diff --git a/hack/cozyreport.bats b/hack/cozyreport.bats index 1b4e2fa63..68622c6b3 100644 --- a/hack/cozyreport.bats +++ b/hack/cozyreport.bats @@ -28,11 +28,13 @@ # - Only [A-Za-z0-9] from the title survives into the function name, so keep # titles distinctive in their alphanumeric run. # -# Run with: hack/cozytest.sh hack/cozyreport.bats -# (or `bats hack/cozyreport.bats` if the bats binary is installed; -# cozytest.sh is the CI path.) +# Run from the repo root with `make bats-unit-tests`, or directly with +# `bats hack/cozyreport.bats`. The legacy `hack/cozytest.sh` runner remains a +# supported compatibility path. # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" SCRIPT="$HACK_DIR/cozyreport.sh" diff --git a/hack/cozystack-controller-secret-rbac.bats b/hack/cozystack-controller-secret-rbac.bats index ec0badf3b..9d7ccf913 100644 --- a/hack/cozystack-controller-secret-rbac.bats +++ b/hack/cozystack-controller-secret-rbac.bats @@ -62,9 +62,11 @@ # Requires: helm, yq (mikefarah v4+), jq -- the same three # hack/admin-kubeconfig-invariant.bats needs to render and inspect a chart. # -# Run with: hack/cozytest.sh hack/cozystack-controller-secret-rbac.bats +# Run with: bats hack/cozystack-controller-secret-rbac.bats # ----------------------------------------------------------------------------- +load test_helper + # What "*" in a rule's verbs means for a namespaced resource such as Secret. VERB_UNIVERSE='["create","delete","deletecollection","get","list","patch","update","watch"]' diff --git a/hack/cozystack-version-stamp.bats b/hack/cozystack-version-stamp.bats index 85db10d8f..f128f37e8 100644 --- a/hack/cozystack-version-stamp.bats +++ b/hack/cozystack-version-stamp.bats @@ -11,14 +11,17 @@ # same manager would strip the label, so the manifest is centralized here and # every migration sources it instead of copy-pasting a heredoc. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run` or `$status`. Assertions are expressed as -# direct shell tests that exit non-zero on failure. Each test runs in its own -# subshell, so NAMESPACE set/unset in one test does not leak into another. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run` and `$status` for that compatibility path and use direct shell +# assertions. Each test runs in its own process, so NAMESPACE does not leak. # -# Run with: hack/cozytest.sh hack/cozystack-version-stamp.bats +# Run with: bats hack/cozystack-version-stamp.bats # ----------------------------------------------------------------------------- +load test_helper + @test "renders parseable YAML for a version" { . packages/core/platform/images/migrations/migrations/lib/cozystack-version.sh render_cozystack_version_manifest 45 | yq . >/dev/null diff --git a/hack/cozytest-capture-gate.bats b/hack/cozytest-capture-gate.bats index 149a086b9..208902caf 100644 --- a/hack/cozytest-capture-gate.bats +++ b/hack/cozytest-capture-gate.bats @@ -37,9 +37,11 @@ # - Only [A-Za-z0-9] from the title survives into the function name, so keep # titles distinctive in their alphanumeric run. # -# Run with: hack/cozytest.sh hack/cozytest-capture-gate.bats +# Run with: bats hack/cozytest-capture-gate.bats # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" RUNNER="$HACK_DIR/cozytest.sh" diff --git a/hack/cozytest-nonascii-title.bats b/hack/cozytest-nonascii-title.bats index a50730343..389d37943 100644 --- a/hack/cozytest-nonascii-title.bats +++ b/hack/cozytest-nonascii-title.bats @@ -17,11 +17,11 @@ # byte scan is exercised the way that box exercises it; under a C ctype the scan # cannot abort and the guard would pass without proving anything. # -# Harness note (see hack/promote-rewrite-tags_test.bats): the CI path is -# hack/cozytest.sh, not real bats -- no `run`, `$status`, `skip` or setup(). Each -# test runs as a shell function under `set -eu -x`; paths are repo-root-relative. +# The outer tests run under Bats; their fixtures exercise cozytest itself. # -# Run with: hack/cozytest.sh hack/cozytest-nonascii-title.bats +# Run with: bats hack/cozytest-nonascii-title.bats + +load test_helper @test "cozytest.sh runs a test whose title contains a multibyte character" { tmp=$(mktemp -d) diff --git a/hack/cozytest-trace-switch.bats b/hack/cozytest-trace-switch.bats index 5208ff2c1..dfe797898 100644 --- a/hack/cozytest-trace-switch.bats +++ b/hack/cozytest-trace-switch.bats @@ -30,6 +30,8 @@ # Run with: hack/cozytest.sh hack/cozytest-trace-switch.bats # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" RUNNER="$HACK_DIR/cozytest.sh" REPO_ROOT="$(cd "$HACK_DIR/.." && pwd)" @@ -162,11 +164,11 @@ count_streamed_marker_lines() { rm -rf "$dir" } -@test "the unit lane recipe asks for the quiet runner" { +@test "the POSIX compatibility lane asks for the quiet runner" { # Without this the 31MB job log comes back the first time somebody tidies the # recipe, and every test above keeps passing while it happens. grep -qE '^COZYTEST_TRACE \?= 0' "$REPO_ROOT/Makefile" - grep -qF 'COZYTEST_TRACE=$(COZYTEST_TRACE) hack/cozytest.sh' "$REPO_ROOT/Makefile" + grep -qF 'COZYTEST_TRACE=$(COZYTEST_TRACE) "$(BATS_POSIX_SHELL)" hack/cozytest.sh' "$REPO_ROOT/Makefile" } @test "the e2e recipes do not silence their runner" { diff --git a/hack/cozytest.sh b/hack/cozytest.sh index 4c1cfe78e..02a40f383 100755 --- a/hack/cozytest.sh +++ b/hack/cozytest.sh @@ -19,7 +19,7 @@ LINE='----------------------------------------------------------------' # the fail handler prints all of it, so quiet mode drops the output of passing # tests only -- which by definition did not establish the result. On the unit # lane that is a ~40x cut (1338 tests: 239k lines/15M of which 98% was trace), -# and it is why the Makefile's bats-unit-tests target sets it. +# and it is why the Makefile's bats-posix-compat-tests target sets it. # # One exception, stated because it is the whole exposure quiet mode adds: the # fail handler runs only after the pipeline returns, and there is no INT/TERM @@ -314,6 +314,16 @@ LC_ALL=C awk ' ' "$TEST_FILE" > "$TMP_SH" [ -f "$TMP_SH" ] || { echo "Failed to generate test functions" >&2; exit 1; } + +# Compatibility with unit files shared with Bats. Only adjacent .bash helpers +# are supported; cozytest already enables nounset when it runs a test body. +load() { + _lib="$(dirname "$TEST_FILE")/$1.bash" + [ -f "$_lib" ] || { echo "load: no such helper: $_lib" >&2; return 1; } + # shellcheck disable=SC1090 + . "$_lib" +} + # shellcheck disable=SC1090 . "$TMP_SH" diff --git a/hack/downstream-trigger-map.bats b/hack/downstream-trigger-map.bats index 2d9bf8175..b26996755 100644 --- a/hack/downstream-trigger-map.bats +++ b/hack/downstream-trigger-map.bats @@ -10,10 +10,9 @@ # sources, and the tests below hold them to each other. # # The three tests pin, in order: every in-repo path the map cites to a file that -# exists; the workflow carve-out that keeps this suite running on a docs-only PR -# to the map's own location; and the map and the PR-template checklist to the same -# repository list, so a repository cannot be listed in one and forgotten in the -# other. +# exists; the workflow guarantee that Bats contracts run on every docs-only PR; +# and the map and the PR-template checklist to the same repository list, so a +# repository cannot be listed in one and forgotten in the other. # # WHAT THIS DOES NOT CHECK, so nobody mistakes a green tick for a correct map: # @@ -35,9 +34,11 @@ # expensive half — is the coupling real, and is it still real over there — stays # a human's job, and reviewers should not assume CI did it for them. # -# A docs-only PR normally skips the unit tests, which would have exempted the map -# from its own guard; the plan step in .github/workflows/pull-requests.yaml carves -# out docs/agents/contributing.md so that editing the map alone still runs these. +# A docs-only PR skips builds and E2E, but the `checks` job still runs the Bats +# lane. Otherwise editing this map, or another document audited by a unit +# contract, would exempt the changed input from its own guard. + +load test_helper REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" MAP_FILE="$REPO_ROOT/docs/agents/contributing.md" @@ -104,65 +105,21 @@ map_section() { echo "All $count in-repo path(s) cited by the trigger map exist" } -@test "the workflow opts the trigger map back into the unit tests" { +@test "the workflow runs Bats contracts for docs-only changes" { WORKFLOW="$REPO_ROOT/.github/workflows/pull-requests.yaml" [ -f "$WORKFLOW" ] || { echo "missing $WORKFLOW" >&2; exit 1; } - # Every check below reads the workflow with comments stripped. A commented-out - # line still contains its own text, so grepping the raw file would accept a - # carve-out that had been disabled and left behind as a TODO — the most likely - # way this actually rots. - code_only="$(sed 's/#.*//' "$WORKFLOW")" + job_if=$(yq -r '.jobs.checks.if' "$WORKFLOW") + code_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Run unit and controller tests") | .if' "$WORKFLOW") + docs_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Run Bats unit tests for docs-only changes") | .if' "$WORKFLOW") + docs_run=$(yq -r '.jobs.checks.steps[] | select(.name == "Run Bats unit tests for docs-only changes") | .run' "$WORKFLOW") + code_run=$(yq -r '.jobs.checks.steps[] | select(.name == "Run unit and controller tests") | .run' "$WORKFLOW") - # The block of the job that runs the unit tests, from its header to the next - # job's. The condition has to sit on THAT job: pinning it file-wide would accept - # the exact expression pasted into some other job while the unit-test job quietly - # loses it. - unit_job="$(printf '%s\n' "$code_only" | awk ' - /^ [a-zA-Z0-9_-]+:[[:space:]]*$/ { inside = ($0 == " checks:") } - inside')" - printf '%s\n' "$unit_job" | grep -q 'make unit-tests' || { - echo "The 'checks' job in .github/workflows/pull-requests.yaml no longer runs 'make unit-tests'." >&2 - echo "This suite pins the trigger-map carve-out to that job. If the unit tests moved, point" >&2 - echo "the checks below at their new job." >&2 - exit 1 - } - - # Three links carry the carve-out: the plan step names the map, the plan job - # exports the flag, and the unit-test job gates on it. Cutting any one leaves the - # other two looking perfectly wired, and none of them fails loudly — an unset or - # unexported output dereferences to an empty string, so the condition is merely - # false and the suite is skipped in silence, on exactly the PRs it guards, with - # every test in this file still green. Pin all three. - rel="${MAP_FILE#"$REPO_ROOT"/}" - printf '%s\n' "$code_only" | grep -qF "'$rel'" || { - echo "The plan step in .github/workflows/pull-requests.yaml does not name '$rel'." >&2 - echo "Without it, a PR that only edits the trigger map is treated as docs-only, the unit" >&2 - echo "tests are skipped, and this file never runs — exactly when it is needed most." >&2 - echo "Fix: point the trigger_map detection at the map's new path." >&2 - exit 1 - } - - printf '%s\n' "$code_only" | grep -qF 'trigger_map: ${{ steps.p.outputs.trigger_map }}' || { - echo "The plan job in .github/workflows/pull-requests.yaml does not export trigger_map." >&2 - echo "The step still computes it and the unit-test job still reads it, so this looks wired" >&2 - echo "up, but an unexported output dereferences to an empty string: the job is skipped in" >&2 - echo "silence on exactly the PRs this suite guards." >&2 - echo "Fix: restore 'trigger_map: \${{ steps.p.outputs.trigger_map }}' to the plan job's outputs." >&2 - exit 1 - } - - condition="(needs.plan.outputs.code == 'true' || needs.plan.outputs.trigger_map == 'true')" - printf '%s\n' "$unit_job" | grep -qF "$condition" || { - echo "The job that runs 'make unit-tests' is not gated on:" >&2 - echo " $condition" >&2 - echo "Mentioning trigger_map in another job, or in a comment, does not count. Without that" >&2 - echo "exact condition on that job, a PR which only edits the trigger map stays docs-only," >&2 - echo "skips the unit tests, and never runs this file — while every test here still passes." >&2 - exit 1 - } - - echo "The workflow names '$rel', exports the flag, and gates the unit tests on it" + [ "$job_if" = "!contains(github.event.pull_request.labels.*.name, 'release')" ] + [ "$code_if" = "needs.plan.outputs.code == 'true'" ] + [ "$docs_if" = "needs.plan.outputs.code == 'false'" ] + [ "$docs_run" = 'make bats-unit-tests' ] + [ "$code_run" = 'make unit-tests test-controllers -j4 -k --output-sync=target' ] } @test "the trigger map and the PR-template checklist list the same repositories" { diff --git a/hack/etcd-probe_test.bats b/hack/etcd-probe_test.bats index 742cb9577..da0d34e17 100644 --- a/hack/etcd-probe_test.bats +++ b/hack/etcd-probe_test.bats @@ -24,16 +24,18 @@ # need a live cluster and are covered by the e2e run, matching how the other # hack/ helpers are tested. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run` or `$status`, and setup()/teardown() are not -# honored. Each test runs under `set -eu -x`; assertions are direct shell tests -# that exit non-zero on failure. Titles are delimited by ASCII double quotes and -# only [A-Za-z0-9] survives into the generated function name, so keep them -# distinctive in their alphanumeric run. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run`, `$status`, setup(), and teardown() for that compatibility path and use +# direct shell assertions. Titles remain distinguishable after stripping all +# but [A-Za-z0-9], as required by the legacy translator. # -# Run with: hack/cozytest.sh hack/etcd-probe_test.bats +# Run with: bats hack/etcd-probe_test.bats # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" # shellcheck source=/dev/null . "$HACK_DIR/e2e-chainsaw/_lib/etcd-probe.sh" diff --git a/hack/fork-pr-checkout_test.bats b/hack/fork-pr-checkout_test.bats index a47b329c1..33bc4655e 100644 --- a/hack/fork-pr-checkout_test.bats +++ b/hack/fork-pr-checkout_test.bats @@ -6,6 +6,8 @@ # refs/pull/... there without the opt-in fails the publish job for every fork # PR, and the required "E2E Tests" status never turns green. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" FORK="$REPO_ROOT/.github/workflows/e2e-fork.yaml" diff --git a/hack/gate-concurrency-contract.bats b/hack/gate-concurrency-contract.bats index a49057991..5133a4cd8 100644 --- a/hack/gate-concurrency-contract.bats +++ b/hack/gate-concurrency-contract.bats @@ -28,6 +28,8 @@ # `plan`, and the extraction below, which matches on `CONCLUSION === '…'`, # will not notice. Whoever adds such a job owns that. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" FORK="$REPO_ROOT/.github/workflows/e2e-fork.yaml" PULL_REQUESTS="$REPO_ROOT/.github/workflows/pull-requests.yaml" diff --git a/hack/go-module-fetch-contract.bats b/hack/go-module-fetch-contract.bats index c6c68ee01..0ffafbad7 100644 --- a/hack/go-module-fetch-contract.bats +++ b/hack/go-module-fetch-contract.bats @@ -23,6 +23,8 @@ # # Requires: awk, make, sed. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" RETRY="$REPO_ROOT/hack/go-mod-download.sh" WORKFLOWS="$REPO_ROOT/.github/workflows" diff --git a/hack/helm-unit-tests.bats b/hack/helm-unit-tests.bats index 563e0aac4..107b1e72a 100644 --- a/hack/helm-unit-tests.bats +++ b/hack/helm-unit-tests.bats @@ -25,16 +25,19 @@ # assertions exact and keeps the real repo's suites, which take minutes, out of # a unit test. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` at column -# 0, and injects `set -e`, so exit statuses are captured explicitly; there is no -# bats `run` or `$status`. setup()/teardown() are not honored, so each test -# creates and removes its own scratch dir inline rather than through a +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` at column 0. The tests therefore +# capture statuses directly and manage their own scratch dirs rather than use +# `run`, `$status`, setup(), teardown(), or a # `trap ... EXIT`: that trap replaces the one the bats binary installs for its # own bookkeeping, and a test failing under it can print no TAP line at all. # -# Run with: hack/cozytest.sh hack/helm-unit-tests.bats +# Run with: bats hack/helm-unit-tests.bats # ----------------------------------------------------------------------------- +load test_helper + # Build a throwaway tree with one package whose `make test` prints $1 and exits # $2. Echoes the tree root. _tree() { diff --git a/hack/helmrelease-readiness.bats b/hack/helmrelease-readiness.bats index b683f4001..8da2926e6 100644 --- a/hack/helmrelease-readiness.bats +++ b/hack/helmrelease-readiness.bats @@ -1,6 +1,8 @@ #!/usr/bin/env bats # Unit coverage for the single post-install HelmRelease readiness gate. +load test_helper + helmrelease_snapshot() { count="$1" not_ready="${2:-0}" diff --git a/hack/image-builders-cross-compile.bats b/hack/image-builders-cross-compile.bats index 7361edb0d..a21027a7a 100644 --- a/hack/image-builders-cross-compile.bats +++ b/hack/image-builders-cross-compile.bats @@ -20,10 +20,10 @@ # per command: a stage with `RUN echo $TARGETARCH` and a bare `go build` would # pass. # -# Written for POSIX sh: hack/cozytest.sh sources this file into /bin/sh. -# # Requires: awk, sed. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" # Outputs with no architecture, one entry per line: a whole file, or diff --git a/hack/image-pin-consistency.bats b/hack/image-pin-consistency.bats index 07c9e3fc5..e3d9508d6 100644 --- a/hack/image-pin-consistency.bats +++ b/hack/image-pin-consistency.bats @@ -20,13 +20,15 @@ # deliberately (backupstrategy-controller reuses platform-migrations as a # curl+jq runner rather than shipping a second one-binary tag). # -# Harness note: the CI path is hack/cozytest.sh, NOT real bats. There is no -# `run`, `$status`, `$output`, `skip`, or setup()/teardown(); each test runs as -# a shell function under `set -eu -x`, so a non-zero exit is the failure. -# Paths are repo-root-relative: BATS_TEST_DIRNAME is unset and would abort the -# whole suite under `set -u`. +# Harness compatibility note: CI runs this file under Bats through +# `make bats-unit-tests`. It also remains compatible with the narrower legacy +# `hack/cozytest.sh` translator, so it avoids `run`, `$status`, `$output`, +# `skip`, and setup()/teardown(). Paths stay repo-root-relative for both +# runners. # -# Run with: hack/cozytest.sh hack/image-pin-consistency.bats +# Run with: bats hack/image-pin-consistency.bats + +load test_helper @test "no image repository is pinned at more than one digest" { tmp=$(mktemp -d) diff --git a/hack/image-targets-pass-buildx-args.bats b/hack/image-targets-pass-buildx-args.bats index 55e417a51..0d5f547a8 100644 --- a/hack/image-targets-pass-buildx-args.bats +++ b/hack/image-targets-pass-buildx-args.bats @@ -16,10 +16,10 @@ # what they build is amd64 by nature; they must yield exactly that platform # whatever the command line says. # -# Written for POSIX sh: hack/cozytest.sh sources this file into /bin/sh. -# # Requires: make. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" PROBE_PLATFORM="linux/riscv64" PROBE_MARKER="--build-arg=BUILDX_ARGS_PROBE=1" diff --git a/hack/installer-image-packages-push_test.bats b/hack/installer-image-packages-push_test.bats index d4c3303f0..7666f38e5 100644 --- a/hack/installer-image-packages-push_test.bats +++ b/hack/installer-image-packages-push_test.bats @@ -5,9 +5,12 @@ # pulls the artifact by digest from the registry, so a build that pushes nothing # must leave the committed ref alone rather than stamp one that resolves nowhere. # -# Run via hack/cozytest.sh (make bats-unit-tests). This is NOT real bats: no -# run/$status/$output, and a `!`-negated pipeline is suppressed under set -e, so -# negative checks use the `if grep -q ...; then ...; false; fi` idiom. +# Run with: bats hack/installer-image-packages-push_test.bats. The bodies avoid +# run/$status/$output so the legacy hack/cozytest.sh translator can still run +# them, and a `!`-negated pipeline is suppressed under set -e, so negative checks +# use the `if grep -q ...; then ...; false; fi` idiom. + +load test_helper REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" diff --git a/hack/isp-hosted-lane_test.bats b/hack/isp-hosted-lane_test.bats index 99a7eab6c..35724ba3a 100644 --- a/hack/isp-hosted-lane_test.bats +++ b/hack/isp-hosted-lane_test.bats @@ -8,6 +8,8 @@ # other than its name says. These checks pin the wiring that makes the run mean # "isp-hosted converged on a cluster that provides its own CNI and storage". +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" WORKFLOW="$REPO_ROOT/.github/workflows/e2e-isp-hosted.yaml" PACKAGE="$REPO_ROOT/hack/e2e-isp-hosted-package.yaml" diff --git a/hack/issue-triage-contract.bats b/hack/issue-triage-contract.bats index f08c75248..054b12db9 100644 --- a/hack/issue-triage-contract.bats +++ b/hack/issue-triage-contract.bats @@ -29,6 +29,8 @@ # would need a JS runtime, and `checks` runs where none is guaranteed, so a # harness would go silently green wherever node is missing. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" WF="$REPO_ROOT/.github/workflows/issue-triage.yaml" STALE="$REPO_ROOT/.github/workflows/stale.yaml" diff --git a/hack/kafka-backup-script.bats b/hack/kafka-backup-script.bats index 6ab5f8736..dff94c2a9 100644 --- a/hack/kafka-backup-script.bats +++ b/hack/kafka-backup-script.bats @@ -23,6 +23,8 @@ # --list, the comma-value bracket grouping, the replication-factor divergence # guard, and the cleanup delete's HTTP handling. +load test_helper + SCRIPT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)/packages/system/backupstrategy-controller/files/kafka-backup.sh" # init_stubs builds a fresh workspace with stub kafka CLIs (the script's $BIN) diff --git a/hack/kafka-metadata-backup-demo-helpers.bats b/hack/kafka-metadata-backup-demo-helpers.bats index 61272e39c..55e1342c2 100644 --- a/hack/kafka-metadata-backup-demo-helpers.bats +++ b/hack/kafka-metadata-backup-demo-helpers.bats @@ -17,6 +17,8 @@ # helper functions close on an indented brace because the runner rewrites a # column-0 `}` into `return 0`, which would mask run_topic_meta's exit status. +load test_helper + make_stubs() { stub=$(mktemp -d) cat > "$stub/kubectl" <<'EOF' diff --git a/hack/kafka-migration-hook.bats b/hack/kafka-migration-hook.bats index 912e8a5bb..bc3ba2b50 100644 --- a/hack/kafka-migration-hook.bats +++ b/hack/kafka-migration-hook.bats @@ -15,11 +15,12 @@ # migration "kafka" pool (collision guard, create-CAS, name-freeing) force the # resolved value with force_kafka_pool. # -# Run via hack/cozytest.sh from the repo root (make bats-unit-tests). No -# setup/teardown, no EXIT traps: each @test builds its own fixture and removes it -# at the end of the body. cozytest.sh rewrites every line that is exactly `}` -# into `return 0` + `}`, the heredoc included, so the stub uses only case/if (no -# shell functions) and every helper's closing brace is indented. +# Run from the repo root with make bats-unit-tests. Direct assertions and +# inline cleanup retain compatibility with cozytest. Its translator rewrites +# lone closing braces even inside heredocs, so the stub avoids shell functions +# and helper closing braces stay indented. + +load test_helper CHART=packages/apps/kafka diff --git a/hack/kafka-pre-delete-hook.bats b/hack/kafka-pre-delete-hook.bats index 123b24ebd..0ab408294 100644 --- a/hack/kafka-pre-delete-hook.bats +++ b/hack/kafka-pre-delete-hook.bats @@ -34,15 +34,11 @@ # that delete exits non-zero, so the uninstall is retried rather than # finishing with the pools still in place. # -# Run via hack/cozytest.sh from the repo root (make bats-unit-tests); relative -# paths resolve against that cwd. That runner implements `@test` and little -# else: it has NO `run` helper and therefore no $status or $output, so every -# test below saves `$?` by hand the way hack/app-cleanup-hooks_test.bats does. -# Under the bats binary `run` would work and the file would pass either way, -# which is exactly how a suite that dies on `run: command not found` under the -# real runner got committed once. docs/agents/e2e-testing.md:66 says it too. -# Each @test builds its own fixture and removes it in the body: no -# setup/teardown, and no EXIT trap. +# Run from the repo root with make bats-unit-tests. Direct status checks +# retain compatibility with cozytest, which has no Bats run/setup helpers. +# Inline cleanup preserves Bats' TAP handler on failed assertions. + +load test_helper KAFKA_CHART=packages/apps/kafka diff --git a/hack/kafka-suite-pvc-cleanup_test.bats b/hack/kafka-suite-pvc-cleanup_test.bats index e2648d8bd..8329c3110 100644 --- a/hack/kafka-suite-pvc-cleanup_test.bats +++ b/hack/kafka-suite-pvc-cleanup_test.bats @@ -6,6 +6,8 @@ # space. These tests read hack/e2e-chainsaw/kafka/chainsaw-test.yaml, because # running it needs a cluster. +load test_helper + SUITE=hack/e2e-chainsaw/kafka/chainsaw-test.yaml @test "kafka test deletes its PVCs in the cleanup of the step that applies the app" { diff --git a/hack/kafka-topic-backup-demo-helpers.bats b/hack/kafka-topic-backup-demo-helpers.bats index df723cdb1..17c1e08f0 100644 --- a/hack/kafka-topic-backup-demo-helpers.bats +++ b/hack/kafka-topic-backup-demo-helpers.bats @@ -12,6 +12,8 @@ # a column-0 `}` into `return 0`, so the helpers below close on an indented # brace to keep run_replication_factor's exit status. +load test_helper + make_stubs() { stub=$(mktemp -d) cat > "$stub/kubectl" <<'EOF' diff --git a/hack/kafka-topic-backup-strategy.bats b/hack/kafka-topic-backup-strategy.bats index 36572fdc4..ce233e0d8 100644 --- a/hack/kafka-topic-backup-strategy.bats +++ b/hack/kafka-topic-backup-strategy.bats @@ -14,6 +14,8 @@ # so each test calls init_stubs itself, and negative assertions go through # expect_fail or an explicit if/return 1. +load test_helper + REPO="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" EXAMPLE="$REPO/examples/backups/kafka" diff --git a/hack/kamaji-crds-schema.bats b/hack/kamaji-crds-schema.bats index 5690a4af4..afcc909a1 100644 --- a/hack/kamaji-crds-schema.bats +++ b/hack/kamaji-crds-schema.bats @@ -10,6 +10,8 @@ # The package path is relative because both runners invoke these from the # repository root, as the neighbouring render tests assume too. +load test_helper + @test "kamaji CRD templates render their schema" { rendered=$(helm template kamaji packages/system/kamaji --namespace cozy-kamaji) diff --git a/hack/kubectl-wait-retry_test.bats b/hack/kubectl-wait-retry_test.bats index dbbf7358a..fefe90a8b 100644 --- a/hack/kubectl-wait-retry_test.bats +++ b/hack/kubectl-wait-retry_test.bats @@ -10,11 +10,15 @@ # is captured into the substitution. These tests execute the wrapper in a # `sh -eu` subshell, exactly as chainsaw runs it, against a fake kubectl. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run`/`$status`. Assertions are direct shell tests. -# Run with: hack/cozytest.sh hack/kubectl-wait-retry_test.bats +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run` and `$status` for that compatibility path and use direct shell assertions. +# Run with: bats hack/kubectl-wait-retry_test.bats # ----------------------------------------------------------------------------- +load test_helper + FAKEBIN="$PWD/hack/testdata/kubectl-wait-retry" prep() { diff --git a/hack/kubernetes-md0-migration.bats b/hack/kubernetes-md0-migration.bats index 1339cd47b..2393d346d 100644 --- a/hack/kubernetes-md0-migration.bats +++ b/hack/kubernetes-md0-migration.bats @@ -20,18 +20,21 @@ # "The stamp ran" is detected by the version stamp's `kubectl apply` appearing # in the fake's call log — equivalently, CURRENT_VERSION was/was not advanced. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line (column 0); there is no bats `run`/`$status`/`setup`/`teardown`. -# Assertions are direct shell tests that exit non-zero on failure, and the +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` at column 0. The tests avoid +# `run`, `$status`, setup(), and teardown() for that compatibility path. Their +# assertions are direct shell tests, and the # fake-kubectl heredocs deliberately keep every `}` off column 0 so the parser # does not mistake one for the end of a test. Each @test runs in its own # subshell, and cleanup is inline (no EXIT/RETURN trap — see # docs/agents/e2e-testing.md §3). # -# Run with: hack/cozytest.sh hack/kubernetes-md0-migration.bats -# (or `bats hack/kubernetes-md0-migration.bats`) +# Run with: bats hack/kubernetes-md0-migration.bats # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" REPO_ROOT="$(cd "$HACK_DIR/.." && pwd)" MIGRATION="$REPO_ROOT/packages/core/platform/images/migrations/migrations/47" diff --git a/hack/kubernetes-nodes-unpin-hook.bats b/hack/kubernetes-nodes-unpin-hook.bats index 9a5661bfb..657d517de 100644 --- a/hack/kubernetes-nodes-unpin-hook.bats +++ b/hack/kubernetes-nodes-unpin-hook.bats @@ -17,11 +17,14 @@ # Requires helm (present in the unit-tests CI toolchain; the same job installs # the helm-unittest plugin). # -# cozytest.sh awk parser: @test blocks only, a bare `}` at column 0 ends a test, -# no run/$status/setup/teardown. Assertions are direct shell tests. -# Run with: hack/cozytest.sh hack/kubernetes-nodes-unpin-hook.bats +# For compatibility with the legacy cozytest.sh translator, tests avoid +# run/$status and file-local setup/teardown, and keep bare `}` lines out of test +# bodies. Assertions are direct shell tests. +# Run with: bats hack/kubernetes-nodes-unpin-hook.bats # ----------------------------------------------------------------------------- +load test_helper + FAKEBIN="$PWD/hack/testdata/kubernetes-nodes-unpin" CHART="$PWD/packages/apps/kubernetes-nodes" diff --git a/hack/kubernetes-pre-delete-hook.bats b/hack/kubernetes-pre-delete-hook.bats index 9f16ffccd..ee3953941 100644 --- a/hack/kubernetes-pre-delete-hook.bats +++ b/hack/kubernetes-pre-delete-hook.bats @@ -6,14 +6,17 @@ # which cannot say whether the closing message is reachable once a backstop has # swallowed a failure. # -# Run via hack/cozytest.sh from the repo root (make bats-unit-tests); relative -# paths resolve against that cwd. The runner has no setup/teardown, so each -# @test builds its own fixture and removes it at the end of the body. No EXIT +# Run from the repo root through `make bats-unit-tests`; relative paths resolve +# against that cwd. The file also remains compatible with the legacy +# `hack/cozytest.sh` translator, so each @test builds and removes its fixture +# inline instead of relying on setup()/teardown(). No EXIT # traps: a test that dies inside one prints neither `not ok` nor a reason, and # the suite reads green. The removal is therefore reached only on a passing run; # a failing one leaves its temp directory behind, which is what you want when # reading the rendered script and the captured output afterwards. +load test_helper + CHART=packages/apps/kubernetes # The stub is driven by two independent lists of globs, matched against the diff --git a/hack/linstor-storage-mode_test.bats b/hack/linstor-storage-mode_test.bats index f8c996558..510da4cd6 100644 --- a/hack/linstor-storage-mode_test.bats +++ b/hack/linstor-storage-mode_test.bats @@ -3,6 +3,8 @@ # registration, management StorageClasses, and downstream Chainsaw fixtures. # The library guard keeps this suite cluster-free under hack/cozytest.sh. +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" POST_PREP="$HACK_DIR/e2e-post-install-prep.sh" # shellcheck source=/dev/null diff --git a/hack/mariadb-first-boot-waits.bats b/hack/mariadb-first-boot-waits.bats index 95c9b2127..036cd4e57 100644 --- a/hack/mariadb-first-boot-waits.bats +++ b/hack/mariadb-first-boot-waits.bats @@ -70,6 +70,8 @@ # Run with: hack/cozytest.sh hack/mariadb-first-boot-waits.bats # ----------------------------------------------------------------------------- +load test_helper + MBW_SCRIPT="examples/backups/mariadb/run-all.sh" MBW_CHART="packages/apps/mariadb/templates/mariadb.yaml" MBW_SUITE="hack/e2e-chainsaw/mariadb/chainsaw-test.yaml" diff --git a/hack/md-no-hardwrap.bats b/hack/md-no-hardwrap.bats index 667b43bf8..54ec7d92a 100644 --- a/hack/md-no-hardwrap.bats +++ b/hack/md-no-hardwrap.bats @@ -26,6 +26,8 @@ # The hook speaks the agent hook protocol: a JSON payload on stdin, exit 0 to # allow, exit 2 to block with an explanation on stderr. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" HOOK="$REPO_ROOT/.claude/hooks/md-no-hardwrap.py" diff --git a/hack/migration-50-etcd-adopt.bats b/hack/migration-50-etcd-adopt.bats index ee2cf66dc..22422ea77 100644 --- a/hack/migration-50-etcd-adopt.bats +++ b/hack/migration-50-etcd-adopt.bats @@ -19,13 +19,16 @@ # always carrying the S3 backup destination, and stays at 0 after a # successful adoption. Only a failed adoption restores it to 1. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run`/`$status`/`setup`. Assertions are direct -# shell tests that exit non-zero on failure. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run`, `$status`, and setup() for that compatibility path. # -# Run with: hack/cozytest.sh hack/migration-50-etcd-adopt.bats +# Run with: bats hack/migration-50-etcd-adopt.bats # ----------------------------------------------------------------------------- +load test_helper + FAKEBIN="$PWD/hack/testdata/migration-50" MIG="$PWD/packages/core/platform/images/migrations/migrations/50" diff --git a/hack/migration-54-redis-adopt.bats b/hack/migration-54-redis-adopt.bats index ab2643634..18fa8ce48 100644 --- a/hack/migration-54-redis-adopt.bats +++ b/hack/migration-54-redis-adopt.bats @@ -52,16 +52,17 @@ # (the only step that touches the network) and run_migration() uses that image, # still --network none. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its own -# line, rewriting the latter into `return 0` + `}`; there is no bats -# `run`/`$status`/`setup`. A helper whose exit status matters must therefore -# capture it and `return` it by hand before its closing brace, or the injected -# `return 0` would mask it (see run_migration below). Assertions are direct shell -# tests that exit non-zero on failure. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# rewrites a bare `}` into `return 0` plus `}`. The tests avoid `run`, `$status`, +# and setup() for that path. A helper whose status matters captures and returns +# it before its closing brace so the injected return cannot mask it. # -# Run with: hack/cozytest.sh hack/migration-54-redis-adopt.bats +# Run with: bats hack/migration-54-redis-adopt.bats # ----------------------------------------------------------------------------- +load test_helper + FAKEBIN="$PWD/hack/testdata/migration-54-redis" MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations" diff --git a/hack/migration-55-fluxcd-orphan.bats b/hack/migration-55-fluxcd-orphan.bats index 5790dc107..92dd8475c 100644 --- a/hack/migration-55-fluxcd-orphan.bats +++ b/hack/migration-55-fluxcd-orphan.bats @@ -47,16 +47,17 @@ # Same base, same busybox ash, plus the one tool the script actually requires; # the selection logic under test is jq's own, run for real. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its own -# line, rewriting the latter into `return 0` + `}`; there is no bats -# `run`/`$status`/`setup`. A helper whose exit status matters must therefore -# capture it and `return` it by hand before its closing brace, or the injected -# `return 0` would mask it (see run_migration below). Assertions are direct shell -# tests that exit non-zero on failure. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# rewrites a bare `}` into `return 0` plus `}`. The tests avoid `run`, `$status`, +# and setup() for that path. A helper whose status matters captures and returns +# it before its closing brace so the injected return cannot mask it. # -# Run with: hack/cozytest.sh hack/migration-55-fluxcd-orphan.bats +# Run with: bats hack/migration-55-fluxcd-orphan.bats # ----------------------------------------------------------------------------- +load test_helper + FAKEBIN="$PWD/hack/testdata/migration-55-fluxcd" MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations" diff --git a/hack/migration-56-adopt-path.bats b/hack/migration-56-adopt-path.bats index 3d2c0b9c9..1869266f1 100644 --- a/hack/migration-56-adopt-path.bats +++ b/hack/migration-56-adopt-path.bats @@ -14,11 +14,14 @@ # skipped, ANY read failure aborts the run before the version stamp, and # stderr never contaminates data. # -# cozytest.sh awk parser: @test blocks only, a bare `}` at column 0 ends a test, -# no run/$status/setup/teardown. Assertions are direct shell tests. -# Run with: hack/cozytest.sh hack/migration-56-adopt-path.bats +# For compatibility with the legacy cozytest.sh translator, tests avoid +# run/$status and file-local setup/teardown, and keep bare `}` lines out of test +# bodies. Assertions are direct shell tests. +# Run with: bats hack/migration-56-adopt-path.bats # ----------------------------------------------------------------------------- +load test_helper + FAKEBIN="$PWD/hack/testdata/migration-56" MIG="$PWD/packages/core/platform/images/migrations/migrations/56" diff --git a/hack/migration-56-adopt-values.bats b/hack/migration-56-adopt-values.bats index 449609c33..21fd3960d 100644 --- a/hack/migration-56-adopt-values.bats +++ b/hack/migration-56-adopt-values.bats @@ -24,13 +24,16 @@ # talos/version/images (plus the group fields and storageClass fallback). It # fails against the buggy `pick(o; ...)` and passes against `pick($o; ...)`. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run`/`$status`/`setup`/`teardown`. Assertions are -# direct shell tests that exit non-zero on failure. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run`, `$status`, setup(), and teardown() for that compatibility path. # -# Run with: hack/cozytest.sh hack/migration-56-adopt-values.bats +# Run with: bats hack/migration-56-adopt-values.bats # ----------------------------------------------------------------------------- +load test_helper + FAKEBIN="$PWD/hack/testdata/migration-56" MIG="$PWD/packages/core/platform/images/migrations/migrations/56" diff --git a/hack/migration-57-seaweedfs-size-limit.bats b/hack/migration-57-seaweedfs-size-limit.bats index e5f2a26bd..d2922def7 100644 --- a/hack/migration-57-seaweedfs-size-limit.bats +++ b/hack/migration-57-seaweedfs-size-limit.bats @@ -59,6 +59,8 @@ # Run with: hack/cozytest.sh hack/migration-57-seaweedfs-size-limit.bats # ----------------------------------------------------------------------------- +load test_helper + FAKEBIN="$PWD/hack/testdata/migration-57-seaweedfs-size-limit" MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations" RUNBOOK="$PWD/docs/operations/seaweedfs-lowering-the-volume-size-limit.md" diff --git a/hack/migration-58-http-cache-freeze.bats b/hack/migration-58-http-cache-freeze.bats index 86134154e..6f07d3b83 100644 --- a/hack/migration-58-http-cache-freeze.bats +++ b/hack/migration-58-http-cache-freeze.bats @@ -18,6 +18,8 @@ # Run with: hack/cozytest.sh hack/migration-58-http-cache-freeze.bats # ----------------------------------------------------------------------------- +load test_helper + FAKEBIN="$PWD/hack/testdata/migration-58-http-cache" MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations" ALPINE=$(sed -n 's/^FROM \(alpine:[^ ]*\).*$/\1/p' \ diff --git a/hack/migration-59-vmi-hotplug-volumes.bats b/hack/migration-59-vmi-hotplug-volumes.bats index cc7d052eb..556061da2 100644 --- a/hack/migration-59-vmi-hotplug-volumes.bats +++ b/hack/migration-59-vmi-hotplug-volumes.bats @@ -14,6 +14,8 @@ # Run with: hack/cozytest.sh hack/migration-59-vmi-hotplug-volumes.bats # ----------------------------------------------------------------------------- +load test_helper + FIXTURES="$PWD/hack/testdata/migration-59-vmi-hotplug" MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations" ALPINE=$(sed -n 's/^FROM \(alpine:[^ ]*\).*$/\1/p' \ diff --git a/hack/migration-seaweedfs-db-adopt.bats b/hack/migration-seaweedfs-db-adopt.bats index 93f61753f..fd17d7e8b 100644 --- a/hack/migration-seaweedfs-db-adopt.bats +++ b/hack/migration-seaweedfs-db-adopt.bats @@ -55,13 +55,16 @@ # the script does any work, while on a developer box /bin/sh is often bash, which # runs green and proves nothing about ash. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run`/`$status`/`setup`. Assertions are direct -# shell tests that exit non-zero on failure. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run`, `$status`, and setup() for that compatibility path. # -# Run with: hack/cozytest.sh hack/migration-seaweedfs-db-adopt.bats +# Run with: bats hack/migration-seaweedfs-db-adopt.bats # ----------------------------------------------------------------------------- +load test_helper + FAKEBIN="$PWD/hack/testdata/migration-seaweedfs-db" MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations" diff --git a/hack/monitoring-pvc-backfill-migration.bats b/hack/monitoring-pvc-backfill-migration.bats index 9c2f3925a..147f7a628 100755 --- a/hack/monitoring-pvc-backfill-migration.bats +++ b/hack/monitoring-pvc-backfill-migration.bats @@ -23,15 +23,16 @@ # - stamps the version (kubectl apply) on a clean run; # - is a safe no-op that still stamps when the CRDs are absent. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line (column 0); assertions are direct shell tests, and the fake-kubectl -# heredoc keeps every `}` off column 0. Each @test cleans up inline (no -# EXIT/RETURN trap — see docs/agents/e2e-testing.md §3). +# For compatibility with the legacy cozytest.sh translator, assertions are +# direct shell tests and the fake-kubectl heredoc keeps every `}` off column 0. +# Each @test cleans up inline (no EXIT/RETURN trap — see +# docs/agents/e2e-testing.md §3). # -# Run with: hack/cozytest.sh hack/monitoring-pvc-backfill-migration.bats -# (or `bats hack/monitoring-pvc-backfill-migration.bats`) +# Run with: bats hack/monitoring-pvc-backfill-migration.bats # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" REPO_ROOT="$(cd "$HACK_DIR/.." && pwd)" MIGRATION="$REPO_ROOT/packages/core/platform/images/migrations/migrations/51" diff --git a/hack/multus-install-cni-plugins.bats b/hack/multus-install-cni-plugins.bats index a99ce1fb4..d5819ea20 100644 --- a/hack/multus-install-cni-plugins.bats +++ b/hack/multus-install-cni-plugins.bats @@ -3,15 +3,17 @@ # extracted from the rendered DaemonSet and run. The helm-unittest cases in # packages/system/multus/tests/multus_test.yaml match its source TEXT instead. # -# Run via hack/cozytest.sh from the repo root (make bats-unit-tests); relative -# paths resolve against that cwd. The runner has no setup/teardown, so each -# @test builds its own fixture and removes it at the last reachable cleanup -# point of its body. +# Run from the repo root through `make bats-unit-tests`; relative paths resolve +# against that cwd. The file also remains compatible with the legacy +# `hack/cozytest.sh` translator, so each @test builds and removes its fixture +# inline instead of relying on setup()/teardown(). # # Test-level EXIT traps replace Bats' own handler and hide failing TAP results. # Cleanup follows aborting assertions; see docs/agents/e2e-testing.md. # Traps in the script under test run in a separate shell and remain in scope. +load test_helper + CHART=packages/system/multus # Extract the init container's script from the rendered chart into $1 and rebind diff --git a/hack/nats-backup-demo-helpers.bats b/hack/nats-backup-demo-helpers.bats index 51c93bb4a..07cc41c0e 100644 --- a/hack/nats-backup-demo-helpers.bats +++ b/hack/nats-backup-demo-helpers.bats @@ -16,6 +16,8 @@ # helper functions close on an indented brace because the runner rewrites a # column-0 `}` into `return 0`, which would mask run_stream_count's exit status. +load test_helper + make_stubs() { stub=$(mktemp -d) cat > "$stub/kubectl" <<'EOF' diff --git a/hack/nightly-mirror_test.bats b/hack/nightly-mirror_test.bats index bb18017fe..376f6b67e 100644 --- a/hack/nightly-mirror_test.bats +++ b/hack/nightly-mirror_test.bats @@ -8,17 +8,13 @@ # tree), and every copy targets the destination registry with the source digest # preserved. # -# Harness note: the CI path is hack/cozytest.sh, NOT real bats — see the same -# note in hack/promote-retag_test.bats. No `run`, `$status`, `$output`, `skip`, -# or setup()/teardown(); each @test is a shell function under `set -eu -x`, so a -# non-zero exit aborts the test (that is the exit-0 assertion). A test that -# expects a non-zero exit must capture it with `|| rc=$?`. mikefarah yq is -# assumed present (provided by the test toolchain). +# The compatibility pass uses cozytest's POSIX translator, so expected errors +# are captured explicitly without Bats-only run/status helpers. +# Test-level EXIT traps hide Bats' failing TAP results; cleanup follows assertions. # -# Test-level EXIT traps replace Bats' own handler and hide failing TAP results. -# Cleanup follows aborting assertions; see docs/agents/e2e-testing.md. -# -# Run with: hack/cozytest.sh hack/nightly-mirror_test.bats +# Run with: bats hack/nightly-mirror_test.bats + +load test_helper # Build a synthetic baked tree exercising all four image-ref shapes plus the # refs that MUST be filtered (third-party host, cozystack-packages artifact). diff --git a/hack/overlay-main-images_test.bats b/hack/overlay-main-images_test.bats index 88a009640..7cff2d842 100644 --- a/hack/overlay-main-images_test.bats +++ b/hack/overlay-main-images_test.bats @@ -3,8 +3,9 @@ # packages a PR did NOT rebuild at the current-main images from the # cozystack-packages:main artifact. # -# Run from the repo root: bats hack/overlay-main-images_test.bats -# (CI runs it via hack/cozytest.sh through `make unit-tests`.) +# Run from the repo root with `make bats-unit-tests`, or directly with +# `bats hack/overlay-main-images_test.bats`. The legacy `hack/cozytest.sh` +# runner remains a supported compatibility path. # # Each fixture test builds a throwaway tree: a `packages/` tree on release # (ghcr/v1.5.0) refs, and a `main/` dir standing in for the extracted @@ -15,6 +16,8 @@ # Cleanup follows aborting assertions; see docs/agents/e2e-testing.md. # Leave the fixture directory before deleting it so the shell retains a cwd. +load test_helper + @test "overlays an unbuilt unit (.tag) to current-main and reports it" { root=$(pwd) w=$(mktemp -d) diff --git a/hack/package-mk-phony.bats b/hack/package-mk-phony.bats index 30bd53b42..51828cb2c 100644 --- a/hack/package-mk-phony.bats +++ b/hack/package-mk-phony.bats @@ -52,9 +52,11 @@ # recognizes only @test blocks and a bare `}` at column 0, and injects `set -e`, # so exit statuses are captured explicitly rather than tested inline. # -# Run with: hack/cozytest.sh hack/package-mk-phony.bats +# Run with: bats hack/package-mk-phony.bats # ----------------------------------------------------------------------------- +load test_helper + @test "every target hack/package.mk declares phony still runs when a file of that name exists" { repo=$(pwd) targets=$(sed -n 's/^\.PHONY:[[:space:]]*//p' "$repo/hack/package.mk") diff --git a/hack/package-test-target-coverage.bats b/hack/package-test-target-coverage.bats index 7fa739e37..4a8d61bea 100644 --- a/hack/package-test-target-coverage.bats +++ b/hack/package-test-target-coverage.bats @@ -73,6 +73,8 @@ # no `$status`. # ----------------------------------------------------------------------------- +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" @test "every package shipping a helm-unittest suite answers the sweep probe" { diff --git a/hack/package-update-templates.bats b/hack/package-update-templates.bats index ee07444e6..4e1806574 100644 --- a/hack/package-update-templates.bats +++ b/hack/package-update-templates.bats @@ -106,6 +106,8 @@ # A second `update:` can fool this in either direction: make runs the last # recipe, this reads whichever ones are adjacent. Nothing in the tree writes # one. +load test_helper + wipe_command() { awk '/^update:/ { r = 1; next } r && /^\t[[:space:]]*@?#/ { next } diff --git a/hack/platform-packages_test.bats b/hack/platform-packages_test.bats index d80a2da2f..fe2a7d5aa 100644 --- a/hack/platform-packages_test.bats +++ b/hack/platform-packages_test.bats @@ -3,6 +3,8 @@ # hack/e2e-platform-packages.sh. This suite runs through hack/cozytest.sh, so it # uses direct shell assertions rather than bats' run/status helpers. +load test_helper + @test "QEMU defaults keep the platform-managed LINSTOR package" { manifest=$(hack/e2e-platform-packages.sh) names=$(printf '%s\n' "$manifest" | yq -N '.metadata.name') diff --git a/hack/pod-label-census_test.bats b/hack/pod-label-census_test.bats index 835ee4202..465e3fef1 100644 --- a/hack/pod-label-census_test.bats +++ b/hack/pod-label-census_test.bats @@ -23,16 +23,18 @@ # on PATH, to prove the streams stay apart end to end rather than only in the # half that never touches kubectl. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run` or `$status`, and setup()/teardown() are not -# honored. Each test runs under `set -eu -x`; assertions are direct shell tests -# that exit non-zero on failure. Titles are delimited by ASCII double quotes and -# only [A-Za-z0-9] survives into the generated function name, so keep them -# distinctive in their alphanumeric run. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run`, `$status`, setup(), and teardown() for that compatibility path and use +# direct shell assertions. Titles remain distinguishable after stripping all +# but [A-Za-z0-9], as required by the legacy translator. # -# Run with: hack/cozytest.sh hack/pod-label-census_test.bats +# Run with: bats hack/pod-label-census_test.bats # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" # shellcheck source=/dev/null . "$HACK_DIR/e2e-chainsaw/_lib/pod-label-census.sh" diff --git a/hack/post-install-prep.bats b/hack/post-install-prep.bats index 6e48d1781..cc34a4a6b 100644 --- a/hack/post-install-prep.bats +++ b/hack/post-install-prep.bats @@ -28,9 +28,11 @@ # - A line that is exactly `}` is rewritten, so stub bodies below use `case` # rather than nested functions. # -# Run with: hack/cozytest.sh hack/post-install-prep.bats +# Run with: bats hack/post-install-prep.bats # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" SCRIPT="$HACK_DIR/e2e-post-install-prep.sh" diff --git a/hack/postgres-backup-app-login_test.bats b/hack/postgres-backup-app-login_test.bats index fb51c4f33..370775bf3 100644 --- a/hack/postgres-backup-app-login_test.bats +++ b/hack/postgres-backup-app-login_test.bats @@ -13,6 +13,8 @@ # /bin/sh (dash in CI). It is sourced first, then the stubs override its # cluster-touching functions. +load test_helper + @test "psql_app_exec reads the credentials Secret named after the cluster, not the app name" { calls=$(mktemp) CALLS="$calls" bash -uc ' diff --git a/hack/pr-labeler-contract.bats b/hack/pr-labeler-contract.bats index d848b0fa1..2c2daa140 100644 --- a/hack/pr-labeler-contract.bats +++ b/hack/pr-labeler-contract.bats @@ -11,6 +11,8 @@ # required, not optional: a suite that skips when the runtime is missing goes # green having run nothing. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" WF="$REPO_ROOT/.github/workflows/pr-labeler.yaml" LABELS="$REPO_ROOT/.github/labels.yml" diff --git a/hack/pre-commit-generate-packages_test.bats b/hack/pre-commit-generate-packages_test.bats index 15cc91118..20e6b5772 100755 --- a/hack/pre-commit-generate-packages_test.bats +++ b/hack/pre-commit-generate-packages_test.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats -# cozytest.sh provides no bats lifecycle or result helpers, so each test owns -# its fixture and uses ordinary shell assertions. + +load test_helper add_generator() { root="$1" diff --git a/hack/promote-gate-contract.bats b/hack/promote-gate-contract.bats index 184214682..65b7916eb 100644 --- a/hack/promote-gate-contract.bats +++ b/hack/promote-gate-contract.bats @@ -4,6 +4,8 @@ # These tests intentionally pin executable/structural workflow lines, not prose: # a commented-out gate, label, or body note must never satisfy the contract. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" PROMOTE="$REPO_ROOT/.github/workflows/promote-rc.yaml" PULL_REQUESTS="$REPO_ROOT/.github/workflows/pull-requests.yaml" diff --git a/hack/promote-packages-artifact_test.bats b/hack/promote-packages-artifact_test.bats index 2924125b5..f885b62d8 100644 --- a/hack/promote-packages-artifact_test.bats +++ b/hack/promote-packages-artifact_test.bats @@ -1,6 +1,8 @@ #!/usr/bin/env bats # Behavioural tests for the stable-candidate packages artifact publisher. -# Run with: hack/cozytest.sh hack/promote-packages-artifact_test.bats +# Run with: bats hack/promote-packages-artifact_test.bats + +load test_helper _test_workspace() { if [ -n "${BATS_TEST_TMPDIR:-}" ]; then diff --git a/hack/promote-packages-contract.bats b/hack/promote-packages-contract.bats index 4d9c9f9aa..a30aa21a2 100644 --- a/hack/promote-packages-contract.bats +++ b/hack/promote-packages-contract.bats @@ -3,6 +3,8 @@ # The registry workflow cannot run in a PR lane, so ordering around the first # write-once stable tag is pinned here. Behaviour lives in the two *_test suites. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" PROMOTE="$REPO_ROOT/.github/workflows/promote-rc.yaml" FINALIZE="$REPO_ROOT/.github/workflows/pull-requests-release.yaml" diff --git a/hack/promote-retag_test.bats b/hack/promote-retag_test.bats index bf96f173d..61d858777 100644 --- a/hack/promote-retag_test.bats +++ b/hack/promote-retag_test.bats @@ -8,18 +8,17 @@ # cannot push to aborted the whole promotion. The selector must emit only # cozystack-owned ($REGISTRY/...) refs. # -# Harness note: the CI path is hack/cozytest.sh, NOT real bats. cozytest.sh's -# awk parser recognizes only @test blocks and a bare `}` on its own line; there -# is no `run`, `$status`, `$output`, `skip`, or setup()/teardown(). Each test -# runs as a shell function under `set -eu -x`, so a non-zero exit aborts the -# test (that is the exit-0 assertion) and other expectations are direct shell -# tests. A test that expects a non-zero exit must capture it with `|| rc=$?` -# so the harness's `set -e` does not abort first. mikefarah yq is assumed -# present (provided by the test toolchain, like the other yq-using bats here). +# Harness compatibility note: CI runs this file under Bats through +# `make bats-unit-tests`. It also remains compatible with the narrower legacy +# `hack/cozytest.sh` translator, whose awk parser recognizes only @test blocks +# and a bare `}` on its own line. The tests therefore avoid `run`, `$status`, +# `$output`, `skip`, and setup()/teardown(), and capture expected non-zero exits +# with `|| rc=$?`. mikefarah yq is assumed present (provided by the test +# toolchain, like the other yq-using Bats files here). # -# Run with: hack/cozytest.sh hack/promote-retag_test.bats -# (or `bats hack/promote-retag_test.bats` if the bats binary is -# installed; cozytest.sh is the CI path.) +# Run with: bats hack/promote-retag_test.bats + +load test_helper _make_registry_mocks() { t="$1" diff --git a/hack/promote-rewrite-tags_test.bats b/hack/promote-rewrite-tags_test.bats index 0ee884a28..7a1be6c74 100644 --- a/hack/promote-rewrite-tags_test.bats +++ b/hack/promote-rewrite-tags_test.bats @@ -17,16 +17,16 @@ # registry prefix finds files it has never heard of, which is the whole failure # mode. Stamp wider than the rewrite, then assert the rewrite cleaned all of it. # -# Harness note: the CI path is hack/cozytest.sh, NOT real bats. cozytest.sh's -# awk parser recognizes only @test blocks and a bare `}` on its own line; there -# is no `run`, `$status`, `$output`, `skip`, or setup()/teardown(). Each test -# runs as a shell function under `set -eu -x`, so a non-zero exit aborts the -# test (that is the exit-0 assertion). A test that expects a non-zero exit must -# capture it with `|| rc=$?` so the harness's `set -e` does not abort first. -# Paths are repo-root-relative: BATS_TEST_DIRNAME is unset here and would abort -# the whole suite under `set -u`. +# Harness compatibility note: CI runs this file under Bats through +# `make bats-unit-tests`. It also remains compatible with the narrower legacy +# `hack/cozytest.sh` translator, whose awk parser recognizes only @test blocks +# and a bare `}` on its own line. The tests therefore avoid `run`, `$status`, +# `$output`, `skip`, and setup()/teardown(), and capture expected non-zero exits +# with `|| rc=$?`. Paths stay repo-root-relative for both runners. # -# Run with: hack/cozytest.sh hack/promote-rewrite-tags_test.bats +# Run with: bats hack/promote-rewrite-tags_test.bats + +load test_helper @test "rewrite leaves no rc reference anywhere in the tree" { tmp=$(mktemp -d) diff --git a/hack/promotion-retention-contract.bats b/hack/promotion-retention-contract.bats index 245162662..70e70268c 100644 --- a/hack/promotion-retention-contract.bats +++ b/hack/promotion-retention-contract.bats @@ -1,6 +1,8 @@ #!/usr/bin/env bats # Structural contract for abandoned stable-candidate artifact retention. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" RETENTION="$REPO_ROOT/.github/workflows/retention.yaml" diff --git a/hack/proxmox-e2e-fixture_test.bats b/hack/proxmox-e2e-fixture_test.bats index eaa4e0a50..0d2df3e5a 100755 --- a/hack/proxmox-e2e-fixture_test.bats +++ b/hack/proxmox-e2e-fixture_test.bats @@ -6,6 +6,8 @@ # integer. So the defaults live in the script, the fixture carries bare # variables, and the render refuses to hand out a fixture with one left in it. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" FIXTURE="$REPO_ROOT/hack/e2e-chainsaw/kubernetes-proxmox/tenant.yaml" RUNNER="$REPO_ROOT/hack/e2e-chainsaw/_lib/run-proxmox.sh" @@ -37,7 +39,8 @@ RUNNER="$REPO_ROOT/hack/e2e-chainsaw/_lib/run-proxmox.sh" # The PVC probe waits for the disk to leave the storage. A listing that fails # must fail the check: read as empty, it would report a leaked disk as freed. -# (hack/cozytest.sh runs these as POSIX sh: no `run`, no [[ ]].) +# (The bodies stay POSIX sh, so the legacy hack/cozytest.sh translator can still +# run them: no `run`, no [[ ]].) @test "disk-freed fails when the storage cannot be listed, instead of calling the disk gone" { stub="$(mktemp -d)" printf '#!/bin/sh\necho "ssh: connect to host pve: Connection refused" >&2\nexit 255\n' > "$stub/ssh" diff --git a/hack/proxmox-step1b-pod-match_test.bats b/hack/proxmox-step1b-pod-match_test.bats index 3eaa62919..d211fa58c 100755 --- a/hack/proxmox-step1b-pod-match_test.bats +++ b/hack/proxmox-step1b-pod-match_test.bats @@ -13,8 +13,11 @@ # and no Pod is deleted; the listing-plus-shell-match form does not. # ----------------------------------------------------------------------------- -# hack/cozytest.sh runs @test bodies as POSIX sh functions and knows no -# setup/teardown, so each test calls prepare_loop and removes $work itself. +load test_helper + +# The legacy hack/cozytest.sh translator runs @test bodies as POSIX sh functions +# and knows no setup/teardown; the tests stay compatible with it, so each test +# calls prepare_loop and removes $work itself. prepare_loop() { work=$(mktemp -d) cat > "$work/vals.yaml" <<'VALS' diff --git a/hack/redis-backup-demo-helpers.bats b/hack/redis-backup-demo-helpers.bats index 7dd9252e1..bbc9fdaa3 100644 --- a/hack/redis-backup-demo-helpers.bats +++ b/hack/redis-backup-demo-helpers.bats @@ -13,6 +13,8 @@ # rewrites a column-0 `}` into `return 0`, which would mask run_redis_cmd's # exit status. +load test_helper + make_stubs() { stub=$(mktemp -d) cat > "$stub/kubectl" <<'EOF' diff --git a/hack/redis-backup-strategy.bats b/hack/redis-backup-strategy.bats index 283c354ec..ec2292654 100644 --- a/hack/redis-backup-strategy.bats +++ b/hack/redis-backup-strategy.bats @@ -11,10 +11,9 @@ # the live master. Blanking the object key or breaking the rfs-redis- sentinel # prefix is caught the same way — by what the stubs are actually called with. # -# Run via hack/cozytest.sh from the repo root (make bats-unit-tests); relative -# paths resolve against that cwd. Each @test builds its own fixture; a -# load-bearing assertion is never the last line (cozytest.sh rewrites a lone -# `}` into `return 0` + `}`, which would swallow the final command's status). +# Run from the repo root with make bats-unit-tests. To retain compatibility +# with cozytest, assertions precede inline cleanup: its translator would +# replace a final assertion's status with return 0 at a lone closing brace. # # The rendered script is run with `bash`, not `sh`: it opens `set -euo # pipefail`, which the production image's busybox ash supports but the CI @@ -22,6 +21,8 @@ # semantics; same reason hack/migration-50-etcd-adopt.bats runs its script # with bash. +load test_helper + CHART=packages/system/backupstrategy-controller # Render the shipped strategy's in-Pod script into $1, substitute the three diff --git a/hack/release-changelog-behaviour.bats b/hack/release-changelog-behaviour.bats index cdea3de15..cb2a22446 100644 --- a/hack/release-changelog-behaviour.bats +++ b/hack/release-changelog-behaviour.bats @@ -31,6 +31,8 @@ # Test-level EXIT traps replace Bats' own handler and hide failing TAP results. # Cleanup follows aborting assertions; see docs/agents/e2e-testing.md. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" PRESERVE="$REPO_ROOT/hack/changelog-preserve.sh" VALIDATE="$REPO_ROOT/hack/validate-changelog.sh" diff --git a/hack/release-changelog-contract.bats b/hack/release-changelog-contract.bats index 1ba3f0454..27596c037 100644 --- a/hack/release-changelog-contract.bats +++ b/hack/release-changelog-contract.bats @@ -74,6 +74,8 @@ # Test-level EXIT traps replace Bats' own handler and hide failing TAP results. # Cleanup follows aborting assertions; see docs/agents/e2e-testing.md. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" PROMOTE="$REPO_ROOT/.github/workflows/promote-rc.yaml" GENERATE="$REPO_ROOT/.github/workflows/changelog-generate.yaml" diff --git a/hack/release-freeze-contract.bats b/hack/release-freeze-contract.bats index 9393ec4f3..30a6e20bb 100644 --- a/hack/release-freeze-contract.bats +++ b/hack/release-freeze-contract.bats @@ -17,6 +17,8 @@ # line silently, and getting the comparator wrong aims every backport at the # wrong branch. Cheap to pin here, expensive to discover there. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" CUT="$REPO_ROOT/.github/workflows/cut-prerelease.yaml" BACKPORT="$REPO_ROOT/.github/workflows/backport.yaml" diff --git a/hack/remediation-guard.bats b/hack/remediation-guard.bats index 54df47c4d..c7a94569c 100644 --- a/hack/remediation-guard.bats +++ b/hack/remediation-guard.bats @@ -12,13 +12,16 @@ # are useless there because flux's ClearFailures zeroes them on successful # reconciliation; .status.history retains the snapshot trail. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on -# its own line; there is no bats `run` or `$status`. Assertions are -# expressed as direct shell tests that exit non-zero on failure. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run` and `$status` for that compatibility path and use direct shell assertions. # -# Run with: hack/cozytest.sh hack/remediation-guard.bats +# Run with: bats hack/remediation-guard.bats # ----------------------------------------------------------------------------- +load test_helper + @test "empty history returns not-detected" { . hack/e2e-chainsaw/_lib/remediation-guard.sh if helmrelease_has_remediation_cycle ""; then diff --git a/hack/route-hostname-policy-version-coverage.bats b/hack/route-hostname-policy-version-coverage.bats index eafe38e0e..736afb6d6 100644 --- a/hack/route-hostname-policy-version-coverage.bats +++ b/hack/route-hostname-policy-version-coverage.bats @@ -33,16 +33,15 @@ # The policy side is read from `helm template` output rather than the template # source, so what is checked is the object the apiserver would receive. # -# Harness note: the CI path is hack/cozytest.sh, NOT real bats. There is no -# `run`, `$status`, `$output`, `skip`, or setup()/teardown(); each test runs -# as a shell function under `set -eu -x`, so a non-zero exit is the failure. -# Paths are repo-root-relative: BATS_TEST_DIRNAME is unset and would abort the -# whole suite under `set -u`. +# Paths are repo-root-relative. Direct assertions retain compatibility with +# cozytest, which does not provide Bats-specific helpers or directory variables. # # Requires: yq (mikefarah v4+), helm. Both are available on the project's CI # image and are already relied on by other hack/*.bats. # -# Run with: hack/cozytest.sh hack/route-hostname-policy-version-coverage.bats +# Run with: bats hack/route-hostname-policy-version-coverage.bats + +load test_helper CRD_BUNDLE=packages/system/gateway-api-crds/templates/crds-experimental.yaml BASICS_CHART=packages/system/cozystack-basics diff --git a/hack/run-kubernetes-block-io_test.bats b/hack/run-kubernetes-block-io_test.bats index b8eeb1e99..b083f9e4f 100644 --- a/hack/run-kubernetes-block-io_test.bats +++ b/hack/run-kubernetes-block-io_test.bats @@ -20,6 +20,8 @@ # filesystem's own counters, which are not this Pod's. A zero read as an idle # disk is the wrong conclusion this suite pins against. +load test_helper + kubectl_calls=/dev/null kubectl_node_names= kubectl_list_rc=0 diff --git a/hack/run-kubernetes-cpu-throttle_test.bats b/hack/run-kubernetes-cpu-throttle_test.bats index 28a0efca8..d254c7d49 100644 --- a/hack/run-kubernetes-cpu-throttle_test.bats +++ b/hack/run-kubernetes-cpu-throttle_test.bats @@ -13,6 +13,8 @@ # the next reader the same ambiguity while looking like it had resolved it. # These tests pin the separation, not the reading. +load test_helper + kubectl_calls=/dev/null kubectl_node_names= kubectl_list_rc=0 diff --git a/hack/run-kubernetes-drain_test.bats b/hack/run-kubernetes-drain_test.bats index 425edbd94..bc0ec6b35 100644 --- a/hack/run-kubernetes-drain_test.bats +++ b/hack/run-kubernetes-drain_test.bats @@ -9,13 +9,16 @@ # is never misread as "the tenant has drained". The function returns 0 (drained) # only when every capture is empty/whitespace, non-zero otherwise. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run` or `$status`. Assertions are expressed as -# direct shell tests that exit non-zero on failure. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run` and `$status` for that compatibility path and use direct shell assertions. # -# Run with: hack/cozytest.sh hack/run-kubernetes-drain_test.bats +# Run with: bats hack/run-kubernetes-drain_test.bats # ----------------------------------------------------------------------------- +load test_helper + @test "all-empty captures report drained" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh if ! cozy_tenant_drained "" "" ""; then diff --git a/hack/run-kubernetes-install-wait_test.bats b/hack/run-kubernetes-install-wait_test.bats index 6c91f6e7c..067ceb474 100644 --- a/hack/run-kubernetes-install-wait_test.bats +++ b/hack/run-kubernetes-install-wait_test.bats @@ -158,6 +158,8 @@ # happens under bats at all -- which is why running this by hand can look green # while CI reds, which is why both are kept green. +load test_helper + REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" TENANT_SCRIPT="${TENANT_SCRIPT:-$REPO_ROOT/hack/e2e-chainsaw/_lib/run-kubernetes.sh}" diff --git a/hack/run-kubernetes-join-timing_test.bats b/hack/run-kubernetes-join-timing_test.bats index b7162baf2..178dc34c0 100644 --- a/hack/run-kubernetes-join-timing_test.bats +++ b/hack/run-kubernetes-join-timing_test.bats @@ -2,6 +2,8 @@ # Successful-node-join timing stays in the normal job log; these mocks keep its # resource/event rendering cluster-free under hack/cozytest.sh. +load test_helper + kubectl() { case "$*" in *"get events.events.k8s.io"*) diff --git a/hack/run-kubernetes-kvm-exits_test.bats b/hack/run-kubernetes-kvm-exits_test.bats index d6306440f..0473e642f 100644 --- a/hack/run-kubernetes-kvm-exits_test.bats +++ b/hack/run-kubernetes-kvm-exits_test.bats @@ -17,6 +17,8 @@ # directory, which reads as a kernel that paid nothing while nothing was ever # asked. +load test_helper + timeout_calls=/dev/null timeout_rc_override= unshare_calls=/dev/null diff --git a/hack/run-kubernetes-network-counters_test.bats b/hack/run-kubernetes-network-counters_test.bats index 977edc61e..73da4c2b2 100644 --- a/hack/run-kubernetes-network-counters_test.bats +++ b/hack/run-kubernetes-network-counters_test.bats @@ -23,6 +23,8 @@ # spelled packets_dropped, and both are invisible to anyone writing the filter # from memory. +load test_helper + kubectl_calls=/dev/null kubectl_node_names= kubectl_list_rc=0 diff --git a/hack/run-kubernetes-node-join_test.bats b/hack/run-kubernetes-node-join_test.bats index 83aeae633..1462b616e 100644 --- a/hack/run-kubernetes-node-join_test.bats +++ b/hack/run-kubernetes-node-join_test.bats @@ -25,15 +25,18 @@ # its budget inside a Pod it creates, and shrinking that budget would change # which image factory the happy path picks. # -# cozytest.sh's awk parser ends an @test block at the first bare closing brace, -# so command mocks stay at top level, and there is no bats `run`/`$status`: -# assertions are direct shell tests that exit non-zero on failure. EXIT-trap +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, which ends an @test +# block at the first bare closing brace, so command mocks stay at top level and +# assertions avoid `run` and `$status`. EXIT-trap # cleanup is banned in hack/*.bats and frozen by a guard in hack/cozyreport.bats, # so each test removes its own temp dir on its last line. # -# Run with: hack/cozytest.sh hack/run-kubernetes-node-join_test.bats +# Run with: bats hack/run-kubernetes-node-join_test.bats # ----------------------------------------------------------------------------- +load test_helper + kubectl_calls=/dev/null unbounded_calls=/dev/null timeout_calls=/dev/null diff --git a/hack/run-kubernetes-oidc_test.bats b/hack/run-kubernetes-oidc_test.bats index c68d60991..a9bcd2837 100644 --- a/hack/run-kubernetes-oidc_test.bats +++ b/hack/run-kubernetes-oidc_test.bats @@ -3,6 +3,8 @@ # Sourcing run-kubernetes.sh only defines functions; every kubectl interaction # below is replaced with a shell stub, so this file needs no cluster. +load test_helper + @test "HelmRelease upgrade wait rejects stale Ready and stale observedGeneration" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh state_file=$(mktemp) diff --git a/hack/run-kubernetes-remediation_test.bats b/hack/run-kubernetes-remediation_test.bats index 8fc7746f0..5f0b18e95 100644 --- a/hack/run-kubernetes-remediation_test.bats +++ b/hack/run-kubernetes-remediation_test.bats @@ -16,15 +16,18 @@ # interprets an empty history, and the failure paths of the reads themselves - # without a cluster. The readings are unit-tested in hack/remediation-guard.bats. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run` or `$status`. Assertions are expressed as +# CI runs this file under Bats. It stays compatible with the legacy cozytest.sh +# translator, whose awk parser recognizes only @test blocks and a bare `}` on its +# own line, so there is no bats `run` or `$status`. Assertions are expressed as # direct shell tests that exit non-zero on failure. Scratch directories are # removed at the end of the test body rather than from a trap (both runners set # -e, so a failed test leaves its fixtures behind for inspection). # -# Run with: hack/cozytest.sh hack/run-kubernetes-remediation_test.bats +# Run with: bats hack/run-kubernetes-remediation_test.bats # ----------------------------------------------------------------------------- +load test_helper + # Writes a fixture-driven kubectl stub into $1/bin and returns with $1/fixtures # created. The stub answers the shapes the guard issues: the namespace listing # (from ALL_HR), one release's install strategy and Ready condition together diff --git a/hack/run-kubernetes-runner-canary_test.bats b/hack/run-kubernetes-runner-canary_test.bats index 1386184c7..7e2f85f90 100644 --- a/hack/run-kubernetes-runner-canary_test.bats +++ b/hack/run-kubernetes-runner-canary_test.bats @@ -18,17 +18,15 @@ # ceiling and an arm that ran to the end are three different things, and only # the last of them is a rate. # -# Run with: hack/cozytest.sh hack/run-kubernetes-runner-canary_test.bats +# Run with: bats hack/run-kubernetes-runner-canary_test.bats # -# cozytest.sh is the canonical runner and the CI path: it runs this file under -# `sh` with `set -eu`, and in CI that `sh` is dash, which is what the collector -# meets in the sandbox image. `bats` runs the same file under bash without those -# flags, so it is a convenience rather than an equal alternative: the guards -# here that exist for dash semantics do not fire under it. Spelling the capture -# truncation `: >` again -- the regression the guard below is written for -- -# leaves bats green, and leaves cozytest green too wherever /bin/sh is not dash, -# which includes macOS, where it is bash in POSIX mode. Run `dash -# hack/cozytest.sh` to exercise those guards off CI. +# CI runs this file under Bats through `make bats-unit-tests`; nounset comes from +# test_helper. The collector itself runs under dash in the sandbox image, so the +# failed-open behavior below is paired with a structural assertion that pins +# the safe `true >` spelling. Replacing it with the POSIX special builtin `: >` +# can exit dash before the caller handles the status even when Bash stays green. + +load test_helper timeout_calls=/dev/null timeout_rc_override= @@ -1324,6 +1322,20 @@ STUB rm -rf "$tmp" } +@test "capture truncation avoids a POSIX special builtin under dash" { + source_file=hack/e2e-chainsaw/_lib/run-kubernetes.sh + body=$(awk ' + /^cozy_capture_runner_canary\(\) \{/ { inside = 1 } + inside { print } + inside && /^}$/ { exit } + ' "$source_file") + + if ! printf '%s\n' "$body" | grep -Fq 'if ! true >"${capture}"; then'; then + echo "FAIL: runner canary no longer opens its capture through non-special true" >&2 + return 1 + fi +} + @test "a capture file that cannot be opened reaches the job log rather than ending the shell" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh tmp=$(mktemp -d) diff --git a/hack/run-kubernetes-runner-cpu_test.bats b/hack/run-kubernetes-runner-cpu_test.bats index 78d81a519..4eca0a3fc 100644 --- a/hack/run-kubernetes-runner-cpu_test.bats +++ b/hack/run-kubernetes-runner-cpu_test.bats @@ -21,9 +21,11 @@ # that spent nothing -- and the legends, which are what stop a reader comparing a # hypervisor's row against its guest's. # -# Run with: hack/cozytest.sh hack/run-kubernetes-runner-cpu_test.bats -# (or `bats hack/run-kubernetes-runner-cpu_test.bats` if the bats -# binary is installed; cozytest.sh is the CI path.) +# Run from the repo root with `make bats-unit-tests`, or directly with +# `bats hack/run-kubernetes-runner-cpu_test.bats`. The legacy +# `hack/cozytest.sh` runner remains a supported compatibility path. + +load test_helper timeout_calls=/dev/null timeout_rc_override= diff --git a/hack/run-kubernetes-sandbox-cpu-time_test.bats b/hack/run-kubernetes-sandbox-cpu-time_test.bats index 1fbf895fa..0cd1bace4 100644 --- a/hack/run-kubernetes-sandbox-cpu-time_test.bats +++ b/hack/run-kubernetes-sandbox-cpu-time_test.bats @@ -14,6 +14,8 @@ # empty directory, which reads as a sandbox that lost no time while nothing was # ever asked. +load test_helper + kubectl_calls=/dev/null kubectl_rows='srv1|192.0.2.11' kubectl_list_rc=0 diff --git a/hack/run-kubernetes-schedulable_test.bats b/hack/run-kubernetes-schedulable_test.bats index 63cdb300b..cfd09bc27 100644 --- a/hack/run-kubernetes-schedulable_test.bats +++ b/hack/run-kubernetes-schedulable_test.bats @@ -16,13 +16,16 @@ # Ready, not SchedulingDisabled, and no NoSchedule/NoExecute taint. # PreferNoSchedule only lowers the node's score and must not block. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run` or `$status`. Assertions are expressed as -# direct shell tests that exit non-zero on failure. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run` and `$status` for that compatibility path and use direct shell assertions. # -# Run with: hack/cozytest.sh hack/run-kubernetes-schedulable_test.bats +# Run with: bats hack/run-kubernetes-schedulable_test.bats # ----------------------------------------------------------------------------- +load test_helper + @test "a Ready node with no taints accepts pods" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh if ! cozy_node_accepts_pods True '' ''; then diff --git a/hack/run-kubernetes-serial-console_test.bats b/hack/run-kubernetes-serial-console_test.bats index 810e6e901..3dd91773d 100644 --- a/hack/run-kubernetes-serial-console_test.bats +++ b/hack/run-kubernetes-serial-console_test.bats @@ -10,6 +10,8 @@ # property, because a capture that grew an in-guest dependency would still look # healthy while being blind to exactly the runs it was written for. +load test_helper + kubectl_calls=/dev/null kubectl_pod_names= kubectl_list_rc=0 diff --git a/hack/run-kubernetes-talos-diagnostics_test.bats b/hack/run-kubernetes-talos-diagnostics_test.bats index 1486cdef5..c5411837f 100644 --- a/hack/run-kubernetes-talos-diagnostics_test.bats +++ b/hack/run-kubernetes-talos-diagnostics_test.bats @@ -12,6 +12,8 @@ # inspection, which is what a failed test wants anyway. See # docs/agents/e2e-testing.md and hack/bats-no-exit-trap.bats. +load test_helper + kubectl_calls=/dev/null kubectl_manifest=/dev/null kubectl_exec_rc=0 diff --git a/hack/run-kubernetes-talos-spec_test.bats b/hack/run-kubernetes-talos-spec_test.bats index eae1fde01..9eabc119d 100644 --- a/hack/run-kubernetes-talos-spec_test.bats +++ b/hack/run-kubernetes-talos-spec_test.bats @@ -18,13 +18,17 @@ # spec.talos", which is the actual contract. A string match would keep passing # on a block indented into the wrong parent. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run`/`$status`. Sourcing run-kubernetes.sh only -# defines functions and touches no cluster. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run` and `$status` for that compatibility path. Sourcing run-kubernetes.sh +# only defines functions and touches no cluster. # -# Run with: hack/cozytest.sh hack/run-kubernetes-talos-spec_test.bats +# Run with: bats hack/run-kubernetes-talos-spec_test.bats # ----------------------------------------------------------------------------- +load test_helper + # Splice a talos_spec_block result under `spec:` of a minimal CR and print the # document, so yq can be asked what the tenant apiserver would see. spec_doc() { diff --git a/hack/run-kubernetes-thread-cpu_test.bats b/hack/run-kubernetes-thread-cpu_test.bats index aec6b8af2..4d9f7c90b 100644 --- a/hack/run-kubernetes-thread-cpu_test.bats +++ b/hack/run-kubernetes-thread-cpu_test.bats @@ -12,6 +12,8 @@ # and it exists nowhere else in this tree. These tests pin the split and the # separation between an unread container and a quiet one, not the reading. +load test_helper + kubectl_calls=/dev/null kubectl_pod_names= kubectl_list_rc=0 diff --git a/hack/run-migrations.bats b/hack/run-migrations.bats index 1e9c38cf9..11ec1b0ba 100644 --- a/hack/run-migrations.bats +++ b/hack/run-migrations.bats @@ -13,9 +13,11 @@ # that keeps the stamped version in a file, so one pod's stamps are what the # next pod reads. The harness shape is hack/migration-58-http-cache-freeze.bats's. # -# Run with: hack/cozytest.sh hack/run-migrations.bats +# Run with: bats hack/run-migrations.bats from the repo root. # ----------------------------------------------------------------------------- +load test_helper + FAKEBIN="$PWD/hack/testdata/run-migrations" RUNNER="$PWD/packages/core/platform/images/migrations/run-migrations.sh" LIB="$PWD/packages/core/platform/images/migrations/migrations/lib" diff --git a/hack/runner-identity.bats b/hack/runner-identity.bats index d69fbd6a8..acfa36e32 100644 --- a/hack/runner-identity.bats +++ b/hack/runner-identity.bats @@ -41,11 +41,13 @@ # - An @test block ends at the first closing brace in column zero, so helpers # and stub heredocs stay at top level. # -# Run with: hack/cozytest.sh hack/runner-identity.bats -# (or `bats hack/runner-identity.bats` if the bats binary is -# installed; cozytest.sh is the CI path.) +# Run from the repo root with `make bats-unit-tests`, or directly with +# `bats hack/runner-identity.bats`. The legacy `hack/cozytest.sh` runner remains +# a supported compatibility path. # ----------------------------------------------------------------------------- +load test_helper + HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" SCRIPT="$HACK_DIR/e2e-runner-identity.sh" REPO_ROOT="$(cd "$HACK_DIR/.." && pwd)" diff --git a/hack/sandbox-cidr-disjoint.bats b/hack/sandbox-cidr-disjoint.bats index fc245c764..44c1e0d14 100644 --- a/hack/sandbox-cidr-disjoint.bats +++ b/hack/sandbox-cidr-disjoint.bats @@ -39,15 +39,17 @@ # legal reformats and all of them turn the guard red with the offending value # in the message, which is the safe direction but not the same promise. # -# Harness note: the CI path is hack/cozytest.sh, NOT real bats. There is no -# `run`, `$status`, `$output`, `skip`, or setup()/teardown(); each test runs as -# a shell function under `set -eu -x`, so a non-zero exit is the failure. A -# `!`-negated command never trips errexit, so every assertion below is written -# as an explicit `if ... return 1`. A top-level helper function would have -# `return 0` injected before its closing brace by the runner's awk, which is -# why the arithmetic is inlined instead. Paths are repo-root-relative. +# Harness compatibility note: CI runs this file under Bats through +# `make bats-unit-tests`. It also remains compatible with the narrower legacy +# `hack/cozytest.sh` translator, so it avoids `run`, `$status`, `$output`, +# `skip`, and setup()/teardown(). A `!`-negated command never trips errexit, so +# every assertion below is written as an explicit `if ... return 1`. The legacy +# translator injects `return 0` before a top-level helper's closing brace, which +# is why the arithmetic is inlined instead. Paths stay repo-root-relative. # -# Run with: hack/cozytest.sh hack/sandbox-cidr-disjoint.bats +# Run with: bats hack/sandbox-cidr-disjoint.bats + +load test_helper @test "sandbox cluster CIDR overlaps no platform range" { talos_file=hack/e2e-prepare-cluster.bats diff --git a/hack/sandbox-runner-headroom.bats b/hack/sandbox-runner-headroom.bats index 37dc7fecb..ce1eec15b 100644 --- a/hack/sandbox-runner-headroom.bats +++ b/hack/sandbox-runner-headroom.bats @@ -99,19 +99,19 @@ # at least one matched job, which turns a half-broken parser red instead of # quietly narrowing what is checked. # -# Harness note: the CI path is hack/cozytest.sh, NOT real bats. There is no -# `run`, `$status`, `$output`, `skip`, or setup()/teardown(); each test runs as -# a shell function under `set -eu -x`, so a non-zero exit is the failure. A -# `!`-negated command never trips errexit, so every assertion below is written -# as an explicit `if ... return 1`. A top-level helper function would have -# `return 0` injected before its closing brace by the runner's awk, so nothing -# is factored into one. The same awk pass injects that line before any bare `}` -# at column 0 and reads any line starting with `@test ` as a block header, so -# the fixtures below are built with `printf` rather than heredocs: a heredoc -# holding either shape would be rewritten as if it were code. Paths are -# repo-root-relative. +# Harness compatibility note: CI runs this file under Bats through +# `make bats-unit-tests`. It also remains compatible with the narrower legacy +# `hack/cozytest.sh` translator, so it avoids `run`, `$status`, `$output`, +# `skip`, and setup()/teardown(). A `!`-negated command never trips errexit, so +# every assertion below is written as an explicit `if ... return 1`. The legacy +# translator injects `return 0` before a top-level helper's closing brace, so +# nothing is factored into one. It also rewrites bare `}` lines and lines that +# begin with `@test `, so the fixtures below use `printf` instead of heredocs. +# Paths stay repo-root-relative. # -# Run with: hack/cozytest.sh hack/sandbox-runner-headroom.bats +# Run with: bats hack/sandbox-runner-headroom.bats + +load test_helper # The guest-side derivation, held in one place so the check that runs against # the live tree and the fixtures that pin its refusals cannot drift apart. diff --git a/hack/seaweedfs-chart-seam.bats b/hack/seaweedfs-chart-seam.bats index 952daa5a5..188ab8539 100644 --- a/hack/seaweedfs-chart-seam.bats +++ b/hack/seaweedfs-chart-seam.bats @@ -15,7 +15,9 @@ # _namespace into it (packages/apps/tenant/templates/namespace.yaml). The live # merge, Secret included, is asserted by hack/e2e-chainsaw/seaweedfs/. # -# Run with: hack/cozytest.sh hack/seaweedfs-chart-seam.bats +# Run with: bats hack/seaweedfs-chart-seam.bats + +load test_helper REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" APP_CHART="$REPO_ROOT/packages/extra/seaweedfs" diff --git a/hack/seaweedfs-guard-parity.bats b/hack/seaweedfs-guard-parity.bats index ed65488fb..afa68a4e9 100644 --- a/hack/seaweedfs-guard-parity.bats +++ b/hack/seaweedfs-guard-parity.bats @@ -25,9 +25,11 @@ # release; extra/ is and derives the child), and that is the ONLY sanctioned # difference. Both are asserted separately below. # -# Run with: hack/cozytest.sh hack/seaweedfs-guard-parity.bats +# Run with: bats hack/seaweedfs-guard-parity.bats # ----------------------------------------------------------------------------- +load test_helper + SYS="$PWD/packages/system/seaweedfs/templates/naming-guard.yaml" EXTRA="$PWD/packages/extra/seaweedfs/templates/seaweedfs.yaml" diff --git a/hack/seaweedfs-naming-audit.bats b/hack/seaweedfs-naming-audit.bats index aedb34577..0efef7334 100644 --- a/hack/seaweedfs-naming-audit.bats +++ b/hack/seaweedfs-naming-audit.bats @@ -20,12 +20,16 @@ # Moving the classifier into a file with tests is the point. These drive it # against a fake kubectl, mocking only the cluster boundary. # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its own -# line; there is no bats `run`/`$status`/`setup`. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run`, `$status`, and setup() for that compatibility path. # -# Run with: hack/cozytest.sh hack/seaweedfs-naming-audit.bats +# Run with: bats hack/seaweedfs-naming-audit.bats # ----------------------------------------------------------------------------- +load test_helper + SEAWEEDFS_AUDIT_LIB=1 export SEAWEEDFS_AUDIT_LIB # shellcheck source=seaweedfs-naming-audit.sh diff --git a/hack/seaweedfs-naming-audit.sh b/hack/seaweedfs-naming-audit.sh index e848080ba..702c0864c 100755 --- a/hack/seaweedfs-naming-audit.sh +++ b/hack/seaweedfs-naming-audit.sh @@ -54,13 +54,15 @@ # reads identically to "nothing to do". # Nothing is ever mutated on either path -- every kubectl call is read-only. # -# POSIX sh, deliberately. hack/seaweedfs-naming-audit.bats sources this file, and -# cozytest.sh sources the converted test into its own /bin/sh — which is dash on -# the CI runner. A bash shebang would not save it: `.` runs the file's contents in -# the caller's shell and the shebang is just a comment. Keeping the script POSIX -# is what makes the tested shell and the executed shell the same one; anything -# bash-only here is untested in CI and unavailable at run time. `pipefail` in -# particular is not POSIX and dash 0.5.12 (Ubuntu) rejects it outright. +# POSIX sh, deliberately. hack/seaweedfs-naming-audit.bats sources the pure +# helpers under Bats, then drives the executable path repeatedly with explicit +# `sh hack/seaweedfs-naming-audit.sh` subprocesses. Those subprocess assertions, +# plus the narrow cozytest.sh compatibility pass in the root Makefile, keep the +# tested shell and the executed shell aligned; retain at least one explicit sh +# invocation if this suite is refactored. A bash shebang would not save sourced +# helpers: `.` runs the file's contents in the caller's shell and the shebang is +# just a comment. `pipefail` in particular is not POSIX and dash 0.5.12 (Ubuntu) +# rejects it outright. set -u # run_kubectl -- run ONE read-only kubectl query diff --git a/hack/select-e2e.sh b/hack/select-e2e.sh index c14782480..53588f9b3 100755 --- a/hack/select-e2e.sh +++ b/hack/select-e2e.sh @@ -132,6 +132,10 @@ full_suite_pattern='^(packages/library/|packages/core/|api/|cmd/|internal/|pkg/| # unit-tests` runs them, gated on pull-requests.yaml's `code` # output, which is "any path outside docs/" and is computed # in the workflow rather than from this script +# - hack/test_helper.bash shared setup for that same unit-only BATS lane. It +# is loaded by BATS_UNIT_FILES and by nothing in the e2e +# sandbox; enumerate it rather than making every future +# top-level *.bash file inert # - packages/tests/ helm-unittest fixture charts. cozy-lib-tests exercises # library/cozy-lib from the outside; changing a test OF # cozy-lib does not change cozy-lib, no PackageSource lists @@ -160,7 +164,7 @@ full_suite_pattern='^(packages/library/|packages/core/|api/|cmd/|internal/|pkg/| # .gitattributes NOT on this list is classified by whatever # rule its path falls under — the graph, or the unclassified # fall-through — both of which fail safe -inert_config_pattern='^(examples/|\.github/|\.claude/|\.gemini/|img/|hack/testdata/|packages/tests/|hack/[^/]+\.bats$|hack/boilerplate\.go\.txt$|hack/dcgm-default-counters\.csv$|LICENSE$|\.gitignore$|\.pre-commit-config\.yaml$|\.coderabbit\.yaml$|packages/system/\.gitattributes$|packages/system/(backup-controller|backupstrategy-controller)/definitions/\.gitattributes$)' +inert_config_pattern='^(examples/|\.github/|\.claude/|\.gemini/|img/|hack/testdata/|packages/tests/|hack/[^/]+\.bats$|hack/test_helper\.bash$|hack/boilerplate\.go\.txt$|hack/dcgm-default-counters\.csv$|LICENSE$|\.gitignore$|\.pre-commit-config\.yaml$|\.coderabbit\.yaml$|packages/system/\.gitattributes$|packages/system/(backup-controller|backupstrategy-controller)/definitions/\.gitattributes$)' # All known Chainsaw suites: every dir directly under hack/e2e-chainsaw/ in # which Chainsaw v0.2.15 would find a Test (pkg/discovery/load.go and step.go). diff --git a/hack/select-e2e_test.bats b/hack/select-e2e_test.bats index c883d091f..eece232c8 100644 --- a/hack/select-e2e_test.bats +++ b/hack/select-e2e_test.bats @@ -2,18 +2,20 @@ # ----------------------------------------------------------------------------- # Unit tests for hack/select-e2e.sh # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run` or `$status`. Each test runs as a shell -# function under `set -eu -x`, so assertions are direct shell tests that exit -# non-zero on failure. setup()/teardown() are not honored — each test creates -# and cleans its own scratch dir. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run`, `$status`, setup(), and teardown() for that compatibility path and use +# direct shell assertions with inline fixture cleanup. # # Test-level EXIT traps replace Bats' own handler and hide failing TAP results. # Cleanup follows aborting assertions; see docs/agents/e2e-testing.md. # -# Run with: hack/cozytest.sh hack/select-e2e_test.bats +# Run with: bats hack/select-e2e_test.bats # ----------------------------------------------------------------------------- +load test_helper + # Assert that a selection is the WHOLE suite list, not merely a long one. # # These asserts used to read `[ "$(echo "$output" | wc -w)" -gt 5 ]`, which is @@ -536,6 +538,24 @@ assert_full_suite() { rm -rf "$tmp" } +@test "the shared unit Bats helper selects nothing" { + # test_helper.bash is loaded only by BATS_UNIT_FILES. Enumerate this exact + # helper as inert rather than matching hack/*.bash: a future production + # helper must still hit the unclassified fail-safe until someone decides + # which e2e lane exercises it. + tmp=$(mktemp -d) + cp -r packages/core/platform/sources "$tmp/sources" + echo "hack/test_helper.bash" > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + [ -z "$output" ] + # Inert means this path contributes nothing; it must not mask a real app + # path beside it. + printf '%s\n' hack/test_helper.bash packages/apps/postgres/values.yaml > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + [ "$output" = "postgres" ] + rm -rf "$tmp" +} + @test "every hack/*.bats file lands on the lane its name says" { # The rule above is a claim about every hack/*.bats, asserted on two. This # pins the claim itself: for every hack/*.bats in the tree, the selector's diff --git a/hack/select-install_test.bats b/hack/select-install_test.bats index 9e6b3ade9..a2f6b90a1 100644 --- a/hack/select-install_test.bats +++ b/hack/select-install_test.bats @@ -2,21 +2,22 @@ # ----------------------------------------------------------------------------- # Unit tests for hack/select-install.sh # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run` or `$status`. Each test runs as a shell -# function under `set -eu -x`, so assertions are direct shell tests that exit -# non-zero on failure. setup()/teardown() are not honored, and cleanup goes on -# the last line of the body rather than in an EXIT trap -- a trap there replaces -# the bats binary's own, and a failing test then prints no TAP line at all. To -# assert a NON-zero exit, invert with `if` so `set -e` doesn't abort the test. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run`, `$status`, setup(), and teardown() for that compatibility path. Cleanup +# stays at the last reachable line rather than in an EXIT trap, which would +# replace Bats' bookkeeping trap and hide a failing TAP result. # # Tests that only READ the production sources call the script directly (it # defaults sources-dir to packages/core/platform/sources); only the tests that # build a synthetic graph or suite tree use a scratch dir. # -# Run with: hack/cozytest.sh hack/select-install_test.bats +# Run with: bats hack/select-install_test.bats # ----------------------------------------------------------------------------- +load test_helper + assert_contains_package() { printf '%s\n' "$1" | tr ' ' '\n' | grep -Fxq "$2" } diff --git a/hack/stitch-multiarch_test.bats b/hack/stitch-multiarch_test.bats index 435804cc2..730a89d64 100644 --- a/hack/stitch-multiarch_test.bats +++ b/hack/stitch-multiarch_test.bats @@ -7,10 +7,13 @@ # real sha256 sums of the stored bytes and an index the docker stub creates # resolves like any other manifest. # -# Harness note: run by hack/cozytest.sh, not real bats; each @test is a shell -# function under `set -eu`, sourced into POSIX sh (dash on CI). +# Harness note: the unit lane runs this file under Bats. The bodies stay POSIX +# sh (no `run`, no [[ ]]; each @test a shell function under `set -eu`), so +# hack/cozytest.sh can still run it under dash by hand. # -# Run with: hack/cozytest.sh hack/stitch-multiarch_test.bats +# Run with: bats hack/stitch-multiarch_test.bats + +load test_helper REG=registry.example/cozy diff --git a/hack/talos-image-cache_test.bats b/hack/talos-image-cache_test.bats index 19be3c623..e3ce75d92 100644 --- a/hack/talos-image-cache_test.bats +++ b/hack/talos-image-cache_test.bats @@ -41,15 +41,17 @@ # Root cause + fix are from cozystack/cozystack#3254 (@lexfrei); this is the port # to the Chainsaw layout (hack/e2e-chainsaw/_lib/). # -# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its -# own line; there is no bats `run` or `$status`, and setup()/teardown() are not -# honored. Each test runs under `set -eu -x`; assertions are direct shell tests -# that exit non-zero on failure. mikefarah yq prints `---` between matched -# documents, so document streams are compared with those separators stripped. +# CI runs this file under Bats through `make bats-unit-tests`. It also remains +# compatible with the legacy `hack/cozytest.sh` translator, whose awk parser +# recognizes only @test blocks and a bare `}` on its own line. The tests avoid +# `run`, `$status`, setup(), and teardown() for that compatibility path and use +# direct shell assertions. mikefarah yq's document separators are stripped. # -# Run with: hack/cozytest.sh hack/talos-image-cache_test.bats +# Run with: bats hack/talos-image-cache_test.bats # ----------------------------------------------------------------------------- +load test_helper + @test "manifest documents partition into pre-Cilium apply plus the Cilium policy" { manifest=hack/e2e-talos-image-cache.yaml total=$(yq '.kind' "$manifest" | grep -vc '^---$') diff --git a/hack/talos-installer-index_test.bats b/hack/talos-installer-index_test.bats index 06eae410d..6bf7d63fb 100644 --- a/hack/talos-installer-index_test.bats +++ b/hack/talos-installer-index_test.bats @@ -4,10 +4,10 @@ # pushed as. A wrong digest or size makes that push fail; a wrong or missing # platform makes a node pull the other arch's installer. # -# Written for POSIX sh: hack/cozytest.sh sources this file into /bin/sh. -# # Requires: jq, sha256sum. +load test_helper + @test "installer-index.sh indexes each arch's manifest under its platform" { layout="$(mktemp -d)" mkdir -p "$layout/blobs/sha256" diff --git a/hack/talos-reconcile-heredoc_test.bats b/hack/talos-reconcile-heredoc_test.bats index d094f42fe..6fd19e850 100644 --- a/hack/talos-reconcile-heredoc_test.bats +++ b/hack/talos-reconcile-heredoc_test.bats @@ -37,10 +37,12 @@ # # A field that is in neither arm is the regression this file exists to catch. # -# Needs `helm` + `yq`; cozytest.sh runs from the repo root. -# Run with: hack/cozytest.sh hack/talos-reconcile-heredoc_test.bats +# Needs `helm` + `yq` and must run from the repo root. +# Run with: bats hack/talos-reconcile-heredoc_test.bats # ----------------------------------------------------------------------------- +load test_helper + @test "kubernetes-nodes refuses hostile Talos image coordinates at render" { work=$(mktemp -d) cat > "$work/vals.yaml" <<'VALS' diff --git a/hack/tenant-pre-delete-hook.bats b/hack/tenant-pre-delete-hook.bats index e926053fc..b142e3904 100644 --- a/hack/tenant-pre-delete-hook.bats +++ b/hack/tenant-pre-delete-hook.bats @@ -11,13 +11,16 @@ # delete which gave up. Those are the two halves below: one test reads the # arguments, the rest inject the non-zero exit an expired delete produces. # -# Run via hack/cozytest.sh from the repo root (make bats-unit-tests); relative -# paths resolve against that cwd. The runner has no setup/teardown, so each -# @test builds its own fixture and removes it at the end of the body. No EXIT +# Run from the repo root through `make bats-unit-tests`; relative paths resolve +# against that cwd. The file also remains compatible with the legacy +# `hack/cozytest.sh` translator, so each @test builds and removes its fixture +# inline instead of relying on setup()/teardown(). No EXIT # traps: a test that dies inside one prints neither `not ok` nor a reason, and # the suite reads green. A failing test therefore leaves its temp directory # behind, which is what you want when reading the rendered script afterwards. +load test_helper + CHART=packages/apps/tenant # The two waves, told apart by their label selector. `*tenantmodule=true*` does diff --git a/hack/test_helper.bash b/hack/test_helper.bash new file mode 100644 index 000000000..f3944c9c0 --- /dev/null +++ b/hack/test_helper.bash @@ -0,0 +1,11 @@ +# Bats supplies errexit but not nounset. Enable nounset in each test process +# without exporting shell options to production scripts launched by the tests. +# A file-local setup() must call strict_setup first; bats-strict-setup.bats +# audits that contract against the Makefile's discovered unit files. +strict_setup() { + set -u +} + +setup() { + strict_setup +} diff --git a/hack/tracing-central-credentials.bats b/hack/tracing-central-credentials.bats index a3602a9af..63b0bfebb 100644 --- a/hack/tracing-central-credentials.bats +++ b/hack/tracing-central-credentials.bats @@ -6,7 +6,9 @@ # nothing. helm-unittest asserts one template at a time and cannot compare them, # so this renders both together and compares. # -# Run via hack/cozytest.sh from the repo root (make bats-unit-tests). +# Run from the repo root: bats hack/tracing-central-credentials.bats. + +load test_helper MONITORING_CHART=packages/system/monitoring diff --git a/hack/unit-test-parallelism.bats b/hack/unit-test-parallelism.bats index 08c615c60..c7cdf0d52 100644 --- a/hack/unit-test-parallelism.bats +++ b/hack/unit-test-parallelism.bats @@ -1,34 +1,30 @@ #!/usr/bin/env bats -# Contract for the bounded parallel unit-test fan-out used by PR CI. +# Bats owns per-test parallelism; Make also schedules the non-Bats targets. -@test "every non-E2E BATS file is an independent Make target" { - # --output-sync arrived in GNU Make 4.0, and macOS still ships 3.81, where - # this invocation dies on an unknown option rather than telling the reader - # why. CI is on 4.x, so the contract is still enforced where it gates a - # merge; skipping here keeps the suite runnable on a contributor's machine - # instead of failing for a reason that has nothing to do with the contract. - make_major=$(make --version 2>/dev/null | sed -n '1s/^GNU Make \([0-9][0-9]*\).*/\1/p') - if [ -z "$make_major" ] || [ "$make_major" -lt 4 ]; then - echo "skipped: this make has no --output-sync (needs GNU Make >= 4.0)" >&2 - return 0 - fi - expected=$(find hack -maxdepth 1 -type f -name '*.bats' ! -name 'e2e-*.bats' | wc -l) - output=$(make -n -j4 --output-sync=target bats-unit-tests) - # The COZYTEST_TRACE prefix is part of the pattern on purpose: the switch is - # per-target now that the serial loop is gone, and a target that loses it - # would go back to streaming full xtrace for every suite. - actual=$(printf '%s\n' "$output" | grep -c '^COZYTEST_TRACE=[^ ]* hack/cozytest.sh "hack/.*\.bats"$') - [ "$actual" -eq "$expected" ] || { - echo "make scheduled $actual BATS files independently, expected $expected" >&2 - exit 1 +load test_helper + +@test "the Bats invocation receives every discovered unit file exactly once" { + tmp=$(mktemp -d) + mkdir "$tmp/bin" + printf '%s\n' '#!/bin/sh' 'printf "%s\n" "$@" >> "$BATS_ARG_LOG"' > "$tmp/bin/bats" + chmod +x "$tmp/bin/bats" + expected=$(MAKEFLAGS= MAKELEVEL= make --no-print-directory -s print-bats-unit-files) + [ -n "$expected" ] + MAKEFLAGS= MAKELEVEL= PATH="$tmp/bin:$PATH" BATS_ARG_LOG="$tmp/args" \ + make --no-print-directory -s BATS_JOBS=4 "BATS_REPORT_DIR=$tmp/reports" bats-unit-tests + actual=$(grep '^hack/.*\.bats$' "$tmp/args") + [ "$actual" = "$expected" ] || { + echo "Bats did not receive the discovered unit files exactly once" >&2 + cat "$tmp/args" >&2 + false } - if printf '%s\n' "$output" | grep -q 'for f in'; then - echo "bats-unit-tests still serializes the suite in a shell loop" >&2 - exit 1 - fi + [ "$(sed -n '1,2p' "$tmp/args")" = "$(printf '%s\n' -j 4)" ] + rm -rf "$tmp" } -@test "PR workflow shares four slots across unit and controller targets" { +@test "PR workflow schedules unit and controller targets with four Make jobs and -k" { + # -k is what lets test-controllers start when a unit-tests prerequisite + # fails first; without it a red Bats file hides the Go controller suite. grep -qF 'run: make unit-tests test-controllers -j4 -k --output-sync=target' \ .github/workflows/pull-requests.yaml || { echo "PR checks do not use the bounded four-slot make invocation" >&2 diff --git a/hack/verify-multiarch_test.bats b/hack/verify-multiarch_test.bats index da641dbc5..655dab8f5 100644 --- a/hack/verify-multiarch_test.bats +++ b/hack/verify-multiarch_test.bats @@ -5,10 +5,13 @@ # The registry is a stub: skopeo on PATH answers `inspect --raw` from one file # per digest, so each test decides what every ref resolves to. # -# Harness note: run by hack/cozytest.sh, not real bats; each @test is a shell -# function under `set -eu`, sourced into POSIX sh (dash on CI). +# Harness note: the unit lane runs this file under Bats. The bodies stay POSIX +# sh (no `run`, no [[ ]]; each @test a shell function under `set -eu`), so +# hack/cozytest.sh can still run it under dash by hand. # -# Run with: hack/cozytest.sh hack/verify-multiarch_test.bats +# Run with: bats hack/verify-multiarch_test.bats + +load test_helper _stub_registry() { MOCK_REG="$1" diff --git a/hack/verify-promoted-packages_test.bats b/hack/verify-promoted-packages_test.bats index 1f6bad367..582fc5e6c 100644 --- a/hack/verify-promoted-packages_test.bats +++ b/hack/verify-promoted-packages_test.bats @@ -1,6 +1,8 @@ #!/usr/bin/env bats # Behavioural tests for the pre-publication packages artifact verification. -# Run with: hack/cozytest.sh hack/verify-promoted-packages_test.bats +# Run with: bats hack/verify-promoted-packages_test.bats + +load test_helper _test_workspace() { if [ -n "${BATS_TEST_TMPDIR:-}" ]; then