#!/usr/bin/env bats # Behavioural tests for the pre-delete cleanup hook of packages/apps/tenant: the # script is extracted from the rendered Job and executed against a stub kubectl. # The helm-unittest cases in packages/apps/tenant/tests/cleanup_role_watch_test.yaml # assert the Job's shape — the deadline and the attempt count — which is what # the script cannot observe about itself. # # What a stub can and cannot show. `--timeout` is implemented inside kubectl, so # no stub can make that deadline actually elapse; what the stub can do is record # whether the flag reached kubectl at all, and answer for the run that follows a # delete which gave up. Those are the two halves below: one test reads the # arguments, the rest inject the non-zero exit an expired delete produces. # # Run from the repo root through `make bats-unit-tests`; relative paths resolve # against that cwd. The file also remains compatible with the legacy # `hack/cozytest.sh` translator, so each @test builds and removes its fixture # inline instead of relying on setup()/teardown(). No EXIT # traps: a test that dies inside one prints neither `not ok` nor a reason, and # the suite reads green. A failing test therefore leaves its temp directory # behind, which is what you want when reading the rendered script afterwards. load test_helper CHART=packages/apps/tenant # The two waves, told apart by their label selector. `*tenantmodule=true*` does # not match the applications wave: that one spells the selector # `tenantmodule!=true`, which contains no `tenantmodule=true` substring. WAVE_APPS='*tenantmodule!=true*' WAVE_MODULES='*tenantmodule=true*' # Render the pre-delete Job into $1. The release name has to start with # `tenant-` and carry no second dash (templates/tenant.yaml enforces it), and # `_cluster` normally arrives from the cozystack-values Secret, so the one key # templates/keycloakgroups.yaml indexes is supplied here as a string. # # The emptiness check must be an early `return 1`: hack/cozytest.sh rewrites # every line matching ^}$ into `return 0` plus `}`, helpers included, so a check # left in last position has its status discarded. helm's stderr is kept rather # than dropped, because it carries the reason a render failed; only the # per-render notice about the charts/cozy-lib symlink is filtered out. render_job() { helm template tenant-test "$CHART" \ --namespace tenant-root \ --set-string '_cluster.oidc-enabled=false' \ --show-only templates/cleanup-job.yaml 2>"$1.err" | yq 'select(.kind == "Job")' - > "$1" if [ ! -s "$1" ]; then echo "FAIL: the chart rendered no pre-delete Job" >&2 grep -v 'found symbolic link in path' "$1.err" >&2 || true return 1 fi } # A stub kubectl in $1. Every invocation is logged; those matching a glob in # $KUBECTL_FAIL exit non-zero with the wording kubectl itself prints when a # `--wait` deadline expires, which is the only outcome the script can tell apart # from success. # # The closing brace of the helper inside the heredoc is indented on purpose: # cozytest.sh rewrites every line matching ^}$ anywhere in this file, the # heredoc included, and a `return 0` spliced into the stub would make every # injected failure disappear. make_kubectl() { mkdir -p "$1" cat > "$1/kubectl" <<'STUB' #!/bin/sh args="$*" echo "$args" >> "${KUBECTL_LOG:-/dev/null}" matches() { [ -n "$1" ] || return 1 oldifs=$IFS IFS=' ' for pat in $1; do case "$args" in $pat) IFS=$oldifs; return 0 ;; esac done IFS=$oldifs return 1 } if matches "${KUBECTL_FAIL:-}"; then echo "error: timed out waiting for the condition on helmreleases.helm.toolkit.fluxcd.io" >&2 exit 1 fi exit 0 STUB chmod 0755 "$1/kubectl" } # Fixture: $tmp holds the rendered Job, its script, and the stub kubectl. setup_case() { tmp=$(mktemp -d) || return 1 render_job "$tmp/job.yaml" || return 1 yq '.spec.template.spec.containers[0].command[2]' "$tmp/job.yaml" > "$tmp/s.sh" || return 1 [ -s "$tmp/s.sh" ] || return 1 make_kubectl "$tmp/bin" || return 1 } # Run the hook with the failures injected by $1, leaving merged output in # $tmp/out, the invocations in $tmp/kubectl.log and the exit status in $rc. The # status must not abort the test: it is itself one of the things under test. # KUBECONFIG is neutered so a stub that stopped being found on PATH cannot reach # a real cluster. run_hook() { rc=0 : > "$tmp/kubectl.log" KUBECONFIG=/dev/null \ KUBECTL_LOG="$tmp/kubectl.log" \ KUBECTL_FAIL="$1" \ PATH="$tmp/bin:$PATH" sh "$tmp/s.sh" > "$tmp/out" 2>&1 || rc=$? # The stub has to have been reached, or the assertions below measured # something other than this script driving kubectl. [ -s "$tmp/kubectl.log" ] || return 1 } @test "a run where both waves completed reports success" { setup_case run_hook "" [ "$rc" = 0 ] grep -qF 'Cleanup completed successfully' "$tmp/out" # Both waves have to have run, or the success claim is about a script that # deleted nothing and the tests below have no baseline to differ from. grep -q 'tenantmodule!=true' "$tmp/kubectl.log" grep -q 'tenantmodule=true' "$tmp/kubectl.log" rm -rf "$tmp" } @test "every delete reaches kubectl with a wall-clock bound" { # The defect this pins. kubectl reads the default `--timeout=0` as wait # forever and substitutes a week, so a delete that arrives without the flag # hangs the hook for as long as any child HelmRelease cannot finalize — and # helm returns from the uninstall before removing anything, leaving the # release in storage to be retried. Read from the arguments the stub actually # received rather than from the template text, so a flag that stops being # passed cannot pass here by still being written somewhere in the file. setup_case run_hook "" deletes=$(grep -c 'delete helmreleases' "$tmp/kubectl.log" || true) [ "$deletes" = 2 ] # A bound of zero is the unbounded case spelled explicitly, so the value has # to start with a non-zero digit rather than merely be present. bounded=$(grep 'delete helmreleases' "$tmp/kubectl.log" | grep -c -- '--timeout=[1-9][0-9]*s' || true) if [ "$bounded" != "$deletes" ]; then echo "FAIL: $bounded of $deletes deletes carried a non-zero --timeout" cat "$tmp/kubectl.log" return 1 fi rm -rf "$tmp" } @test "the waits, the Job deadline and the uninstall budget stay in order" { # Four numbers, one ordering, and both ends of it are load-bearing. # # Waves under the deadline: activeDeadlineSeconds is the outer stop, and only # a backstop while the waits inside it can finish first. Raise a wave past the # remainder and the deadline starts cutting runs that were going to succeed. # # Deadline plus termination under the uninstall budget: flux takes that budget # from spec.uninstall.timeout / spec.timeout, and cozystack-api generates the # tenant HelmRelease with neither, so it is helm-controller's 300s default. The # deadline is measured from the Job's startTime and helm's budget from just # before it creates the Job, so at 300 helm gives up first — on a Job that is # still running, which is the state this whole change exists to end. # # The termination period belongs on the same side of that comparison as the # deadline. Since 1.31 the Job controller withholds the terminal condition # until every pod has finished terminating, so what helm waits for is the sum # rather than the deadline alone — and the sum is what is actually spent here, # because PID 1 is a shell running a multi-command script: it never execs into # kubectl, and a default disposition sent to PID 1 is discarded by the kernel # with no handler installed to catch it. Comparing the deadline alone would # call a Job that reports its failure at 300 correctly bounded at 270. UNINSTALL_BUDGET=300 setup_case waits=0 for seconds in $(grep -o -- '--timeout=[0-9]*s' "$tmp/s.sh" | sed -e 's|--timeout=||' -e 's|s$||'); do waits=$((waits + seconds)) done # Non-zero, or the comparison below is satisfied by a script with no waits at # all to measure. [ "$waits" -gt 0 ] deadline=$(yq '.spec.activeDeadlineSeconds' "$tmp/job.yaml") # Checked before it is compared. Both comparisons below sit inside `if`, where # a status is a verdict rather than an error, so `[ 180 -ge null ]` reads as # "false" and a Job with no deadline at all passes every ordering check here. case "$deadline" in '' | *[!0-9]*) echo "FAIL: the Job carries no numeric activeDeadlineSeconds (read: '$deadline')" echo " and then nothing bounds a run whose kubectl call never returns" return 1 ;; esac if [ "$waits" -ge "$deadline" ]; then echo "FAIL: ${waits}s of bounded waits against an activeDeadlineSeconds of ${deadline}s" echo " leaves nothing for scheduling, the image pull and the delete calls" return 1 fi grace=$(yq '.spec.template.spec.terminationGracePeriodSeconds' "$tmp/job.yaml") # Read before it is added, for the same reason the deadline is: `null` here # would make the sum below a string comparison away from meaning nothing, and # an absent period is the 30s default rather than zero — the case that puts # the failure at exactly the budget. case "$grace" in '' | *[!0-9]*) echo "FAIL: the pod carries no numeric terminationGracePeriodSeconds (read: '$grace')" echo " so it defaults to 30s and the Job reports its failure at ${deadline}s+30s" return 1 ;; esac if [ "$((deadline + grace))" -ge "$UNINSTALL_BUDGET" ]; then echo "FAIL: an activeDeadlineSeconds of ${deadline}s plus a termination period of" echo " ${grace}s reaches ${UNINSTALL_BUDGET}s of uninstall budget, so helm gives up" echo " before the Job it is waiting on reports the failure" return 1 fi rm -rf "$tmp" } @test "an applications wave that did not finish ends the run" { # Nothing stands behind these deletes, so a wave that gave up must not be # reported as done. The closing line is unreachable under `set -e`, and it has # to stay that way: a `|| echo` appended here would reinstate the exact defect # this file exists to stop — a cleanup that did not happen, claimed as one # that did, after which helm goes on to tear the tenant namespace down over # HelmReleases that are still there. setup_case run_hook "$WAVE_APPS" if [ "$rc" = 0 ]; then echo "FAIL: a wave that could not finish left the hook reporting success" cat "$tmp/out" return 1 fi if grep -qF 'Cleanup completed successfully' "$tmp/out"; then echo "FAIL: the hook claimed success with applications still undeleted" cat "$tmp/out" return 1 fi rm -rf "$tmp" } @test "an applications wave that did not finish does not go on to the modules" { # The ordering the two waves exist for. Tenant modules — etcd, monitoring, # ingress, seaweedfs, the gateway — are what the applications above run on, # and deleting them out from under applications that are still terminating # makes the namespace harder to remove, not easier. setup_case run_hook "$WAVE_APPS" # The first wave has to have been attempted, or the absence below is the # absence of a script that ran nothing. grep -q 'tenantmodule!=true' "$tmp/kubectl.log" if grep -q 'tenantmodule=true' "$tmp/kubectl.log"; then echo "FAIL: the tenant modules were deleted after the applications wave gave up" cat "$tmp/kubectl.log" return 1 fi rm -rf "$tmp" } @test "a tenant modules wave that did not finish ends the run" { # The second wave is the last thing the script does, so a guard appended to it # would be invisible to every assertion about what runs afterwards. Only the # exit status and the closing line separate a swallowed failure from a real # completion here. setup_case run_hook "$WAVE_MODULES" # Both waves ran: the failure being injected is the second one, not the first. grep -q 'tenantmodule!=true' "$tmp/kubectl.log" grep -q 'tenantmodule=true' "$tmp/kubectl.log" if [ "$rc" = 0 ]; then echo "FAIL: a tenant modules wave that could not finish left the hook reporting success" cat "$tmp/out" return 1 fi if grep -qF 'Cleanup completed successfully' "$tmp/out"; then echo "FAIL: the hook claimed success with tenant modules still undeleted" cat "$tmp/out" return 1 fi rm -rf "$tmp" }