cozystack/hack/kubernetes-pre-delete-hook.bats

#!/usr/bin/env bats
# Behavioural tests for the pre-delete cleanup hook of packages/apps/kubernetes:
# the script is extracted from the rendered Job and executed against a stub
# kubectl. The helm-unittest cases in
# packages/apps/kubernetes/tests/delete_hook_test.yaml match its source TEXT,
# which cannot say whether the closing message is reachable once a backstop has
# swallowed a failure.
#
# Run from the repo root through `make bats-unit-tests`; relative paths resolve
# against that cwd. The file also remains compatible with the legacy
# `hack/cozytest.sh` translator, so each @test builds and removes its fixture
# inline instead of relying on setup()/teardown(). No EXIT
# traps: a test that dies inside one prints neither `not ok` nor a reason, and
# the suite reads green. The removal is therefore reached only on a passing run;
# a failing one leaves its temp directory behind, which is what you want when
# reading the rendered script and the captured output afterwards.

load test_helper

CHART=packages/apps/kubernetes

# The stub is driven by two independent lists of globs, matched against the
# joined arguments, and keeping them separate is the point: the script under
# test distinguishes "this object is not there" from "I could not ask", so a
# fixture that spelled both as one failing call could not tell whether an
# assertion passed for the right reason.
#
#   $KUBECTL_FAIL    -- these invocations exit non-zero: the apiserver refused,
#                       timed out, or was unreachable.
#   $KUBECTL_PRESENT -- these `get ... -o name` invocations print an object.
#                       Anything not listed exits 0 with no output, which is
#                       what `--ignore-not-found` does for an absent object.

# The ordinary end state after Step 4 waited: the TenantControlPlane is gone and
# the datastore Secret is still there for Step 4b to clear.
PRESENT_SECRET_ONLY='*get secret*'

# Step 4's delete timed out with the TenantControlPlane still alive, so Step 4b
# must decline to touch a finalizer Kamaji may still own.
FAIL_TCP_DELETE='*delete tenantcontrolplanes*'
PRESENT_TCP_AND_SECRET='*get tenantcontrolplanes*
*get secret*'

# The Secret is there, and the patch that would clear its finalizer is rejected.
FAIL_SECRET_PATCH='*patch secret*'

# Step 2's bounded delete times out, so the run takes the force-clear route --
# and every force-clear then succeeds. A changed route is not residue: nothing
# survives this run, so it must still be reported as a clean one.
FAIL_HR_DELETE='*delete helmreleases*'

# ... and the re-list driving that force-clear fails too. The loop body never
# runs, so no patch is attempted and no per-object backstop can fire.
FAIL_HR_DELETE_AND_LIST='*delete helmreleases*
*get helmreleases*'

# Each of Step 4b's two probes, failing on its own. Neither says the object is
# absent -- they say the script could not find out, which is a third state.
FAIL_TCP_PROBE='*get tenantcontrolplanes*'
FAIL_SECRET_PROBE='*get secret*'

# Every failure at once. The two HelmRelease globs are distinguished by the
# patch body: Step 1 suspends (no "finalizers" in its payload) and must keep
# working, because it is not backstopped and `set -e` would end the run there.
FAIL_ALL='*delete helmreleases*
*patch helmrelease*finalizers*
*delete tenantcontrolplanes*
*get tenantcontrolplanes*
*patch secret*'

# Extract the pre-delete Job's script from the rendered chart into $1. The
# non-empty check must be an early `return 1`: hack/cozytest.sh rewrites every
# line matching ^}$ into `return 0` plus `}`, helpers included, so a check left
# in last position has its status discarded.
#
# helm's stderr is kept rather than discarded, because it carries the reason a
# render failed and without it this returns 1 with nothing to read. Only the
# per-render notice about the charts/cozy-lib symlink is dropped, and only when
# there is a failure to report.
render_script() {
  helm template test-k8s "$CHART" \
    --namespace tenant-test \
    --values "$CHART/tests/values-ci.yaml" \
    --show-only templates/delete.yaml 2>"$1.err" |
    yq 'select(.kind == "Job") | .spec.template.spec.containers[0].command[2]' - > "$1"
  if [ ! -s "$1" ]; then
    echo "FAIL: the chart rendered no pre-delete Job script" >&2
    grep -v 'found symbolic link in path' "$1.err" >&2 || true
    return 1
  fi
}

# A stub kubectl for $1/kubectl. `get ... -o name` answers for the three object
# kinds the script probes; the HelmRelease list returns TWO children so that a
# per-object failure in the force-clear loop hits the same backstop label twice
# and the once-per-step property has something to be true about.
#
# The closing brace of the helper inside the heredoc is indented on purpose:
# hack/cozytest.sh rewrites every line matching ^}$ anywhere in this file, the
# heredoc included, and a `return 0` spliced into the stub would make every
# injected failure disappear.
make_kubectl() {
  mkdir -p "$1"
  cat > "$1/kubectl" <<'STUB'
#!/bin/sh
args="$*"
echo "$args" >> "${KUBECTL_LOG:-/dev/null}"

matches() {
  [ -n "$1" ] || return 1
  oldifs=$IFS
  IFS='
'
  for pat in $1; do
    case "$args" in
      $pat) IFS=$oldifs; return 0 ;;
    esac
  done
  IFS=$oldifs
  return 1
  }

if matches "${KUBECTL_FAIL:-}"; then
  echo "stub kubectl: injected failure: $args" >&2
  exit 1
fi

case "$args" in
  *"get helmreleases"*"-o name"*)
    echo "helmrelease.helm.toolkit.fluxcd.io/cilium"
    echo "helmrelease.helm.toolkit.fluxcd.io/coredns"
    ;;
  *"get tenantcontrolplanes"*"-o name"*)
    matches "${KUBECTL_PRESENT:-}" && echo "tenantcontrolplane.kamaji.clastix.io/test-k8s"
    ;;
  *"get secret"*"-o name"*)
    matches "${KUBECTL_PRESENT:-}" && echo "secret/test-k8s-datastore-config"
    ;;
esac
exit 0
STUB
  chmod 0755 "$1/kubectl"
}

# Fixture: $tmp holds the rendered script and the stub kubectl.
setup_case() {
  tmp=$(mktemp -d) || return 1
  render_script "$tmp/s.sh" || return 1
  make_kubectl "$tmp/bin" || return 1
}

# Run the hook with the injected failures in $1 and the existing objects in $2,
# leaving merged output in $tmp/out and the exit status in $rc. The status must
# not abort the test: it is itself one of the things under test. KUBECONFIG is
# neutered so that a stub that stopped being found on PATH cannot reach a real
# cluster.
run_hook() {
  rc=0
  # Truncated per call, not appended to: the "stub was reached" check below is
  # vacuous on a second run against a log the first run already filled.
  : > "$tmp/kubectl.log"
  KUBECONFIG=/dev/null \
  KUBECTL_LOG="$tmp/kubectl.log" \
  KUBECTL_FAIL="$1" \
  KUBECTL_PRESENT="$2" \
  PATH="$tmp/bin:$PATH" sh "$tmp/s.sh" > "$tmp/out" 2>&1 || rc=$?
  # The stub has to have been reached, or the assertions below measured
  # something other than this script driving kubectl.
  [ -s "$tmp/kubectl.log" ] || return 1
}

@test "claims success only when nothing was left behind" {
  setup_case
  run_hook "" "$PRESENT_SECRET_ONLY"

  [ "$rc" = 0 ]
  grep -qF 'Cleanup completed successfully' "$tmp/out"
  # The clean run must exercise the Step 4b patch rather than skip it, or the
  # success claim is about a script that did almost nothing.
  grep -qF 'Removing finalizers from secret/test-k8s-datastore-config' "$tmp/out"
  if grep -qF 'residue left by' "$tmp/out"; then
    echo "FAIL: a clean run reported residue"
    cat "$tmp/out"
    return 1
  fi
  if grep -qF 'WARNING' "$tmp/out"; then
    echo "FAIL: a clean run emitted a warning"
    cat "$tmp/out"
    return 1
  fi

  rm -rf "$tmp"
}

@test "a rejected finalizer patch withdraws the success claim and names the step" {
  setup_case
  run_hook "$FAIL_SECRET_PATCH" "$PRESENT_SECRET_ONLY"

  [ "$rc" = 0 ]
  # The defect this pins: the closing line used to assert success even here.
  if grep -qF 'Cleanup completed successfully' "$tmp/out"; then
    echo "FAIL: cleanup claimed success while the datastore-secret finalizer was left in place"
    cat "$tmp/out"
    return 1
  fi
  grep -qF 'Cleanup incomplete; residue left by: step 4b (datastore-secret finalizer)' "$tmp/out"
  # And only that step: a summary that names every backstop unconditionally
  # tells an operator no more than the old unconditional success line did.
  if grep -qF 'step 2 (' "$tmp/out"; then
    echo "FAIL: a step 2 backstop was reported without firing"
    cat "$tmp/out"
    return 1
  fi

  rm -rf "$tmp"
}

@test "a fallback route that succeeded is reported as a clean run" {
  # Step 2's bounded delete times out and every force-clear that follows works,
  # which is the ordinary outcome for a tenant with no working nodes -- the case
  # the wait is bounded for in the first place. Nothing survives the run, so
  # calling it incomplete would send an operator hunting for objects that are
  # not there: the same class of untrue closing line this file exists to stop,
  # only pointing the other way.
  setup_case
  run_hook "$FAIL_HR_DELETE" "$PRESENT_SECRET_ONLY"

  [ "$rc" = 0 ]
  # The route really was taken, or this passes for a run that never fell back.
  grep -qF 'Force-clearing finalizers on helmrelease.helm.toolkit.fluxcd.io/cilium' "$tmp/out"
  if grep -qF 'residue left by' "$tmp/out"; then
    echo "FAIL: a fallback that left nothing behind was reported as incomplete"
    cat "$tmp/out"
    return 1
  fi
  grep -qF 'Cleanup completed successfully' "$tmp/out"

  rm -rf "$tmp"
}

@test "an enumeration that failed is reported, not taken for an empty one" {
  setup_case
  run_hook "$FAIL_HR_DELETE_AND_LIST" "$PRESENT_SECRET_ONLY"

  [ "$rc" = 0 ]
  if grep -qF 'Cleanup completed successfully' "$tmp/out"; then
    echo "FAIL: cleanup claimed success after it could not list what it had to force-clear"
    cat "$tmp/out"
    return 1
  fi
  grep -qF 'Cleanup incomplete; residue left by: step 2 (child HelmRelease enumeration)' "$tmp/out"
  # And not the per-object label: that one means a patch was attempted and
  # rejected. Here the loop never ran, so claiming it would invent a failure.
  if grep -qF 'step 2 (child HelmRelease finalizers)' "$tmp/out"; then
    echo "FAIL: a per-object patch failure was reported although the loop never ran"
    cat "$tmp/out"
    return 1
  fi

  rm -rf "$tmp"
}

@test "a TenantControlPlane outliving Step 4 is reported even though no command failed" {
  # Step 4 times out with the TCP still alive, so Step 4b declines to strip a
  # finalizer Kamaji may still own. Every kubectl call in Step 4b succeeds, so
  # nothing is caught by a `||` -- and the namespace is still wedged. This is
  # the one residue the script reports from a branch rather than from a failure.
  setup_case
  run_hook "$FAIL_TCP_DELETE" "$PRESENT_TCP_AND_SECRET"

  [ "$rc" = 0 ]
  if grep -qF 'Cleanup completed successfully' "$tmp/out"; then
    echo "FAIL: cleanup claimed success with the datastore-secret finalizer left in place"
    cat "$tmp/out"
    return 1
  fi
  # The label names the TenantControlPlane, not the Secret. Both branches of
  # Step 4b leave the finalizer in place, but only the other one wants it
  # cleared by hand; here it is Kamaji's, and a summary pointing at the Secret
  # would send the reader to strip a finalizer its owner is still using.
  grep -qF 'Cleanup incomplete; residue left by: step 4b (TenantControlPlane still deleting)' "$tmp/out"
  if grep -qF 'step 4b (datastore-secret finalizer)' "$tmp/out"; then
    echo "FAIL: the summary blamed the Secret for a TenantControlPlane that is still deleting"
    cat "$tmp/out"
    return 1
  fi
  grep -qF 'is still present, so Kamaji may still own' "$tmp/out"

  rm -rf "$tmp"
}

@test "a TenantControlPlane probe that failed is not read as absent" {
  # The probe is the gate that stops the finalizer being stripped from an owner
  # that still exists, so reading its failure as "confirmed gone" defeats it in
  # the one direction that does damage. The script must neither strip nor claim
  # the run was clean.
  setup_case
  run_hook "$FAIL_TCP_PROBE" ""

  [ "$rc" = 0 ]
  if grep -qF 'Cleanup completed successfully' "$tmp/out"; then
    echo "FAIL: cleanup claimed success without establishing whether the TenantControlPlane was gone"
    cat "$tmp/out"
    return 1
  fi
  grep -qF 'Cleanup incomplete; residue left by: step 4b (datastore-secret state unknown)' "$tmp/out"
  # And it must NOT have gone on to patch: that is the damage the gate prevents.
  if grep -qF 'Removing finalizers from secret/' "$tmp/out"; then
    echo "FAIL: the finalizer was stripped although the TenantControlPlane state was unknown"
    cat "$tmp/out"
    return 1
  fi

  rm -rf "$tmp"
}

@test "a Secret probe that failed is not read as absent" {
  # The TCP is confirmed gone, so Step 4b is entitled to act -- but it cannot
  # find out whether the Secret is there. "not present; nothing to do" would be
  # an assertion the script never checked.
  setup_case
  run_hook "$FAIL_SECRET_PROBE" ""

  [ "$rc" = 0 ]
  if grep -qF 'Cleanup completed successfully' "$tmp/out"; then
    echo "FAIL: cleanup claimed success without establishing whether the Secret was still there"
    cat "$tmp/out"
    return 1
  fi
  if grep -qF 'not present; nothing to do' "$tmp/out"; then
    echo "FAIL: a failed probe was reported as the Secret being absent"
    cat "$tmp/out"
    return 1
  fi
  grep -qF 'Cleanup incomplete; residue left by: step 4b (datastore-secret state unknown)' "$tmp/out"

  rm -rf "$tmp"
}

@test "every step that left residue is named in the closing message" {
  setup_case
  run_hook "$FAIL_ALL" ""

  [ "$rc" = 0 ]
  if grep -qF 'Cleanup completed successfully' "$tmp/out"; then
    echo "FAIL: cleanup claimed success with every remedy having failed"
    cat "$tmp/out"
    return 1
  fi
  # Read the summary as one line: asserting the labels against the whole output
  # would pass on the individual WARNINGs, which were already there. Two labels,
  # not five: the timeouts that changed route left nothing of their own, and
  # what survived them is what the per-object failures below record.
  summary=$(grep -F 'residue left by:' "$tmp/out")
  for label in \
    'step 2 (child HelmRelease finalizers)' \
    'step 4b (datastore-secret state unknown)'; do
    case "$summary" in
      *"$label"*) ;;
      *)
        echo "FAIL: the closing message did not name: $label"
        echo "$summary"
        return 1
        ;;
    esac
  done
  # Once each. The force-clear loop fails on both child HelmReleases, and a
  # summary that listed its step twice would read as two separate failures.
  dupes=$(printf '%s\n' "$summary" | tr ',' '\n' |
    grep -cF 'step 2 (child HelmRelease finalizers)' || true)
  if [ "$dupes" != 1 ]; then
    echo "FAIL: the step 2 finalizer backstop is named $dupes times, expected once"
    echo "$summary"
    return 1
  fi

  rm -rf "$tmp"
}

@test "a step with no backstop ends the run instead of claiming success" {
  # The closing message is honest only while the steps that are NOT backstopped
  # still end the run under `set -e`. Nothing but `set -e` enforces that, so a
  # `|| true` appended to one of them later would reinstate the exact defect
  # this file pins: a cleanup that did not happen, reported as success. Step 3
  # stands in for Steps 3, 5, 6, 7 and the patch inside Step 1. Step 1's
  # enumeration is NOT among them: it carries `|| true` on purpose, because
  # Step 2 deletes by label rather than from that list, so a failure there
  # leaves nothing behind to report.
  setup_case
  run_hook '*delete kamajicontrolplanes*' "$PRESENT_SECRET_ONLY"

  if [ "$rc" = 0 ]; then
    echo "FAIL: a failed Step 3 did not end the run"
    cat "$tmp/out"
    return 1
  fi
  if grep -qF 'Cleanup completed successfully' "$tmp/out"; then
    echo "FAIL: cleanup claimed success after Step 3 failed"
    cat "$tmp/out"
    return 1
  fi

  rm -rf "$tmp"
}

@test "exits 0 whether cleanup was clean or every remedy failed" {
  # Pinned on its own because it is the constraint the honesty fix must not buy
  # its honesty with. This is a pre-delete hook: a non-zero exit aborts the Helm
  # uninstall before the release is removed, so the release stays in storage and
  # the same teardown is retried forever. Reporting failure through the exit
  # code is what the closing message exists to avoid.
  setup_case

  run_hook "" "$PRESENT_SECRET_ONLY"
  if [ "$rc" != 0 ]; then
    echo "FAIL: a clean run exited $rc"
    cat "$tmp/out"
    return 1
  fi

  run_hook "$FAIL_ALL" ""
  if [ "$rc" != 0 ]; then
    echo "FAIL: an all-failures run exited $rc, which would wedge the release"
    cat "$tmp/out"
    return 1
  fi

  rm -rf "$tmp"
}