diff --git a/hack/kafka-pre-delete-hook.bats b/hack/kafka-pre-delete-hook.bats index e8af58aec..90e852670 100644 --- a/hack/kafka-pre-delete-hook.bats +++ b/hack/kafka-pre-delete-hook.bats @@ -32,7 +32,11 @@ # the hook skips keeps its brokers running after the release is gone; # * a KafkaNodePool CRD that is absent exits 0, and any other failure of # that delete exits non-zero, so the uninstall is retried rather than -# finishing with the pools still in place. +# finishing with the pools still in place; +# * the release's StrimziPodSets are deleted after the pools, with the same +# exit codes. Garbage collection does not take them on a cluster upgraded +# from Strimzi 0.x until kube-controller-manager restarts +# (kubernetes/kubernetes#102641), so the hook is what stops the brokers. # # Run from the repo root with make bats-unit-tests. Direct status checks # retain compatibility with cozytest, which has no Bats run/setup helpers. @@ -65,7 +69,8 @@ render_hook() { # The KafkaNodePool delete answers from POOL_OUT / POOL_RC instead, so a test # can fail one call without failing the other. POOL_RC defaults to a clean # delete, because the topic-path tests above are about the topic call; every -# pool-path test sets it explicitly. +# pool-path test sets it explicitly. The StrimziPodSet delete answers from +# SPS_OUT / SPS_RC the same way. # No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone. write_fake_kubectl() { cat > "$1/kubectl" <<'KEOF' @@ -75,6 +80,9 @@ case "$*" in *kafkanodepools*) printf '%s\n' "${POOL_OUT:-kafkanodepool.kafka.strimzi.io \"b-0\" deleted}" exit "${POOL_RC:-0}" ;; + *strimzipodsets*) + printf '%s\n' "${SPS_OUT:-strimzipodset.core.strimzi.io \"kafka-test-b-0\" deleted}" + exit "${SPS_RC:-0}" ;; esac if [ -z "${DELETE_RC:-}" ]; then echo "fake kubectl: DELETE_RC unset; the test would assert nothing" >&2 @@ -284,6 +292,11 @@ KEOF cat "$tmp/calls" false fi + if ! grep -q -- "delete strimzipodsets.core.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls"; then + echo "FAIL: the StrimziPodSets were skipped after the topic wait timed out" + cat "$tmp/calls" + false + fi rm -rf "$tmp" } @@ -342,3 +355,96 @@ KEOF fi rm -rf "$tmp" } + +@test "kafka pre-delete hook deletes this release's StrimziPodSets after its node pools" { + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh" + + export KLOG="$tmp/calls" + export DELETE_RC=0 + export POOL_RC=0 + export SPS_RC=0 + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc on a clean delete" + cat "$tmp/out" "$tmp/err" + false + fi + # Anchored like the selectors above. + sps_line=$(grep -n -- "delete strimzipodsets.core.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls" | cut -d: -f1) + pool_line=$(grep -n -- "delete kafkanodepools.kafka.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls" | cut -d: -f1) + if [ -z "$sps_line" ] || [ -z "$pool_line" ]; then + echo "FAIL: expected one release-scoped delete of each kind" + cat "$tmp/calls" + false + fi + # Deleting the sets while their pools exist lets the operator recreate them. + if [ "$sps_line" -le "$pool_line" ]; then + echo "FAIL: the StrimziPodSets were deleted before the node pools" + cat "$tmp/calls" + false + fi + if ! grep -q -- "strimzipodsets.* --wait=false" "$tmp/calls"; then + echo "FAIL: the StrimziPodSet delete waits, which spends the Job's deadline" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "kafka pre-delete hook exits 0 when the StrimziPodSet CRD is absent" { + for msg in 'error: the server doesn'"'"'t have a resource type "strimzipodsets"' \ + 'error: no matches for kind "StrimziPodSet" in version "core.strimzi.io/v1"'; do + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh" + + export KLOG="$tmp/calls" + export DELETE_RC=0 + export POOL_RC=0 + export SPS_RC=1 + export SPS_OUT="$msg" + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc on an absent StrimziPodSet CRD, wedging the uninstall" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q 'StrimziPodSet CRD absent' "$tmp/out"; then + echo "FAIL: the absent CRD was not reported as such" + cat "$tmp/out" + false + fi + rm -rf "$tmp" + done +} + +@test "kafka pre-delete hook exits non-zero when the StrimziPodSet delete fails" { + # A set left here keeps its brokers running after the release is gone, and + # Helm never runs this hook for the release again. + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh" + + export KLOG="$tmp/calls" + export DELETE_RC=0 + export POOL_RC=0 + export SPS_RC=1 + export SPS_OUT='Error from server (Forbidden): strimzipodsets.core.strimzi.io is forbidden' + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -eq 0 ]; then + echo "FAIL: hook exited 0 on a Forbidden, so the uninstall would finish and leave the brokers" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q '^ERROR: deleting StrimziPodSets failed' "$tmp/err"; then + echo "FAIL: the failure was not reported as an error" + cat "$tmp/err" + false + fi + rm -rf "$tmp" +} diff --git a/packages/apps/kafka/templates/hooks/delete.yaml b/packages/apps/kafka/templates/hooks/delete.yaml index e54c5b716..e37778919 100644 --- a/packages/apps/kafka/templates/hooks/delete.yaml +++ b/packages/apps/kafka/templates/hooks/delete.yaml @@ -126,6 +126,29 @@ spec: else printf '%s\n' "${out}" fi + + # Garbage collection should take the StrimziPodSets with their + # pools, but on a cluster upgraded from Strimzi 0.x it does not + # until kube-controller-manager restarts: its collector still + # tracks them under v1beta2, which is no longer served, and + # retries them forever (kubernetes/kubernetes#102641). Deleting + # them here does not depend on that. Fails like the pool call, + # for the same reason. + echo "Deleting StrimziPodSets for {{ .Release.Name }}" + if ! out=$(kubectl -n {{ .Release.Namespace }} delete strimzipodsets.core.strimzi.io \ + -l 'strimzi.io/cluster={{ .Release.Name }}' \ + --ignore-not-found=true \ + --wait=false 2>&1); then + printf '%s\n' "${out}" + case "${out}" in + *"the server doesn't have a resource type"*|*"no matches for kind"*) + echo "StrimziPodSet CRD absent; nothing to delete" ;; + *) + echo "ERROR: deleting StrimziPodSets failed: ${out}" >&2; exit 1 ;; + esac + else + printf '%s\n' "${out}" + fi volumeMounts: - name: tmp mountPath: /tmp @@ -166,6 +189,9 @@ rules: - apiGroups: ["kafka.strimzi.io"] resources: ["kafkanodepools"] verbs: ["list", "delete"] + - apiGroups: ["core.strimzi.io"] + resources: ["strimzipodsets"] + verbs: ["list", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/packages/apps/kafka/tests/delete_test.yaml b/packages/apps/kafka/tests/delete_test.yaml index d6feced7a..ff09d465f 100644 --- a/packages/apps/kafka/tests/delete_test.yaml +++ b/packages/apps/kafka/tests/delete_test.yaml @@ -154,8 +154,34 @@ tests: documentIndex: 2 - lengthEqual: path: rules - count: 2 - documentIndex: 2 + count: 3 + documentSelector: + path: kind + value: Role + + - it: deletes release-scoped StrimziPodSets after the KafkaNodePools + asserts: + # After an upgrade from Strimzi 0.x, kube-controller-manager's garbage + # collector still tracks each StrimziPodSet under v1beta2, a version no + # longer served, and never deletes it when its pool goes + # (kubernetes/kubernetes#102641). The brokers keep running until + # kube-controller-manager restarts, so the hook deletes the sets itself. + # Same selector and no wait, as for the pools. + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: "(?s)delete kafkanodepools\\.kafka\\.strimzi\\.io .*delete strimzipodsets\\.core\\.strimzi\\.io \\\\\\s+-l 'strimzi\\.io/cluster=kafka-test' \\\\\\s+--ignore-not-found=true \\\\\\s+--wait=false" + documentSelector: + path: kind + value: Job + - contains: + path: rules + content: + apiGroups: ["core.strimzi.io"] + resources: ["strimzipodsets"] + verbs: ["list", "delete"] + documentSelector: + path: kind + value: Role - it: prefixes the hook image with the cluster images registry when set set: