fix(kafka): delete the release's StrimziPodSets on uninstall

On a cluster upgraded from Strimzi 0.x, deleting a Kafka application
leaves its brokers running. kube-controller-manager's garbage collector
first saw the StrimziPodSets under v1beta2 and never updates that
record, so once v1beta2 is no longer served it retries them forever
instead of deleting them, until kube-controller-manager restarts
(kubernetes/kubernetes#102641).

The pre-delete hook now deletes the release's StrimziPodSets after its
node pools, so uninstall no longer depends on garbage collection for
the brokers. An absent CRD is not a failure; any other error fails the
hook like the pool delete does, because Helm never runs it for the
release again.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
Aleksei Sviridkin committed 2026-11-17 12:22:55 +03:00
1 parent a909b733ea
commit f293d64e84
3 files changed
+162 -4

No files matched your search

+108 -2
View File
@@ -32,7 +32,11 @@
# the hook skips keeps its brokers running after the release is gone;
# * a KafkaNodePool CRD that is absent exits 0, and any other failure of
# that delete exits non-zero, so the uninstall is retried rather than
# finishing with the pools still in place.
# finishing with the pools still in place;
# * the release's StrimziPodSets are deleted after the pools, with the same
# exit codes. Garbage collection does not take them on a cluster upgraded
# from Strimzi 0.x until kube-controller-manager restarts
# (kubernetes/kubernetes#102641), so the hook is what stops the brokers.
#
# Run from the repo root with make bats-unit-tests. Direct status checks
# retain compatibility with cozytest, which has no Bats run/setup helpers.
@@ -65,7 +69,8 @@ render_hook() {
# The KafkaNodePool delete answers from POOL_OUT / POOL_RC instead, so a test
# can fail one call without failing the other. POOL_RC defaults to a clean
# delete, because the topic-path tests above are about the topic call; every
# pool-path test sets it explicitly.
# pool-path test sets it explicitly. The StrimziPodSet delete answers from
# SPS_OUT / SPS_RC the same way.
# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone.
write_fake_kubectl() {
cat > "$1/kubectl" <<'KEOF'
@@ -75,6 +80,9 @@ case "$*" in
*kafkanodepools*)
printf '%s\n' "${POOL_OUT:-kafkanodepool.kafka.strimzi.io \"b-0\" deleted}"
exit "${POOL_RC:-0}" ;;
*strimzipodsets*)
printf '%s\n' "${SPS_OUT:-strimzipodset.core.strimzi.io \"kafka-test-b-0\" deleted}"
exit "${SPS_RC:-0}" ;;
esac
if [ -z "${DELETE_RC:-}" ]; then
echo "fake kubectl: DELETE_RC unset; the test would assert nothing" >&2
@@ -284,6 +292,11 @@ KEOF
cat "$tmp/calls"
false
fi
if ! grep -q -- "delete strimzipodsets.core.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls"; then
echo "FAIL: the StrimziPodSets were skipped after the topic wait timed out"
cat "$tmp/calls"
false
fi
rm -rf "$tmp"
}
@@ -342,3 +355,96 @@ KEOF
fi
rm -rf "$tmp"
}
@test "kafka pre-delete hook deletes this release's StrimziPodSets after its node pools" {
tmp=$(mktemp -d)
write_fake_kubectl "$tmp"
render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh"
export KLOG="$tmp/calls"
export DELETE_RC=0
export POOL_RC=0
export SPS_RC=0
rc=0
PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$?
if [ "$rc" -ne 0 ]; then
echo "FAIL: hook exited $rc on a clean delete"
cat "$tmp/out" "$tmp/err"
false
fi
# Anchored like the selectors above.
sps_line=$(grep -n -- "delete strimzipodsets.core.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls" | cut -d: -f1)
pool_line=$(grep -n -- "delete kafkanodepools.kafka.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls" | cut -d: -f1)
if [ -z "$sps_line" ] || [ -z "$pool_line" ]; then
echo "FAIL: expected one release-scoped delete of each kind"
cat "$tmp/calls"
false
fi
# Deleting the sets while their pools exist lets the operator recreate them.
if [ "$sps_line" -le "$pool_line" ]; then
echo "FAIL: the StrimziPodSets were deleted before the node pools"
cat "$tmp/calls"
false
fi
if ! grep -q -- "strimzipodsets.* --wait=false" "$tmp/calls"; then
echo "FAIL: the StrimziPodSet delete waits, which spends the Job's deadline"
cat "$tmp/calls"
false
fi
rm -rf "$tmp"
}
@test "kafka pre-delete hook exits 0 when the StrimziPodSet CRD is absent" {
for msg in 'error: the server doesn'"'"'t have a resource type "strimzipodsets"' \
'error: no matches for kind "StrimziPodSet" in version "core.strimzi.io/v1"'; do
tmp=$(mktemp -d)
write_fake_kubectl "$tmp"
render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh"
export KLOG="$tmp/calls"
export DELETE_RC=0
export POOL_RC=0
export SPS_RC=1
export SPS_OUT="$msg"
rc=0
PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$?
if [ "$rc" -ne 0 ]; then
echo "FAIL: hook exited $rc on an absent StrimziPodSet CRD, wedging the uninstall"
cat "$tmp/out" "$tmp/err"
false
fi
if ! grep -q 'StrimziPodSet CRD absent' "$tmp/out"; then
echo "FAIL: the absent CRD was not reported as such"
cat "$tmp/out"
false
fi
rm -rf "$tmp"
done
}
@test "kafka pre-delete hook exits non-zero when the StrimziPodSet delete fails" {
# A set left here keeps its brokers running after the release is gone, and
# Helm never runs this hook for the release again.
tmp=$(mktemp -d)
write_fake_kubectl "$tmp"
render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh"
export KLOG="$tmp/calls"
export DELETE_RC=0
export POOL_RC=0
export SPS_RC=1
export SPS_OUT='Error from server (Forbidden): strimzipodsets.core.strimzi.io is forbidden'
rc=0
PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$?
if [ "$rc" -eq 0 ]; then
echo "FAIL: hook exited 0 on a Forbidden, so the uninstall would finish and leave the brokers"
cat "$tmp/out" "$tmp/err"
false
fi
if ! grep -q '^ERROR: deleting StrimziPodSets failed' "$tmp/err"; then
echo "FAIL: the failure was not reported as an error"
cat "$tmp/err"
false
fi
rm -rf "$tmp"
}
@@ -126,6 +126,29 @@ spec:
else
printf '%s\n' "${out}"
fi
# Garbage collection should take the StrimziPodSets with their
# pools, but on a cluster upgraded from Strimzi 0.x it does not
# until kube-controller-manager restarts: its collector still
# tracks them under v1beta2, which is no longer served, and
# retries them forever (kubernetes/kubernetes#102641). Deleting
# them here does not depend on that. Fails like the pool call,
# for the same reason.
echo "Deleting StrimziPodSets for {{ .Release.Name }}"
if ! out=$(kubectl -n {{ .Release.Namespace }} delete strimzipodsets.core.strimzi.io \
-l 'strimzi.io/cluster={{ .Release.Name }}' \
--ignore-not-found=true \
--wait=false 2>&1); then
printf '%s\n' "${out}"
case "${out}" in
*"the server doesn't have a resource type"*|*"no matches for kind"*)
echo "StrimziPodSet CRD absent; nothing to delete" ;;
*)
echo "ERROR: deleting StrimziPodSets failed: ${out}" >&2; exit 1 ;;
esac
else
printf '%s\n' "${out}"
fi
volumeMounts:
- name: tmp
mountPath: /tmp
@@ -166,6 +189,9 @@ rules:
- apiGroups: ["kafka.strimzi.io"]
resources: ["kafkanodepools"]
verbs: ["list", "delete"]
- apiGroups: ["core.strimzi.io"]
resources: ["strimzipodsets"]
verbs: ["list", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
+28 -2
View File
@@ -154,8 +154,34 @@ tests:
documentIndex: 2
- lengthEqual:
path: rules
count: 2
documentIndex: 2
count: 3
documentSelector:
path: kind
value: Role
- it: deletes release-scoped StrimziPodSets after the KafkaNodePools
asserts:
# After an upgrade from Strimzi 0.x, kube-controller-manager's garbage
# collector still tracks each StrimziPodSet under v1beta2, a version no
# longer served, and never deletes it when its pool goes
# (kubernetes/kubernetes#102641). The brokers keep running until
# kube-controller-manager restarts, so the hook deletes the sets itself.
# Same selector and no wait, as for the pools.
- matchRegex:
path: spec.template.spec.containers[0].command[2]
pattern: "(?s)delete kafkanodepools\\.kafka\\.strimzi\\.io .*delete strimzipodsets\\.core\\.strimzi\\.io \\\\\\s+-l 'strimzi\\.io/cluster=kafka-test' \\\\\\s+--ignore-not-found=true \\\\\\s+--wait=false"
documentSelector:
path: kind
value: Job
- contains:
path: rules
content:
apiGroups: ["core.strimzi.io"]
resources: ["strimzipodsets"]
verbs: ["list", "delete"]
documentSelector:
path: kind
value: Role
- it: prefixes the hook image with the cluster images registry when set
set: