mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(kafka): delete the release's StrimziPodSets on uninstall
On a cluster upgraded from Strimzi 0.x, deleting a Kafka application leaves its brokers running. kube-controller-manager's garbage collector first saw the StrimziPodSets under v1beta2 and never updates that record, so once v1beta2 is no longer served it retries them forever instead of deleting them, until kube-controller-manager restarts (kubernetes/kubernetes#102641). The pre-delete hook now deletes the release's StrimziPodSets after its node pools, so uninstall no longer depends on garbage collection for the brokers. An absent CRD is not a failure; any other error fails the hook like the pool delete does, because Helm never runs it for the release again. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
3 files changed
+162
-4
No files matched your search
@@ -32,7 +32,11 @@
|
||||
# the hook skips keeps its brokers running after the release is gone;
|
||||
# * a KafkaNodePool CRD that is absent exits 0, and any other failure of
|
||||
# that delete exits non-zero, so the uninstall is retried rather than
|
||||
# finishing with the pools still in place.
|
||||
# finishing with the pools still in place;
|
||||
# * the release's StrimziPodSets are deleted after the pools, with the same
|
||||
# exit codes. Garbage collection does not take them on a cluster upgraded
|
||||
# from Strimzi 0.x until kube-controller-manager restarts
|
||||
# (kubernetes/kubernetes#102641), so the hook is what stops the brokers.
|
||||
#
|
||||
# Run from the repo root with make bats-unit-tests. Direct status checks
|
||||
# retain compatibility with cozytest, which has no Bats run/setup helpers.
|
||||
@@ -65,7 +69,8 @@ render_hook() {
|
||||
# The KafkaNodePool delete answers from POOL_OUT / POOL_RC instead, so a test
|
||||
# can fail one call without failing the other. POOL_RC defaults to a clean
|
||||
# delete, because the topic-path tests above are about the topic call; every
|
||||
# pool-path test sets it explicitly.
|
||||
# pool-path test sets it explicitly. The StrimziPodSet delete answers from
|
||||
# SPS_OUT / SPS_RC the same way.
|
||||
# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone.
|
||||
write_fake_kubectl() {
|
||||
cat > "$1/kubectl" <<'KEOF'
|
||||
@@ -75,6 +80,9 @@ case "$*" in
|
||||
*kafkanodepools*)
|
||||
printf '%s\n' "${POOL_OUT:-kafkanodepool.kafka.strimzi.io \"b-0\" deleted}"
|
||||
exit "${POOL_RC:-0}" ;;
|
||||
*strimzipodsets*)
|
||||
printf '%s\n' "${SPS_OUT:-strimzipodset.core.strimzi.io \"kafka-test-b-0\" deleted}"
|
||||
exit "${SPS_RC:-0}" ;;
|
||||
esac
|
||||
if [ -z "${DELETE_RC:-}" ]; then
|
||||
echo "fake kubectl: DELETE_RC unset; the test would assert nothing" >&2
|
||||
@@ -284,6 +292,11 @@ KEOF
|
||||
cat "$tmp/calls"
|
||||
false
|
||||
fi
|
||||
if ! grep -q -- "delete strimzipodsets.core.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls"; then
|
||||
echo "FAIL: the StrimziPodSets were skipped after the topic wait timed out"
|
||||
cat "$tmp/calls"
|
||||
false
|
||||
fi
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
@@ -342,3 +355,96 @@ KEOF
|
||||
fi
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
@test "kafka pre-delete hook deletes this release's StrimziPodSets after its node pools" {
|
||||
tmp=$(mktemp -d)
|
||||
write_fake_kubectl "$tmp"
|
||||
render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh"
|
||||
|
||||
export KLOG="$tmp/calls"
|
||||
export DELETE_RC=0
|
||||
export POOL_RC=0
|
||||
export SPS_RC=0
|
||||
rc=0
|
||||
PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "FAIL: hook exited $rc on a clean delete"
|
||||
cat "$tmp/out" "$tmp/err"
|
||||
false
|
||||
fi
|
||||
# Anchored like the selectors above.
|
||||
sps_line=$(grep -n -- "delete strimzipodsets.core.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls" | cut -d: -f1)
|
||||
pool_line=$(grep -n -- "delete kafkanodepools.kafka.strimzi.io -l strimzi.io/cluster=kafka-test " "$tmp/calls" | cut -d: -f1)
|
||||
if [ -z "$sps_line" ] || [ -z "$pool_line" ]; then
|
||||
echo "FAIL: expected one release-scoped delete of each kind"
|
||||
cat "$tmp/calls"
|
||||
false
|
||||
fi
|
||||
# Deleting the sets while their pools exist lets the operator recreate them.
|
||||
if [ "$sps_line" -le "$pool_line" ]; then
|
||||
echo "FAIL: the StrimziPodSets were deleted before the node pools"
|
||||
cat "$tmp/calls"
|
||||
false
|
||||
fi
|
||||
if ! grep -q -- "strimzipodsets.* --wait=false" "$tmp/calls"; then
|
||||
echo "FAIL: the StrimziPodSet delete waits, which spends the Job's deadline"
|
||||
cat "$tmp/calls"
|
||||
false
|
||||
fi
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
@test "kafka pre-delete hook exits 0 when the StrimziPodSet CRD is absent" {
|
||||
for msg in 'error: the server doesn'"'"'t have a resource type "strimzipodsets"' \
|
||||
'error: no matches for kind "StrimziPodSet" in version "core.strimzi.io/v1"'; do
|
||||
tmp=$(mktemp -d)
|
||||
write_fake_kubectl "$tmp"
|
||||
render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh"
|
||||
|
||||
export KLOG="$tmp/calls"
|
||||
export DELETE_RC=0
|
||||
export POOL_RC=0
|
||||
export SPS_RC=1
|
||||
export SPS_OUT="$msg"
|
||||
rc=0
|
||||
PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "FAIL: hook exited $rc on an absent StrimziPodSet CRD, wedging the uninstall"
|
||||
cat "$tmp/out" "$tmp/err"
|
||||
false
|
||||
fi
|
||||
if ! grep -q 'StrimziPodSet CRD absent' "$tmp/out"; then
|
||||
echo "FAIL: the absent CRD was not reported as such"
|
||||
cat "$tmp/out"
|
||||
false
|
||||
fi
|
||||
rm -rf "$tmp"
|
||||
done
|
||||
}
|
||||
|
||||
@test "kafka pre-delete hook exits non-zero when the StrimziPodSet delete fails" {
|
||||
# A set left here keeps its brokers running after the release is gone, and
|
||||
# Helm never runs this hook for the release again.
|
||||
tmp=$(mktemp -d)
|
||||
write_fake_kubectl "$tmp"
|
||||
render_hook "$KAFKA_CHART" kafka-test tenant-test kafka-test-pre-delete "$tmp/hook.sh"
|
||||
|
||||
export KLOG="$tmp/calls"
|
||||
export DELETE_RC=0
|
||||
export POOL_RC=0
|
||||
export SPS_RC=1
|
||||
export SPS_OUT='Error from server (Forbidden): strimzipodsets.core.strimzi.io is forbidden'
|
||||
rc=0
|
||||
PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$?
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
echo "FAIL: hook exited 0 on a Forbidden, so the uninstall would finish and leave the brokers"
|
||||
cat "$tmp/out" "$tmp/err"
|
||||
false
|
||||
fi
|
||||
if ! grep -q '^ERROR: deleting StrimziPodSets failed' "$tmp/err"; then
|
||||
echo "FAIL: the failure was not reported as an error"
|
||||
cat "$tmp/err"
|
||||
false
|
||||
fi
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
Reference in new issue
Block a user