admin/cozystack
1
0 Fork 0
mirror of https://github.com/cozystack/cozystack.git synced 2026-10-23 03:27:50 +09:00
Code Issues Packages Projects Releases Wiki Activity

fix(fluxcd): keep the hook waiting on a release it cannot read

Browse Source 
The pre-delete hook read a release's state once, so a single failed
read was taken as the release being handled: its fence came off before
helm-controller's uninstall, and the report counted it as released
cleanly. A state read is now retried within the release's share of the
window, a release whose state still cannot be read keeps its fence for
the wait like one helm-controller has not handled, and the end of the
wait reads each again before naming what the release was left to.

A release still suspended after the sweep is named as suspended but not
by flux-plunger, which also holds when no manager is recorded for the
field at all.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-10-13 15:09:00 +03:00
1 parent 568e6f5e18
commit 7011510456
2 files changed
+66 -19
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

hack/flux-plunger-fence-hook_test.bats
+42 -2
View File
@@ -118,7 +118,8 @@ fences_left() {
# for a reason other than pods (wait), the listing
# (list), a listing after the first (relist), the
# pod listing (pods), a release's ownership read
# (<ns>@<name>.ownedread), unsuspend
# (<ns>@<name>.ownedread), a release's state read
# (<ns>@<name>.stateread), unsuspend
# (<ns>@<name>.apply), unsuspend racing another
# write (<ns>@<name>.applyconflict) or finalizer patch
# (<ns>@<name>.patch);
@@ -269,6 +270,7 @@ if [ "$verb" = get ]; then
case "$tmpl" in *'ne .manager "flux-plunger"'*) [ -e "$rel.coowned" ] && echo other ;; esac
exit 0 ;;
*deletionTimestamp*)
if take_fail "$rel.stateread"; then echo "fake: the state read timed out" >&2; exit 1; fi
[ -e "$rel.deleting" ] || exit 0
case "$tmpl" in *"$SUSPENDED_DELETING"*) if [ -e "$rel.suspended" ]; then printf 'suspended'; exit 0; fi ;; esac
case "$tmpl" in *"$UNHANDLED"*) ;; *) exit 0 ;; esac
@@ -492,7 +494,7 @@ run_hook() {
[ "$status" -eq 0 ]
[ "$(fences_left "$STATE")" -eq 0 ]
grep -q -x "released $FENCE from tenant-x/app-x without an uninstall: another manager holds its suspension" "$tmp/out"
grep -q -x "released $FENCE from tenant-x/app-x without an uninstall: it is still suspended, not by flux-plunger" "$tmp/out"
grep -q -x 'released 1 of 1 fenced HelmReleases, 1 without an uninstall' "$tmp/out"
rm -rf "$tmp"
}
@@ -573,6 +575,44 @@ run_hook() {
rm -rf "$tmp"
}
@test "a state read that fails once does not end the wait for helm-controller" {
tmp=$(mktemp -d)
prepare "$tmp"
add_release "$STATE" tenant-a app-a "$FENCE"
touch "$STATE/tenant-a@app-a.deleting"
echo yes > "$STATE/tenant-a@app-a.owned"
echo 3 > "$STATE/tenant-a@app-a.hcdelay"
echo 1 > "$STATE/tenant-a@app-a.stateread.fail"
run_hook "$tmp"
[ "$status" -eq 0 ]
[ "$(fences_left "$STATE")" -eq 0 ]
[ "$(cat "$STATE/tenant-a@app-a.obs")" = "$(cat "$STATE/tenant-a@app-a.gen")" ]
grep -q -x "released $FENCE from tenant-a/app-a" "$tmp/out"
grep -q -x 'released 1 of 1 fenced HelmReleases, 0 without an uninstall' "$tmp/out"
rm -rf "$tmp"
}
@test "a release whose state can never be read is still released inside the deadline, and says so" {
tmp=$(mktemp -d)
prepare "$tmp"
add_release "$STATE" tenant-a app-a "$FENCE"
touch "$STATE/tenant-a@app-a.deleting"
echo yes > "$STATE/tenant-a@app-a.owned"
echo 9999 > "$STATE/tenant-a@app-a.stateread.fail"
run_hook "$tmp"
[ "$status" -eq 0 ]
[ "$(fences_left "$STATE")" -eq 0 ]
grep -q -x "released $FENCE from tenant-a/app-a; whether helm-controller had handled it could not be read" "$tmp/out"
# It kept the fence for the whole wait, as one helm-controller had not handled.
[ "$(cat "$CLOCK")" -ge $((WINDOW - 30)) ]
[ "$(cat "$CLOCK")" -lt "$DEADLINE" ]
rm -rf "$tmp"
}
@test "a release helm-controller handles while the hook waits keeps its fence until the uninstall has run" {
tmp=$(mktemp -d)
prepare "$tmp"
packages/system/flux-plunger/templates/uninstall-fence-cleanup.yaml
+24 -17
View File
@@ -347,12 +347,13 @@ spec:
# read_state <ns> <name>: how helm-controller stands with a release
# being deleted. "suspended": it skips the uninstall of a suspended
# release, and the suspension is another manager's. "unhandled":
# release, and the suspension is not flux-plunger's. "unhandled":
# its own finalizer no longer holds the release and it has not
# observed the latest generation, so the fence is all that keeps
# the release until it runs the uninstall. Empty otherwise.
# the release until it runs the uninstall. Empty otherwise. Fails
# when the release cannot be read.
read_state() {
kubectl --namespace "$1" get "$hr" "$2" --ignore-not-found --output go-template='{{`{{if .metadata.deletionTimestamp}}{{if .spec.suspend}}suspended{{else}}{{$held := false}}{{range .metadata.finalizers}}{{if eq . "finalizers.fluxcd.io"}}{{$held = true}}{{end}}{{end}}{{if and (not $held) (ne (printf "%v" .status.observedGeneration) (printf "%v" .metadata.generation))}}unhandled{{end}}{{end}}{{end}}`}}' || echo unknown
kubectl --namespace "$1" get "$hr" "$2" --ignore-not-found --output go-template='{{`{{if .metadata.deletionTimestamp}}{{if .spec.suspend}}suspended{{else}}{{$held := false}}{{range .metadata.finalizers}}{{if eq . "finalizers.fluxcd.io"}}{{$held = true}}{{end}}{{end}}{{if and (not $held) (ne (printf "%v" .status.observedGeneration) (printf "%v" .metadata.generation))}}unhandled{{end}}{{end}}{{end}}`}}'
}
# report_release <ref> <state>: remove the fence and name what that
@@ -371,7 +372,7 @@ spec:
echo "released $fence from $1 without an uninstall: helm-controller had not handled it" ;;
suspended)
unhandled=$((unhandled + 1))
echo "released $fence from $1 without an uninstall: another manager holds its suspension" ;;
echo "released $fence from $1 without an uninstall: it is still suspended, not by flux-plunger" ;;
unknown)
echo "released $fence from $1; whether helm-controller had handled it could not be read" ;;
*)
@@ -408,11 +409,15 @@ spec:
echo "NOT released $ref: flux-plunger's suspension could not be lifted" >&2
continue ;;
esac
state=$(read_state "$hns" "$name")
if [ "$state" = unhandled ]; then
pending="$pending $ref"
continue
fi
# A release whose state cannot be read waits like one helm-controller
# has not handled: unfencing it on a failed read could delete it
# before its uninstall.
state=$(retry read_state "$hns" "$name") || state=unknown
case "$state" in
unhandled|unknown)
pending="$pending $ref"
continue ;;
esac
report_release "$ref" "$state"
done
# The last fences go well before the window closes, so their
@@ -422,19 +427,21 @@ spec:
sleep 2
still=
for ref in $pending; do
state=$(read_state "${ref%%/*}" "${ref#*/}")
if [ "$state" = unhandled ]; then
still="$still $ref"
else
begin_release
report_release "$ref" "$state"
fi
state=$(read_state "${ref%%/*}" "${ref#*/}") || state=unknown
case "$state" in
unhandled|unknown)
still="$still $ref" ;;
*)
begin_release
report_release "$ref" "$state" ;;
esac
done
pending=$still
done
for ref in $pending; do
begin_release
report_release "$ref" unhandled
state=$(read_state "${ref%%/*}" "${ref#*/}") || state=unknown
report_release "$ref" "$state"
done
stop_at=$retry_until