mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-22 12:43:32 +09:00
Two ways the gate did not do what its arguments promise. The list read carried no bound. kubectl defaults to no request timeout, and the read sits above the deadline test at the bottom of the loop, so against a wedged apiserver it blocks there and timeout_seconds stops being a ceiling: what arrives is the job timeout as a kill, instead of the not_ready dump this function exists to print. Bounded with kubectl's own --request-timeout rather than a `timeout` wrapper, because the unit suite substitutes a kubectl shell function and `timeout` execs a binary. The final diagnostic read is bounded the same way. `now` was sampled before the read, so a read costing more than the poll interval was priced at zero against the deadline and bought the loop another whole iteration past it. Sampled after, on both branches. And the count >= minimum_count half of the gate was pinned by nothing: every fixture returned 11 or 12 items against a minimum of 11, so the condition could be deleted outright and all five tests stayed green. That threshold is the direct replacement for the `wc -l` existence backstop this branch removes from hack/e2e-install-cozystack.bats -- the only thing separating "everything is Ready" from "almost nothing got created", since three HelmReleases that exist are all Ready. Both new cases were checked by reverting the code they cover. Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com> Assisted-By: LLM
182 lines
6.2 KiB
Bash
182 lines
6.2 KiB
Bash
#!/usr/bin/env bats
|
|
# Unit coverage for the single post-install HelmRelease readiness gate.
|
|
|
|
helmrelease_snapshot() {
|
|
count="$1"
|
|
not_ready="${2:-0}"
|
|
stalled="${3:-0}"
|
|
stale="${4:-0}"
|
|
jq -nc --argjson count "$count" --argjson notReady "$not_ready" --argjson stalled "$stalled" --argjson stale "$stale" '
|
|
{items: [range(1; $count + 1) as $i |
|
|
{metadata: {namespace: "ns", name: ("hr-" + ($i | tostring)), generation: 2},
|
|
status: {observedGeneration: (if $i == $stale then 1 else 2 end), conditions:
|
|
([{type: "Ready", status: (if $i == $notReady then "False" else "True" end), reason: "Progressing", message: "working"}]
|
|
+ (if $i == $stalled then [{type: "Stalled", status: "True", message: "terminal failure"}] else [] end))}}
|
|
]}
|
|
'
|
|
}
|
|
|
|
@test "stale Ready stays closed until the current generation is observed" {
|
|
. hack/e2e-wait-helmreleases.sh
|
|
clock=$(mktemp)
|
|
printf '0\n' > "$clock"
|
|
date() { sed -n '1p' "$clock"; }
|
|
sleep() {
|
|
now=$(sed -n '1p' "$clock")
|
|
printf '%s\n' "$(( now + $1 ))" > "$clock"
|
|
}
|
|
kubectl() {
|
|
now=$(sed -n '1p' "$clock")
|
|
if [ "$now" -lt 4 ]; then
|
|
helmrelease_snapshot 11 0 0 3
|
|
else
|
|
helmrelease_snapshot 11
|
|
fi
|
|
}
|
|
|
|
cozy_wait_all_helmreleases_ready 30 11 4 2 >/dev/null
|
|
[ "$(sed -n '1p' "$clock")" -ge 8 ] || {
|
|
echo "the gate accepted Ready=True from an unobserved generation" >&2
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
@test "a late HelmRelease resets stability and is included before success" {
|
|
. hack/e2e-wait-helmreleases.sh
|
|
clock=$(mktemp)
|
|
calls=$(mktemp)
|
|
printf '0\n' > "$clock"
|
|
date() { sed -n '1p' "$clock"; }
|
|
sleep() {
|
|
now=$(sed -n '1p' "$clock")
|
|
printf '%s\n' "$(( now + $1 ))" > "$clock"
|
|
}
|
|
kubectl() {
|
|
printf '%s\n' "$*" >> "$calls"
|
|
now=$(sed -n '1p' "$clock")
|
|
if [ "$now" -lt 2 ]; then
|
|
helmrelease_snapshot 11
|
|
elif [ "$now" -lt 4 ]; then
|
|
helmrelease_snapshot 12 12
|
|
else
|
|
helmrelease_snapshot 12
|
|
fi
|
|
}
|
|
|
|
cozy_wait_all_helmreleases_ready 30 11 5 2 >/dev/null
|
|
[ "$(sed -n '1p' "$clock")" -ge 10 ] || {
|
|
echo "the gate returned before the late release became Ready and stabilized" >&2
|
|
exit 1
|
|
}
|
|
[ "$(grep -c -- '-o json' "$calls")" -ge 6 ]
|
|
}
|
|
|
|
@test "Stalled HelmRelease fails immediately without sleeping to timeout" {
|
|
. hack/e2e-wait-helmreleases.sh
|
|
calls=$(mktemp)
|
|
date() { printf '100\n'; }
|
|
sleep() { echo "sleep must not run after Stalled=True" >&2; return 1; }
|
|
kubectl() {
|
|
printf '%s\n' "$*" >> "$calls"
|
|
helmrelease_snapshot 11 3 3
|
|
}
|
|
|
|
if cozy_wait_all_helmreleases_ready 900 11 5 2 >/dev/null 2>&1; then
|
|
echo "a terminal Stalled HelmRelease passed readiness" >&2
|
|
exit 1
|
|
fi
|
|
[ "$(wc -l < "$calls")" -eq 1 ] || {
|
|
echo "the Stalled path kept polling instead of failing immediately" >&2
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
@test "a transient list error keeps the gate closed" {
|
|
. hack/e2e-wait-helmreleases.sh
|
|
clock=$(mktemp)
|
|
calls=$(mktemp)
|
|
printf '0\n' > "$clock"
|
|
date() { sed -n '1p' "$clock"; }
|
|
sleep() {
|
|
now=$(sed -n '1p' "$clock")
|
|
printf '%s\n' "$(( now + $1 ))" > "$clock"
|
|
}
|
|
kubectl() {
|
|
count=$(wc -l < "$calls")
|
|
printf '%s\n' "$*" >> "$calls"
|
|
if [ "$count" -eq 0 ]; then
|
|
echo 'temporary apiserver error' >&2
|
|
return 1
|
|
fi
|
|
helmrelease_snapshot 11
|
|
}
|
|
|
|
cozy_wait_all_helmreleases_ready 30 11 4 2 >/dev/null 2>&1
|
|
[ "$(sed -n '1p' "$clock")" -ge 6 ] || {
|
|
echo "the list error was mistaken for an all-Ready snapshot" >&2
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
@test "a Ready but short release set keeps the gate closed" {
|
|
# The minimum-count half of the gate is what replaced the `wc -l` existence
|
|
# backstop removed from hack/e2e-install-cozystack.bats, and it is the only
|
|
# thing separating "everything is Ready" from "almost nothing got created":
|
|
# an install that produced three HelmReleases and stopped has every one of
|
|
# them Ready. Every other case here passes 11 or 12 items against a minimum
|
|
# of 11, so without this one the condition can be deleted outright and the
|
|
# suite stays green.
|
|
. hack/e2e-wait-helmreleases.sh
|
|
clock=$(mktemp)
|
|
printf '0\n' > "$clock"
|
|
date() { sed -n '1p' "$clock"; }
|
|
sleep() {
|
|
now=$(sed -n '1p' "$clock")
|
|
printf '%s\n' "$(( now + $1 ))" > "$clock"
|
|
}
|
|
kubectl() { helmrelease_snapshot 3; }
|
|
|
|
if cozy_wait_all_helmreleases_ready 30 11 4 2 >/dev/null 2>&1; then
|
|
echo "the gate opened on 3 Ready HelmReleases against a minimum of 11" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
@test "a read slower than the poll interval is charged against the deadline" {
|
|
# `now` is sampled after the list read, not before it. Sampled before, a read
|
|
# that costs more than the poll interval is priced at zero and the loop buys
|
|
# another full iteration past the deadline -- so the ceiling the caller asked
|
|
# for is not one. Counted in kubectl calls: one snapshot plus the final
|
|
# diagnostic read is the whole run; a third call means a second poll happened
|
|
# after the deadline had already passed.
|
|
. hack/e2e-wait-helmreleases.sh
|
|
clock=$(mktemp)
|
|
calls=$(mktemp)
|
|
printf '0\n' > "$clock"
|
|
date() { sed -n '1p' "$clock"; }
|
|
sleep() {
|
|
now=$(sed -n '1p' "$clock")
|
|
printf '%s\n' "$(( now + $1 ))" > "$clock"
|
|
}
|
|
kubectl() {
|
|
printf 'call\n' >> "$calls"
|
|
now=$(sed -n '1p' "$clock")
|
|
printf '%s\n' "$(( now + 40 ))" > "$clock"
|
|
helmrelease_snapshot 11 1
|
|
}
|
|
|
|
cozy_wait_all_helmreleases_ready 30 11 4 2 >/dev/null 2>&1 || true
|
|
if [ "$(wc -l < "$calls")" -ne 2 ]; then
|
|
echo "expected one snapshot read and one diagnostic read, got $(wc -l < "$calls")" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
@test "install suite has one unmasked HelmRelease gate" {
|
|
[ "$(grep -c 'cozy_wait_all_helmreleases_ready 900 11 5 2' hack/e2e-install-cozystack.bats)" -eq 1 ]
|
|
if grep -q 'kubectl wait hr --all -A.*|| true' hack/e2e-install-cozystack.bats; then
|
|
echo "the old masked HelmRelease wait survived" >&2
|
|
exit 1
|
|
fi
|
|
}
|