#!/bin/bash # Readiness gate for the platform HelmReleases created during E2E install. cozy_wait_all_helmreleases_ready() { local timeout_seconds="${1:-900}" local minimum_count="${2:-11}" local quiet_seconds="${3:-5}" local poll_seconds="${4:-2}" local deadline local snapshot= count= stalled= not_ready= fingerprint= local stable_fingerprint= stable_since=0 now case "${timeout_seconds}:${minimum_count}:${quiet_seconds}:${poll_seconds}" in *[!0-9:]*) echo "HelmRelease readiness timing arguments must be integers" >&2 return 1 ;; esac if [ "${minimum_count}" -eq 0 ] || [ "${poll_seconds}" -eq 0 ]; then echo "HelmRelease readiness minimum count and poll interval must be positive" >&2 return 1 fi deadline=$(( $(date +%s) + timeout_seconds )) echo "Waiting for at least ${minimum_count} HelmReleases to become Ready and remain a stable set for ${quiet_seconds}s..." while :; do # Bounded by kubectl's own request timeout rather than by a `timeout` # wrapper: the unit suite substitutes a `kubectl` shell function, and # `timeout` execs a binary, so a wrapper here would run the real client # against whatever cluster the runner is pointed at. kubectl defaults to no # request timeout at all, and this read sits above the deadline test at the # bottom of the loop -- so unbounded, a wedged apiserver blocks here and # ${timeout_seconds} stops being a ceiling. if ! snapshot=$(kubectl get helmreleases.helm.toolkit.fluxcd.io -A --request-timeout=30s -o json); then now=$(date +%s) echo "HelmRelease readiness list failed; keeping the gate closed" >&2 stable_fingerprint= stable_since=0 else # Sampled after the read, not before it: a read that costs more than the # poll interval would otherwise be priced at zero against the deadline and # buy the loop another whole iteration past it. now=$(date +%s) if ! count=$(printf '%s\n' "${snapshot}" | jq -r '.items | length'); then echo "Could not parse the HelmRelease readiness list" >&2 return 1 fi stalled=$(printf '%s\n' "${snapshot}" | jq -r ' .items[] | select((.status.observedGeneration // 0) == .metadata.generation) | select([.status.conditions[]? | select(.type == "Stalled" and .status == "True")] | length > 0) | "\(.metadata.namespace)/\(.metadata.name): " + (([.status.conditions[]? | select(.type == "Stalled") | .message][0]) // "Stalled=True") ') if [ -n "${stalled}" ]; then echo "HelmRelease reconciliation reached a terminal Stalled condition:" >&2 printf '%s\n' "${stalled}" | sed 's/^/ /' >&2 return 1 fi not_ready=$(printf '%s\n' "${snapshot}" | jq -r ' .items[] | (.status.observedGeneration // 0) as $observed | .metadata.generation as $generation | ([.status.conditions[]? | select(.type == "Ready" and .status == "True")] | length) as $ready | select($observed != $generation or $ready == 0) | "\(.metadata.namespace)/\(.metadata.name): " + (if $observed != $generation then "generation \($generation) not observed (status.observedGeneration=\($observed))" else (([.status.conditions[]? | select(.type == "Ready") | "\(.reason): \(.message)"][0]) // "Ready condition absent") end) ') if [ "${count}" -ge "${minimum_count}" ] && [ -z "${not_ready}" ]; then fingerprint=$(printf '%s\n' "${snapshot}" | jq -r ' .items | map("\(.metadata.namespace)/\(.metadata.name)") | sort | join("\n") ') if [ "${fingerprint}" != "${stable_fingerprint}" ]; then stable_fingerprint="${fingerprint}" stable_since="${now}" elif [ $(( now - stable_since )) -ge "${quiet_seconds}" ]; then echo "All ${count} HelmReleases are Ready; the release set stayed stable for ${quiet_seconds}s." return 0 fi else stable_fingerprint= stable_since=0 fi fi if [ "${now}" -ge "${deadline}" ]; then echo "HelmReleases did not all become Ready within ${timeout_seconds}s" >&2 if [ -n "${not_ready}" ]; then printf '%s\n' "${not_ready}" | sed 's/^/ /' >&2 fi if ! kubectl get helmreleases.helm.toolkit.fluxcd.io -A --request-timeout=30s >&2; then echo "The final HelmRelease diagnostic list also failed" >&2 fi return 1 fi sleep "${poll_seconds}" done }