#!/bin/bash # Wait helpers shared by every backup walkthrough under examples/backups/. # Each walkthrough's 00-helpers.sh sources this file after defining the log_* # functions these call; wait_hr_ready and wait_deleted read $NAMESPACE. # hack/backup-examples-shared-helpers.bats fails if a walkthrough grows its own # copy again. # Wait until a JSONPath value on a resource matches the desired string. # # The optional 7th argument is a TERMINAL value to bail on: a BackupJob or # RestoreJob that reaches Failed never reaches Succeeded, so polling out the # remaining budget only delays the report, lets the TTL reaper take the Job's # Pod log, and under Chainsaw risks the step being SIGKILLed at its op timeout # instead of failing cleanly. # # kubectl's stderr is left attached: a read that fails for RBAC or transport # reasons says so instead of reading as an empty value. --ignore-not-found # keeps a resource that does not exist yet, the usual case at the start of a # wait, from printing an error on every poll. wait_for_field() { local resource_type="$1" local resource_name="$2" local jsonpath="$3" local desired="$4" local namespace="${5:-}" local timeout="${6:-300}" local fail_value="${7:-}" log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..." local elapsed=0 local ns_flag=() [[ -n "$namespace" ]] && ns_flag=(-n "$namespace") while true; do local current current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" --ignore-not-found -o jsonpath="$jsonpath" || true) if [[ "$current" == "$desired" ]]; then log_success "$resource_type/$resource_name reached '$desired'" return 0 fi if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')" return 1 fi if [[ $elapsed -ge $timeout ]]; then log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')" return 1 fi sleep 5 elapsed=$((elapsed + 5)) done } # Wait for a HelmRelease to become Ready, with an existence backstop (the apps # controller creates the HR asynchronously, so a bare `kubectl wait` right after # `kubectl apply` races it) and a fail-fast on Stalled=True: a stalled HR has # exhausted its remediation retries and will never turn Ready, so polling to the # timeout only hides the real error. wait_hr_ready() { local name="$1" local timeout="${2:-300}" local elapsed=0 local lookup state seen=0 log_substep "Waiting for HelmRelease/$name to become Ready..." while true; do # --ignore-not-found makes an absent release exit 0, so a non-zero exit # is kubectl failing to answer. Presence is read off the name line, # because a warning on stderr lands in the same string. if ! lookup=$(kubectl -n "$NAMESPACE" get hr "$name" --ignore-not-found -o name 2>&1); then state=error elif [[ $'\n'"$lookup"$'\n' != *"/$name"$'\n'* ]]; then state=absent else state=present seen=1 local ready stalled ready=$(kubectl -n "$NAMESPACE" get hr "$name" \ -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' || true) if [[ "$ready" == "True" ]]; then log_success "HelmRelease/$name is Ready" return 0 fi stalled=$(kubectl -n "$NAMESPACE" get hr "$name" \ -o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' || true) if [[ "$stalled" == "True" ]]; then log_error "HelmRelease/$name is Stalled (terminal):" break fi fi if [[ $elapsed -ge $timeout ]]; then log_error "Timeout waiting for HelmRelease/$name to become Ready:" case "$state" in error) echo " could not look up HelmRelease/$name: $lookup" >&2 # Seen earlier: the reads below may answer where this lookup # did not. [[ $seen -eq 1 ]] || return 1 ;; absent) if [[ $seen -eq 1 ]]; then echo " HelmRelease/$name was deleted from $NAMESPACE while waiting" >&2 else echo " HelmRelease/$name never appeared in $NAMESPACE" >&2 fi return 1 ;; esac break fi sleep 5 elapsed=$((elapsed + 5)) done # Reached on Stalled or on timeout. Conditions and history rather than the # Ready message alone: a failed install is retried on an interval, so Ready # may describe the retry rather than the failure behind it. kubectl -n "$NAMESPACE" get hr "$name" \ -o jsonpath='{range .status.conditions[*]} {.type}={.status} ({.reason}): {.message}{"\n"}{end}' >&2 || true local hist if ! hist=$(kubectl -n "$NAMESPACE" get hr "$name" \ -o jsonpath='{range .status.history[*]} history: {.status} {.chartVersion} {.lastDeployed}{"\n"}{end}'); then echo " history: kubectl could not read it" >&2 elif [[ -n "${hist//[[:space:]]/}" ]]; then printf '%s\n' "$hist" >&2 else # A release with no history prints nothing and exits 0; say so, # because a blank in a failure dump reads as a dump that broke. echo " history: (none recorded)" >&2 fi return 1 } # Wait until a namespaced resource is really gone. # # `kubectl wait --for=delete` is not used: it errors out when the resource is # already absent, which is the normal case on a clean namespace (cleanup.sh is # idempotent and runs as a pre-clean too), and swallowing that error would also # swallow a genuine failure. Polling `get --ignore-not-found` treats "already # gone" and "gone now" as the same success (empty output, exit 0) while a # retrieval failure — RBAC, API-server, transport — is a non-zero exit that # must NOT be read as deletion, or cleanup settles on a false success and leaves # half-uninstalled resources for the next run. Such an error just keeps us # polling until the resource is confirmed gone or the timeout fires. wait_deleted() { local resource_type="$1" local resource_name="$2" local timeout="${3:-300}" local elapsed=0 local got while true; do if got=$(kubectl -n "$NAMESPACE" get "$resource_type" "$resource_name" --ignore-not-found 2>/dev/null) && [[ -z "$got" ]]; then [[ $elapsed -gt 0 ]] && log_success "$resource_type/$resource_name is gone" return 0 fi if [[ $elapsed -ge $timeout ]]; then log_error "Timeout waiting for $resource_type/$resource_name to be deleted; it is still present after ${timeout}s:" kubectl -n "$NAMESPACE" get "$resource_type" "$resource_name" -o wide >&2 || true return 1 fi [[ $elapsed -eq 0 ]] && log_substep "Waiting for $resource_type/$resource_name to be deleted..." sleep 5 elapsed=$((elapsed + 5)) done }