admin/cozystack
1
0 Fork 0
mirror of https://github.com/cozystack/cozystack.git synced 2026-10-23 03:27:50 +09:00
Code Issues Packages Projects Releases Wiki Activity

refactor(backups): define the walkthrough wait helpers once

Browse Source 
Every backup walkthrough carried its own wait_for_field, and most their
own wait_hr_ready and wait_deleted: twelve, eight and two copies, which
had already drifted into five, three and two bodies. A fix made in one
copy did not reach the others, and a diagnostics change carried by hand
between two of them once went into the wrong function and was caught
only by reading the diff.

The helpers now live in examples/backups/_lib/wait-helpers.sh, sourced
by each walkthrough's 00-helpers.sh. Where copies differed, the shared
body keeps the diagnostics any of them had: wait_for_field leaves
kubectl's stderr attached, and wait_hr_ready prints the conditions and
history dump on Stalled or timeout. Call sites and budgets are unchanged.

A new unit test fails when any walkthrough defines one of these helpers
again, which replaces the byte-identity pin between copies.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
Aleksei Sviridkin committed 2026-10-01 02:21:01 +03:00
1 parent 5318b5e507
commit ead7048b35
15 files changed
+313 -1100
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

examples/backups/_lib/wait-helpers.sh
+165
View File
@@ -0,0 +1,165 @@
#!/bin/bash
# Wait helpers shared by every backup walkthrough under examples/backups/.
# Each walkthrough's 00-helpers.sh sources this file after defining the log_*
# functions these call; wait_hr_ready and wait_deleted read $NAMESPACE.
# hack/backup-examples-shared-helpers.bats fails if a walkthrough grows its own
# copy again.
# Wait until a JSONPath value on a resource matches the desired string.
#
# The optional 7th argument is a TERMINAL value to bail on: a BackupJob or
# RestoreJob that reaches Failed never reaches Succeeded, so polling out the
# remaining budget only delays the report, lets the TTL reaper take the Job's
# Pod log, and under Chainsaw risks the step being SIGKILLed at its op timeout
# instead of failing cleanly.
#
# kubectl's stderr is left attached: a read that fails for RBAC or transport
# reasons says so instead of reading as an empty value. --ignore-not-found
# keeps a resource that does not exist yet, the usual case at the start of a
# wait, from printing an error on every poll.
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
local fail_value="${7:-}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" --ignore-not-found -o jsonpath="$jsonpath" || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then
log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')"
return 1
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# Wait for a HelmRelease to become Ready, with an existence backstop (the apps
# controller creates the HR asynchronously, so a bare `kubectl wait` right after
# `kubectl apply` races it) and a fail-fast on Stalled=True: a stalled HR has
# exhausted its remediation retries and will never turn Ready, so polling to the
# timeout only hides the real error.
wait_hr_ready() {
local name="$1"
local timeout="${2:-300}"
local elapsed=0
local lookup state seen=0
log_substep "Waiting for HelmRelease/$name to become Ready..."
while true; do
# --ignore-not-found makes an absent release exit 0, so a non-zero exit
# is kubectl failing to answer. Presence is read off the name line,
# because a warning on stderr lands in the same string.
if ! lookup=$(kubectl -n "$NAMESPACE" get hr "$name" --ignore-not-found -o name 2>&1); then
state=error
elif [[ $'\n'"$lookup"$'\n' != *"/$name"$'\n'* ]]; then
state=absent
else
state=present
seen=1
local ready stalled
ready=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' || true)
if [[ "$ready" == "True" ]]; then
log_success "HelmRelease/$name is Ready"
return 0
fi
stalled=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' || true)
if [[ "$stalled" == "True" ]]; then
log_error "HelmRelease/$name is Stalled (terminal):"
break
fi
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for HelmRelease/$name to become Ready:"
case "$state" in
error)
echo " could not look up HelmRelease/$name: $lookup" >&2
# Seen earlier: the reads below may answer where this lookup
# did not.
[[ $seen -eq 1 ]] || return 1
;;
absent)
if [[ $seen -eq 1 ]]; then
echo " HelmRelease/$name was deleted from $NAMESPACE while waiting" >&2
else
echo " HelmRelease/$name never appeared in $NAMESPACE" >&2
fi
return 1
;;
esac
break
fi
sleep 5
elapsed=$((elapsed + 5))
done
# Reached on Stalled or on timeout. Conditions and history rather than the
# Ready message alone: a failed install is retried on an interval, so Ready
# may describe the retry rather than the failure behind it.
kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.conditions[*]} {.type}={.status} ({.reason}): {.message}{"\n"}{end}' >&2 || true
local hist
if ! hist=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.history[*]} history: {.status} {.chartVersion} {.lastDeployed}{"\n"}{end}'); then
echo " history: kubectl could not read it" >&2
elif [[ -n "${hist//[[:space:]]/}" ]]; then
printf '%s\n' "$hist" >&2
else
# A release with no history prints nothing and exits 0; say so,
# because a blank in a failure dump reads as a dump that broke.
echo " history: (none recorded)" >&2
fi
return 1
}
# Wait until a namespaced resource is really gone.
#
# `kubectl wait --for=delete` is not used: it errors out when the resource is
# already absent, which is the normal case on a clean namespace (cleanup.sh is
# idempotent and runs as a pre-clean too), and swallowing that error would also
# swallow a genuine failure. Polling `get --ignore-not-found` treats "already
# gone" and "gone now" as the same success (empty output, exit 0) while a
# retrieval failure — RBAC, API-server, transport — is a non-zero exit that
# must NOT be read as deletion, or cleanup settles on a false success and leaves
# half-uninstalled resources for the next run. Such an error just keeps us
# polling until the resource is confirmed gone or the timeout fires.
wait_deleted() {
local resource_type="$1"
local resource_name="$2"
local timeout="${3:-300}"
local elapsed=0
local got
while true; do
if got=$(kubectl -n "$NAMESPACE" get "$resource_type" "$resource_name" --ignore-not-found 2>/dev/null) && [[ -z "$got" ]]; then
[[ $elapsed -gt 0 ]] && log_success "$resource_type/$resource_name is gone"
return 0
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name to be deleted; it is still present after ${timeout}s:"
kubectl -n "$NAMESPACE" get "$resource_type" "$resource_name" -o wide >&2 || true
return 1
fi
[[ $elapsed -eq 0 ]] && log_substep "Waiting for $resource_type/$resource_name to be deleted..."
sleep 5
elapsed=$((elapsed + 5))
done
}
examples/backups/clickhouse/00-helpers.sh
+4 -156
View File
@@ -79,128 +79,10 @@ print_header() {
echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2
}
# Wait until a JSONPath value on a resource matches the desired string.
#
# The optional 7th argument is a TERMINAL value to bail on: a BackupJob or
# RestoreJob that reaches Failed will never reach Succeeded, so polling out the
# remaining budget only delays the report and — when this runs under Chainsaw —
# risks the step being SIGKILLed at its op timeout with nothing collected
# instead of failing cleanly. Same contract as the postgres/mariadb helpers.
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
local fail_value="${7:-}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then
log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')"
return 1
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# Wait for a HelmRelease to become Ready, failing fast on Stalled=True.
#
# `kubectl wait hr ... --for=condition=ready` cannot do either half of this: it
# errors out if the HelmRelease does not exist yet (the app CR is reconciled
# asynchronously, so the HR appears a moment after `kubectl apply` returns), and
# it keeps waiting through a terminal Stalled — a chart that will never install
# burns the whole timeout instead of reporting the reason. Same contract as the
# postgres/mariadb helpers.
wait_hr_ready() {
local name="$1"
local timeout="${2:-300}"
local elapsed=0
local lookup state seen=0
log_substep "Waiting for HelmRelease/$name to become Ready..."
while true; do
# --ignore-not-found makes an absent release exit 0, so a non-zero exit
# is kubectl failing to answer. Presence is read off the name line,
# because a warning on stderr lands in the same string.
if ! lookup=$(kubectl -n "$NAMESPACE" get hr "$name" --ignore-not-found -o name 2>&1); then
state=error
elif [[ $'\n'"$lookup"$'\n' != *"/$name"$'\n'* ]]; then
state=absent
else
state=present
seen=1
local ready stalled
ready=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' || true)
if [[ "$ready" == "True" ]]; then
log_success "HelmRelease/$name is Ready"
return 0
fi
stalled=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' || true)
if [[ "$stalled" == "True" ]]; then
log_error "HelmRelease/$name is Stalled (terminal):"
break
fi
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for HelmRelease/$name to become Ready:"
case "$state" in
error)
echo " could not look up HelmRelease/$name: $lookup" >&2
# Seen earlier: the reads below may answer where this lookup
# did not.
[[ $seen -eq 1 ]] || return 1
;;
absent)
if [[ $seen -eq 1 ]]; then
echo " HelmRelease/$name was deleted from $NAMESPACE while waiting" >&2
else
echo " HelmRelease/$name never appeared in $NAMESPACE" >&2
fi
return 1
;;
esac
break
fi
sleep 5
elapsed=$((elapsed + 5))
done
# Reached on Stalled or on timeout. Conditions and history rather than the
# Ready message alone: a failed install is retried on an interval, so Ready
# may describe the retry rather than the failure behind it.
kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.conditions[*]} {.type}={.status} ({.reason}): {.message}{"\n"}{end}' >&2 || true
local hist
if ! hist=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.history[*]} history: {.status} {.chartVersion} {.lastDeployed}{"\n"}{end}'); then
echo " history: kubectl could not read it" >&2
elif [[ -n "${hist//[[:space:]]/}" ]]; then
printf '%s\n' "$hist" >&2
else
# A release with no history prints nothing and exits 0; say so,
# because a blank in a failure dump reads as a dump that broke.
echo " history: (none recorded)" >&2
fi
return 1
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# Wait for a StatefulSet to report a ready replica, tolerating its asynchronous
# creation (the ClickHouse operator creates it after the CHI is reconciled).
@@ -237,40 +119,6 @@ wait_sts_ready() {
done
}
# Wait until a namespaced resource is really gone.
#
# `kubectl wait --for=delete` is not used: it errors out when the resource is
# already absent, which is the normal case on a clean namespace (cleanup.sh is
# idempotent and runs as a pre-clean too), and swallowing that error would also
# swallow a genuine failure. Polling `get --ignore-not-found` treats "already
# gone" and "gone now" as the same success (empty output, exit 0) while a
# retrieval failure — RBAC, API-server, transport — is a non-zero exit that
# must NOT be read as deletion, or cleanup settles on a false success and leaves
# half-uninstalled resources for the next run. Such an error just keeps us
# polling until the resource is confirmed gone or the timeout fires.
wait_deleted() {
local resource_type="$1"
local resource_name="$2"
local timeout="${3:-300}"
local elapsed=0
local got
while true; do
if got=$(kubectl -n "$NAMESPACE" get "$resource_type" "$resource_name" --ignore-not-found 2>/dev/null) && [[ -z "$got" ]]; then
[[ $elapsed -gt 0 ]] && log_success "$resource_type/$resource_name is gone"
return 0
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name to be deleted; it is still present after ${timeout}s:"
kubectl -n "$NAMESPACE" get "$resource_type" "$resource_name" -o wide >&2 || true
return 1
fi
[[ $elapsed -eq 0 ]] && log_substep "Waiting for $resource_type/$resource_name to be deleted..."
sleep 5
elapsed=$((elapsed + 5))
done
}
# Run a SQL statement against the ClickHouse cluster of the given app instance.
# Args: <release-name> <sql>
# Note: the Cozystack ClickHouse RD prefixes Helm release names with
examples/backups/etcd/00-helpers.sh
+4 -40
View File
@@ -43,46 +43,10 @@ print_header() {
echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2
}
# Wait until a JSONPath value on a resource matches the desired string.
# Optional 7th arg is a TERMINAL failure value: once the field reaches it the
# wait returns 1 immediately instead of polling to the timeout. BackupJob and
# RestoreJob settle on a terminal phase=Failed that never becomes Succeeded, so
# waiting the full timeout only wastes wall-clock AND risks the snapshot Job's
# Pod being TTL-reaped (TTLSecondsAfterFinished=600s) before diagnostics can read
# its log. Failing fast on the terminal phase keeps that log in reach.
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
local fail_value="${7:-}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then
log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')"
return 1
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# Ephemeral etcdctl pod name. Created on first etcdctl_exec call; reused
# for subsequent calls in the same script run. cleanup.sh removes it.
examples/backups/foundationdb/00-helpers.sh
+4 -29
View File
@@ -47,35 +47,10 @@ print_header() {
echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2
}
# Wait until a JSONPath value on a resource matches the desired string.
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# Run an fdbcli statement against a FoundationDB cluster of the given app
# instance. Args: <app-name> <fdbcli-script>
examples/backups/kafka-metadata/00-helpers.sh
+4 -59
View File
@@ -134,65 +134,10 @@ provision_demo_strategy() {
kubectl apply -f "$SCRIPT_DIR/03-backupclass.yaml" >&2
}
# Wait until a JSONPath value on a resource matches the desired string. Optional
# 7th arg is a TERMINAL failure value: once the field reaches it the wait returns
# 1 immediately instead of polling to the timeout.
wait_for_field() {
local resource_type="$1" resource_name="$2" jsonpath="$3" desired="$4"
local namespace="${5:-}" timeout="${6:-300}" fail_value="${7:-}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then
log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')"
return 1
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# Wait for a HelmRelease to become Ready, with an existence backstop and a
# fail-fast on Stalled=True.
wait_hr_ready() {
local name="$1" timeout="${2:-300}" elapsed=0
log_substep "Waiting for HelmRelease/$name to become Ready..."
while true; do
if kubectl -n "$NAMESPACE" get hr "$name" >/dev/null 2>&1; then
local ready stalled
ready=$(kubectl -n "$NAMESPACE" get hr "$name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
if [[ "$ready" == "True" ]]; then
log_success "HelmRelease/$name is Ready"
return 0
fi
stalled=$(kubectl -n "$NAMESPACE" get hr "$name" -o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' 2>/dev/null || true)
if [[ "$stalled" == "True" ]]; then
log_error "HelmRelease/$name is Stalled: $(kubectl -n "$NAMESPACE" get hr "$name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null)"
return 1
fi
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for HelmRelease/$name to become Ready"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# Wait until the Strimzi Kafka cluster kafka-<app> reports Ready=True — the same
# precondition the driver gates on.
examples/backups/kafka/00-helpers.sh
+4 -152
View File
@@ -119,158 +119,10 @@ print_header() {
echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2
}
# Wait until a JSONPath value on a resource matches the desired string.
# Optional 7th arg is a TERMINAL failure value: once the field reaches it the
# wait returns 1 immediately instead of polling to the timeout. BackupJob and
# RestoreJob settle on a terminal phase=Failed that never becomes Succeeded (the
# strategy Job exhausts its backoffLimit), so failing fast on it keeps
# wall-clock - and the strategy Pod's log - in reach instead of burning the full
# timeout and then mislabelling a deterministic failure as "Timeout".
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
local fail_value="${7:-}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then
log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')"
return 1
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# Wait for a HelmRelease to become Ready, with an existence backstop (the apps
# controller creates the HR asynchronously, so a bare `kubectl wait` right after
# `kubectl apply` races it) and a fail-fast on Stalled=True - a stalled HR has
# exhausted its remediation retries and will never turn Ready, so polling to the
# timeout only hides the real error.
wait_hr_ready() {
local name="$1"
local timeout="${2:-300}"
local elapsed=0
local lookup state seen=0
log_substep "Waiting for HelmRelease/$name to become Ready..."
while true; do
# --ignore-not-found makes an absent release exit 0, so a non-zero exit
# is kubectl failing to answer. Presence is read off the name line,
# because a warning on stderr lands in the same string.
if ! lookup=$(kubectl -n "$NAMESPACE" get hr "$name" --ignore-not-found -o name 2>&1); then
state=error
elif [[ $'\n'"$lookup"$'\n' != *"/$name"$'\n'* ]]; then
state=absent
else
state=present
seen=1
local ready stalled
ready=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' || true)
if [[ "$ready" == "True" ]]; then
log_success "HelmRelease/$name is Ready"
return 0
fi
stalled=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' || true)
if [[ "$stalled" == "True" ]]; then
log_error "HelmRelease/$name is Stalled (terminal):"
break
fi
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for HelmRelease/$name to become Ready:"
case "$state" in
error)
echo " could not look up HelmRelease/$name: $lookup" >&2
# Seen earlier: the reads below may answer where this lookup
# did not.
[[ $seen -eq 1 ]] || return 1
;;
absent)
if [[ $seen -eq 1 ]]; then
echo " HelmRelease/$name was deleted from $NAMESPACE while waiting" >&2
else
echo " HelmRelease/$name never appeared in $NAMESPACE" >&2
fi
return 1
;;
esac
break
fi
sleep 5
elapsed=$((elapsed + 5))
done
# Reached on Stalled or on timeout. Conditions and history rather than the
# Ready message alone: a failed install is retried on an interval, so Ready
# may describe the retry rather than the failure behind it.
kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.conditions[*]} {.type}={.status} ({.reason}): {.message}{"\n"}{end}' >&2 || true
local hist
if ! hist=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.history[*]} history: {.status} {.chartVersion} {.lastDeployed}{"\n"}{end}'); then
echo " history: kubectl could not read it" >&2
elif [[ -n "${hist//[[:space:]]/}" ]]; then
printf '%s\n' "$hist" >&2
else
# A release with no history prints nothing and exits 0; say so,
# because a blank in a failure dump reads as a dump that broke.
echo " history: (none recorded)" >&2
fi
return 1
}
# Wait until a namespaced resource is really gone.
#
# `kubectl wait --for=delete` is not used: it errors when the resource is
# already absent, the normal case on a clean namespace (cleanup.sh is
# idempotent and runs as a pre-clean too), and swallowing that error would also
# swallow a genuine failure. Polling `get --ignore-not-found` treats "already
# gone" and "gone now" alike (empty output, exit 0), while a retrieval failure -
# RBAC, API-server, transport - is a non-zero exit that must NOT be read as
# deletion, or the teardown settles on a false success and leaves
# half-uninstalled resources for the next run.
wait_deleted() {
local resource_type="$1"
local resource_name="$2"
local timeout="${3:-300}"
local elapsed=0
local got
while true; do
if got=$(kubectl -n "$NAMESPACE" get "$resource_type" "$resource_name" --ignore-not-found 2>/dev/null) && [[ -z "$got" ]]; then
[[ $elapsed -gt 0 ]] && log_success "$resource_type/$resource_name is gone"
return 0
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name to be deleted; still present after ${timeout}s:"
kubectl -n "$NAMESPACE" get "$resource_type" "$resource_name" -o wide >&2 || true
return 1
fi
[[ $elapsed -eq 0 ]] && log_substep "Waiting for $resource_type/$resource_name to be deleted..."
sleep 5
elapsed=$((elapsed + 5))
done
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# In-cluster bootstrap address for a Kafka application instance. The Cozystack
# chart names the Strimzi Kafka cluster "kafka-<app>", and Strimzi names the
examples/backups/mariadb/00-helpers.sh
+4 -118
View File
@@ -60,124 +60,10 @@ print_header() {
echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2
}
# Wait until a JSONPath value on a resource matches the desired string.
# Optional 7th arg is a TERMINAL failure value: once the field reaches it the
# wait returns 1 immediately instead of polling to the timeout. BackupJob and
# RestoreJob settle on a terminal phase=Failed that never becomes Succeeded, so
# failing fast on it keeps wall-clock (and the operator Job's log, before its
# TTL reaper fires) in reach.
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
local fail_value="${7:-}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then
log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')"
return 1
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# Wait for a HelmRelease to become Ready, with an existence backstop (the
# apps controller creates the HR asynchronously, so a bare `kubectl wait`
# right after `kubectl apply` races it) and a fail-fast on Stalled=True —
# a stalled HR has exhausted its remediation retries and will never turn
# Ready, so polling to the timeout only hides the real error.
wait_hr_ready() {
local name="$1"
local timeout="${2:-300}"
local elapsed=0
local lookup state seen=0
log_substep "Waiting for HelmRelease/$name to become Ready..."
while true; do
# --ignore-not-found makes an absent release exit 0, so a non-zero exit
# is kubectl failing to answer. Presence is read off the name line,
# because a warning on stderr lands in the same string.
if ! lookup=$(kubectl -n "$NAMESPACE" get hr "$name" --ignore-not-found -o name 2>&1); then
state=error
elif [[ $'\n'"$lookup"$'\n' != *"/$name"$'\n'* ]]; then
state=absent
else
state=present
seen=1
local ready stalled
ready=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' || true)
if [[ "$ready" == "True" ]]; then
log_success "HelmRelease/$name is Ready"
return 0
fi
stalled=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' || true)
if [[ "$stalled" == "True" ]]; then
log_error "HelmRelease/$name is Stalled (terminal):"
break
fi
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for HelmRelease/$name to become Ready:"
case "$state" in
error)
echo " could not look up HelmRelease/$name: $lookup" >&2
# Seen earlier: the reads below may answer where this lookup
# did not.
[[ $seen -eq 1 ]] || return 1
;;
absent)
if [[ $seen -eq 1 ]]; then
echo " HelmRelease/$name was deleted from $NAMESPACE while waiting" >&2
else
echo " HelmRelease/$name never appeared in $NAMESPACE" >&2
fi
return 1
;;
esac
break
fi
sleep 5
elapsed=$((elapsed + 5))
done
# Reached on Stalled or on timeout. Conditions and history rather than the
# Ready message alone: a failed install is retried on an interval, so Ready
# may describe the retry rather than the failure behind it.
kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.conditions[*]} {.type}={.status} ({.reason}): {.message}{"\n"}{end}' >&2 || true
local hist
if ! hist=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.history[*]} history: {.status} {.chartVersion} {.lastDeployed}{"\n"}{end}'); then
echo " history: kubectl could not read it" >&2
elif [[ -n "${hist//[[:space:]]/}" ]]; then
printf '%s\n' "$hist" >&2
else
# A release with no history prints nothing and exits 0; say so,
# because a blank in a failure dump reads as a dump that broke.
echo " history: (none recorded)" >&2
fi
return 1
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# Name of the primary pod of a k8s.mariadb.com/MariaDB CR (the writer).
# mariadb-operator 25.10.x reports the primary identity on
examples/backups/mongodb/00-helpers.sh
+4 -118
View File
@@ -43,124 +43,10 @@ print_header() {
echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2
}
# Wait until a JSONPath value on a resource matches the desired string.
# Optional 7th arg is a TERMINAL failure value: once the field reaches it the
# wait returns 1 immediately instead of polling to the timeout. BackupJob and
# RestoreJob settle on a terminal phase=Failed that never becomes Succeeded, so
# failing fast on it keeps wall-clock (and the operator's pbm log, before its
# TTL reaper fires) in reach.
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
local fail_value="${7:-}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then
log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')"
return 1
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# Wait for a HelmRelease to become Ready, with an existence backstop (the apps
# controller creates the HR asynchronously, so a bare `kubectl wait` right after
# `kubectl apply` races it) and a fail-fast on Stalled=True — a stalled HR has
# exhausted its remediation retries and will never turn Ready, so polling to the
# timeout only hides the real error.
wait_hr_ready() {
local name="$1"
local timeout="${2:-300}"
local elapsed=0
local lookup state seen=0
log_substep "Waiting for HelmRelease/$name to become Ready..."
while true; do
# --ignore-not-found makes an absent release exit 0, so a non-zero exit
# is kubectl failing to answer. Presence is read off the name line,
# because a warning on stderr lands in the same string.
if ! lookup=$(kubectl -n "$NAMESPACE" get hr "$name" --ignore-not-found -o name 2>&1); then
state=error
elif [[ $'\n'"$lookup"$'\n' != *"/$name"$'\n'* ]]; then
state=absent
else
state=present
seen=1
local ready stalled
ready=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' || true)
if [[ "$ready" == "True" ]]; then
log_success "HelmRelease/$name is Ready"
return 0
fi
stalled=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' || true)
if [[ "$stalled" == "True" ]]; then
log_error "HelmRelease/$name is Stalled (terminal):"
break
fi
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for HelmRelease/$name to become Ready:"
case "$state" in
error)
echo " could not look up HelmRelease/$name: $lookup" >&2
# Seen earlier: the reads below may answer where this lookup
# did not.
[[ $seen -eq 1 ]] || return 1
;;
absent)
if [[ $seen -eq 1 ]]; then
echo " HelmRelease/$name was deleted from $NAMESPACE while waiting" >&2
else
echo " HelmRelease/$name never appeared in $NAMESPACE" >&2
fi
return 1
;;
esac
break
fi
sleep 5
elapsed=$((elapsed + 5))
done
# Reached on Stalled or on timeout. Conditions and history rather than the
# Ready message alone: a failed install is retried on an interval, so Ready
# may describe the retry rather than the failure behind it.
kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.conditions[*]} {.type}={.status} ({.reason}): {.message}{"\n"}{end}' >&2 || true
local hist
if ! hist=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.history[*]} history: {.status} {.chartVersion} {.lastDeployed}{"\n"}{end}'); then
echo " history: kubectl could not read it" >&2
elif [[ -n "${hist//[[:space:]]/}" ]]; then
printf '%s\n' "$hist" >&2
else
# A release with no history prints nothing and exits 0; say so,
# because a blank in a failure dump reads as a dump that broke.
echo " history: (none recorded)" >&2
fi
return 1
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# Wait until the psmdb.percona.com/PerconaServerMongoDB CR reports state=ready
# (the operator's own readiness signal — the rs0 members are up and initialised
examples/backups/nats/00-helpers.sh
+4 -29
View File
@@ -50,35 +50,10 @@ print_header() {
echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2
}
# Wait until a JSONPath value on a resource matches the desired string.
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# In-cluster NATS client URL for the given application instance. The NATS app
# names its client Service after the application (fullnameOverride=<name>) and
examples/backups/postgres/00-helpers.sh
+4 -118
View File
@@ -58,124 +58,10 @@ print_header() {
echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2
}
# Wait until a JSONPath value on a resource matches the desired string.
# Optional 7th arg is a TERMINAL failure value: once the field reaches it the
# wait returns 1 immediately instead of polling to the timeout. BackupJob and
# RestoreJob settle on a terminal phase=Failed that never becomes Succeeded, so
# failing fast on it keeps wall-clock (and the snapshot Pod's log, before its
# TTL reaper fires) in reach.
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
local fail_value="${7:-}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then
log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')"
return 1
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# Wait for a HelmRelease to become Ready, with an existence backstop (the
# apps controller creates the HR asynchronously, so a bare `kubectl wait`
# right after `kubectl apply` races it) and a fail-fast on Stalled=True —
# a stalled HR has exhausted its remediation retries and will never turn
# Ready, so polling to the timeout only hides the real error.
wait_hr_ready() {
local name="$1"
local timeout="${2:-300}"
local elapsed=0
local lookup state seen=0
log_substep "Waiting for HelmRelease/$name to become Ready..."
while true; do
# --ignore-not-found makes an absent release exit 0, so a non-zero exit
# is kubectl failing to answer. Presence is read off the name line,
# because a warning on stderr lands in the same string.
if ! lookup=$(kubectl -n "$NAMESPACE" get hr "$name" --ignore-not-found -o name 2>&1); then
state=error
elif [[ $'\n'"$lookup"$'\n' != *"/$name"$'\n'* ]]; then
state=absent
else
state=present
seen=1
local ready stalled
ready=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' || true)
if [[ "$ready" == "True" ]]; then
log_success "HelmRelease/$name is Ready"
return 0
fi
stalled=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' || true)
if [[ "$stalled" == "True" ]]; then
log_error "HelmRelease/$name is Stalled (terminal):"
break
fi
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for HelmRelease/$name to become Ready:"
case "$state" in
error)
echo " could not look up HelmRelease/$name: $lookup" >&2
# Seen earlier: the reads below may answer where this lookup
# did not.
[[ $seen -eq 1 ]] || return 1
;;
absent)
if [[ $seen -eq 1 ]]; then
echo " HelmRelease/$name was deleted from $NAMESPACE while waiting" >&2
else
echo " HelmRelease/$name never appeared in $NAMESPACE" >&2
fi
return 1
;;
esac
break
fi
sleep 5
elapsed=$((elapsed + 5))
done
# Reached on Stalled or on timeout. Conditions and history rather than the
# Ready message alone: a failed install is retried on an interval, so Ready
# may describe the retry rather than the failure behind it.
kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.conditions[*]} {.type}={.status} ({.reason}): {.message}{"\n"}{end}' >&2 || true
local hist
if ! hist=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.history[*]} history: {.status} {.chartVersion} {.lastDeployed}{"\n"}{end}'); then
echo " history: kubectl could not read it" >&2
elif [[ -n "${hist//[[:space:]]/}" ]]; then
printf '%s\n' "$hist" >&2
else
# A release with no history prints nothing and exits 0; say so,
# because a blank in a failure dump reads as a dump that broke.
echo " history: (none recorded)" >&2
fi
return 1
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# Name of the primary pod of a cnpg.io Cluster (the one accepting writes).
cnpg_primary_pod() {
examples/backups/rabbitmq/00-helpers.sh
+4 -114
View File
@@ -64,120 +64,10 @@ print_header() {
echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2
}
# Wait until a JSONPath value on a resource matches the desired string.
# Optional 7th arg is a TERMINAL failure value: once the field reaches it the
# wait returns 1 immediately instead of polling to the timeout. BackupJob and
# RestoreJob settle on a terminal phase=Failed that never becomes Succeeded, so
# failing fast on it keeps wall-clock in reach.
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
local fail_value="${7:-}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then
log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')"
return 1
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
# Wait for a HelmRelease to become Ready, with an existence backstop (the apps
# controller creates the HR asynchronously) and a fail-fast on Stalled=True.
wait_hr_ready() {
local name="$1"
local timeout="${2:-300}"
local elapsed=0
local lookup state seen=0
log_substep "Waiting for HelmRelease/$name to become Ready..."
while true; do
# --ignore-not-found makes an absent release exit 0, so a non-zero exit
# is kubectl failing to answer. Presence is read off the name line,
# because a warning on stderr lands in the same string.
if ! lookup=$(kubectl -n "$NAMESPACE" get hr "$name" --ignore-not-found -o name 2>&1); then
state=error
elif [[ $'\n'"$lookup"$'\n' != *"/$name"$'\n'* ]]; then
state=absent
else
state=present
seen=1
local ready stalled
ready=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' || true)
if [[ "$ready" == "True" ]]; then
log_success "HelmRelease/$name is Ready"
return 0
fi
stalled=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' || true)
if [[ "$stalled" == "True" ]]; then
log_error "HelmRelease/$name is Stalled (terminal):"
break
fi
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for HelmRelease/$name to become Ready:"
case "$state" in
error)
echo " could not look up HelmRelease/$name: $lookup" >&2
# Seen earlier: the reads below may answer where this lookup
# did not.
[[ $seen -eq 1 ]] || return 1
;;
absent)
if [[ $seen -eq 1 ]]; then
echo " HelmRelease/$name was deleted from $NAMESPACE while waiting" >&2
else
echo " HelmRelease/$name never appeared in $NAMESPACE" >&2
fi
return 1
;;
esac
break
fi
sleep 5
elapsed=$((elapsed + 5))
done
# Reached on Stalled or on timeout. Conditions and history rather than the
# Ready message alone: a failed install is retried on an interval, so Ready
# may describe the retry rather than the failure behind it.
kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.conditions[*]} {.type}={.status} ({.reason}): {.message}{"\n"}{end}' >&2 || true
local hist
if ! hist=$(kubectl -n "$NAMESPACE" get hr "$name" \
-o jsonpath='{range .status.history[*]} history: {.status} {.chartVersion} {.lastDeployed}{"\n"}{end}'); then
echo " history: kubectl could not read it" >&2
elif [[ -n "${hist//[[:space:]]/}" ]]; then
printf '%s\n' "$hist" >&2
else
# A release with no history prints nothing and exits 0; say so,
# because a blank in a failure dump reads as a dump that broke.
echo " history: (none recorded)" >&2
fi
return 1
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# Wait until the rabbitmq.com/RabbitmqCluster CR reports AllReplicasReady=True
# (the cluster-operator's own readiness signal — every server pod is up and the
examples/backups/redis/00-helpers.sh
+4 -40
View File
@@ -64,46 +64,10 @@ print_header() {
echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2
}
# Wait until a JSONPath value on a resource matches the desired string. An
# optional 7th arg is a TERMINAL failure value: once the field reaches it the
# wait returns 1 immediately instead of polling to the timeout (a BackupJob /
# RestoreJob settles on phase=Failed that never becomes Succeeded).
wait_for_field() {
local resource_type="$1" resource_name="$2" jsonpath="$3" desired="$4"
local namespace="${5:-}" timeout="${6:-300}" fail_value="${7:-}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0 ns_flag=()
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
while true; do
local current
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" || true)
[[ "$current" == "$desired" ]] && { log_success "$resource_type/$resource_name reached '$desired'"; return 0; }
[[ -n "$fail_value" && "$current" == "$fail_value" ]] && { log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')"; return 1; }
(( elapsed >= timeout )) && { log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"; return 1; }
sleep 5
elapsed=$((elapsed + 5))
done
}
# Wait for a HelmRelease to become Ready, with an existence backstop (the apps
# controller creates the HR asynchronously) and a fail-fast on Stalled=True.
wait_hr_ready() {
local name="$1" timeout="${2:-300}" elapsed=0
log_substep "Waiting for HelmRelease/$name to become Ready..."
while true; do
if kubectl -n "$NAMESPACE" get hr "$name" >/dev/null 2>&1; then
local ready stalled
ready=$(kubectl -n "$NAMESPACE" get hr "$name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' || true)
[[ "$ready" == "True" ]] && { log_success "HelmRelease/$name is Ready"; return 0; }
stalled=$(kubectl -n "$NAMESPACE" get hr "$name" -o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' || true)
[[ "$stalled" == "True" ]] && { log_error "HelmRelease/$name is Stalled (terminal)"; return 1; }
fi
(( elapsed >= timeout )) && { log_error "Timeout waiting for HelmRelease/$name to become Ready"; return 1; }
sleep 5
elapsed=$((elapsed + 5))
done
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# The cozystack redis chart names its RedisFailover (hence the operator's rfr-/
# rfs- Services and the -auth Secret) redis-<app>, mirroring the strategy driver
examples/backups/vmi/00-helpers.sh
+4 -32
View File
@@ -65,38 +65,10 @@ resource_exists() {
fi
}
# Wait for a resource field to reach a desired value
wait_for_field() {
local resource_type="$1"
local resource_name="$2"
local jsonpath="$3"
local desired="$4"
local namespace="${5:-}"
local timeout="${6:-300}"
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
local elapsed=0
local ns_flag=""
[[ -n "$namespace" ]] && ns_flag="-n $namespace"
while true; do
local current
# shellcheck disable=SC2086
current=$(kubectl get "$resource_type" "$resource_name" $ns_flag -o jsonpath="$jsonpath" 2>/dev/null || true)
if [[ "$current" == "$desired" ]]; then
log_success "$resource_type/$resource_name reached '$desired'"
return 0
fi
if [[ $elapsed -ge $timeout ]]; then
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
echo -n "." >&2
done
}
# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by
# every backup walkthrough, so a fix to one reaches all of them.
# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh
source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh"
# Print a separator line
separator() {
hack/backup-examples-shared-helpers.bats
+93
View File
@@ -0,0 +1,93 @@
#!/usr/bin/env bats
# Contract for the wait helpers the backup walkthroughs share: each is defined
# once, in examples/backups/_lib/wait-helpers.sh, so a fix made there reaches
# every walkthrough; each walkthrough's 00-helpers.sh provides them; and
# wait_for_field reports why kubectl could not read a field while staying quiet
# about a resource that does not exist yet.
SHARED=examples/backups/_lib/wait-helpers.sh
SHARED_HELPERS="wait_for_field wait_hr_ready wait_deleted"
@test "each shared wait helper is defined once, in the shared file" {
fail=
for fn in $SHARED_HELPERS; do
definers=$(grep -rlE --include='*.sh' "^[[:space:]]*(function[[:space:]]+)?${fn}[[:space:]]*\(\)" examples packages | sort)
if [ "$definers" != "$SHARED" ]; then
fail="$fail
${fn} is defined in:
${definers:- (nowhere)}"
fi
done
[ -z "$fail" ] || {
echo "a walkthrough carries its own copy of a shared helper; use the one in $SHARED:$fail" >&2
exit 1
}
}
@test "every walkthrough's helpers provide the shared wait helpers" {
# Some helpers read the cluster while being sourced (kafka resolves its
# image from the operator); a stub keeps this test off any real cluster.
stub=$(mktemp -d)
printf '#!/bin/sh\nexit 1\n' > "$stub/kubectl"
chmod +x "$stub/kubectl"
for helpers in examples/backups/*/00-helpers.sh; do
PATH="$stub:$PATH" bash -c '
set -euo pipefail
. "$1"
for fn in $2; do
declare -F "$fn" >/dev/null || { echo "$1 does not provide $fn" >&2; exit 1; }
done
' _ "$helpers" "$SHARED_HELPERS"
done
}
@test "wait_for_field shows why kubectl could not read the field" {
# A Forbidden read has to reach the caller; swallowed, it looks like a
# field that is still empty.
stub=$(mktemp -d)
printf '#!/bin/sh\necho "Error from server (Forbidden): backupjobs is forbidden" >&2\nexit 1\n' > "$stub/kubectl"
chmod +x "$stub/kubectl"
err=$(mktemp)
if PATH="$stub:$PATH" bash -c '
set -euo pipefail
. examples/backups/postgres/00-helpers.sh
wait_for_field backupjob demo "{.status.phase}" Succeeded tenant-test 0
' 2>"$err"; then
echo "wait_for_field succeeded against a kubectl that cannot read" >&2
exit 1
fi
grep -q 'backupjobs is forbidden' "$err" || {
echo "wait_for_field dropped kubectl's reason:" >&2
cat "$err" >&2
exit 1
}
}
@test "wait_for_field stays quiet while the resource does not exist yet" {
# The stub answers like kubectl: NotFound on stderr and exit 1, unless the
# read asks for --ignore-not-found, which makes an absent resource empty.
stub=$(mktemp -d)
cat > "$stub/kubectl" <<'EOF'
#!/bin/sh
case " $* " in
*" --ignore-not-found "*) exit 0 ;;
esac
echo 'Error from server (NotFound): backupjobs.backups.cozystack.io "demo" not found' >&2
exit 1
EOF
chmod +x "$stub/kubectl"
err=$(mktemp)
if PATH="$stub:$PATH" bash -c '
set -euo pipefail
. examples/backups/postgres/00-helpers.sh
wait_for_field backupjob demo "{.status.phase}" Succeeded tenant-test 0
' 2>"$err"; then
echo "wait_for_field succeeded against a resource that does not exist" >&2
exit 1
fi
if grep -q 'NotFound' "$err"; then
echo "wait_for_field printed kubectl's NotFound for a resource it is waiting on:" >&2
cat "$err" >&2
exit 1
fi
}
hack/mariadb-first-boot-waits.bats
+7 -95
View File
@@ -38,17 +38,10 @@
# wait would sum ceilings the script reaches only on the happy path. Both sit at
# exactly their constant, so raising a wait without raising the op reds.
#
# Two tests reach beyond mariadb: several backup examples share one
# wait_hr_ready body, and those tests run its exit branches in every copy and
# hold the copies byte-identical. The population is every definition in the
# tree except the named exceptions below (mbw_hr_copies), and each exception is
# held to still differ, so it goes red once its copy converges:
# examples/backups/redis/00-helpers.sh -- written in the compact one-line
# style of the rest of that file, so converging it rewrites its shape
# rather than applying the shared body.
# examples/backups/kafka-metadata/00-helpers.sh -- a helper of its own rather
# than a copy, with the same diagnostic gap on timeout; closing that is a
# change to kafka-metadata, not to this population.
# The errexit test reaches beyond mariadb: wait_hr_ready is defined once, in
# examples/backups/_lib/wait-helpers.sh, for every backup example, and that
# test runs its exit branches through the helpers run-all.sh sources.
# hack/backup-examples-shared-helpers.bats holds it to that one definition.
#
# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its
# own line; there is no bats `run` or `$status`, and no setup/teardown.
@@ -90,33 +83,6 @@ mbw_mariadb_waits() {
}'
}
# wait_hr_ready definitions deliberately left out of the byte-identity pin; the
# header says why for each.
MBW_HR_EXCEPTIONS="examples/backups/kafka-metadata/00-helpers.sh examples/backups/redis/00-helpers.sh"
# Print, sorted, every shell file under examples/ and packages/ that defines
# wait_hr_ready, exceptions included. Found rather than listed, so a new copy
# joins the identity pin without anyone remembering to add it.
mbw_hr_definers() {
grep -rlE --include='*.sh' '^wait_hr_ready\(\)' examples packages | sort
}
# Print the definers minus the named exceptions: the population pinned
# byte-identical to the mariadb copy.
mbw_hr_copies() {
for mbw_f in $(mbw_hr_definers); do
case " $MBW_HR_EXCEPTIONS " in
*" $mbw_f "*) ;;
*) printf '%s\n' "$mbw_f" ;;
esac
done
}
# Print the wait_hr_ready body defined in $1.
mbw_hr_body() {
sed -n '/^wait_hr_ready()/,/^}/p' "$1"
}
# Print every release argument wait_hr_ready is called with in $1, mariadb or
# not. Used to show the extractor above is selecting rather than matching
# everything.
@@ -565,15 +531,9 @@ MBW_FIXTURE
# stub that returns the same string to every call makes the conditions dump
# emit it too, and an assertion on it would then pass with the branch it
# names deleted.
# Every copy, not just the mariadb one: the identity check below compares
# text, and only a run shows that each copy still behaves under errexit.
mbw_copies=$(mbw_hr_copies)
printf '%s\n' "$mbw_copies" | grep -qx 'examples/backups/mariadb/00-helpers.sh' || {
echo "the wait_hr_ready population no longer contains the mariadb copy:" >&2
printf '%s\n' "$mbw_copies" >&2
exit 1
}
for mbw_helper in $mbw_copies; do
# Sourced the way run-all.sh sources it, so the shared body is reached
# through the walkthrough's own helpers rather than loaded directly.
for mbw_helper in examples/backups/mariadb/00-helpers.sh; do
# Run under bash EXPLICITLY, not under the runner's shell. run-all.sh and
# the helpers are #!/bin/bash; cozytest.sh is #!/bin/sh and sources this
# file, so on the CI runner the body would execute under dash, where
@@ -819,54 +779,6 @@ MBW_FIXTURE
}
@test "every wait_hr_ready copy in the tree matches the mariadb one" {
# A separate contract from the errexit one above, and it fails for a
# separate reason, so it gets a name of its own: a divergence introduced by
# editing one helper would otherwise surface only when another example
# ran. Compare the function bodies rather than the files, because the files
# legitimately differ in their app-specific blocks.
mbw_ref=examples/backups/mariadb/00-helpers.sh
mbw_a=$(mbw_hr_body "$mbw_ref")
[ -n "$mbw_a" ] || { echo "wait_hr_ready not found in $mbw_ref" >&2; exit 1; }
mbw_copies=$(mbw_hr_copies)
mbw_n=$(printf '%s\n' "$mbw_copies" | grep -c . || true)
# A population of one compares mariadb with itself and passes on nothing.
[ "$mbw_n" -ge 2 ] || {
echo "found ${mbw_n} wait_hr_ready copies outside the exceptions, expected more than the mariadb one:" >&2
printf '%s\n' "$mbw_copies" >&2
exit 1
}
for mbw_f in $mbw_copies; do
[ "$(mbw_hr_body "$mbw_f")" = "$mbw_a" ] || {
echo "$mbw_f: wait_hr_ready has diverged from $mbw_ref" >&2
echo "carry the change to every copy, or name the file in MBW_HR_EXCEPTIONS with its reason in the header" >&2
mbw_tmp=$(mktemp)
mbw_hr_body "$mbw_f" > "$mbw_tmp"
printf '%s\n' "$mbw_a" | diff - "$mbw_tmp" >&2 || true
rm -f "$mbw_tmp"
exit 1
}
done
}
@test "each wait_hr_ready exception still defines a body of its own" {
# An exception is a statement that a copy differs. Once it converges, or
# moves, the entry exempts nothing and the reason in the header is false,
# so both states red rather than leaving a stale name behind.
mbw_a=$(mbw_hr_body examples/backups/mariadb/00-helpers.sh)
mbw_definers=$(mbw_hr_definers)
for mbw_f in $MBW_HR_EXCEPTIONS; do
printf '%s\n' "$mbw_definers" | grep -qx "$mbw_f" || {
echo "$mbw_f is a named exception but defines no wait_hr_ready; drop it from MBW_HR_EXCEPTIONS and the header" >&2
exit 1
}
[ "$(mbw_hr_body "$mbw_f")" != "$mbw_a" ] || {
echo "$mbw_f now matches the mariadb wait_hr_ready; drop it from MBW_HR_EXCEPTIONS and the header so the identity pin covers it" >&2
exit 1
}
done
}
@test "a trailing space does not turn a literal budget into skip" {
# The extractor splits on an explicit regex, so whitespace at end of line
# becomes an empty final field. Pinned against a fixture rather than the