mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-09 05:25:14 +09:00
Alexey Artamonov
f02ca8cbc2
The backup examples and e2e suites no longer set passwords. They read the generated ones from <release>-credentials with jq, since a username may contain a dot that jsonpath treats as a path step, and log in as an application user after each restore, so a restore that leaves the credentials unusable fails the run. The MariaDB run also checks that an account created out of band on the source is absent from the copy. The unreachable-target restore step asserts a terminal Failed attributed to the recovery window, since the reason itself is best effort. Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
259 lines
12 KiB
Bash
Executable File
259 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
# Shared helpers for the PostgreSQL backup/restore demo.
|
|
# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh"
|
|
|
|
export RED='\033[0;31m'
|
|
export GREEN='\033[0;32m'
|
|
export YELLOW='\033[1;33m'
|
|
export BLUE='\033[0;34m'
|
|
export MAGENTA='\033[0;35m'
|
|
export CYAN='\033[0;36m'
|
|
export WHITE='\033[1;37m'
|
|
export NC='\033[0m'
|
|
export BOLD='\033[1m'
|
|
|
|
# Default settings (override via environment).
|
|
export NAMESPACE="${NAMESPACE:-tenant-root}"
|
|
export BUCKET_NAME="${BUCKET_NAME:-pg-backups}"
|
|
export PG_SRC_NAME="${PG_SRC_NAME:-pg-src}"
|
|
export PG_TARGET_NAME="${PG_TARGET_NAME:-pg-target}"
|
|
# The apps/postgres chart names the cnpg.io Cluster (and its Pods'
|
|
# cnpg.io/cluster label) after the Helm release, which is postgres-<app>.
|
|
export PG_SRC_CLUSTER="postgres-${PG_SRC_NAME}"
|
|
export PG_TARGET_CLUSTER="postgres-${PG_TARGET_NAME}"
|
|
export STRATEGY_NAME="${STRATEGY_NAME:-cnpg-strategy-default}"
|
|
export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-postgres-cnpg}"
|
|
export BACKUPJOB_NAME="${BACKUPJOB_NAME:-pg-src-adhoc}"
|
|
# A second ad-hoc BackupJob taken after the PITR marker writes; its completion
|
|
# is the deterministic "WAL past the recovery target is archived" gate for the
|
|
# point-in-time restore (see run-all.sh step 45).
|
|
export BACKUPJOB_POSTMARKER_NAME="${BACKUPJOB_POSTMARKER_NAME:-pg-src-postmarker}"
|
|
export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-pg-src-to-pg-target}"
|
|
export RESTOREJOB_PITR_NAME="${RESTOREJOB_PITR_NAME:-pg-src-to-pg-target-pitr}"
|
|
export RESTOREJOB_UNREACHABLE_NAME="${RESTOREJOB_UNREACHABLE_NAME:-pg-src-to-pg-target-unreachable}"
|
|
export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-pg-src-in-place}"
|
|
export PLAN_NAME="${PLAN_NAME:-pg-src-daily}"
|
|
|
|
# S3 endpoint CA. cozystack's default seaweedfs serves its S3 endpoint with a
|
|
# self-signed certificate whose CA lives in this Secret; the demo copies its
|
|
# ca.crt into a per-app Secret the barman-cloud ObjectStore trusts via
|
|
# endpointCA. The name follows the seaweedfs chart's fullnameOverride
|
|
# ("seaweedfs" -> "seaweedfs-ca-cert"); run-all.sh auto-discovers the CA
|
|
# Certificate's actual secret when this default is absent, so an upstream
|
|
# fullname change does not silently break the copy. On a cluster whose S3
|
|
# endpoint is signed by a publicly-trusted CA, set S3_CA_SECRET="" to skip the
|
|
# copy and drop endpointCA from the manifests.
|
|
export S3_CA_SECRET="${S3_CA_SECRET:-seaweedfs-ca-cert}"
|
|
export S3_CA_NAMESPACE="${S3_CA_NAMESPACE:-tenant-root}"
|
|
export S3_CA_KEY="${S3_CA_KEY:-ca.crt}"
|
|
|
|
log_info() { echo -e "${BLUE}i${NC} $*" >&2; }
|
|
log_success() { echo -e "${GREEN}OK${NC} $*" >&2; }
|
|
log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; }
|
|
log_error() { echo -e "${RED}x${NC} $*" >&2; }
|
|
log_step() { echo -e "\n${MAGENTA}${BOLD}> $*${NC}" >&2; }
|
|
log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; }
|
|
|
|
print_header() {
|
|
echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2
|
|
}
|
|
|
|
# Wait until a JSONPath value on a resource matches the desired string.
|
|
# Optional 7th arg is a TERMINAL failure value: once the field reaches it the
|
|
# wait returns 1 immediately instead of polling to the timeout. BackupJob and
|
|
# RestoreJob settle on a terminal phase=Failed that never becomes Succeeded, so
|
|
# failing fast on it keeps wall-clock (and the snapshot Pod's log, before its
|
|
# TTL reaper fires) in reach.
|
|
wait_for_field() {
|
|
local resource_type="$1"
|
|
local resource_name="$2"
|
|
local jsonpath="$3"
|
|
local desired="$4"
|
|
local namespace="${5:-}"
|
|
local timeout="${6:-300}"
|
|
local fail_value="${7:-}"
|
|
|
|
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
|
|
local elapsed=0
|
|
local ns_flag=()
|
|
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
|
|
|
|
while true; do
|
|
local current
|
|
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
|
|
if [[ "$current" == "$desired" ]]; then
|
|
log_success "$resource_type/$resource_name reached '$desired'"
|
|
return 0
|
|
fi
|
|
if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then
|
|
log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')"
|
|
return 1
|
|
fi
|
|
if [[ $elapsed -ge $timeout ]]; then
|
|
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
|
|
return 1
|
|
fi
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
done
|
|
}
|
|
|
|
# Wait for a HelmRelease to become Ready, with an existence backstop (the
|
|
# apps controller creates the HR asynchronously, so a bare `kubectl wait`
|
|
# right after `kubectl apply` races it) and a fail-fast on Stalled=True —
|
|
# a stalled HR has exhausted its remediation retries and will never turn
|
|
# Ready, so polling to the timeout only hides the real error.
|
|
wait_hr_ready() {
|
|
local name="$1"
|
|
local timeout="${2:-300}"
|
|
local elapsed=0
|
|
local lookup state seen=0
|
|
|
|
log_substep "Waiting for HelmRelease/$name to become Ready..."
|
|
while true; do
|
|
# --ignore-not-found makes an absent release exit 0, so a non-zero exit
|
|
# is kubectl failing to answer. Presence is read off the name line,
|
|
# because a warning on stderr lands in the same string.
|
|
if ! lookup=$(kubectl -n "$NAMESPACE" get hr "$name" --ignore-not-found -o name 2>&1); then
|
|
state=error
|
|
elif [[ $'\n'"$lookup"$'\n' != *"/$name"$'\n'* ]]; then
|
|
state=absent
|
|
else
|
|
state=present
|
|
seen=1
|
|
local ready stalled
|
|
ready=$(kubectl -n "$NAMESPACE" get hr "$name" \
|
|
-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' || true)
|
|
if [[ "$ready" == "True" ]]; then
|
|
log_success "HelmRelease/$name is Ready"
|
|
return 0
|
|
fi
|
|
stalled=$(kubectl -n "$NAMESPACE" get hr "$name" \
|
|
-o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' || true)
|
|
if [[ "$stalled" == "True" ]]; then
|
|
log_error "HelmRelease/$name is Stalled (terminal):"
|
|
break
|
|
fi
|
|
fi
|
|
if [[ $elapsed -ge $timeout ]]; then
|
|
log_error "Timeout waiting for HelmRelease/$name to become Ready:"
|
|
case "$state" in
|
|
error)
|
|
echo " could not look up HelmRelease/$name: $lookup" >&2
|
|
# Seen earlier: the reads below may answer where this lookup
|
|
# did not.
|
|
[[ $seen -eq 1 ]] || return 1
|
|
;;
|
|
absent)
|
|
if [[ $seen -eq 1 ]]; then
|
|
echo " HelmRelease/$name was deleted from $NAMESPACE while waiting" >&2
|
|
else
|
|
echo " HelmRelease/$name never appeared in $NAMESPACE" >&2
|
|
fi
|
|
return 1
|
|
;;
|
|
esac
|
|
break
|
|
fi
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
done
|
|
# Reached on Stalled or on timeout. Conditions and history rather than the
|
|
# Ready message alone: a failed install is retried on an interval, so Ready
|
|
# may describe the retry rather than the failure behind it.
|
|
kubectl -n "$NAMESPACE" get hr "$name" \
|
|
-o jsonpath='{range .status.conditions[*]} {.type}={.status} ({.reason}): {.message}{"\n"}{end}' >&2 || true
|
|
local hist
|
|
if ! hist=$(kubectl -n "$NAMESPACE" get hr "$name" \
|
|
-o jsonpath='{range .status.history[*]} history: {.status} {.chartVersion} {.lastDeployed}{"\n"}{end}'); then
|
|
echo " history: kubectl could not read it" >&2
|
|
elif [[ -n "${hist//[[:space:]]/}" ]]; then
|
|
printf '%s\n' "$hist" >&2
|
|
else
|
|
# A release with no history prints nothing and exits 0; say so,
|
|
# because a blank in a failure dump reads as a dump that broke.
|
|
echo " history: (none recorded)" >&2
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# Name of the primary pod of a cnpg.io Cluster (the one accepting writes).
|
|
cnpg_primary_pod() {
|
|
local cluster="$1"
|
|
kubectl -n "$NAMESPACE" get pod \
|
|
-l "cnpg.io/cluster=${cluster},cnpg.io/instanceRole=primary" \
|
|
-o name 2>/dev/null | head -n1
|
|
}
|
|
|
|
# Run a psql statement on a cnpg.io Cluster's primary. Args: <cluster> <db> <sql>
|
|
psql_exec() {
|
|
local cluster="$1" db="$2" sql="$3"
|
|
local pod
|
|
pod=$(cnpg_primary_pod "$cluster")
|
|
[[ -n "$pod" ]] || { log_error "no primary pod for cnpg cluster '$cluster'"; return 1; }
|
|
kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- \
|
|
psql -U postgres -d "$db" -tAc "$sql"
|
|
}
|
|
|
|
# Run a psql statement AS the application user over TCP (via the cluster's -rw
|
|
# Service), forcing password authentication. psql_exec above connects as the
|
|
# in-pod postgres superuser through the local socket (peer auth) and so never
|
|
# exercises a user's password; only this path proves the password the
|
|
# <release>-credentials Secret advertises actually authenticates. The password is
|
|
# read from that chart-managed Secret with jq, not a jsonpath: a username may
|
|
# contain a '.', and jsonpath's `{.data['a.b']}` treats the dot as a path step
|
|
# and returns nothing, so a dotted user would read an empty password and time out
|
|
# on a credential that is fine. jq indexes the key as a literal string. The chart
|
|
# names the CNPG Cluster, the -rw Service and the credentials Secret all after the
|
|
# release, so the cluster IS the release here - derive the Secret from it rather
|
|
# than taking a separate name that a caller can get wrong (a release/app-name
|
|
# mix-up read a Secret that never exists and made every login time out).
|
|
# Args: <cluster> <user> <db> <sql>
|
|
psql_app_exec() {
|
|
local cluster="$1" user="$2" db="$3" sql="$4"
|
|
local pod pw
|
|
pod=$(cnpg_primary_pod "$cluster")
|
|
[[ -n "$pod" ]] || { log_error "no primary pod for cnpg cluster '$cluster'"; return 1; }
|
|
pw=$(kubectl -n "$NAMESPACE" get secret "${cluster}-credentials" -o json \
|
|
| jq -r --arg u "$user" '.data[$u] // empty' | base64 -d)
|
|
[[ -n "$pw" ]] || { log_error "no password for user '${user}' in ${cluster}-credentials"; return 1; }
|
|
kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- \
|
|
env PGPASSWORD="$pw" psql -h "${cluster}-rw" -U "$user" -d "$db" -tAc "$sql"
|
|
}
|
|
|
|
# Block until the application user can authenticate against a cluster, or fail
|
|
# after <timeout>s. Needed after a to-copy restore: the driver clears
|
|
# bootstrap.enabled once recovery converges so the chart's init-job re-runs
|
|
# ALTER ROLE ... WITH PASSWORD, converging the recovered roles (which carry the
|
|
# source's password hashes) onto the freshly generated <release>-credentials
|
|
# Secret. That convergence is asynchronous - a HelmRelease upgrade plus its
|
|
# post-upgrade init-job hook - so this is a readiness wait on an eventually
|
|
# consistent property, not a retry over a deterministic step.
|
|
# Args: <cluster> <user> <db> <timeout>
|
|
wait_for_app_login() {
|
|
local cluster="$1" user="$2" db="$3" timeout="${4:-300}"
|
|
local elapsed=0 errfile
|
|
errfile=$(mktemp)
|
|
log_substep "Waiting for user '${user}' to authenticate against ${cluster}..."
|
|
while true; do
|
|
# Success is decided on stdout alone (the '1' from SELECT 1), so a stray
|
|
# notice on stderr cannot read as a failure. stderr is kept in errfile,
|
|
# not discarded: a transient not-yet-converged attempt stays quiet, but
|
|
# the LAST attempt's stderr is what names the cause on timeout (a missing
|
|
# password, or the wrong Secret) - without it a real failure reads as a
|
|
# generic timeout.
|
|
if [[ "$(psql_app_exec "$cluster" "$user" "$db" 'SELECT 1;' 2>"$errfile" | tr -d '[:space:]')" == "1" ]]; then
|
|
rm -f "$errfile"
|
|
log_success "user '${user}' authenticated against ${cluster}"
|
|
return 0
|
|
fi
|
|
if [[ $elapsed -ge $timeout ]]; then
|
|
log_error "Timeout waiting for user '${user}' to authenticate against ${cluster}: the credentials Secret advertises a password the roles never received (bootstrap not cleared, or the init-job did not reconcile). Last attempt: $(cat "$errfile")"
|
|
rm -f "$errfile"
|
|
return 1
|
|
fi
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
done
|
|
}
|