Files
cozystack/examples/backups/mongodb/run-all.sh
T
Aleksei Sviridkin 04d469fc85 fix(tests): let the backup walkthrough waits outlive their installs
The backup round-trips drive examples/backups, whose bucket waits gave
up at 300s and whose postgres waits at 360s, under the 600s install
timeout cozystack-api gives each of those releases. The kafka example's
bucket wait passed no budget and took the helper's 300s default. A
failed install there reported only that the wait ran out.

Raise each to 660s, and grow the Chainsaw op around every walkthrough
by what its waits gained, so a failure at any later wait keeps the
headroom it had and is still reported by the script rather than cut
off by the op. The kafka and kafka-metadata Kafka waits stay: their
7h install timeout cannot be cleared inside the E2E job cap.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
2026-10-01 17:40:22 +03:00

133 lines
6.7 KiB
Bash
Executable File

#!/bin/bash
# Convenience runner + e2e harness for the MongoDB backup/restore demo.
#
# It applies the SAME numbered manifests a human reads (00-bucket.yaml ..
# 30-restorejob-to-copy.yaml), filling the REPLACE_WITH_* placeholders from the
# provisioned Bucket so the documented flow and the automated test can never
# drift. Stops on the first failure.
#
# Flow: Bucket -> source MongoDB with backup.enabled (+ a sentinel document) ->
# ad-hoc BackupJob against the cozy-default BackupClass (wait Succeeded) ->
# empty target MongoDB + to-copy RestoreJob (wait Succeeded) -> assert the
# sentinel round-tripped through S3 into the restored copy while the source
# stays untouched.
#
# Why to-copy and not in-place: in-place replays the dump straight into the live
# source cluster, so it cannot witness the restore on a separate instance.
# To-copy leaves the source running and lands the marker document on a distinct
# target, a stronger restore proof; the in-place dispatch is covered at the unit
# level by the controller tests.
#
# Override NAMESPACE / endpoint via the environment; see 00-helpers.sh.
# hack/e2e-chainsaw/mongodb/ drives this file as mongodb-2-backup-roundtrip.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/00-helpers.sh"
# App user password baked into the source spec; the demo writes the sentinel as
# the operator databaseAdmin (see mongosh_exec), so this is just to satisfy the
# chart's user declaration.
MONGODB_PASSWORD="${MONGODB_PASSWORD:-Xai7Wepo0aeThie8}"
# Substitute the manifest placeholders. $BUCKET / $S3_ENDPOINT / the access keys
# are resolved from the Bucket below.
subst() {
sed \
-e "s|REPLACE_WITH_COSI_BUCKET_NAME|${BUCKET}|g" \
-e "s|REPLACE_WITH_S3_ENDPOINT|${S3_ENDPOINT}|g" \
-e "s|REPLACE_WITH_S3_ACCESS_KEY|${S3_ACCESS_KEY}|g" \
-e "s|REPLACE_WITH_S3_SECRET_KEY|${S3_SECRET_KEY}|g" \
-e "s|REPLACE_WITH_PASSWORD|${MONGODB_PASSWORD}|g" \
"$SCRIPT_DIR/$1"
}
print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}"
kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml"
# 660s, above this generated release's 600s install timeout; see "Sizing an
# HR-Ready budget" in docs/agents/e2e-testing.md.
wait_hr_ready "bucket-${BUCKET_NAME}" 660
wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \
'{.status.bucketReady}' true "$NAMESPACE" 300
wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \
'{.status.accessGranted}' true "$NAMESPACE" 300
if [[ "${COZY_E2E_BACKUP_PREFLIGHT:-0}" == "1" ]]; then
source "$SCRIPT_DIR/../../../hack/e2e-chainsaw/_lib/backup-access-preflight.sh"
cozy_backup_access_preflight "$NAMESPACE" "bucket-${BUCKET_NAME}-${BUCKET_USER}" 90
fi
log_substep "Reading bucket coordinates from BucketInfo Secret..."
TMP=$(mktemp)
trap 'rm -f "$TMP"' EXIT
kubectl -n "$NAMESPACE" get secret "bucket-${BUCKET_NAME}-${BUCKET_USER}" \
-o jsonpath='{.data.BucketInfo}' | base64 -d > "$TMP"
S3_ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' "$TMP")
S3_SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' "$TMP")
# S3_ENDPOINT can be overridden via the environment: BucketInfo advertises the
# EXTERNAL ingress endpoint, which in-cluster Pods (the pbm agent runs inside
# the mongod pod) cannot always reach or TLS-validate; the in-cluster
# alternative is https://seaweedfs-s3.<ns>:8333, reached with
# insecureSkipTLSVerify=true (psmdb s3 storage has no CA-bundle field).
S3_ENDPOINT="${S3_ENDPOINT:-$(jq -r '.spec.secretS3.endpoint' "$TMP")}"
BUCKET=$(jq -r '.spec.bucketName' "$TMP")
for v in S3_ACCESS_KEY S3_SECRET_KEY S3_ENDPOINT BUCKET; do
[[ -n "${!v}" && "${!v}" != "null" ]] || { log_error "BucketInfo missing required field: ${v}"; exit 1; }
done
log_success "Bucket '${BUCKET}' at endpoint '${S3_ENDPOINT}'."
print_header "Step 05: Deploy source MongoDB '${MONGODB_SRC_NAME}' with backups enabled"
subst 05-mongodb-src.yaml | kubectl -n "$NAMESPACE" apply -f -
# 660s, above this generated release's 600s install timeout; see "Sizing an
# HR-Ready budget" in docs/agents/e2e-testing.md.
wait_hr_ready "$MONGODB_SRC_CR" 660
wait_psmdb_ready "$MONGODB_SRC_CR" 600
print_header "Step 05b: Write a sentinel document so the backup has something to prove"
SENTINEL_TOKEN="e2e-$(date +%s)-$$"
mongosh_exec "$MONGODB_SRC_CR" \
"db.getSiblingDB('demo').e2e_sentinel.replaceOne({_id:1},{_id:1,token:'${SENTINEL_TOKEN}'},{upsert:true})"
log_success "Sentinel token: ${SENTINEL_TOKEN}"
print_header "Step 10: Submit ad-hoc BackupJob '${BACKUPJOB_NAME}' and wait for Succeeded"
kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/10-backupjob-adhoc.yaml"
wait_for_field backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \
'{.status.phase}' Succeeded "$NAMESPACE" 900 Failed
BACKUP_NAME=$(kubectl -n "$NAMESPACE" get backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \
-o jsonpath='{.status.backupRef.name}')
[[ -n "$BACKUP_NAME" ]] || { log_error "BackupJob succeeded but reported no backupRef"; exit 1; }
log_success "Backup artefact: ${BACKUP_NAME}"
if [[ "${SKIP_RESTORE:-0}" == "1" ]]; then
# hack/e2e-chainsaw/mongodb/ asserts this line to prove the skip really
# happened, so rewording it reds that suite with a bare match failure.
log_warning "SKIP_RESTORE=1: stopping after a successful backup."
exit 0
fi
print_header "Step 20/30: Restore to a copy '${MONGODB_TARGET_NAME}' and wait for Succeeded"
subst 20-mongodb-target.yaml | kubectl -n "$NAMESPACE" apply -f -
wait_hr_ready "$MONGODB_TARGET_CR" 660
wait_psmdb_ready "$MONGODB_TARGET_CR" 600
kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/30-restorejob-to-copy.yaml"
wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_TOCOPY_NAME" \
'{.status.phase}' Succeeded "$NAMESPACE" 900 Failed
print_header "Step 30 verify: the sentinel round-tripped through S3 into the copy"
GOT=$(mongosh_exec "$MONGODB_TARGET_CR" \
"print(db.getSiblingDB('demo').e2e_sentinel.findOne({_id:1}).token)" | tr -d '[:space:]')
if [[ "$GOT" != "$SENTINEL_TOKEN" ]]; then
log_error "sentinel mismatch: target has '${GOT}', expected '${SENTINEL_TOKEN}'"
exit 1
fi
log_success "Round-trip verified: '${MONGODB_TARGET_NAME}' restored sentinel '${GOT}' from S3."
# To-copy must not mutate the source. Regressing into a source-touching restore
# would corrupt the running instance, so assert the source still reads back.
SRC_GOT=$(mongosh_exec "$MONGODB_SRC_CR" \
"print(db.getSiblingDB('demo').e2e_sentinel.findOne({_id:1}).token)" | tr -d '[:space:]')
if [[ "$SRC_GOT" != "$SENTINEL_TOKEN" ]]; then
log_error "source sentinel changed after to-copy restore: source has '${SRC_GOT}', expected '${SENTINEL_TOKEN}'"
exit 1
fi
log_success "Source '${MONGODB_SRC_NAME}' left untouched by the to-copy restore."