mirror of
https://github.com/cozystack/cozystack.git
synced 2026-10-23 03:27:50 +09:00
fix(tests): let the backup walkthrough waits outlive their installs
The backup round-trips drive examples/backups, whose bucket waits gave up at 300s and whose postgres waits at 360s, under the 600s install timeout cozystack-api gives each of those releases. The kafka example's bucket wait passed no budget and took the helper's 300s default. A failed install there reported only that the wait ran out. Raise each to 660s, and grow the Chainsaw op around every walkthrough by what its waits gained, so a failure at any later wait keeps the headroom it had and is still reported by the script rather than cut off by the op. The kafka and kafka-metadata Kafka waits stay: their 7h install timeout cannot be cleared inside the E2E job cap. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
14 files changed
+52
-31
No files matched your search
@@ -27,7 +27,9 @@ spec:
|
||||
EOF
|
||||
|
||||
log_substep "Waiting for bucket HelmRelease to be Ready..."
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 300
|
||||
# 660s, above this generated release's 600s install timeout; see "Sizing an
|
||||
# HR-Ready budget" in docs/agents/e2e-testing.md.
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 660
|
||||
kubectl -n "$NAMESPACE" wait bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" --for=jsonpath='{.status.bucketReady}'=true --timeout=180s
|
||||
# Cozystack's bucket app provisions a BucketAccess named "<bucket-name>-backup"
|
||||
# (the "-backup" suffix is the BucketAccessClass name); the BucketInfo Secret
|
||||
|
||||
@@ -26,7 +26,9 @@ spec:
|
||||
readonly: false
|
||||
EOF
|
||||
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}"
|
||||
# 660s, above this generated release's 600s install timeout; see "Sizing an
|
||||
# HR-Ready budget" in docs/agents/e2e-testing.md.
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 660
|
||||
kubectl -n "$NAMESPACE" wait bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" --for=jsonpath='{.status.bucketReady}'=true --timeout=300s
|
||||
# Cozystack's bucket app provisions a BucketAccess named "<bucket-name>-backup"
|
||||
# (the "-backup" suffix is the BucketAccessClass name); the BucketInfo Secret
|
||||
|
||||
@@ -48,7 +48,9 @@ wait_app_grant_ready() {
|
||||
|
||||
print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}"
|
||||
kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml"
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 300
|
||||
# 660s, above this generated release's 600s install timeout; see "Sizing an
|
||||
# HR-Ready budget" in docs/agents/e2e-testing.md.
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 660
|
||||
wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \
|
||||
'{.status.bucketReady}' true "$NAMESPACE" 300
|
||||
wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \
|
||||
|
||||
@@ -44,7 +44,9 @@ subst() {
|
||||
|
||||
print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}"
|
||||
kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml"
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 300
|
||||
# 660s, above this generated release's 600s install timeout; see "Sizing an
|
||||
# HR-Ready budget" in docs/agents/e2e-testing.md.
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 660
|
||||
wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \
|
||||
'{.status.bucketReady}' true "$NAMESPACE" 300
|
||||
wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \
|
||||
|
||||
@@ -38,7 +38,9 @@ subst() {
|
||||
|
||||
print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}"
|
||||
kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml"
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 300
|
||||
# 660s, above this generated release's 600s install timeout; see "Sizing an
|
||||
# HR-Ready budget" in docs/agents/e2e-testing.md.
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 660
|
||||
wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \
|
||||
'{.status.bucketReady}' true "$NAMESPACE" 300
|
||||
wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-backup" \
|
||||
@@ -123,7 +125,9 @@ materialise_backup_secrets "$PG_SRC_NAME"
|
||||
|
||||
print_header "Step 05b: Deploy source Postgres '${PG_SRC_NAME}' and wait for it to be healthy"
|
||||
subst 05-postgres-src.yaml | kubectl -n "$NAMESPACE" apply -f -
|
||||
wait_hr_ready "postgres-${PG_SRC_NAME}" 360
|
||||
# 660s, above this generated release's 600s install timeout; see "Sizing an
|
||||
# HR-Ready budget" in docs/agents/e2e-testing.md.
|
||||
wait_hr_ready "postgres-${PG_SRC_NAME}" 660
|
||||
wait_for_field clusters.postgresql.cnpg.io "$PG_SRC_CLUSTER" \
|
||||
'{.status.phase}' 'Cluster in healthy state' "$NAMESPACE" 360
|
||||
# The 'demo' database and 'app' user are created by the chart's init Job, not
|
||||
@@ -167,7 +171,8 @@ materialise_backup_secrets "$PG_TARGET_NAME"
|
||||
kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/30-postgres-target.yaml"
|
||||
# Let the target's first install settle before the RestoreJob driver suspends
|
||||
# its HelmRelease — suspending an HR mid-install races helm-controller.
|
||||
wait_hr_ready "postgres-${PG_TARGET_NAME}" 360
|
||||
# 660s, as for the source above.
|
||||
wait_hr_ready "postgres-${PG_TARGET_NAME}" 660
|
||||
kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/40-restorejob-to-copy.yaml"
|
||||
wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_TOCOPY_NAME" \
|
||||
'{.status.phase}' Succeeded "$NAMESPACE" 1200 Failed
|
||||
|
||||
@@ -43,7 +43,9 @@ print_header "RabbitMQ definitions backup/restore demo (namespace: $NAMESPACE)"
|
||||
# --- Bucket ------------------------------------------------------------------
|
||||
print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}"
|
||||
kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml"
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 300
|
||||
# 660s, above this generated release's 600s install timeout; see "Sizing an
|
||||
# HR-Ready budget" in docs/agents/e2e-testing.md.
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 660
|
||||
wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \
|
||||
'{.status.bucketReady}' true "$NAMESPACE" 300
|
||||
wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \
|
||||
|
||||
@@ -41,7 +41,9 @@ print_header "Redis backup/restore demo (namespace: $NAMESPACE)"
|
||||
# --- Bucket ------------------------------------------------------------------
|
||||
print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}"
|
||||
kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/bucket.yaml"
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 300
|
||||
# 660s, above this generated release's 600s install timeout; see "Sizing an
|
||||
# HR-Ready budget" in docs/agents/e2e-testing.md.
|
||||
wait_hr_ready "bucket-${BUCKET_NAME}" 660
|
||||
wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \
|
||||
'{.status.bucketReady}' true "$NAMESPACE" 300
|
||||
wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \
|
||||
|
||||
@@ -339,13 +339,13 @@ spec:
|
||||
# Inner budgets, sized against what CI actually takes rather than guessed
|
||||
# generously — the whole bucket suite completes in ~37s and a backup-enabled
|
||||
# ClickHouse plus Keeper reaches Ready in ~98s (measured on run 30353894332):
|
||||
# 03 750s = 300 bucket HR + 180 claim + 180 access + 90 S3 preflight
|
||||
# 03 1110s = 660 bucket HR + 180 claim + 180 access + 90 S3 preflight
|
||||
# 04 960s = 660 HR + 300 StatefulSet
|
||||
# 05 480s = BackupJob to Succeeded
|
||||
# 06 480s = in-place RestoreJob
|
||||
# 07 1440s = 660 HR + 300 StatefulSet + 480 to-copy RestoreJob
|
||||
# sum = 4110s (68m30s)
|
||||
# 72m therefore leaves ~3m30s of headroom for the kubectl apply / exec SQL
|
||||
# sum = 4470s (74m30s)
|
||||
# 78m therefore leaves ~3m30s of headroom for the kubectl apply / exec SQL
|
||||
# calls interleaved between those waits (convention 6: justified in-line).
|
||||
# The 660s HR waits are sized in "Sizing an HR-Ready budget",
|
||||
# docs/agents/e2e-testing.md.
|
||||
@@ -355,7 +355,7 @@ spec:
|
||||
# wait_hr_ready on Stalled=True — so a genuinely broken backup reports in
|
||||
# seconds. The happy path is minutes.
|
||||
- script:
|
||||
timeout: 72m
|
||||
timeout: 78m
|
||||
content: |
|
||||
set -eu
|
||||
cd ../../..
|
||||
|
||||
@@ -264,16 +264,17 @@ spec:
|
||||
NAMESPACE="$NAMESPACE" examples/backups/kafka/cleanup.sh
|
||||
check:
|
||||
($error == null): true
|
||||
# The 40m budget is the whole documented flow: bucket provisioning, TWO
|
||||
# The 46m budget is the whole documented flow: bucket provisioning, TWO
|
||||
# two-broker Strimzi clusters (source + to-copy target, each pulls the
|
||||
# Kafka image, boots ZooKeeper then the brokers, and passes readiness), the
|
||||
# freeze-end-offset drain to S3, the in-place restore, the refused restore
|
||||
# into the populated topic (three strategy Pod attempts, each stopping at
|
||||
# the emptiness check), and the to-copy restore with its verification.
|
||||
# Kafka image, boots its KRaft controllers and brokers, and passes
|
||||
# readiness), the freeze-end-offset drain to S3, the in-place restore, the
|
||||
# refused restore into the populated topic (three strategy Pod attempts,
|
||||
# each stopping at the emptiness check), and the to-copy restore with its
|
||||
# verification, sized with the 660s bucket HR wait in 03-create-bucket.sh.
|
||||
# run-all.sh fails fast on terminal BackupJob/RestoreJob phases and stalled
|
||||
# HelmReleases rather than polling out the budget.
|
||||
- script:
|
||||
timeout: 40m
|
||||
timeout: 46m
|
||||
content: |
|
||||
set -eu
|
||||
cd ../../..
|
||||
|
||||
@@ -454,18 +454,18 @@ spec:
|
||||
NAMESPACE="$NAMESPACE" examples/backups/mariadb/cleanup.sh
|
||||
check:
|
||||
($error == null): true
|
||||
# 42m runs the whole documented flow: bucket provisioning, a 2-replica
|
||||
# 48m runs the whole documented flow: bucket provisioning, a 2-replica
|
||||
# source MariaDB, the logical dump to S3, an empty 2-replica target, and
|
||||
# the to-copy restore + verify (convention 6: justified in-line).
|
||||
#
|
||||
# Sized so a failed install at a 660s HelmRelease wait in run-all.sh is
|
||||
# Sized so a failed install at any 660s HelmRelease wait in run-all.sh is
|
||||
# reported by the script rather than cut off here, which SIGKILLs the
|
||||
# process group and loses the script's error. hack/mariadb-first-boot-waits.bats
|
||||
# holds this op against those waits, so raising a wait in run-all.sh means
|
||||
# raising this op too. A bound on the whole flow needs a phase budget
|
||||
# (docs/agents/e2e-testing.md), not a larger literal.
|
||||
- script:
|
||||
timeout: 42m
|
||||
timeout: 48m
|
||||
content: |
|
||||
set -eu
|
||||
cd ../../..
|
||||
|
||||
@@ -425,16 +425,16 @@ spec:
|
||||
NAMESPACE="$NAMESPACE" examples/backups/mongodb/cleanup.sh
|
||||
check:
|
||||
($error == null): true
|
||||
# The 36m budget covers what this step drives: bucket provisioning, a
|
||||
# The 42m budget covers what this step drives: bucket provisioning, a
|
||||
# single-replica source MongoDB coming up (psmdb reconcile, PVC bind, image
|
||||
# pull, pbm-agent sidecars) and the logical dump to S3 (convention 6:
|
||||
# justified in-line). It is not lowered for the skipped restore because
|
||||
# run-all.sh's own per-wait budgets up to the backup already exceed it, so
|
||||
# this is the ceiling that fires either way. run-all.sh fails fast on a
|
||||
# terminal BackupJob phase and stalled HelmReleases rather than polling out
|
||||
# the budget. Sized with the 660s source HR wait in run-all.sh.
|
||||
# the budget. Sized with the 660s bucket and source HR waits in run-all.sh.
|
||||
- script:
|
||||
timeout: 36m
|
||||
timeout: 42m
|
||||
content: |
|
||||
set -eu
|
||||
cd ../../..
|
||||
|
||||
@@ -312,7 +312,7 @@ spec:
|
||||
steps:
|
||||
- name: run
|
||||
try:
|
||||
# The 40m budget is the whole documented flow: bucket provisioning, a
|
||||
# The 66m budget is the whole documented flow: bucket provisioning, a
|
||||
# 2-replica source Postgres, the plugin base backup to S3, an empty target
|
||||
# app, and THREE sequential purge + re-bootstrap restores against it - the
|
||||
# latest-point to-copy restore, the point-in-time restore (step 45), and
|
||||
@@ -321,11 +321,12 @@ spec:
|
||||
# to the PITR target), and the in-place restore (step 50, the
|
||||
# regression guard for the archive-vs-recovery serverName collision)
|
||||
# (convention 6: justified in-line). The old 25m budget predated the PITR
|
||||
# phases; four restores plus the negative phase need the larger window.
|
||||
# phases; four restores plus the negative phase need the larger window,
|
||||
# and the 660s bucket and Postgres HR waits in run-all.sh add 16m to it.
|
||||
# run-all.sh fails fast on terminal BackupJob/RestoreJob phases and stalled
|
||||
# HelmReleases rather than polling out the budget.
|
||||
- script:
|
||||
timeout: 50m
|
||||
timeout: 66m
|
||||
content: |
|
||||
set -eu
|
||||
cd ../../..
|
||||
|
||||
@@ -152,10 +152,10 @@ spec:
|
||||
($error == null): true
|
||||
# Honest round-trip: seed a sentinel definition, back it up through
|
||||
# cozy-default, drop it, restore in place, then restore into a copy -
|
||||
# asserting the sentinel survives object storage each time. 42m: sized
|
||||
# asserting the sentinel survives object storage each time. 48m: sized
|
||||
# with the 660s HR waits in run-all.sh (convention 6: justified in-line).
|
||||
- script:
|
||||
timeout: 42m
|
||||
timeout: 48m
|
||||
content: |
|
||||
set -eu
|
||||
cd ../../..
|
||||
|
||||
@@ -702,8 +702,10 @@ spec:
|
||||
# Honest round-trip: seed a sentinel key, back it up through this demo's own
|
||||
# Bucket, restore it in place, then restore into a copy - asserting the exact
|
||||
# bytes survive object storage each time (04 prints the to-copy success).
|
||||
# 36m: sized with the 660s bucket HR wait in run-all.sh (convention 6:
|
||||
# justified in-line).
|
||||
- script:
|
||||
timeout: 30m
|
||||
timeout: 36m
|
||||
content: |
|
||||
set -eu
|
||||
cd ../../..
|
||||
|
||||
Reference in new issue
Block a user