fix(tests): let the backup walkthrough waits outlive their installs

The backup round-trips drive examples/backups, whose bucket waits gave
up at 300s and whose postgres waits at 360s, under the 600s install
timeout cozystack-api gives each of those releases. The kafka example's
bucket wait passed no budget and took the helper's 300s default. A
failed install there reported only that the wait ran out.

Raise each to 660s, and grow the Chainsaw op around every walkthrough
by what its waits gained, so a failure at any later wait keeps the
headroom it had and is still reported by the script rather than cut
off by the op. The kafka and kafka-metadata Kafka waits stay: their
7h install timeout cannot be cleared inside the E2E job cap.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>

examples/backups/clickhouse/03-create-bucket.sh

@@ -27,7 +27,9 @@ spec:
 EOF
 
 log_substep "Waiting for bucket HelmRelease to be Ready..."
-wait_hr_ready "bucket-${BUCKET_NAME}" 300
+# 660s, above this generated release's 600s install timeout; see "Sizing an
+# HR-Ready budget" in docs/agents/e2e-testing.md.
+wait_hr_ready "bucket-${BUCKET_NAME}" 660
 kubectl -n "$NAMESPACE" wait bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" --for=jsonpath='{.status.bucketReady}'=true --timeout=180s
 # Cozystack's bucket app provisions a BucketAccess named "<bucket-name>-backup"
 # (the "-backup" suffix is the BucketAccessClass name); the BucketInfo Secret

examples/backups/kafka/03-create-bucket.sh

@@ -26,7 +26,9 @@ spec:
       readonly: false
 EOF
 
-wait_hr_ready "bucket-${BUCKET_NAME}"
+# 660s, above this generated release's 600s install timeout; see "Sizing an
+# HR-Ready budget" in docs/agents/e2e-testing.md.
+wait_hr_ready "bucket-${BUCKET_NAME}" 660
 kubectl -n "$NAMESPACE" wait bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" --for=jsonpath='{.status.bucketReady}'=true --timeout=300s
 # Cozystack's bucket app provisions a BucketAccess named "<bucket-name>-backup"
 # (the "-backup" suffix is the BucketAccessClass name); the BucketInfo Secret

examples/backups/mariadb/run-all.sh

@@ -48,7 +48,9 @@ wait_app_grant_ready() {
 
 print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}"
 kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml"
-wait_hr_ready "bucket-${BUCKET_NAME}" 300
+# 660s, above this generated release's 600s install timeout; see "Sizing an
+# HR-Ready budget" in docs/agents/e2e-testing.md.
+wait_hr_ready "bucket-${BUCKET_NAME}" 660
 wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \
     '{.status.bucketReady}' true "$NAMESPACE" 300
 wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \

examples/backups/mongodb/run-all.sh

@@ -44,7 +44,9 @@ subst() {
 
 print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}"
 kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml"
-wait_hr_ready "bucket-${BUCKET_NAME}" 300
+# 660s, above this generated release's 600s install timeout; see "Sizing an
+# HR-Ready budget" in docs/agents/e2e-testing.md.
+wait_hr_ready "bucket-${BUCKET_NAME}" 660
 wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \
     '{.status.bucketReady}' true "$NAMESPACE" 300
 wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \

examples/backups/postgres/run-all.sh

@@ -38,7 +38,9 @@ subst() {
 
 print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}"
 kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml"
-wait_hr_ready "bucket-${BUCKET_NAME}" 300
+# 660s, above this generated release's 600s install timeout; see "Sizing an
+# HR-Ready budget" in docs/agents/e2e-testing.md.
+wait_hr_ready "bucket-${BUCKET_NAME}" 660
 wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \
     '{.status.bucketReady}' true "$NAMESPACE" 300
 wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-backup" \
@@ -123,7 +125,9 @@ materialise_backup_secrets "$PG_SRC_NAME"
 
 print_header "Step 05b: Deploy source Postgres '${PG_SRC_NAME}' and wait for it to be healthy"
 subst 05-postgres-src.yaml | kubectl -n "$NAMESPACE" apply -f -
-wait_hr_ready "postgres-${PG_SRC_NAME}" 360
+# 660s, above this generated release's 600s install timeout; see "Sizing an
+# HR-Ready budget" in docs/agents/e2e-testing.md.
+wait_hr_ready "postgres-${PG_SRC_NAME}" 660
 wait_for_field clusters.postgresql.cnpg.io "$PG_SRC_CLUSTER" \
     '{.status.phase}' 'Cluster in healthy state' "$NAMESPACE" 360
 # The 'demo' database and 'app' user are created by the chart's init Job, not
@@ -167,7 +171,8 @@ materialise_backup_secrets "$PG_TARGET_NAME"
 kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/30-postgres-target.yaml"
 # Let the target's first install settle before the RestoreJob driver suspends
 # its HelmRelease — suspending an HR mid-install races helm-controller.
-wait_hr_ready "postgres-${PG_TARGET_NAME}" 360
+# 660s, as for the source above.
+wait_hr_ready "postgres-${PG_TARGET_NAME}" 660
 kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/40-restorejob-to-copy.yaml"
 wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_TOCOPY_NAME" \
     '{.status.phase}' Succeeded "$NAMESPACE" 1200 Failed

examples/backups/rabbitmq/run-all.sh

@@ -43,7 +43,9 @@ print_header "RabbitMQ definitions backup/restore demo (namespace: $NAMESPACE)"
 # --- Bucket ------------------------------------------------------------------
 print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}"
 kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml"
-wait_hr_ready "bucket-${BUCKET_NAME}" 300
+# 660s, above this generated release's 600s install timeout; see "Sizing an
+# HR-Ready budget" in docs/agents/e2e-testing.md.
+wait_hr_ready "bucket-${BUCKET_NAME}" 660
 wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \
     '{.status.bucketReady}' true "$NAMESPACE" 300
 wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \

examples/backups/redis/run-all.sh

@@ -41,7 +41,9 @@ print_header "Redis backup/restore demo (namespace: $NAMESPACE)"
 # --- Bucket ------------------------------------------------------------------
 print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}"
 kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/bucket.yaml"
-wait_hr_ready "bucket-${BUCKET_NAME}" 300
+# 660s, above this generated release's 600s install timeout; see "Sizing an
+# HR-Ready budget" in docs/agents/e2e-testing.md.
+wait_hr_ready "bucket-${BUCKET_NAME}" 660
 wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \
     '{.status.bucketReady}' true "$NAMESPACE" 300
 wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \

hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml

@@ -339,13 +339,13 @@ spec:
     # Inner budgets, sized against what CI actually takes rather than guessed
     # generously — the whole bucket suite completes in ~37s and a backup-enabled
     # ClickHouse plus Keeper reaches Ready in ~98s (measured on run 30353894332):
-    #   03  750s = 300 bucket HR + 180 claim + 180 access + 90 S3 preflight
+    #   03 1110s = 660 bucket HR + 180 claim + 180 access + 90 S3 preflight
     #   04  960s = 660 HR + 300 StatefulSet
     #   05  480s = BackupJob to Succeeded
     #   06  480s = in-place RestoreJob
     #   07 1440s = 660 HR + 300 StatefulSet + 480 to-copy RestoreJob
-    #        sum = 4110s (68m30s)
-    # 72m therefore leaves ~3m30s of headroom for the kubectl apply / exec SQL
+    #        sum = 4470s (74m30s)
+    # 78m therefore leaves ~3m30s of headroom for the kubectl apply / exec SQL
     # calls interleaved between those waits (convention 6: justified in-line).
     # The 660s HR waits are sized in "Sizing an HR-Ready budget",
     # docs/agents/e2e-testing.md.
@@ -355,7 +355,7 @@ spec:
     # wait_hr_ready on Stalled=True — so a genuinely broken backup reports in
     # seconds. The happy path is minutes.
     - script:
-        timeout: 72m
+        timeout: 78m
         content: |
           set -eu
           cd ../../..

hack/e2e-chainsaw/kafka/chainsaw-test.yaml

@@ -264,16 +264,17 @@ spec:
           NAMESPACE="$NAMESPACE" examples/backups/kafka/cleanup.sh
         check:
           ($error == null): true
-    # The 40m budget is the whole documented flow: bucket provisioning, TWO
+    # The 46m budget is the whole documented flow: bucket provisioning, TWO
     # two-broker Strimzi clusters (source + to-copy target, each pulls the
-    # Kafka image, boots ZooKeeper then the brokers, and passes readiness), the
-    # freeze-end-offset drain to S3, the in-place restore, the refused restore
-    # into the populated topic (three strategy Pod attempts, each stopping at
-    # the emptiness check), and the to-copy restore with its verification.
+    # Kafka image, boots its KRaft controllers and brokers, and passes
+    # readiness), the freeze-end-offset drain to S3, the in-place restore, the
+    # refused restore into the populated topic (three strategy Pod attempts,
+    # each stopping at the emptiness check), and the to-copy restore with its
+    # verification, sized with the 660s bucket HR wait in 03-create-bucket.sh.
     # run-all.sh fails fast on terminal BackupJob/RestoreJob phases and stalled
     # HelmReleases rather than polling out the budget.
     - script:
-        timeout: 40m
+        timeout: 46m
         content: |
           set -eu
           cd ../../..

hack/e2e-chainsaw/mariadb/chainsaw-test.yaml

@@ -454,18 +454,18 @@ spec:
           NAMESPACE="$NAMESPACE" examples/backups/mariadb/cleanup.sh
         check:
           ($error == null): true
-    # 42m runs the whole documented flow: bucket provisioning, a 2-replica
+    # 48m runs the whole documented flow: bucket provisioning, a 2-replica
     # source MariaDB, the logical dump to S3, an empty 2-replica target, and
     # the to-copy restore + verify (convention 6: justified in-line).
     #
-    # Sized so a failed install at a 660s HelmRelease wait in run-all.sh is
+    # Sized so a failed install at any 660s HelmRelease wait in run-all.sh is
     # reported by the script rather than cut off here, which SIGKILLs the
     # process group and loses the script's error. hack/mariadb-first-boot-waits.bats
     # holds this op against those waits, so raising a wait in run-all.sh means
     # raising this op too. A bound on the whole flow needs a phase budget
     # (docs/agents/e2e-testing.md), not a larger literal.
     - script:
-        timeout: 42m
+        timeout: 48m
         content: |
           set -eu
           cd ../../..

hack/e2e-chainsaw/mongodb/chainsaw-test.yaml

@@ -425,16 +425,16 @@ spec:
           NAMESPACE="$NAMESPACE" examples/backups/mongodb/cleanup.sh
         check:
           ($error == null): true
-    # The 36m budget covers what this step drives: bucket provisioning, a
+    # The 42m budget covers what this step drives: bucket provisioning, a
     # single-replica source MongoDB coming up (psmdb reconcile, PVC bind, image
     # pull, pbm-agent sidecars) and the logical dump to S3 (convention 6:
     # justified in-line). It is not lowered for the skipped restore because
     # run-all.sh's own per-wait budgets up to the backup already exceed it, so
     # this is the ceiling that fires either way. run-all.sh fails fast on a
     # terminal BackupJob phase and stalled HelmReleases rather than polling out
-    # the budget. Sized with the 660s source HR wait in run-all.sh.
+    # the budget. Sized with the 660s bucket and source HR waits in run-all.sh.
     - script:
-        timeout: 36m
+        timeout: 42m
         content: |
           set -eu
           cd ../../..

hack/e2e-chainsaw/postgres/chainsaw-test.yaml

@@ -312,7 +312,7 @@ spec:
   steps:
   - name: run
     try:
-    # The 40m budget is the whole documented flow: bucket provisioning, a
+    # The 66m budget is the whole documented flow: bucket provisioning, a
     # 2-replica source Postgres, the plugin base backup to S3, an empty target
     # app, and THREE sequential purge + re-bootstrap restores against it - the
     # latest-point to-copy restore, the point-in-time restore (step 45), and
@@ -321,11 +321,12 @@ spec:
     # to the PITR target), and the in-place restore (step 50, the
     # regression guard for the archive-vs-recovery serverName collision)
     # (convention 6: justified in-line). The old 25m budget predated the PITR
-    # phases; four restores plus the negative phase need the larger window.
+    # phases; four restores plus the negative phase need the larger window,
+    # and the 660s bucket and Postgres HR waits in run-all.sh add 16m to it.
     # run-all.sh fails fast on terminal BackupJob/RestoreJob phases and stalled
     # HelmReleases rather than polling out the budget.
     - script:
-        timeout: 50m
+        timeout: 66m
         content: |
           set -eu
           cd ../../..

hack/e2e-chainsaw/rabbitmq/chainsaw-test.yaml

@@ -152,10 +152,10 @@ spec:
           ($error == null): true
     # Honest round-trip: seed a sentinel definition, back it up through
     # cozy-default, drop it, restore in place, then restore into a copy -
-    # asserting the sentinel survives object storage each time. 42m: sized
+    # asserting the sentinel survives object storage each time. 48m: sized
     # with the 660s HR waits in run-all.sh (convention 6: justified in-line).
     - script:
-        timeout: 42m
+        timeout: 48m
         content: |
           set -eu
           cd ../../..

hack/e2e-chainsaw/redis/chainsaw-test.yaml

@@ -702,8 +702,10 @@ spec:
     # Honest round-trip: seed a sentinel key, back it up through this demo's own
     # Bucket, restore it in place, then restore into a copy - asserting the exact
     # bytes survive object storage each time (04 prints the to-copy success).
+    # 36m: sized with the 660s bucket HR wait in run-all.sh (convention 6:
+    # justified in-line).
     - script:
-        timeout: 30m
+        timeout: 36m
         content: |
           set -eu
           cd ../../..