diff --git a/hack/clickhouse-system-bucket-shared-state.bats b/hack/clickhouse-system-bucket-shared-state.bats index 2f7a79197..e76cc0a1d 100644 --- a/hack/clickhouse-system-bucket-shared-state.bats +++ b/hack/clickhouse-system-bucket-shared-state.bats @@ -91,3 +91,35 @@ restore() { } jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null } + +# The Altinity driver names the Backup after the BackupJob and adopts one that +# already exists under that name, so a Backup left behind would stand in for +# the next run's backup. It is reclaimed by name, whether or not the BackupJob +# recorded it in status.backupRef. +@test "the round-trip reclaims its Backup by name before and after the run" { + suite=hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml + test='select(.metadata.name == "clickhouse-3-system-bucket-roundtrip") | .spec.steps[]' + calls="$STUB/calls" + # Nothing is left over from an earlier run, and no BackupJob recorded a + # backupRef. + cat >"$STUB/kubectl" <<'STUBEOF' +#!/bin/bash +echo "$*" >>"$STUB/calls" +case "$*" in + *" get secret "*) exit 1 ;; +esac +STUBEOF + for selector in \ + 'select(.name == "preconditions") | .try[].script.content' \ + 'select(.name == "run") | .finally[] | select(.description == "system-bucket round-trip cleanup") | .script.content'; do + : >"$calls" + script=$(yq "$test | $selector" "$suite") + [ -n "$script" ] || { echo "no script at: $selector" >&2; false; } + PATH="$STUB:$PATH" NAMESPACE=tenant-root sh -c "$script" + grep -qE '^-n tenant-root delete backup sysbucket-backup( |$)' "$calls" || { + echo "the script at '$selector' does not delete Backup sysbucket-backup by name; kubectl calls:" >&2 + cat "$calls" >&2 + false + } + done +} diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index 03130cfb3..c48e32f24 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -439,7 +439,7 @@ spec: - name: preconditions try: - script: - timeout: 2m + timeout: 3m content: | set -eu if kubectl -n "$NAMESPACE" get secret seaweedfs-s3-ingress-tls -o name >/dev/null 2>&1; then @@ -451,9 +451,11 @@ spec: exit 1 fi # Fixed names in a persistent namespace: a leftover Succeeded job - # would satisfy the waits in the run step without a new attempt. + # would satisfy the waits in the run step without a new attempt, and + # the driver adopts a leftover Backup of the BackupJob's name. kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s + kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s kubectl -n "$NAMESPACE" delete secret cozy-backups-creds --ignore-not-found --wait=true --timeout=30s check: ($error == null): true @@ -562,10 +564,9 @@ spec: timeout: 3m content: | set -eu - backup=$(kubectl -n "$NAMESPACE" get backupjob sysbucket-backup --ignore-not-found -o jsonpath='{.status.backupRef.name}') kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s - if [ -n "$backup" ]; then - kubectl -n "$NAMESPACE" delete backup "$backup" --ignore-not-found --wait=true --timeout=60s - fi + # The driver names the Backup after the BackupJob; by name it is + # found even when status.backupRef was never written. + kubectl -n "$NAMESPACE" delete backup sysbucket-backup --ignore-not-found --wait=true --timeout=60s kubectl -n "$NAMESPACE" delete secret e2e-system-bucket-s3-tls --ignore-not-found --wait=true --timeout=30s