mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-09 05:25:14 +09:00
Andrey Kolkov
5bd029c46a
Record each topic's ReplicationFactor in the backup manifest, off the same --describe line the partition count already comes from, and recreate the topic with it on restore. The value used to be a BackupClass parameter defaulting to 1, and the Cozystack Kafka chart sets the broker-level min.insync.replicas to 2 as soon as kafka.replicas reaches 2: kafka-console-producer writes with acks=all, the broker rejects every write into a topic with fewer replicas than that, and the producer logs the rejection and exits 0. A restore onto any multi-broker cluster therefore landed zero records and failed the post-replay count with a message naming the wrong cause; in place, the documented delete had already taken the live topic with it. replicationFactor stays as an override for restoring into a smaller cluster, and a value the target cannot host now fails at --create in Kafka's own words. Hold the pre-replay emptiness check to the topic's partition count, as the backup side already does. kafka-get-offsets skips a partition it could not read and still exits 0, so a leader election in flight dropped the populated partitions from the listing, the surviving empty ones summed to zero, and the replay appended a full copy into a live topic - the exact outcome the check was moved ahead of the replay to prevent. Both restore-side counts now go through one helper that refuses a short listing. The data-file-versus-manifest check runs both ways before anything is created, and the replay walks the manifest topic by topic, announcing each one, so a retried Pod's log says which topics landed. The demo now runs two brokers with two-replica topics, which is the shape the chart puts min.insync.replicas=2 on; the restore steps assert the restored topic's replication factor, and step 06 submits a second RestoreJob against the populated topic and requires it to be refused with the topic untouched. Smaller findings from the same round: the topics parameter is held to Kafka's name charset (a literal "*" no longer path-expands) and refused on a repeated name, which produced a Succeeded backup no restore could balance; the strategy verifies S3 TLS against the seaweedfs CA projected into the <app>-backup-s3 Secret instead of curl -k, following the rabbitmq example, so the e2e endpoint moves to the .svc name the serving cert covers; the image lookup keeps digest-pinned references; the default namespace is tenant-root, the only one the demo ever ran in; step 05 caches the resolved Backup name for the restore steps; the decoy count subtracts begin offsets like the topic count; the bats test name carries its count again; and the scenario walk-throughs the sibling examples ship are added. Signed-off-by: Andrey Kolkov <andrey.kolkov@aenix.io> Assisted-By: LLM
273 lines
10 KiB
Bash
273 lines
10 KiB
Bash
#!/usr/bin/env bats
|
|
# Unit tests for the E2E-only S3 access gate that runs before expensive backup
|
|
# workloads. All cluster and S3 calls are shell stubs.
|
|
|
|
@test "preflight helper remains sourceable by the POSIX test runner" {
|
|
helper=hack/e2e-chainsaw/_lib/backup-access-preflight.sh
|
|
[ "$(sed -n '1p' "$helper")" = '#!/bin/sh' ]
|
|
if grep -nE '^[[:space:]]*(set[[:space:]].*pipefail|local([[:space:]]|$))' "$helper"; then
|
|
echo "backup preflight is sourced by /bin/sh and must not contain Bash-only shell state" >&2
|
|
exit 1
|
|
fi
|
|
sh -n "$helper"
|
|
}
|
|
|
|
@test "preflight fails closed when the BucketInfo Secret cannot be read" {
|
|
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
|
|
out=$(mktemp)
|
|
kubectl() {
|
|
case "$*" in
|
|
*" get secret bucket-demo-backup "*) return 1 ;;
|
|
*) echo "preflight continued after the Secret read failed: $*" >&2; return 1 ;;
|
|
esac
|
|
}
|
|
|
|
if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then
|
|
echo "failed BucketInfo Secret read passed the S3 preflight" >&2
|
|
exit 1
|
|
fi
|
|
grep -q 'failed to read BucketInfo Secret tenant-test/bucket-demo-backup' "$out"
|
|
if grep -q 'lacks access key' "$out"; then
|
|
echo "preflight continued into credential parsing after the Secret read failed" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
@test "preflight retries IAM convergence then proves PUT GET compare and DELETE" {
|
|
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
|
|
attempts=$(mktemp)
|
|
uploaded_source=$(mktemp)
|
|
calls=$(mktemp)
|
|
printf '0\n' > "$attempts"
|
|
|
|
kubectl() {
|
|
printf 'kubectl %s\n' "$*" >> "$calls"
|
|
case "$*" in
|
|
*" get secret bucket-demo-backup "*)
|
|
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
|
|
;;
|
|
*" port-forward "*)
|
|
while :; do command sleep 1; done
|
|
;;
|
|
*) echo "unexpected kubectl call: $*" >&2; return 1 ;;
|
|
esac
|
|
}
|
|
timeout() {
|
|
shift
|
|
if [ "$1" = sh ]; then
|
|
return 0
|
|
fi
|
|
"$@"
|
|
}
|
|
sleep() { :; }
|
|
mc() {
|
|
printf 'mc %s\n' "$*" >> "$calls"
|
|
case "$1" in
|
|
alias) return 0 ;;
|
|
cp)
|
|
case "$3" in
|
|
backup-preflight/*)
|
|
command cp "$(sed -n '1p' "$uploaded_source")" "$4"
|
|
;;
|
|
*)
|
|
count=$(sed -n '1p' "$attempts")
|
|
count=$(( count + 1 ))
|
|
printf '%s\n' "$count" > "$attempts"
|
|
if [ "$count" -eq 1 ]; then
|
|
echo 'AccessDenied: IAM identity not loaded yet' >&2
|
|
return 1
|
|
fi
|
|
printf '%s\n' "$3" > "$uploaded_source"
|
|
;;
|
|
esac
|
|
;;
|
|
rm) return 0 ;;
|
|
*) echo "unexpected mc call: $*" >&2; return 1 ;;
|
|
esac
|
|
}
|
|
|
|
cozy_backup_access_preflight tenant-test bucket-demo-backup 5 >/dev/null
|
|
[ "$(sed -n '1p' "$attempts")" -eq 2 ] || {
|
|
echo "preflight did not retry the not-yet-loaded IAM identity" >&2
|
|
cat "$calls" >&2
|
|
exit 1
|
|
}
|
|
[ "$(grep -c '^mc rm ' "$calls")" -eq 1 ] || {
|
|
echo "preflight did not delete its successfully verified object exactly once" >&2
|
|
cat "$calls" >&2
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
@test "preflight fails before backup work when granted credentials never reach S3" {
|
|
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
|
|
out=$(mktemp)
|
|
kubectl() {
|
|
case "$*" in
|
|
*" get secret bucket-demo-backup "*)
|
|
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
|
|
;;
|
|
*" port-forward "*) while :; do command sleep 1; done ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
timeout() {
|
|
shift
|
|
if [ "$1" = sh ]; then return 0; fi
|
|
"$@"
|
|
}
|
|
mc() {
|
|
case "$1" in
|
|
alias) return 0 ;;
|
|
cp) echo 'AccessDenied' >&2; return 1 ;;
|
|
*) return 0 ;;
|
|
esac
|
|
}
|
|
|
|
if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then
|
|
echo "unusable credentials passed the S3 preflight" >&2
|
|
exit 1
|
|
fi
|
|
grep -q 'granted but did not pass S3 PUT/GET/DELETE within 0s' "$out"
|
|
grep -q 's3-preflight: AccessDenied' "$out"
|
|
grep -q 's3-preflight: attempt 1 failed at PUT' "$out"
|
|
}
|
|
|
|
@test "preflight timeout reports the port-forward and how every attempt ended" {
|
|
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
|
|
out=$(mktemp)
|
|
now=$(mktemp)
|
|
pf_ready=$(mktemp)
|
|
attempts=$(mktemp)
|
|
uploaded_source=$(mktemp)
|
|
printf '100\n' > "$now"
|
|
printf '0\n' > "$attempts"
|
|
|
|
kubectl() {
|
|
case "$*" in
|
|
*" get secret bucket-demo-backup "*)
|
|
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
|
|
;;
|
|
*" port-forward "*)
|
|
echo 'Forwarding from 127.0.0.1:18333 -> 8333'
|
|
printf 'ready\n' > "$pf_ready"
|
|
while :; do command sleep 1; done
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
# A stubbed clock keeps the retry budget independent of how fast the
|
|
# stubs run: only a completed PUT advances it, so the loop ends after a
|
|
# known number of attempts.
|
|
date() {
|
|
if [ "${1:-}" = +%s ]; then sed -n '1p' "$now"; else command date "$@"; fi
|
|
}
|
|
timeout() {
|
|
shift
|
|
if [ "$1" = sh ]; then
|
|
waited=0
|
|
while [ ! -s "$pf_ready" ] && [ "$waited" -lt 100 ]; do
|
|
waited=$(( waited + 1 ))
|
|
command sleep 0.1
|
|
done
|
|
return 0
|
|
fi
|
|
"$@"
|
|
}
|
|
sleep() { :; }
|
|
# The first three attempts break at a later stage than the one before, so
|
|
# the stages that follow a completed upload are told apart rather than
|
|
# merged.
|
|
mc() {
|
|
case "$1" in
|
|
alias) return 0 ;;
|
|
cp)
|
|
case "$3" in
|
|
backup-preflight/*)
|
|
case "$(sed -n '1p' "$attempts")" in
|
|
1)
|
|
echo 'mc: <ERROR> Unable to download: connection reset by peer' >&2
|
|
return 1
|
|
;;
|
|
2) printf 'truncated\n' > "$4" ;;
|
|
*) command cp "$(sed -n '1p' "$uploaded_source")" "$4" ;;
|
|
esac
|
|
;;
|
|
*)
|
|
printf '%s\n' "$3" > "$uploaded_source"
|
|
count=$(sed -n '1p' "$attempts")
|
|
printf '%s\n' "$(( count + 1 ))" > "$attempts"
|
|
stamp=$(sed -n '1p' "$now")
|
|
printf '%s\n' "$(( stamp + 10 ))" > "$now"
|
|
# Do not tidy this arm away into a clean ascending
|
|
# ladder. A ladder where every attempt fails later than
|
|
# the last can never catch a missing per-iteration
|
|
# stage reset, because the stale value is overwritten
|
|
# before anything reads it.
|
|
if [ "$count" -eq 3 ]; then
|
|
echo 'mc: <ERROR> Unable to upload: connection reset by peer' >&2
|
|
return 1
|
|
fi
|
|
echo 'source -> backup-preflight/bucket-real/object'
|
|
;;
|
|
esac
|
|
;;
|
|
rm) return 1 ;;
|
|
*) return 0 ;;
|
|
esac
|
|
}
|
|
|
|
if cozy_backup_access_preflight tenant-test bucket-demo-backup 35 >"$out" 2>&1; then
|
|
echo "an S3 round trip that never completed passed the preflight" >&2
|
|
exit 1
|
|
fi
|
|
grep -q 'port-forward: Forwarding from 127.0.0.1:18333' "$out" || {
|
|
echo "timeout path dropped the port-forward log" >&2
|
|
cat "$out" >&2
|
|
exit 1
|
|
}
|
|
for expected in \
|
|
'attempt 1 failed at GET' \
|
|
'attempt 2 failed at compare' \
|
|
'attempt 3 failed at DELETE' \
|
|
'attempt 4 failed at PUT'; do
|
|
grep -q "s3-preflight: $expected" "$out" || {
|
|
echo "timeout path lost '$expected'" >&2
|
|
cat "$out" >&2
|
|
exit 1
|
|
}
|
|
done
|
|
}
|
|
|
|
@test "all active database backup flows gate immediately after accessGranted" {
|
|
for file in \
|
|
examples/backups/postgres/run-all.sh \
|
|
examples/backups/mariadb/run-all.sh \
|
|
examples/backups/mongodb/run-all.sh \
|
|
examples/backups/clickhouse/03-create-bucket.sh \
|
|
examples/backups/kafka/03-create-bucket.sh; do
|
|
granted=$(grep -n 'accessGranted' "$file" | head -n 1 | cut -d: -f1)
|
|
preflight=$(grep -n 'cozy_backup_access_preflight' "$file" | head -n 1 | cut -d: -f1)
|
|
coordinates=$(grep -n 'Reading bucket coordinates' "$file" | head -n 1 | cut -d: -f1)
|
|
if [ -z "$granted" ] || [ -z "$preflight" ] || [ -z "$coordinates" ] \
|
|
|| [ "$granted" -ge "$preflight" ] || [ "$preflight" -ge "$coordinates" ]; then
|
|
echo "$file must run the S3 preflight after accessGranted and before consuming bucket coordinates" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
}
|
|
|
|
@test "the five preflight-wired Chainsaw backup roundtrips enable the E2E preflight" {
|
|
for file in \
|
|
hack/e2e-chainsaw/postgres/chainsaw-test.yaml \
|
|
hack/e2e-chainsaw/mariadb/chainsaw-test.yaml \
|
|
hack/e2e-chainsaw/mongodb/chainsaw-test.yaml \
|
|
hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml \
|
|
hack/e2e-chainsaw/kafka/chainsaw-test.yaml; do
|
|
grep -q 'COZY_E2E_BACKUP_PREFLIGHT=1' "$file" || {
|
|
echo "$file does not enable the backup access preflight" >&2
|
|
exit 1
|
|
}
|
|
done
|
|
}
|