test(clickhouse): require the system-bucket round-trip to run on the sidecar that started without credentials

A Pod restart during the round-trip would hand the sidecar fresh
credentials at start, which is how the old environment-based sidecar
got them, and the Test would pass without the running sidecar ever
picking them up. The Pod uid and the sidecar's restart count must stay
the same across both jobs.

The probe deadline is now measured on the clock: counting only its
pauses let a hanging exec stretch it past the step budget, so the step
timeout would fire before the probe could say what stalled.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-04 19:43:55 +03:00
1 parent 6345b939d2
commit dbcc2af679
2 files changed
+22 -8

No files matched your search

@@ -521,13 +521,14 @@ spec:
- name: run
try:
# Inner budgets of system-bucket-roundtrip.sh:
# 240s probe: CoreDNS picking up the rewrite and the Ingress serving
# the Test certificate
# 280s probe: a 240s deadline for CoreDNS picking up the rewrite and
# the Ingress serving the Test certificate, plus one last 30s
# attempt and its 10s pause
# 480s BackupJob to Succeeded, including the driver's 2m hold
# 480s RestoreJob to Succeeded, same hold
# 90s the EXIT trap: Corefile replace plus a 60s Secret delete
# 1290s (21m30s)
# 25m leaves 3m30s for the SQL and kubectl calls between those waits, so
# 1330s (22m10s)
# 25m leaves 2m50s for the SQL and kubectl calls between those waits, so
# an inner wait is what fires and names what stalled.
- script:
timeout: 25m
@@ -111,11 +111,11 @@ log_step "Checking that ${S3_HOST} serves the Test certificate from inside ${NAM
kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.ca\.crt}' | base64 -d >"$workdir/ca.crt"
want=$(kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.tls\.crt}' | base64 -d \
| openssl x509 -noout -fingerprint -sha256)
elapsed=0
deadline=$((SECONDS + PROBE_TIMEOUT))
while true; do
# --cacert makes curl verify both the chain and the host name; %{certs}
# prints the chain it was served, leaf first.
if served=$(kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \
if served=$(timeout 30 kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \
"cat >/tmp/ca.crt && curl -sS -o /dev/null --max-time 10 --cacert /tmp/ca.crt -w '%{certs}' https://${S3_HOST}/" \
<"$workdir/ca.crt" 2>"$workdir/probe.err"); then
got=$(printf '%s\n' "$served" | sed -n '/BEGIN CERTIFICATE/,/END CERTIFICATE/p;/END CERTIFICATE/q' \
@@ -126,16 +126,26 @@ while true; do
fi
echo "served ${got}, want ${want}" >"$workdir/probe.err"
fi
if [ "$elapsed" -ge "$PROBE_TIMEOUT" ]; then
if [ "$SECONDS" -ge "$deadline" ]; then
log_error "${S3_HOST} did not serve the Test certificate within ${PROBE_TIMEOUT}s: $(cat "$workdir/probe.err")"
kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' >&2 || true
kubectl -n "$NAMESPACE" describe ingress ingress-seaweedfs-s3 >&2 || true
exit 1
fi
sleep 10
elapsed=$((elapsed + 10))
done
# A restarted Pod reads its credentials afresh at start, which is what the
# old environment-based sidecar needed; the round-trip proves the fix only if
# the sidecar that started without them is the one that serves both jobs.
sidecar_identity() {
kubectl -n "$NAMESPACE" get pod "chi-clickhouse-${CLICKHOUSE_NAME}-clickhouse-0-0-0" \
-o jsonpath='{.metadata.uid} {.status.containerStatuses[?(@.name=="clickhouse-backup")].restartCount}'
}
sidecar_before=$(sidecar_identity)
[[ "$sidecar_before" =~ ^[0-9a-f-]+\ [0-9]+$ ]] \
|| { log_error "could not read the ClickHouse Pod uid and clickhouse-backup restart count (got '${sidecar_before}')"; exit 1; }
log_step "Writing the sentinel row"
clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id"
clickhouse_query "$CLICKHOUSE_NAME" "INSERT INTO default.sentinel VALUES (1, 'before-backup')"
@@ -182,4 +192,7 @@ wait_for_field restorejob "$RESTOREJOB_NAME" '{.status.phase}' Succeeded "$NAMES
count=$(clickhouse_query "$CLICKHOUSE_NAME" "SELECT count() FROM default.sentinel" | tr -d '[:space:]')
[[ "$count" =~ ^[0-9]+$ ]] || { log_error "non-numeric sentinel count: '${count}'"; exit 1; }
(( count >= 1 )) || { log_error "sentinel row count after restore is ${count}; expected >= 1"; exit 1; }
sidecar_after=$(sidecar_identity)
[ "$sidecar_after" = "$sidecar_before" ] \
|| { log_error "the ClickHouse Pod or its clickhouse-backup container restarted during the round-trip (uid restarts: '${sidecar_before}' then '${sidecar_after}')"; exit 1; }
log_success "System-bucket round-trip verified: ${count} sentinel row(s) restored"