From dbcc2af6791c320a60b80662e78074bcd23b87cf Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 19:43:55 +0300 Subject: [PATCH] test(clickhouse): require the system-bucket round-trip to run on the sidecar that started without credentials A Pod restart during the round-trip would hand the sidecar fresh credentials at start, which is how the old environment-based sidecar got them, and the Test would pass without the running sidecar ever picking them up. The Pod uid and the sidecar's restart count must stay the same across both jobs. The probe deadline is now measured on the clock: counting only its pauses let a hanging exec stretch it past the step budget, so the step timeout would fire before the probe could say what stalled. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../clickhouse/chainsaw-test.yaml | 9 ++++---- .../clickhouse/system-bucket-roundtrip.sh | 21 +++++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml index f5b444a84..d96013140 100644 --- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml @@ -521,13 +521,14 @@ spec: - name: run try: # Inner budgets of system-bucket-roundtrip.sh: - # 240s probe: CoreDNS picking up the rewrite and the Ingress serving - # the Test certificate + # 280s probe: a 240s deadline for CoreDNS picking up the rewrite and + # the Ingress serving the Test certificate, plus one last 30s + # attempt and its 10s pause # 480s BackupJob to Succeeded, including the driver's 2m hold # 480s RestoreJob to Succeeded, same hold # 90s the EXIT trap: Corefile replace plus a 60s Secret delete - # 1290s (21m30s) - # 25m leaves 3m30s for the SQL and kubectl calls between those waits, so + # 1330s (22m10s) + # 25m leaves 2m50s for the SQL and kubectl calls between those waits, so # an inner wait is what fires and names what stalled. - script: timeout: 25m diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh index 4ca52024b..fe1361418 100755 --- a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh +++ b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh @@ -111,11 +111,11 @@ log_step "Checking that ${S3_HOST} serves the Test certificate from inside ${NAM kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.ca\.crt}' | base64 -d >"$workdir/ca.crt" want=$(kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.tls\.crt}' | base64 -d \ | openssl x509 -noout -fingerprint -sha256) -elapsed=0 +deadline=$((SECONDS + PROBE_TIMEOUT)) while true; do # --cacert makes curl verify both the chain and the host name; %{certs} # prints the chain it was served, leaf first. - if served=$(kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \ + if served=$(timeout 30 kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \ "cat >/tmp/ca.crt && curl -sS -o /dev/null --max-time 10 --cacert /tmp/ca.crt -w '%{certs}' https://${S3_HOST}/" \ <"$workdir/ca.crt" 2>"$workdir/probe.err"); then got=$(printf '%s\n' "$served" | sed -n '/BEGIN CERTIFICATE/,/END CERTIFICATE/p;/END CERTIFICATE/q' \ @@ -126,16 +126,26 @@ while true; do fi echo "served ${got}, want ${want}" >"$workdir/probe.err" fi - if [ "$elapsed" -ge "$PROBE_TIMEOUT" ]; then + if [ "$SECONDS" -ge "$deadline" ]; then log_error "${S3_HOST} did not serve the Test certificate within ${PROBE_TIMEOUT}s: $(cat "$workdir/probe.err")" kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' >&2 || true kubectl -n "$NAMESPACE" describe ingress ingress-seaweedfs-s3 >&2 || true exit 1 fi sleep 10 - elapsed=$((elapsed + 10)) done +# A restarted Pod reads its credentials afresh at start, which is what the +# old environment-based sidecar needed; the round-trip proves the fix only if +# the sidecar that started without them is the one that serves both jobs. +sidecar_identity() { + kubectl -n "$NAMESPACE" get pod "chi-clickhouse-${CLICKHOUSE_NAME}-clickhouse-0-0-0" \ + -o jsonpath='{.metadata.uid} {.status.containerStatuses[?(@.name=="clickhouse-backup")].restartCount}' +} +sidecar_before=$(sidecar_identity) +[[ "$sidecar_before" =~ ^[0-9a-f-]+\ [0-9]+$ ]] \ + || { log_error "could not read the ClickHouse Pod uid and clickhouse-backup restart count (got '${sidecar_before}')"; exit 1; } + log_step "Writing the sentinel row" clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id" clickhouse_query "$CLICKHOUSE_NAME" "INSERT INTO default.sentinel VALUES (1, 'before-backup')" @@ -182,4 +192,7 @@ wait_for_field restorejob "$RESTOREJOB_NAME" '{.status.phase}' Succeeded "$NAMES count=$(clickhouse_query "$CLICKHOUSE_NAME" "SELECT count() FROM default.sentinel" | tr -d '[:space:]') [[ "$count" =~ ^[0-9]+$ ]] || { log_error "non-numeric sentinel count: '${count}'"; exit 1; } (( count >= 1 )) || { log_error "sentinel row count after restore is ${count}; expected >= 1"; exit 1; } +sidecar_after=$(sidecar_identity) +[ "$sidecar_after" = "$sidecar_before" ] \ + || { log_error "the ClickHouse Pod or its clickhouse-backup container restarted during the round-trip (uid restarts: '${sidecar_before}' then '${sidecar_after}')"; exit 1; } log_success "System-bucket round-trip verified: ${count} sentinel row(s) restored"