test(clickhouse): fail the round-trip when CoreDNS cannot be restored

The round-trip puts the host CoreDNS Corefile back from its EXIT trap,
which runs with errexit off. A refused kubectl replace, for instance a
resourceVersion conflict with another writer, fell through to the
success message and returned 0, so the run stayed green while the
rewrite remained in a cluster-wide object and the next run refused to
start. The restore now returns the failure.

The restore moves into a file of its own so a unit test can run it
against a stubbed kubectl and so other callers can share it.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-05 21:06:53 +03:00
1 parent 0ad642e68d
commit 003bc3ed0b
3 files changed
+132 -33

No files matched your search

@@ -0,0 +1,75 @@
#!/usr/bin/env bats
# The clickhouse-3-system-bucket-roundtrip Test edits the host CoreDNS Corefile
# and must put it back. Its restore runs from an EXIT trap under `set +e`, so a
# restore that fails without saying so leaves the rewrite in a cluster-wide
# object and still reports a green run; the next run then refuses to start.
# These run the restore against a stubbed kubectl.
load test_helper
LIB=hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh
HELPERS=examples/backups/clickhouse/00-helpers.sh
setup() {
strict_setup
STUB=$(mktemp -d)
# A Corefile carrying the Test's marked block, as the run step leaves it.
jq -n '{apiVersion: "v1", kind: "ConfigMap",
metadata: {name: "coredns", namespace: "kube-system", resourceVersion: "7"},
data: {Corefile: ".:53 {\n # === BEGIN e2e-clickhouse-system-bucket ===\n rewrite name exact s3.example.org root-ingress-controller.tenant-root.svc.cozy.local\n # === END e2e-clickhouse-system-bucket ===\n errors\n}\n"}}' \
>"$STUB/coredns.json"
cat >"$STUB/kubectl" <<'EOF'
#!/bin/bash
case "$*" in
*"get configmap coredns -o json"*) cat "$STUB/coredns.json" ;;
replace*)
cp "${@: -1}" "$STUB/replaced.json"
if [ -n "${REPLACE_FAILS:-}" ]; then
echo 'Error from server (Conflict): Operation cannot be fulfilled on configmaps "coredns": the object has been modified' >&2
exit 1
fi
;;
*"get secret"*) ;;
*) echo "unexpected kubectl $*" >&2; exit 2 ;;
esac
EOF
chmod +x "$STUB/kubectl"
export STUB
}
restore() {
PATH="$STUB:$PATH" NAMESPACE=tenant-root bash -c '
set -euo pipefail
source "$1"
source "$2"
set +e
restore_shared_state
' _ "$HELPERS" "$LIB"
}
@test "a CoreDNS restore the API server refuses fails the restore" {
export REPLACE_FAILS=1
err=$(mktemp)
if restore 2>"$err"; then
echo "restore_shared_state returned 0 although kubectl replace was refused" >&2
cat "$err" >&2
false
fi
if grep -q 'Removed the s3.example.org rewrite' "$err"; then
echo "restore_shared_state reported the rewrite removed although kubectl replace was refused:" >&2
cat "$err" >&2
false
fi
}
@test "a CoreDNS restore sends the Corefile back without the marked block" {
restore
jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null || {
echo "restored Corefile:" >&2
jq -r '.data.Corefile' "$STUB/replaced.json" >&2
false
}
# The resourceVersion read travels with the replace, so a concurrent
# writer makes it fail rather than be overwritten.
[ "$(jq -r '.metadata.resourceVersion' "$STUB/replaced.json")" = 7 ]
}
@@ -24,18 +24,14 @@
set -euo pipefail
source examples/backups/clickhouse/00-helpers.sh
source hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh
: "${NAMESPACE:?}"
CLICKHOUSE_NAME=sysbucket
BACKUPJOB_NAME=sysbucket-backup
RESTOREJOB_NAME=sysbucket-restore
S3_HOST=s3.example.org
INGRESS_SECRET=seaweedfs-s3-ingress-tls
CERT_SECRET=e2e-system-bucket-s3-tls
PROBE_POD=e2e-system-bucket-probe
OWNER_LABEL=cozystack.io/e2e-owner=clickhouse-system-bucket
DNS_BEGIN=' # === BEGIN e2e-clickhouse-system-bucket ==='
DNS_END=' # === END e2e-clickhouse-system-bucket ==='
DNS_TARGET="root-ingress-controller.${NAMESPACE}.svc.cozy.local"
# Kubelet refreshes a ConfigMap volume within about a minute and CoreDNS's
# reload plugin polls every 30s; the rest is the 30s answer cache.
@@ -46,35 +42,10 @@ JOB_TIMEOUT=480
workdir=$(mktemp -d)
corefile_has_block() {
kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF -- "$DNS_BEGIN"
}
remove_dns_block() {
kubectl -n kube-system get configmap coredns -o json >"$workdir/coredns.json"
local begins
begins=$(jq -r '.data.Corefile' "$workdir/coredns.json" | grep -cF -- "$DNS_BEGIN" || true)
if [ "$begins" = 0 ]; then
return 0
fi
if [ "$begins" != 1 ] || ! jq -r '.data.Corefile' "$workdir/coredns.json" | grep -qF -- "$DNS_END"; then
log_error "kube-system/coredns holds ${begins} '${DNS_BEGIN# *}' markers or no END marker; reconcile the Corefile by hand"
return 1
fi
jq --arg b "$DNS_BEGIN" --arg e "$DNS_END" \
'.data.Corefile |= (split("\n") | (index($b)) as $i | (index($e)) as $j | .[:$i] + .[$j+1:] | join("\n"))' \
"$workdir/coredns.json" >"$workdir/coredns-clean.json"
kubectl replace -f "$workdir/coredns-clean.json" >/dev/null
log_success "Removed the ${S3_HOST} rewrite from kube-system/coredns"
}
restore_shared_state() {
on_exit() {
local rc=$?
set +e
remove_dns_block || rc=1
if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -l "$OWNER_LABEL" -o name 2>/dev/null | grep -q .; then
kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1
fi
restore_shared_state || rc=1
rm -rf "$workdir"
exit "$rc"
}
@@ -87,7 +58,7 @@ if corefile_has_block; then
log_error "kube-system/coredns already carries the ${S3_HOST} rewrite block, left by an earlier run; remove it by hand"
exit 1
fi
trap restore_shared_state EXIT
trap on_exit EXIT
log_step "Serving the Test certificate for ${S3_HOST} through the platform S3 Ingress"
kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o json \
@@ -0,0 +1,53 @@
#!/bin/bash
# The two shared objects system-bucket-roundtrip.sh changes for the duration
# of its run, and how to put them back. Sourced by that script, whose EXIT trap
# restores them. Needs NAMESPACE and log_success/log_error from
# examples/backups/clickhouse/00-helpers.sh.
S3_HOST=s3.example.org
INGRESS_SECRET=seaweedfs-s3-ingress-tls
OWNER_LABEL=cozystack.io/e2e-owner=clickhouse-system-bucket
DNS_BEGIN=' # === BEGIN e2e-clickhouse-system-bucket ==='
DNS_END=' # === END e2e-clickhouse-system-bucket ==='
corefile_has_block() {
kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF -- "$DNS_BEGIN"
}
remove_dns_block() {
local dir
dir=$(mktemp -d)
kubectl -n kube-system get configmap coredns -o json >"$dir/coredns.json" || { rm -rf "$dir"; return 1; }
local begins
begins=$(jq -r '.data.Corefile' "$dir/coredns.json" | grep -cF -- "$DNS_BEGIN" || true)
if [ "$begins" = 0 ]; then
rm -rf "$dir"
return 0
fi
if [ "$begins" != 1 ] || ! jq -r '.data.Corefile' "$dir/coredns.json" | grep -qF -- "$DNS_END"; then
log_error "kube-system/coredns holds ${begins} '${DNS_BEGIN# *}' markers or no END marker; reconcile the Corefile by hand"
rm -rf "$dir"
return 1
fi
jq --arg b "$DNS_BEGIN" --arg e "$DNS_END" \
'.data.Corefile |= (split("\n") | (index($b)) as $i | (index($e)) as $j | .[:$i] + .[$j+1:] | join("\n"))' \
"$dir/coredns.json" >"$dir/coredns-clean.json"
# Callers run this with errexit off, so a refused replace has to return.
if ! kubectl replace -f "$dir/coredns-clean.json" >/dev/null; then
log_error "could not remove the ${S3_HOST} rewrite from kube-system/coredns; remove the marked block by hand"
rm -rf "$dir"
return 1
fi
log_success "Removed the ${S3_HOST} rewrite from kube-system/coredns"
rm -rf "$dir"
}
# Returns non-zero when either object could not be put back.
restore_shared_state() {
local rc=0
remove_dns_block || rc=1
if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -l "$OWNER_LABEL" -o name 2>/dev/null | grep -q .; then
kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1
fi
return "$rc"
}