mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
test(clickhouse): fail the round-trip when CoreDNS cannot be restored
The round-trip puts the host CoreDNS Corefile back from its EXIT trap, which runs with errexit off. A refused kubectl replace, for instance a resourceVersion conflict with another writer, fell through to the success message and returned 0, so the run stayed green while the rewrite remained in a cluster-wide object and the next run refused to start. The restore now returns the failure. The restore moves into a file of its own so a unit test can run it against a stubbed kubectl and so other callers can share it. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
0ad642e68d
commit
003bc3ed0b
3 files changed
+132
-33
No files matched your search
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bats
|
||||
# The clickhouse-3-system-bucket-roundtrip Test edits the host CoreDNS Corefile
|
||||
# and must put it back. Its restore runs from an EXIT trap under `set +e`, so a
|
||||
# restore that fails without saying so leaves the rewrite in a cluster-wide
|
||||
# object and still reports a green run; the next run then refuses to start.
|
||||
# These run the restore against a stubbed kubectl.
|
||||
|
||||
load test_helper
|
||||
|
||||
LIB=hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh
|
||||
HELPERS=examples/backups/clickhouse/00-helpers.sh
|
||||
|
||||
setup() {
|
||||
strict_setup
|
||||
STUB=$(mktemp -d)
|
||||
# A Corefile carrying the Test's marked block, as the run step leaves it.
|
||||
jq -n '{apiVersion: "v1", kind: "ConfigMap",
|
||||
metadata: {name: "coredns", namespace: "kube-system", resourceVersion: "7"},
|
||||
data: {Corefile: ".:53 {\n # === BEGIN e2e-clickhouse-system-bucket ===\n rewrite name exact s3.example.org root-ingress-controller.tenant-root.svc.cozy.local\n # === END e2e-clickhouse-system-bucket ===\n errors\n}\n"}}' \
|
||||
>"$STUB/coredns.json"
|
||||
cat >"$STUB/kubectl" <<'EOF'
|
||||
#!/bin/bash
|
||||
case "$*" in
|
||||
*"get configmap coredns -o json"*) cat "$STUB/coredns.json" ;;
|
||||
replace*)
|
||||
cp "${@: -1}" "$STUB/replaced.json"
|
||||
if [ -n "${REPLACE_FAILS:-}" ]; then
|
||||
echo 'Error from server (Conflict): Operation cannot be fulfilled on configmaps "coredns": the object has been modified' >&2
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
*"get secret"*) ;;
|
||||
*) echo "unexpected kubectl $*" >&2; exit 2 ;;
|
||||
esac
|
||||
EOF
|
||||
chmod +x "$STUB/kubectl"
|
||||
export STUB
|
||||
}
|
||||
|
||||
restore() {
|
||||
PATH="$STUB:$PATH" NAMESPACE=tenant-root bash -c '
|
||||
set -euo pipefail
|
||||
source "$1"
|
||||
source "$2"
|
||||
set +e
|
||||
restore_shared_state
|
||||
' _ "$HELPERS" "$LIB"
|
||||
}
|
||||
|
||||
@test "a CoreDNS restore the API server refuses fails the restore" {
|
||||
export REPLACE_FAILS=1
|
||||
err=$(mktemp)
|
||||
if restore 2>"$err"; then
|
||||
echo "restore_shared_state returned 0 although kubectl replace was refused" >&2
|
||||
cat "$err" >&2
|
||||
false
|
||||
fi
|
||||
if grep -q 'Removed the s3.example.org rewrite' "$err"; then
|
||||
echo "restore_shared_state reported the rewrite removed although kubectl replace was refused:" >&2
|
||||
cat "$err" >&2
|
||||
false
|
||||
fi
|
||||
}
|
||||
|
||||
@test "a CoreDNS restore sends the Corefile back without the marked block" {
|
||||
restore
|
||||
jq -e '.data.Corefile == ".:53 {\n errors\n}\n"' "$STUB/replaced.json" >/dev/null || {
|
||||
echo "restored Corefile:" >&2
|
||||
jq -r '.data.Corefile' "$STUB/replaced.json" >&2
|
||||
false
|
||||
}
|
||||
# The resourceVersion read travels with the replace, so a concurrent
|
||||
# writer makes it fail rather than be overwritten.
|
||||
[ "$(jq -r '.metadata.resourceVersion' "$STUB/replaced.json")" = 7 ]
|
||||
}
|
||||
@@ -24,18 +24,14 @@
|
||||
set -euo pipefail
|
||||
|
||||
source examples/backups/clickhouse/00-helpers.sh
|
||||
source hack/e2e-chainsaw/clickhouse/system-bucket-shared-state.sh
|
||||
|
||||
: "${NAMESPACE:?}"
|
||||
CLICKHOUSE_NAME=sysbucket
|
||||
BACKUPJOB_NAME=sysbucket-backup
|
||||
RESTOREJOB_NAME=sysbucket-restore
|
||||
S3_HOST=s3.example.org
|
||||
INGRESS_SECRET=seaweedfs-s3-ingress-tls
|
||||
CERT_SECRET=e2e-system-bucket-s3-tls
|
||||
PROBE_POD=e2e-system-bucket-probe
|
||||
OWNER_LABEL=cozystack.io/e2e-owner=clickhouse-system-bucket
|
||||
DNS_BEGIN=' # === BEGIN e2e-clickhouse-system-bucket ==='
|
||||
DNS_END=' # === END e2e-clickhouse-system-bucket ==='
|
||||
DNS_TARGET="root-ingress-controller.${NAMESPACE}.svc.cozy.local"
|
||||
# Kubelet refreshes a ConfigMap volume within about a minute and CoreDNS's
|
||||
# reload plugin polls every 30s; the rest is the 30s answer cache.
|
||||
@@ -46,35 +42,10 @@ JOB_TIMEOUT=480
|
||||
|
||||
workdir=$(mktemp -d)
|
||||
|
||||
corefile_has_block() {
|
||||
kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF -- "$DNS_BEGIN"
|
||||
}
|
||||
|
||||
remove_dns_block() {
|
||||
kubectl -n kube-system get configmap coredns -o json >"$workdir/coredns.json"
|
||||
local begins
|
||||
begins=$(jq -r '.data.Corefile' "$workdir/coredns.json" | grep -cF -- "$DNS_BEGIN" || true)
|
||||
if [ "$begins" = 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
if [ "$begins" != 1 ] || ! jq -r '.data.Corefile' "$workdir/coredns.json" | grep -qF -- "$DNS_END"; then
|
||||
log_error "kube-system/coredns holds ${begins} '${DNS_BEGIN# *}' markers or no END marker; reconcile the Corefile by hand"
|
||||
return 1
|
||||
fi
|
||||
jq --arg b "$DNS_BEGIN" --arg e "$DNS_END" \
|
||||
'.data.Corefile |= (split("\n") | (index($b)) as $i | (index($e)) as $j | .[:$i] + .[$j+1:] | join("\n"))' \
|
||||
"$workdir/coredns.json" >"$workdir/coredns-clean.json"
|
||||
kubectl replace -f "$workdir/coredns-clean.json" >/dev/null
|
||||
log_success "Removed the ${S3_HOST} rewrite from kube-system/coredns"
|
||||
}
|
||||
|
||||
restore_shared_state() {
|
||||
on_exit() {
|
||||
local rc=$?
|
||||
set +e
|
||||
remove_dns_block || rc=1
|
||||
if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -l "$OWNER_LABEL" -o name 2>/dev/null | grep -q .; then
|
||||
kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1
|
||||
fi
|
||||
restore_shared_state || rc=1
|
||||
rm -rf "$workdir"
|
||||
exit "$rc"
|
||||
}
|
||||
@@ -87,7 +58,7 @@ if corefile_has_block; then
|
||||
log_error "kube-system/coredns already carries the ${S3_HOST} rewrite block, left by an earlier run; remove it by hand"
|
||||
exit 1
|
||||
fi
|
||||
trap restore_shared_state EXIT
|
||||
trap on_exit EXIT
|
||||
|
||||
log_step "Serving the Test certificate for ${S3_HOST} through the platform S3 Ingress"
|
||||
kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o json \
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/bin/bash
|
||||
# The two shared objects system-bucket-roundtrip.sh changes for the duration
|
||||
# of its run, and how to put them back. Sourced by that script, whose EXIT trap
|
||||
# restores them. Needs NAMESPACE and log_success/log_error from
|
||||
# examples/backups/clickhouse/00-helpers.sh.
|
||||
|
||||
S3_HOST=s3.example.org
|
||||
INGRESS_SECRET=seaweedfs-s3-ingress-tls
|
||||
OWNER_LABEL=cozystack.io/e2e-owner=clickhouse-system-bucket
|
||||
DNS_BEGIN=' # === BEGIN e2e-clickhouse-system-bucket ==='
|
||||
DNS_END=' # === END e2e-clickhouse-system-bucket ==='
|
||||
|
||||
corefile_has_block() {
|
||||
kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF -- "$DNS_BEGIN"
|
||||
}
|
||||
|
||||
remove_dns_block() {
|
||||
local dir
|
||||
dir=$(mktemp -d)
|
||||
kubectl -n kube-system get configmap coredns -o json >"$dir/coredns.json" || { rm -rf "$dir"; return 1; }
|
||||
local begins
|
||||
begins=$(jq -r '.data.Corefile' "$dir/coredns.json" | grep -cF -- "$DNS_BEGIN" || true)
|
||||
if [ "$begins" = 0 ]; then
|
||||
rm -rf "$dir"
|
||||
return 0
|
||||
fi
|
||||
if [ "$begins" != 1 ] || ! jq -r '.data.Corefile' "$dir/coredns.json" | grep -qF -- "$DNS_END"; then
|
||||
log_error "kube-system/coredns holds ${begins} '${DNS_BEGIN# *}' markers or no END marker; reconcile the Corefile by hand"
|
||||
rm -rf "$dir"
|
||||
return 1
|
||||
fi
|
||||
jq --arg b "$DNS_BEGIN" --arg e "$DNS_END" \
|
||||
'.data.Corefile |= (split("\n") | (index($b)) as $i | (index($e)) as $j | .[:$i] + .[$j+1:] | join("\n"))' \
|
||||
"$dir/coredns.json" >"$dir/coredns-clean.json"
|
||||
# Callers run this with errexit off, so a refused replace has to return.
|
||||
if ! kubectl replace -f "$dir/coredns-clean.json" >/dev/null; then
|
||||
log_error "could not remove the ${S3_HOST} rewrite from kube-system/coredns; remove the marked block by hand"
|
||||
rm -rf "$dir"
|
||||
return 1
|
||||
fi
|
||||
log_success "Removed the ${S3_HOST} rewrite from kube-system/coredns"
|
||||
rm -rf "$dir"
|
||||
}
|
||||
|
||||
# Returns non-zero when either object could not be put back.
|
||||
restore_shared_state() {
|
||||
local rc=0
|
||||
remove_dns_block || rc=1
|
||||
if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -l "$OWNER_LABEL" -o name 2>/dev/null | grep -q .; then
|
||||
kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1
|
||||
fi
|
||||
return "$rc"
|
||||
}
|
||||
Reference in new issue
Block a user