From a25db5aeb6469d1fe5df473479108dd2a84e6911 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 4 Nov 2026 19:26:23 +0300 Subject: [PATCH] fix(backups): wait for an idle ClickHouse backup sidecar before starting The sidecar refuses a new operation with 423 while another one is in progress, but the strategy only waited for the API to answer, and went on to start its operation even when it never did. An operation already running in the sidecar therefore failed the attempt within seconds. Poll the action log until nothing is in progress, and fail the Pod instead of proceeding when the sidecar stays busy or unreachable, so the failure names the cause rather than surfacing as a 423 or a connection error from the operation itself. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/altinity-backup-strategy.bats | 149 ++++++++++++++++++ .../templates/strategy-altinity-default.yaml | 13 +- 2 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 hack/altinity-backup-strategy.bats diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats new file mode 100644 index 000000000..61729bf00 --- /dev/null +++ b/hack/altinity-backup-strategy.bats @@ -0,0 +1,149 @@ +#!/usr/bin/env bats +# Behavioural tests for the shipped ClickHouse (Altinity) strategy script, +# extracted from the rendered cozy-default-altinity Strategy CR and EXECUTED +# against a stubbed clickhouse-backup sidecar API. +# +# The contract pinned here is the start gate: the sidecar serialises its +# operations and answers any new one with HTTP 423 while another is "in +# progress", so a strategy Pod that starts its operation without waiting for +# an idle sidecar burns a Job attempt in seconds. The gate has to fail the Pod +# rather than proceed when the sidecar never becomes idle. +# +# Run from the repo root with make bats-unit-tests. The rendered script is run +# with `bash`, not `sh`: it opens `set -euo pipefail`, which the production +# image's busybox ash supports but the CI runner's /bin/sh (dash) rejects. + +load test_helper + +CHART=packages/system/backupstrategy-controller + +# Render the shipped strategy's in-Pod script into $1 with the controller-render +# placeholders substituted ($2 mode, $3 application name). An empty render, or +# a placeholder left unsubstituted, is an early failure: these tests EXECUTE it. +render_script() { + out=$1; mode=$2; app=$3 + helm template t "$CHART" -n cozy-backup-controller \ + --set backupStorage.bucketNameOverride=test-bucket 2>/dev/null \ + | yq eval 'select(.kind == "Altinity") | .spec.template.spec.containers[0].args[0]' - \ + | sed -e "s#{{ .Mode }}#${mode}#g" \ + -e "s#{{ .Release.Name }}#${app}#g" \ + -e "s#{{ .Release.Namespace }}#tenant-x#g" \ + -e "s#{{ .Backup.ApplicationRef.Name }}#${app}#g" > "$out" + [ -s "$out" ] || return 1 + if grep -q '{{' "$out"; then + echo "rendered script still holds a controller placeholder" >&2 + return 1 + fi +} + +# A curl stub standing in for the sidecar API, put first on PATH next to a +# no-op sleep. State lives in $1: +# unreachable present: every request fails as a refused connection (exit 7) +# busy a number: that many GET /backup/actions answers still list an +# operation "in progress"; while it is positive a POST is +# refused the way the sidecar refuses it (423, curl -f exit 22) +# posted the operation the script started, once it started one +# requests one line per request: method and URL +make_stubs() { + bin=$1; state=$2 + mkdir -p "$bin" "$state" + cat > "$bin/curl" <> "\$state/requests" +if [ -e "\$state/unreachable" ]; then exit 7; fi +busy=\$(cat "\$state/busy" 2>/dev/null || echo 0) +case "\$method \$url" in + "POST "*) + if [ "\$busy" -gt 0 ]; then exit 22; fi + echo "\${url##*/}" > "\$state/posted" + ;; + "GET "*/backup/actions) + echo '{"command":"create_remote clickhouse-myapp-20260101T000000Z","status":"error","start":"2026-01-01 00:00:00","error":"old"}' + if [ "\$busy" -gt 0 ]; then + echo \$((busy - 1)) > "\$state/busy" + echo '{"command":"upload --resumable=1 clickhouse-myapp-20260101T000000Z","status":"in progress","start":"2026-01-01 00:01:00"}' + elif [ -s "\$state/posted" ]; then + echo "{\"command\":\"\$(cat "\$state/posted")\",\"status\":\"success\",\"start\":\"2026-01-01 00:02:00\"}" + fi + ;; + "GET "*/backup/list/remote) + echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}' + ;; +esac +exit 0 +STUB + cat > "$bin/sleep" <<'STUB' +#!/bin/sh +exit 0 +STUB + chmod 0755 "$bin/curl" "$bin/sleep" +} + +run_script() { + tmp=$1 + API_USERNAME=u API_PASSWORD=p PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" +} + +@test "an empty render fails the fixture instead of passing as a no-op" { + tmp=$(mktemp -d) + CHART=$tmp/no-such-chart + if render_script "$tmp/s.sh" backup myapp 2>/dev/null; then + echo "FAIL: render_script reported success with no chart" + false + fi +} + +@test "backup on an idle sidecar starts create_remote and waits for its success" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp + run_script "$tmp" + grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests" +} + +@test "backup waits until the running operation finishes before create_remote" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/busy" + render_script "$tmp/s.sh" backup myapp + run_script "$tmp" + grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests" + # All three busy answers were consumed before the operation was started. + [ "$(cat "$tmp/state/busy")" = 0 ] +} + +@test "backup fails without starting create_remote when the sidecar stays busy" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 1000 > "$tmp/state/busy" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on a busy sidecar"; false; fi +} + +@test "backup fails without starting create_remote when the sidecar never answers" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + touch "$tmp/state/unreachable" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: an unreachable sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on an unreachable sidecar"; false; fi +} + +@test "restore fails without starting restore_remote when the sidecar stays busy" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 1000 > "$tmp/state/busy" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no restore may start on a busy sidecar"; false; fi +} diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml index 3397d6d0d..560b1e597 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml @@ -51,10 +51,21 @@ spec: host="chi-clickhouse-{{ printf "{{ .Release.Name }}" }}-clickhouse-0-0.{{ printf "{{ .Release.Namespace }}" }}.svc" api="http://${host}:7171" auth="-u ${API_USERNAME}:${API_PASSWORD}" + # The sidecar answers 423 to a new operation while any other is + # in progress, so starting before it is idle burns the attempt. + idle= for _ in $(seq 1 60); do - if curl -fsS ${auth} --max-time 2 "${api}/backup/status" >/dev/null 2>&1; then break; fi + if actions=$(curl -fsS ${auth} --max-time 5 "${api}/backup/actions") \ + && [ "$(printf '%s\n' "${actions}" | jq -rs '[.[] | select(.status == "in progress")] | length')" = 0 ]; then + idle=1 + break + fi sleep 5 done + if [ -z "${idle}" ]; then + echo "clickhouse-backup sidecar at ${api} did not become idle" >&2 + exit 1 + fi if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-" target="${prefix}$(date -u +%Y%m%dT%H%M%SZ)"