From 993b79bb370c20969c45fbcb51c01271f452fac6 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 11:23:28 +0300 Subject: [PATCH] fix(backups): take over a running ClickHouse restore on Job retry restore_remote drops every table it restores before recreating it, so running it twice discards whatever was written after the first run finished. The strategy Job retries a failed Pod up to twice, and with the start gate waiting for an idle sidecar instead of hitting 423, a retry whose predecessor died after its POST waited for that restore to finish and then started a second one. A retry now takes over a restore_remote of the same archive that is still running. A finished row is not taken over: the sidecar never trims its action log, so a success row of the archive may belong to an earlier RestoreJob, and treating it as this one would report a restore that never ran. A retry that arrives after the restore finished therefore still runs it again. Once the operation is accepted, one failed poll no longer ends the attempt under errexit; six in a row do. An operation that ends cancelled now fails the Pod instead of being polled forever. A retried backup still starts a new create_remote, because its archive name changes on every attempt; taking it over needs a name fixed for the whole BackupJob. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/altinity-backup-strategy.bats | 174 ++++++++++++++++-- .../templates/strategy-altinity-default.yaml | 82 +++++++-- 2 files changed, 218 insertions(+), 38 deletions(-) diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats index 7c5fdb476..2db1f33c6 100644 --- a/hack/altinity-backup-strategy.bats +++ b/hack/altinity-backup-strategy.bats @@ -10,6 +10,9 @@ # rather than proceed when the sidecar never becomes idle. Once started, an # operation the sidecar no longer lists (its action log lives in memory and a # restart empties it) has to fail the Pod too, since the Job has no deadline. +# A Job retry must not run a restore twice: it takes over the restore_remote +# the attempt before it left running, and a few lost polls do not end an +# attempt in the first place. # # Run from the repo root with make bats-unit-tests. The rendered script is run # with `bash`, not `sh`: it opens `set -euo pipefail`, which the production @@ -40,17 +43,31 @@ render_script() { # A curl stub standing in for the sidecar API, put first on PATH next to a # no-op sleep. State lives in $1: -# unreachable present: every request fails as a refused connection (exit 7) -# busy a number: that many GET /backup/actions answers still list an -# operation "in progress"; while it is positive a POST is -# refused the way the sidecar refuses it (423, curl -f exit 22) -# posted the operation the script started, once it started one -# restarted present: the action log no longer lists the started operation, -# as after the sidecar restarted and lost its in-memory log -# requests one line per request: method and URL -# runaway created when the script exceeds 200 requests; that request -# and every later one fails, so a polling loop that never ends -# stops the test instead of hanging it +# unreachable present: every request fails as a refused connection (7) +# down a number: that many next requests fail the same way, as +# before the sidecar has started +# busy a number: that many GET /backup/actions answers still list +# an operation of someone else "in progress" +# op the command line of the sidecar's own operation, written +# by an accepted POST or seeded by a test as the operation +# an earlier attempt of the same Job started +# op_left how many more GET /backup/actions answers list op as +# "in progress" before it ends with op_result +# op_duration op_left given to an operation a POST starts (default 0) +# op_result the status op ends with (default success) +# prior extra action-log rows, verbatim, older than op +# restarted present: the action log no longer lists op, as after the +# sidecar restarted and lost its in-memory log +# fail_on_post a POST that is accepted makes that many following +# GET /backup/actions requests fail (curl exit 7) +# idle created by an answer that lists nothing in progress; a +# POST is refused (423, curl -f exit 22) unless one was seen +# since the last accepted POST, and while anything runs +# accepted one line per POST the sidecar accepted +# requests one line per request: method and URL +# runaway created when the script exceeds 200 requests; that request +# and every later one fails, so a polling loop that never ends +# stops the test instead of hanging it make_stubs() { bin=$1; state=$2 mkdir -p "$bin" "$state" @@ -68,20 +85,49 @@ done echo "\$method \$url" >> "\$state/requests" if [ "\$(wc -l < "\$state/requests")" -gt 200 ]; then touch "\$state/runaway"; exit 7; fi if [ -e "\$state/unreachable" ]; then exit 7; fi -busy=\$(cat "\$state/busy" 2>/dev/null || echo 0) +num() { cat "\$state/\$1" 2>/dev/null || echo 0; } +down=\$(num down) +if [ "\$down" -gt 0 ]; then echo \$((down - 1)) > "\$state/down"; exit 7; fi +running=0 +if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ] && [ "\$(num op_left)" -gt 0 ]; then running=1; fi case "\$method \$url" in "POST "*) - if [ "\$busy" -gt 0 ]; then exit 22; fi - echo "\${url##*/}" > "\$state/posted" + if [ "\$(num busy)" -gt 0 ] || [ "\$running" = 1 ] || [ ! -e "\$state/idle" ]; then exit 22; fi + case "\$url" in + */create_remote\?name=*) cmd="create_remote \${url##*=}" ;; + */restore_remote/*) cmd="restore_remote \${url##*/}" ;; + *) exit 22 ;; + esac + echo "\$cmd" > "\$state/op" + echo "\$cmd" >> "\$state/accepted" + num op_duration > "\$state/op_left" + num fail_on_post > "\$state/fail_polls" + rm -f "\$state/idle" ;; "GET "*/backup/actions) + fail=\$(num fail_polls) + if [ "\$fail" -gt 0 ]; then echo \$((fail - 1)) > "\$state/fail_polls"; exit 7; fi echo '{"command":"create_remote clickhouse-myapp-20260101T000000Z","status":"error","start":"2026-01-01 00:00:00","error":"old"}' + cat "\$state/prior" 2>/dev/null || true + inprog=0 + busy=\$(num busy) if [ "\$busy" -gt 0 ]; then echo \$((busy - 1)) > "\$state/busy" echo '{"command":"upload --resumable=1 clickhouse-myapp-20260101T000000Z","status":"in progress","start":"2026-01-01 00:01:00"}' - elif [ -s "\$state/posted" ] && [ ! -e "\$state/restarted" ]; then - echo "{\"command\":\"\$(cat "\$state/posted")\",\"status\":\"success\",\"start\":\"2026-01-01 00:02:00\"}" + inprog=1 fi + if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ]; then + left=\$(num op_left) + if [ "\$left" -gt 0 ]; then + echo \$((left - 1)) > "\$state/op_left" + status="in progress" + inprog=1 + else + status=\$(cat "\$state/op_result" 2>/dev/null || echo success) + fi + echo "{\"command\":\"\$(cat "\$state/op")\",\"status\":\"\$status\",\"start\":\"2026-01-01 00:02:00\",\"error\":\"boom\"}" + fi + if [ "\$inprog" = 0 ]; then touch "\$state/idle"; fi ;; "GET "*/backup/list/remote) echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}' @@ -96,6 +142,11 @@ STUB chmod 0755 "$bin/curl" "$bin/sleep" } +# The number of operations the sidecar accepted. +accepted() { + if [ -e "$1/state/accepted" ]; then wc -l < "$1/state/accepted" | tr -d ' '; else echo 0; fi +} + run_script() { tmp=$1 API_USERNAME=u API_PASSWORD=p PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" @@ -124,7 +175,7 @@ run_script() { echo 3 > "$tmp/state/busy" render_script "$tmp/s.sh" backup myapp run_script "$tmp" - grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests" + grep -q '^create_remote clickhouse-myapp-' "$tmp/state/accepted" [ "$(cat "$tmp/state/busy")" = 0 ] } @@ -168,7 +219,94 @@ run_script() { make_stubs "$tmp/bin" "$tmp/state" touch "$tmp/state/restarted" render_script "$tmp/s.sh" backup myapp - if run_script "$tmp"; then echo "FAIL: an operation lost from the action log must fail the Pod"; false; fi + if run_script "$tmp" 2> "$tmp/err"; then echo "FAIL: an operation lost from the action log must fail the Pod"; false; fi + grep -q 'no longer in the sidecar action log' "$tmp/err" grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests" if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling for an operation the sidecar no longer lists"; false; fi } + +@test "restore waits for a sidecar that has not started yet" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 10 > "$tmp/state/down" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "restore waits until another operation finishes, then starts its own" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/busy" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted" +} + +@test "failed polls after restore_remote, short of six in a row, do not fail the attempt" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/op_duration" + echo 5 > "$tmp/state/fail_on_post" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "a retry after an attempt lost its polls joins the running restore instead of starting another" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 20 > "$tmp/state/op_duration" + echo 6 > "$tmp/state/fail_on_post" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: six failed polls in a row must fail the attempt"; false; fi + [ "$(accepted "$tmp")" = 1 ] + echo 0 > "$tmp/state/fail_polls" + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: the retry started restore_remote a second time"; false; fi + [ "$(cat "$tmp/state/op_left")" = 0 ] +} + +@test "restore does not join an operation running on another archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'restore_remote clickhouse-myapp-20251231T000000Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted" +} + +# The sidecar never trims its action log, so a finished restore_remote row of +# the same archive may belong to an earlier RestoreJob; only a running one may +# be taken over. +@test "restore starts restore_remote again when the log holds a finished one of the same archive" { + for status in success error cancel; do + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo "{\"command\":\"restore_remote clickhouse-myapp-20260101T000000Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier restore must not stand in for this one"; false; fi + done +} + +@test "a restore that ends in error fails the Pod" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo error > "$tmp/state/op_result" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: a failed restore must fail the Pod"; false; fi + [ "$(accepted "$tmp")" = 1 ] +} + +@test "a cancelled backup fails the Pod instead of polling forever" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo cancel > "$tmp/state/op_result" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: a cancelled operation must fail the Pod"; false; fi + if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling a cancelled operation"; false; fi +} diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml index cc7275f30..bbae43349 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml @@ -51,42 +51,83 @@ spec: host="chi-clickhouse-{{ printf "{{ .Release.Name }}" }}-clickhouse-0-0.{{ printf "{{ .Release.Namespace }}" }}.svc" api="http://${host}:7171" auth="-u ${API_USERNAME}:${API_PASSWORD}" + # Every request but the POST is retried on failure, every 5s, + # up to $2 times: once the operation is accepted, a dropped poll + # must not fail the attempt, since the Job retry it causes cannot + # tell a lost poll from a lost operation. + api_get() { + n=0 + while true; do + if out=$(curl -fsS ${auth} --max-time 10 "${api}$1"); then + printf '%s\n' "${out}" + return 0 + fi + n=$((n + 1)) + if [ "${n}" -ge "$2" ]; then + echo "GET ${api}$1 failed ${n} times in a row" >&2 + return 1 + fi + sleep 5 + done + } # The sidecar answers 423 to a new operation while any other is # in progress, so starting before it is idle burns the attempt. - idle= - for _ in $(seq 1 60); do - if actions=$(curl -fsS ${auth} --max-time 5 "${api}/backup/actions") \ - && [ "$(printf '%s\n' "${actions}" | jq -rs '[.[] | select(.status == "in progress")] | length')" = 0 ]; then - idle=1 - break - fi - sleep 5 - done - if [ -z "${idle}" ]; then + # The exception is the operation $1 itself already running: an + # earlier attempt of this Job started it and died, and the retry + # takes it over rather than starting it twice. + wait_turn() { + for _ in $(seq 1 60); do + if actions=$(curl -fsS ${auth} --max-time 10 "${api}/backup/actions"); then + turn=$(printf '%s\n' "${actions}" | jq -rs --arg cmd "$1" ' + if any(.[]; .status == "in progress" and .command == $cmd) then "join" + elif any(.[]; .status == "in progress") then "busy" + else "idle" end') || turn= + case "${turn}" in join|idle) return 0 ;; esac + fi + sleep 5 + done echo "clickhouse-backup sidecar at ${api} did not become idle" >&2 exit 1 - fi + } if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-" + # The name is new on every attempt, so a retry never takes over + # the create_remote of the attempt it replaces; that needs a + # name fixed for the whole BackupJob. target="${prefix}$(date -u +%Y%m%dT%H%M%SZ)" - echo "create_remote ${target}" + cmd="create_remote ${target}" + wait_turn "${cmd}" + echo "${cmd}" curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}" >/dev/null else prefix="clickhouse-{{ printf "{{ .Backup.ApplicationRef.Name }}" }}-" - target=$(curl -fsS ${auth} "${api}/backup/list/remote" \ + listing=$(api_get /backup/list/remote 60) + target=$(printf '%s\n' "${listing}" \ | jq -rs --arg prefix "${prefix}" \ '[.[] | select((.name // "") | startswith($prefix))] | sort_by(.created // "") | last.name // ""') if [ -z "${target}" ] || [ "${target}" = "null" ]; then echo "no remote backup found" >&2 exit 1 fi - echo "restore_remote ${target}" - curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null + # restore_remote drops every table it restores before + # recreating it, so a second run discards whatever was written + # after the first one finished. Only a running one is taken + # over: the action log is never trimmed, and a finished row of + # this archive may be left by an earlier RestoreJob. + cmd="restore_remote ${target}" + wait_turn "${cmd}" + if [ "${turn}" = "join" ]; then + echo "${cmd} is already running; waiting for it" + else + echo "${cmd}" + curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null + fi fi while true; do - row=$(curl -fsS ${auth} "${api}/backup/actions" \ - | jq -rs --arg suffix "${target}" \ - '[.[] | select((.command // "") | endswith($suffix))] | sort_by(.start) | last') + actions=$(api_get /backup/actions 6) + row=$(printf '%s\n' "${actions}" \ + | jq -rs --arg cmd "${cmd}" \ + '[.[] | select(.command == $cmd)] | sort_by(.start) | last') # The sidecar logs the operation before acknowledging the POST # and keeps that log only in memory, so a missing row means it # restarted and the operation is gone; nothing will resume it. @@ -97,9 +138,10 @@ spec: status=$(echo "${row}" | jq -r '.status // ""') case "${status}" in success) echo "operation succeeded" ; exit 0 ;; - error) + "in progress") ;; + *) err=$(echo "${row}" | jq -r '.error // "unknown"') - echo "operation failed: ${err}" >&2 + echo "operation ended with status ${status}: ${err}" >&2 exit 1 ;; esac