fix(backups): take over a running ClickHouse restore on Job retry

restore_remote drops every table it restores before recreating it, so
running it twice discards whatever was written after the first run
finished. The strategy Job retries a failed Pod up to twice, and with
the start gate waiting for an idle sidecar instead of hitting 423, a
retry whose predecessor died after its POST waited for that restore to
finish and then started a second one. A retry now takes over a
restore_remote of the same archive that is still running.

A finished row is not taken over: the sidecar never trims its action
log, so a success row of the archive may belong to an earlier
RestoreJob, and treating it as this one would report a restore that
never ran. A retry that arrives after the restore finished therefore
still runs it again.

Once the operation is accepted, one failed poll no longer ends the
attempt under errexit; six in a row do. An operation that ends
cancelled now fails the Pod instead of being polled forever.

A retried backup still starts a new create_remote, because its archive
name changes on every attempt; taking it over needs a name fixed for
the whole BackupJob.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-06 11:23:28 +03:00
1 parent d0a73302d9
commit 993b79bb37
2 files changed
+218 -38

No files matched your search

+156 -18
View File
@@ -10,6 +10,9 @@
# rather than proceed when the sidecar never becomes idle. Once started, an
# operation the sidecar no longer lists (its action log lives in memory and a
# restart empties it) has to fail the Pod too, since the Job has no deadline.
# A Job retry must not run a restore twice: it takes over the restore_remote
# the attempt before it left running, and a few lost polls do not end an
# attempt in the first place.
#
# Run from the repo root with make bats-unit-tests. The rendered script is run
# with `bash`, not `sh`: it opens `set -euo pipefail`, which the production
@@ -40,17 +43,31 @@ render_script() {
# A curl stub standing in for the sidecar API, put first on PATH next to a
# no-op sleep. State lives in $1:
# unreachable present: every request fails as a refused connection (exit 7)
# busy a number: that many GET /backup/actions answers still list an
# operation "in progress"; while it is positive a POST is
# refused the way the sidecar refuses it (423, curl -f exit 22)
# posted the operation the script started, once it started one
# restarted present: the action log no longer lists the started operation,
# as after the sidecar restarted and lost its in-memory log
# requests one line per request: method and URL
# runaway created when the script exceeds 200 requests; that request
# and every later one fails, so a polling loop that never ends
# stops the test instead of hanging it
# unreachable present: every request fails as a refused connection (7)
# down a number: that many next requests fail the same way, as
# before the sidecar has started
# busy a number: that many GET /backup/actions answers still list
# an operation of someone else "in progress"
# op the command line of the sidecar's own operation, written
# by an accepted POST or seeded by a test as the operation
# an earlier attempt of the same Job started
# op_left how many more GET /backup/actions answers list op as
# "in progress" before it ends with op_result
# op_duration op_left given to an operation a POST starts (default 0)
# op_result the status op ends with (default success)
# prior extra action-log rows, verbatim, older than op
# restarted present: the action log no longer lists op, as after the
# sidecar restarted and lost its in-memory log
# fail_on_post a POST that is accepted makes that many following
# GET /backup/actions requests fail (curl exit 7)
# idle created by an answer that lists nothing in progress; a
# POST is refused (423, curl -f exit 22) unless one was seen
# since the last accepted POST, and while anything runs
# accepted one line per POST the sidecar accepted
# requests one line per request: method and URL
# runaway created when the script exceeds 200 requests; that request
# and every later one fails, so a polling loop that never ends
# stops the test instead of hanging it
make_stubs() {
bin=$1; state=$2
mkdir -p "$bin" "$state"
@@ -68,20 +85,49 @@ done
echo "\$method \$url" >> "\$state/requests"
if [ "\$(wc -l < "\$state/requests")" -gt 200 ]; then touch "\$state/runaway"; exit 7; fi
if [ -e "\$state/unreachable" ]; then exit 7; fi
busy=\$(cat "\$state/busy" 2>/dev/null || echo 0)
num() { cat "\$state/\$1" 2>/dev/null || echo 0; }
down=\$(num down)
if [ "\$down" -gt 0 ]; then echo \$((down - 1)) > "\$state/down"; exit 7; fi
running=0
if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ] && [ "\$(num op_left)" -gt 0 ]; then running=1; fi
case "\$method \$url" in
"POST "*)
if [ "\$busy" -gt 0 ]; then exit 22; fi
echo "\${url##*/}" > "\$state/posted"
if [ "\$(num busy)" -gt 0 ] || [ "\$running" = 1 ] || [ ! -e "\$state/idle" ]; then exit 22; fi
case "\$url" in
*/create_remote\?name=*) cmd="create_remote \${url##*=}" ;;
*/restore_remote/*) cmd="restore_remote \${url##*/}" ;;
*) exit 22 ;;
esac
echo "\$cmd" > "\$state/op"
echo "\$cmd" >> "\$state/accepted"
num op_duration > "\$state/op_left"
num fail_on_post > "\$state/fail_polls"
rm -f "\$state/idle"
;;
"GET "*/backup/actions)
fail=\$(num fail_polls)
if [ "\$fail" -gt 0 ]; then echo \$((fail - 1)) > "\$state/fail_polls"; exit 7; fi
echo '{"command":"create_remote clickhouse-myapp-20260101T000000Z","status":"error","start":"2026-01-01 00:00:00","error":"old"}'
cat "\$state/prior" 2>/dev/null || true
inprog=0
busy=\$(num busy)
if [ "\$busy" -gt 0 ]; then
echo \$((busy - 1)) > "\$state/busy"
echo '{"command":"upload --resumable=1 clickhouse-myapp-20260101T000000Z","status":"in progress","start":"2026-01-01 00:01:00"}'
elif [ -s "\$state/posted" ] && [ ! -e "\$state/restarted" ]; then
echo "{\"command\":\"\$(cat "\$state/posted")\",\"status\":\"success\",\"start\":\"2026-01-01 00:02:00\"}"
inprog=1
fi
if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ]; then
left=\$(num op_left)
if [ "\$left" -gt 0 ]; then
echo \$((left - 1)) > "\$state/op_left"
status="in progress"
inprog=1
else
status=\$(cat "\$state/op_result" 2>/dev/null || echo success)
fi
echo "{\"command\":\"\$(cat "\$state/op")\",\"status\":\"\$status\",\"start\":\"2026-01-01 00:02:00\",\"error\":\"boom\"}"
fi
if [ "\$inprog" = 0 ]; then touch "\$state/idle"; fi
;;
"GET "*/backup/list/remote)
echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}'
@@ -96,6 +142,11 @@ STUB
chmod 0755 "$bin/curl" "$bin/sleep"
}
# The number of operations the sidecar accepted.
accepted() {
if [ -e "$1/state/accepted" ]; then wc -l < "$1/state/accepted" | tr -d ' '; else echo 0; fi
}
run_script() {
tmp=$1
API_USERNAME=u API_PASSWORD=p PATH="$tmp/bin:$PATH" bash "$tmp/s.sh"
@@ -124,7 +175,7 @@ run_script() {
echo 3 > "$tmp/state/busy"
render_script "$tmp/s.sh" backup myapp
run_script "$tmp"
grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests"
grep -q '^create_remote clickhouse-myapp-' "$tmp/state/accepted"
[ "$(cat "$tmp/state/busy")" = 0 ]
}
@@ -168,7 +219,94 @@ run_script() {
make_stubs "$tmp/bin" "$tmp/state"
touch "$tmp/state/restarted"
render_script "$tmp/s.sh" backup myapp
if run_script "$tmp"; then echo "FAIL: an operation lost from the action log must fail the Pod"; false; fi
if run_script "$tmp" 2> "$tmp/err"; then echo "FAIL: an operation lost from the action log must fail the Pod"; false; fi
grep -q 'no longer in the sidecar action log' "$tmp/err"
grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests"
if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling for an operation the sidecar no longer lists"; false; fi
}
@test "restore waits for a sidecar that has not started yet" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 10 > "$tmp/state/down"
render_script "$tmp/s.sh" restore myapp
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
}
@test "restore waits until another operation finishes, then starts its own" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 3 > "$tmp/state/busy"
render_script "$tmp/s.sh" restore myapp
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted"
}
@test "failed polls after restore_remote, short of six in a row, do not fail the attempt" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 3 > "$tmp/state/op_duration"
echo 5 > "$tmp/state/fail_on_post"
render_script "$tmp/s.sh" restore myapp
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
}
@test "a retry after an attempt lost its polls joins the running restore instead of starting another" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 20 > "$tmp/state/op_duration"
echo 6 > "$tmp/state/fail_on_post"
render_script "$tmp/s.sh" restore myapp
if run_script "$tmp"; then echo "FAIL: six failed polls in a row must fail the attempt"; false; fi
[ "$(accepted "$tmp")" = 1 ]
echo 0 > "$tmp/state/fail_polls"
run_script "$tmp"
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: the retry started restore_remote a second time"; false; fi
[ "$(cat "$tmp/state/op_left")" = 0 ]
}
@test "restore does not join an operation running on another archive" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 'restore_remote clickhouse-myapp-20251231T000000Z' > "$tmp/state/op"
echo 3 > "$tmp/state/op_left"
render_script "$tmp/s.sh" restore myapp
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted"
}
# The sidecar never trims its action log, so a finished restore_remote row of
# the same archive may belong to an earlier RestoreJob; only a running one may
# be taken over.
@test "restore starts restore_remote again when the log holds a finished one of the same archive" {
for status in success error cancel; do
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo "{\"command\":\"restore_remote clickhouse-myapp-20260101T000000Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior"
render_script "$tmp/s.sh" restore myapp
run_script "$tmp"
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier restore must not stand in for this one"; false; fi
done
}
@test "a restore that ends in error fails the Pod" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo error > "$tmp/state/op_result"
render_script "$tmp/s.sh" restore myapp
if run_script "$tmp"; then echo "FAIL: a failed restore must fail the Pod"; false; fi
[ "$(accepted "$tmp")" = 1 ]
}
@test "a cancelled backup fails the Pod instead of polling forever" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo cancel > "$tmp/state/op_result"
render_script "$tmp/s.sh" backup myapp
if run_script "$tmp"; then echo "FAIL: a cancelled operation must fail the Pod"; false; fi
if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling a cancelled operation"; false; fi
}
@@ -51,42 +51,83 @@ spec:
host="chi-clickhouse-{{ printf "{{ .Release.Name }}" }}-clickhouse-0-0.{{ printf "{{ .Release.Namespace }}" }}.svc"
api="http://${host}:7171"
auth="-u ${API_USERNAME}:${API_PASSWORD}"
# Every request but the POST is retried on failure, every 5s,
# up to $2 times: once the operation is accepted, a dropped poll
# must not fail the attempt, since the Job retry it causes cannot
# tell a lost poll from a lost operation.
api_get() {
n=0
while true; do
if out=$(curl -fsS ${auth} --max-time 10 "${api}$1"); then
printf '%s\n' "${out}"
return 0
fi
n=$((n + 1))
if [ "${n}" -ge "$2" ]; then
echo "GET ${api}$1 failed ${n} times in a row" >&2
return 1
fi
sleep 5
done
}
# The sidecar answers 423 to a new operation while any other is
# in progress, so starting before it is idle burns the attempt.
idle=
for _ in $(seq 1 60); do
if actions=$(curl -fsS ${auth} --max-time 5 "${api}/backup/actions") \
&& [ "$(printf '%s\n' "${actions}" | jq -rs '[.[] | select(.status == "in progress")] | length')" = 0 ]; then
idle=1
break
fi
sleep 5
done
if [ -z "${idle}" ]; then
# The exception is the operation $1 itself already running: an
# earlier attempt of this Job started it and died, and the retry
# takes it over rather than starting it twice.
wait_turn() {
for _ in $(seq 1 60); do
if actions=$(curl -fsS ${auth} --max-time 10 "${api}/backup/actions"); then
turn=$(printf '%s\n' "${actions}" | jq -rs --arg cmd "$1" '
if any(.[]; .status == "in progress" and .command == $cmd) then "join"
elif any(.[]; .status == "in progress") then "busy"
else "idle" end') || turn=
case "${turn}" in join|idle) return 0 ;; esac
fi
sleep 5
done
echo "clickhouse-backup sidecar at ${api} did not become idle" >&2
exit 1
fi
}
if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then
prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-"
# The name is new on every attempt, so a retry never takes over
# the create_remote of the attempt it replaces; that needs a
# name fixed for the whole BackupJob.
target="${prefix}$(date -u +%Y%m%dT%H%M%SZ)"
echo "create_remote ${target}"
cmd="create_remote ${target}"
wait_turn "${cmd}"
echo "${cmd}"
curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}" >/dev/null
else
prefix="clickhouse-{{ printf "{{ .Backup.ApplicationRef.Name }}" }}-"
target=$(curl -fsS ${auth} "${api}/backup/list/remote" \
listing=$(api_get /backup/list/remote 60)
target=$(printf '%s\n' "${listing}" \
| jq -rs --arg prefix "${prefix}" \
'[.[] | select((.name // "") | startswith($prefix))] | sort_by(.created // "") | last.name // ""')
if [ -z "${target}" ] || [ "${target}" = "null" ]; then
echo "no remote backup found" >&2
exit 1
fi
echo "restore_remote ${target}"
curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null
# restore_remote drops every table it restores before
# recreating it, so a second run discards whatever was written
# after the first one finished. Only a running one is taken
# over: the action log is never trimmed, and a finished row of
# this archive may be left by an earlier RestoreJob.
cmd="restore_remote ${target}"
wait_turn "${cmd}"
if [ "${turn}" = "join" ]; then
echo "${cmd} is already running; waiting for it"
else
echo "${cmd}"
curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null
fi
fi
while true; do
row=$(curl -fsS ${auth} "${api}/backup/actions" \
| jq -rs --arg suffix "${target}" \
'[.[] | select((.command // "") | endswith($suffix))] | sort_by(.start) | last')
actions=$(api_get /backup/actions 6)
row=$(printf '%s\n' "${actions}" \
| jq -rs --arg cmd "${cmd}" \
'[.[] | select(.command == $cmd)] | sort_by(.start) | last')
# The sidecar logs the operation before acknowledging the POST
# and keeps that log only in memory, so a missing row means it
# restarted and the operation is gone; nothing will resume it.
@@ -97,9 +138,10 @@ spec:
status=$(echo "${row}" | jq -r '.status // ""')
case "${status}" in
success) echo "operation succeeded" ; exit 0 ;;
error)
"in progress") ;;
*)
err=$(echo "${row}" | jq -r '.error // "unknown"')
echo "operation failed: ${err}" >&2
echo "operation ended with status ${status}: ${err}" >&2
exit 1
;;
esac