mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(backups): take over a running ClickHouse restore on Job retry
restore_remote drops every table it restores before recreating it, so running it twice discards whatever was written after the first run finished. The strategy Job retries a failed Pod up to twice, and with the start gate waiting for an idle sidecar instead of hitting 423, a retry whose predecessor died after its POST waited for that restore to finish and then started a second one. A retry now takes over a restore_remote of the same archive that is still running. A finished row is not taken over: the sidecar never trims its action log, so a success row of the archive may belong to an earlier RestoreJob, and treating it as this one would report a restore that never ran. A retry that arrives after the restore finished therefore still runs it again. Once the operation is accepted, one failed poll no longer ends the attempt under errexit; six in a row do. An operation that ends cancelled now fails the Pod instead of being polled forever. A retried backup still starts a new create_remote, because its archive name changes on every attempt; taking it over needs a name fixed for the whole BackupJob. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
d0a73302d9
commit
993b79bb37
2 files changed
+218
-38
No files matched your search
@@ -10,6 +10,9 @@
|
||||
# rather than proceed when the sidecar never becomes idle. Once started, an
|
||||
# operation the sidecar no longer lists (its action log lives in memory and a
|
||||
# restart empties it) has to fail the Pod too, since the Job has no deadline.
|
||||
# A Job retry must not run a restore twice: it takes over the restore_remote
|
||||
# the attempt before it left running, and a few lost polls do not end an
|
||||
# attempt in the first place.
|
||||
#
|
||||
# Run from the repo root with make bats-unit-tests. The rendered script is run
|
||||
# with `bash`, not `sh`: it opens `set -euo pipefail`, which the production
|
||||
@@ -40,17 +43,31 @@ render_script() {
|
||||
|
||||
# A curl stub standing in for the sidecar API, put first on PATH next to a
|
||||
# no-op sleep. State lives in $1:
|
||||
# unreachable present: every request fails as a refused connection (exit 7)
|
||||
# busy a number: that many GET /backup/actions answers still list an
|
||||
# operation "in progress"; while it is positive a POST is
|
||||
# refused the way the sidecar refuses it (423, curl -f exit 22)
|
||||
# posted the operation the script started, once it started one
|
||||
# restarted present: the action log no longer lists the started operation,
|
||||
# as after the sidecar restarted and lost its in-memory log
|
||||
# requests one line per request: method and URL
|
||||
# runaway created when the script exceeds 200 requests; that request
|
||||
# and every later one fails, so a polling loop that never ends
|
||||
# stops the test instead of hanging it
|
||||
# unreachable present: every request fails as a refused connection (7)
|
||||
# down a number: that many next requests fail the same way, as
|
||||
# before the sidecar has started
|
||||
# busy a number: that many GET /backup/actions answers still list
|
||||
# an operation of someone else "in progress"
|
||||
# op the command line of the sidecar's own operation, written
|
||||
# by an accepted POST or seeded by a test as the operation
|
||||
# an earlier attempt of the same Job started
|
||||
# op_left how many more GET /backup/actions answers list op as
|
||||
# "in progress" before it ends with op_result
|
||||
# op_duration op_left given to an operation a POST starts (default 0)
|
||||
# op_result the status op ends with (default success)
|
||||
# prior extra action-log rows, verbatim, older than op
|
||||
# restarted present: the action log no longer lists op, as after the
|
||||
# sidecar restarted and lost its in-memory log
|
||||
# fail_on_post a POST that is accepted makes that many following
|
||||
# GET /backup/actions requests fail (curl exit 7)
|
||||
# idle created by an answer that lists nothing in progress; a
|
||||
# POST is refused (423, curl -f exit 22) unless one was seen
|
||||
# since the last accepted POST, and while anything runs
|
||||
# accepted one line per POST the sidecar accepted
|
||||
# requests one line per request: method and URL
|
||||
# runaway created when the script exceeds 200 requests; that request
|
||||
# and every later one fails, so a polling loop that never ends
|
||||
# stops the test instead of hanging it
|
||||
make_stubs() {
|
||||
bin=$1; state=$2
|
||||
mkdir -p "$bin" "$state"
|
||||
@@ -68,20 +85,49 @@ done
|
||||
echo "\$method \$url" >> "\$state/requests"
|
||||
if [ "\$(wc -l < "\$state/requests")" -gt 200 ]; then touch "\$state/runaway"; exit 7; fi
|
||||
if [ -e "\$state/unreachable" ]; then exit 7; fi
|
||||
busy=\$(cat "\$state/busy" 2>/dev/null || echo 0)
|
||||
num() { cat "\$state/\$1" 2>/dev/null || echo 0; }
|
||||
down=\$(num down)
|
||||
if [ "\$down" -gt 0 ]; then echo \$((down - 1)) > "\$state/down"; exit 7; fi
|
||||
running=0
|
||||
if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ] && [ "\$(num op_left)" -gt 0 ]; then running=1; fi
|
||||
case "\$method \$url" in
|
||||
"POST "*)
|
||||
if [ "\$busy" -gt 0 ]; then exit 22; fi
|
||||
echo "\${url##*/}" > "\$state/posted"
|
||||
if [ "\$(num busy)" -gt 0 ] || [ "\$running" = 1 ] || [ ! -e "\$state/idle" ]; then exit 22; fi
|
||||
case "\$url" in
|
||||
*/create_remote\?name=*) cmd="create_remote \${url##*=}" ;;
|
||||
*/restore_remote/*) cmd="restore_remote \${url##*/}" ;;
|
||||
*) exit 22 ;;
|
||||
esac
|
||||
echo "\$cmd" > "\$state/op"
|
||||
echo "\$cmd" >> "\$state/accepted"
|
||||
num op_duration > "\$state/op_left"
|
||||
num fail_on_post > "\$state/fail_polls"
|
||||
rm -f "\$state/idle"
|
||||
;;
|
||||
"GET "*/backup/actions)
|
||||
fail=\$(num fail_polls)
|
||||
if [ "\$fail" -gt 0 ]; then echo \$((fail - 1)) > "\$state/fail_polls"; exit 7; fi
|
||||
echo '{"command":"create_remote clickhouse-myapp-20260101T000000Z","status":"error","start":"2026-01-01 00:00:00","error":"old"}'
|
||||
cat "\$state/prior" 2>/dev/null || true
|
||||
inprog=0
|
||||
busy=\$(num busy)
|
||||
if [ "\$busy" -gt 0 ]; then
|
||||
echo \$((busy - 1)) > "\$state/busy"
|
||||
echo '{"command":"upload --resumable=1 clickhouse-myapp-20260101T000000Z","status":"in progress","start":"2026-01-01 00:01:00"}'
|
||||
elif [ -s "\$state/posted" ] && [ ! -e "\$state/restarted" ]; then
|
||||
echo "{\"command\":\"\$(cat "\$state/posted")\",\"status\":\"success\",\"start\":\"2026-01-01 00:02:00\"}"
|
||||
inprog=1
|
||||
fi
|
||||
if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ]; then
|
||||
left=\$(num op_left)
|
||||
if [ "\$left" -gt 0 ]; then
|
||||
echo \$((left - 1)) > "\$state/op_left"
|
||||
status="in progress"
|
||||
inprog=1
|
||||
else
|
||||
status=\$(cat "\$state/op_result" 2>/dev/null || echo success)
|
||||
fi
|
||||
echo "{\"command\":\"\$(cat "\$state/op")\",\"status\":\"\$status\",\"start\":\"2026-01-01 00:02:00\",\"error\":\"boom\"}"
|
||||
fi
|
||||
if [ "\$inprog" = 0 ]; then touch "\$state/idle"; fi
|
||||
;;
|
||||
"GET "*/backup/list/remote)
|
||||
echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}'
|
||||
@@ -96,6 +142,11 @@ STUB
|
||||
chmod 0755 "$bin/curl" "$bin/sleep"
|
||||
}
|
||||
|
||||
# The number of operations the sidecar accepted.
|
||||
accepted() {
|
||||
if [ -e "$1/state/accepted" ]; then wc -l < "$1/state/accepted" | tr -d ' '; else echo 0; fi
|
||||
}
|
||||
|
||||
run_script() {
|
||||
tmp=$1
|
||||
API_USERNAME=u API_PASSWORD=p PATH="$tmp/bin:$PATH" bash "$tmp/s.sh"
|
||||
@@ -124,7 +175,7 @@ run_script() {
|
||||
echo 3 > "$tmp/state/busy"
|
||||
render_script "$tmp/s.sh" backup myapp
|
||||
run_script "$tmp"
|
||||
grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests"
|
||||
grep -q '^create_remote clickhouse-myapp-' "$tmp/state/accepted"
|
||||
[ "$(cat "$tmp/state/busy")" = 0 ]
|
||||
}
|
||||
|
||||
@@ -168,7 +219,94 @@ run_script() {
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
touch "$tmp/state/restarted"
|
||||
render_script "$tmp/s.sh" backup myapp
|
||||
if run_script "$tmp"; then echo "FAIL: an operation lost from the action log must fail the Pod"; false; fi
|
||||
if run_script "$tmp" 2> "$tmp/err"; then echo "FAIL: an operation lost from the action log must fail the Pod"; false; fi
|
||||
grep -q 'no longer in the sidecar action log' "$tmp/err"
|
||||
grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests"
|
||||
if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling for an operation the sidecar no longer lists"; false; fi
|
||||
}
|
||||
|
||||
@test "restore waits for a sidecar that has not started yet" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo 10 > "$tmp/state/down"
|
||||
render_script "$tmp/s.sh" restore myapp
|
||||
run_script "$tmp"
|
||||
[ "$(accepted "$tmp")" = 1 ]
|
||||
}
|
||||
|
||||
@test "restore waits until another operation finishes, then starts its own" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo 3 > "$tmp/state/busy"
|
||||
render_script "$tmp/s.sh" restore myapp
|
||||
run_script "$tmp"
|
||||
[ "$(accepted "$tmp")" = 1 ]
|
||||
grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted"
|
||||
}
|
||||
|
||||
@test "failed polls after restore_remote, short of six in a row, do not fail the attempt" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo 3 > "$tmp/state/op_duration"
|
||||
echo 5 > "$tmp/state/fail_on_post"
|
||||
render_script "$tmp/s.sh" restore myapp
|
||||
run_script "$tmp"
|
||||
[ "$(accepted "$tmp")" = 1 ]
|
||||
}
|
||||
|
||||
@test "a retry after an attempt lost its polls joins the running restore instead of starting another" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo 20 > "$tmp/state/op_duration"
|
||||
echo 6 > "$tmp/state/fail_on_post"
|
||||
render_script "$tmp/s.sh" restore myapp
|
||||
if run_script "$tmp"; then echo "FAIL: six failed polls in a row must fail the attempt"; false; fi
|
||||
[ "$(accepted "$tmp")" = 1 ]
|
||||
echo 0 > "$tmp/state/fail_polls"
|
||||
run_script "$tmp"
|
||||
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: the retry started restore_remote a second time"; false; fi
|
||||
[ "$(cat "$tmp/state/op_left")" = 0 ]
|
||||
}
|
||||
|
||||
@test "restore does not join an operation running on another archive" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo 'restore_remote clickhouse-myapp-20251231T000000Z' > "$tmp/state/op"
|
||||
echo 3 > "$tmp/state/op_left"
|
||||
render_script "$tmp/s.sh" restore myapp
|
||||
run_script "$tmp"
|
||||
[ "$(accepted "$tmp")" = 1 ]
|
||||
grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted"
|
||||
}
|
||||
|
||||
# The sidecar never trims its action log, so a finished restore_remote row of
|
||||
# the same archive may belong to an earlier RestoreJob; only a running one may
|
||||
# be taken over.
|
||||
@test "restore starts restore_remote again when the log holds a finished one of the same archive" {
|
||||
for status in success error cancel; do
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo "{\"command\":\"restore_remote clickhouse-myapp-20260101T000000Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior"
|
||||
render_script "$tmp/s.sh" restore myapp
|
||||
run_script "$tmp"
|
||||
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier restore must not stand in for this one"; false; fi
|
||||
done
|
||||
}
|
||||
|
||||
@test "a restore that ends in error fails the Pod" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo error > "$tmp/state/op_result"
|
||||
render_script "$tmp/s.sh" restore myapp
|
||||
if run_script "$tmp"; then echo "FAIL: a failed restore must fail the Pod"; false; fi
|
||||
[ "$(accepted "$tmp")" = 1 ]
|
||||
}
|
||||
|
||||
@test "a cancelled backup fails the Pod instead of polling forever" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo cancel > "$tmp/state/op_result"
|
||||
render_script "$tmp/s.sh" backup myapp
|
||||
if run_script "$tmp"; then echo "FAIL: a cancelled operation must fail the Pod"; false; fi
|
||||
if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling a cancelled operation"; false; fi
|
||||
}
|
||||
+62
-20
@@ -51,42 +51,83 @@ spec:
|
||||
host="chi-clickhouse-{{ printf "{{ .Release.Name }}" }}-clickhouse-0-0.{{ printf "{{ .Release.Namespace }}" }}.svc"
|
||||
api="http://${host}:7171"
|
||||
auth="-u ${API_USERNAME}:${API_PASSWORD}"
|
||||
# Every request but the POST is retried on failure, every 5s,
|
||||
# up to $2 times: once the operation is accepted, a dropped poll
|
||||
# must not fail the attempt, since the Job retry it causes cannot
|
||||
# tell a lost poll from a lost operation.
|
||||
api_get() {
|
||||
n=0
|
||||
while true; do
|
||||
if out=$(curl -fsS ${auth} --max-time 10 "${api}$1"); then
|
||||
printf '%s\n' "${out}"
|
||||
return 0
|
||||
fi
|
||||
n=$((n + 1))
|
||||
if [ "${n}" -ge "$2" ]; then
|
||||
echo "GET ${api}$1 failed ${n} times in a row" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
# The sidecar answers 423 to a new operation while any other is
|
||||
# in progress, so starting before it is idle burns the attempt.
|
||||
idle=
|
||||
for _ in $(seq 1 60); do
|
||||
if actions=$(curl -fsS ${auth} --max-time 5 "${api}/backup/actions") \
|
||||
&& [ "$(printf '%s\n' "${actions}" | jq -rs '[.[] | select(.status == "in progress")] | length')" = 0 ]; then
|
||||
idle=1
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
if [ -z "${idle}" ]; then
|
||||
# The exception is the operation $1 itself already running: an
|
||||
# earlier attempt of this Job started it and died, and the retry
|
||||
# takes it over rather than starting it twice.
|
||||
wait_turn() {
|
||||
for _ in $(seq 1 60); do
|
||||
if actions=$(curl -fsS ${auth} --max-time 10 "${api}/backup/actions"); then
|
||||
turn=$(printf '%s\n' "${actions}" | jq -rs --arg cmd "$1" '
|
||||
if any(.[]; .status == "in progress" and .command == $cmd) then "join"
|
||||
elif any(.[]; .status == "in progress") then "busy"
|
||||
else "idle" end') || turn=
|
||||
case "${turn}" in join|idle) return 0 ;; esac
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "clickhouse-backup sidecar at ${api} did not become idle" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then
|
||||
prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-"
|
||||
# The name is new on every attempt, so a retry never takes over
|
||||
# the create_remote of the attempt it replaces; that needs a
|
||||
# name fixed for the whole BackupJob.
|
||||
target="${prefix}$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
echo "create_remote ${target}"
|
||||
cmd="create_remote ${target}"
|
||||
wait_turn "${cmd}"
|
||||
echo "${cmd}"
|
||||
curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}" >/dev/null
|
||||
else
|
||||
prefix="clickhouse-{{ printf "{{ .Backup.ApplicationRef.Name }}" }}-"
|
||||
target=$(curl -fsS ${auth} "${api}/backup/list/remote" \
|
||||
listing=$(api_get /backup/list/remote 60)
|
||||
target=$(printf '%s\n' "${listing}" \
|
||||
| jq -rs --arg prefix "${prefix}" \
|
||||
'[.[] | select((.name // "") | startswith($prefix))] | sort_by(.created // "") | last.name // ""')
|
||||
if [ -z "${target}" ] || [ "${target}" = "null" ]; then
|
||||
echo "no remote backup found" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "restore_remote ${target}"
|
||||
curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null
|
||||
# restore_remote drops every table it restores before
|
||||
# recreating it, so a second run discards whatever was written
|
||||
# after the first one finished. Only a running one is taken
|
||||
# over: the action log is never trimmed, and a finished row of
|
||||
# this archive may be left by an earlier RestoreJob.
|
||||
cmd="restore_remote ${target}"
|
||||
wait_turn "${cmd}"
|
||||
if [ "${turn}" = "join" ]; then
|
||||
echo "${cmd} is already running; waiting for it"
|
||||
else
|
||||
echo "${cmd}"
|
||||
curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null
|
||||
fi
|
||||
fi
|
||||
while true; do
|
||||
row=$(curl -fsS ${auth} "${api}/backup/actions" \
|
||||
| jq -rs --arg suffix "${target}" \
|
||||
'[.[] | select((.command // "") | endswith($suffix))] | sort_by(.start) | last')
|
||||
actions=$(api_get /backup/actions 6)
|
||||
row=$(printf '%s\n' "${actions}" \
|
||||
| jq -rs --arg cmd "${cmd}" \
|
||||
'[.[] | select(.command == $cmd)] | sort_by(.start) | last')
|
||||
# The sidecar logs the operation before acknowledging the POST
|
||||
# and keeps that log only in memory, so a missing row means it
|
||||
# restarted and the operation is gone; nothing will resume it.
|
||||
@@ -97,9 +138,10 @@ spec:
|
||||
status=$(echo "${row}" | jq -r '.status // ""')
|
||||
case "${status}" in
|
||||
success) echo "operation succeeded" ; exit 0 ;;
|
||||
error)
|
||||
"in progress") ;;
|
||||
*)
|
||||
err=$(echo "${row}" | jq -r '.error // "unknown"')
|
||||
echo "operation failed: ${err}" >&2
|
||||
echo "operation ended with status ${status}: ${err}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
Reference in new issue
Block a user