fix(backups): keep a ClickHouse archive an earlier attempt finished

A backup Pod that loses sight of its upload, by dying or by losing six
polls in a row, is retried while the sidecar finishes that upload. The
retry found it in the action log as a success, took the sidecar for
idle and asked for create_remote with resume again. clickhouse-backup
then rebuilds the local backup and uploads it once more over the
archive already in the bucket, which is left broken if that second
upload dies halfway.

The archive name belongs to one BackupJob, so a success row of exactly
this upload can only be an earlier attempt's, and the retry now takes
it as its result. An error or cancel row still leads to a resumed
upload, and a restore still runs again, since a restore is not named
per RestoreJob.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-09 16:39:46 +03:00
1 parent 89f8cc02ee
commit d530ca8cf5
2 files changed
+47 -6

No files matched your search

+39 -5
View File
@@ -483,15 +483,49 @@ run_script() {
grep -qx 'create_remote --resume clickhouse-myapp-20260304T050607Z' "$tmp/state/accepted"
}
# The action log is never trimmed, so a finished row of the same archive may
# be an earlier run's and is not taken for this attempt's result.
@test "backup starts create_remote again when the log holds a finished one of the same archive" {
for status in success error cancel; do
# The archive is named per BackupJob, so a success row of exactly this
# upload is an earlier attempt of the same BackupJob that finished while it
# could not see it. Uploading again with resume would re-upload over the
# finished archive and break it if that second upload dies halfway.
@test "a retried backup takes the upload of its archive an earlier attempt finished as its result" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
if grep -q '^POST ' "$tmp/state/requests"; then
echo "FAIL: uploaded again over a finished archive: $(cat "$tmp/state/requests")"
false
fi
}
@test "a retried backup resumes an upload of its archive that ended in error or was cancelled" {
for status in error cancel; do
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo "{\"command\":\"create_remote --resume clickhouse-myapp-20260304T050607Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier upload must not stand in for this one"; false; fi
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row must lead to a resumed upload"; false; fi
done
}
@test "a retried backup resumes when its latest upload failed after an earlier one succeeded" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
printf '%s\n' \
'{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:00"}' \
'{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
}
@test "backup does not take another archive's finished upload as its own" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo '{"command":"create_remote --resume clickhouse-myapp-20260304T040000Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
}
@@ -104,11 +104,18 @@ spec:
# so a local or partial remote backup an earlier attempt left is
# resumed rather than refused as existing, whatever
# use_resumable_state the sidecar runs with. The sidecar logs
# the request as "create_remote --resume <name>".
# the request as "create_remote --resume <name>". The name is
# this BackupJob's alone, so an upload of it that last ended in
# success is an earlier attempt's finished result: resuming
# over it would upload the archive again and leave it broken if
# that upload died halfway.
cmd="create_remote --resume ${target}"
wait_turn "${cmd}"
if [ "${turn}" = "join" ]; then
echo "${cmd} is already running; waiting for it"
elif printf '%s\n' "${actions}" | jq -es --arg cmd "${cmd}" \
'[.[] | select(.command == $cmd)] | sort_by(.start) | last | .status == "success"' >/dev/null; then
echo "${cmd} already succeeded in an earlier attempt"
else
echo "${cmd}"
curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}&resume=true" >/dev/null