From d530ca8cf534eb4387934eccae6a94402ddfe01b Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 17:51:39 +0300 Subject: [PATCH] fix(backups): keep a ClickHouse archive an earlier attempt finished A backup Pod that loses sight of its upload, by dying or by losing six polls in a row, is retried while the sidecar finishes that upload. The retry found it in the action log as a success, took the sidecar for idle and asked for create_remote with resume again. clickhouse-backup then rebuilds the local backup and uploads it once more over the archive already in the bucket, which is left broken if that second upload dies halfway. The archive name belongs to one BackupJob, so a success row of exactly this upload can only be an earlier attempt's, and the retry now takes it as its result. An error or cancel row still leads to a resumed upload, and a restore still runs again, since a restore is not named per RestoreJob. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- hack/altinity-backup-strategy.bats | 44 ++++++++++++++++--- .../templates/strategy-altinity-default.yaml | 9 +++- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats index 2597338a2..46cf62d9a 100644 --- a/hack/altinity-backup-strategy.bats +++ b/hack/altinity-backup-strategy.bats @@ -483,15 +483,49 @@ run_script() { grep -qx 'create_remote --resume clickhouse-myapp-20260304T050607Z' "$tmp/state/accepted" } -# The action log is never trimmed, so a finished row of the same archive may -# be an earlier run's and is not taken for this attempt's result. -@test "backup starts create_remote again when the log holds a finished one of the same archive" { - for status in success error cancel; do +# The archive is named per BackupJob, so a success row of exactly this +# upload is an earlier attempt of the same BackupJob that finished while it +# could not see it. Uploading again with resume would re-upload over the +# finished archive and break it if that second upload dies halfway. +@test "a retried backup takes the upload of its archive an earlier attempt finished as its result" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: uploaded again over a finished archive: $(cat "$tmp/state/requests")" + false + fi +} + +@test "a retried backup resumes an upload of its archive that ended in error or was cancelled" { + for status in error cancel; do tmp=$(mktemp -d) make_stubs "$tmp/bin" "$tmp/state" echo "{\"command\":\"create_remote --resume clickhouse-myapp-20260304T050607Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior" render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z run_script "$tmp" - if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier upload must not stand in for this one"; false; fi + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row must lead to a resumed upload"; false; fi done } + +@test "a retried backup resumes when its latest upload failed after an earlier one succeeded" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + printf '%s\n' \ + '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:00"}' \ + '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "backup does not take another archive's finished upload as its own" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote --resume clickhouse-myapp-20260304T040000Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml index 18174d61a..331a89be8 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml @@ -104,11 +104,18 @@ spec: # so a local or partial remote backup an earlier attempt left is # resumed rather than refused as existing, whatever # use_resumable_state the sidecar runs with. The sidecar logs - # the request as "create_remote --resume ". + # the request as "create_remote --resume ". The name is + # this BackupJob's alone, so an upload of it that last ended in + # success is an earlier attempt's finished result: resuming + # over it would upload the archive again and leave it broken if + # that upload died halfway. cmd="create_remote --resume ${target}" wait_turn "${cmd}" if [ "${turn}" = "join" ]; then echo "${cmd} is already running; waiting for it" + elif printf '%s\n' "${actions}" | jq -es --arg cmd "${cmd}" \ + '[.[] | select(.command == $cmd)] | sort_by(.start) | last | .status == "success"' >/dev/null; then + echo "${cmd} already succeeded in an earlier attempt" else echo "${cmd}" curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}&resume=true" >/dev/null