mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(backups): keep a ClickHouse archive an earlier attempt finished
A backup Pod that loses sight of its upload, by dying or by losing six polls in a row, is retried while the sidecar finishes that upload. The retry found it in the action log as a success, took the sidecar for idle and asked for create_remote with resume again. clickhouse-backup then rebuilds the local backup and uploads it once more over the archive already in the bucket, which is left broken if that second upload dies halfway. The archive name belongs to one BackupJob, so a success row of exactly this upload can only be an earlier attempt's, and the retry now takes it as its result. An error or cancel row still leads to a resumed upload, and a restore still runs again, since a restore is not named per RestoreJob. Assisted-by: LLM Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
89f8cc02ee
commit
d530ca8cf5
2 files changed
+47
-6
No files matched your search
@@ -483,15 +483,49 @@ run_script() {
|
||||
grep -qx 'create_remote --resume clickhouse-myapp-20260304T050607Z' "$tmp/state/accepted"
|
||||
}
|
||||
|
||||
# The action log is never trimmed, so a finished row of the same archive may
|
||||
# be an earlier run's and is not taken for this attempt's result.
|
||||
@test "backup starts create_remote again when the log holds a finished one of the same archive" {
|
||||
for status in success error cancel; do
|
||||
# The archive is named per BackupJob, so a success row of exactly this
|
||||
# upload is an earlier attempt of the same BackupJob that finished while it
|
||||
# could not see it. Uploading again with resume would re-upload over the
|
||||
# finished archive and break it if that second upload dies halfway.
|
||||
@test "a retried backup takes the upload of its archive an earlier attempt finished as its result" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior"
|
||||
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
|
||||
run_script "$tmp"
|
||||
if grep -q '^POST ' "$tmp/state/requests"; then
|
||||
echo "FAIL: uploaded again over a finished archive: $(cat "$tmp/state/requests")"
|
||||
false
|
||||
fi
|
||||
}
|
||||
|
||||
@test "a retried backup resumes an upload of its archive that ended in error or was cancelled" {
|
||||
for status in error cancel; do
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo "{\"command\":\"create_remote --resume clickhouse-myapp-20260304T050607Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior"
|
||||
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
|
||||
run_script "$tmp"
|
||||
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier upload must not stand in for this one"; false; fi
|
||||
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row must lead to a resumed upload"; false; fi
|
||||
done
|
||||
}
|
||||
|
||||
@test "a retried backup resumes when its latest upload failed after an earlier one succeeded" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
printf '%s\n' \
|
||||
'{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:00"}' \
|
||||
'{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior"
|
||||
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
|
||||
run_script "$tmp"
|
||||
[ "$(accepted "$tmp")" = 1 ]
|
||||
}
|
||||
|
||||
@test "backup does not take another archive's finished upload as its own" {
|
||||
tmp=$(mktemp -d)
|
||||
make_stubs "$tmp/bin" "$tmp/state"
|
||||
echo '{"command":"create_remote --resume clickhouse-myapp-20260304T040000Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior"
|
||||
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
|
||||
run_script "$tmp"
|
||||
[ "$(accepted "$tmp")" = 1 ]
|
||||
}
|
||||
@@ -104,11 +104,18 @@ spec:
|
||||
# so a local or partial remote backup an earlier attempt left is
|
||||
# resumed rather than refused as existing, whatever
|
||||
# use_resumable_state the sidecar runs with. The sidecar logs
|
||||
# the request as "create_remote --resume <name>".
|
||||
# the request as "create_remote --resume <name>". The name is
|
||||
# this BackupJob's alone, so an upload of it that last ended in
|
||||
# success is an earlier attempt's finished result: resuming
|
||||
# over it would upload the archive again and leave it broken if
|
||||
# that upload died halfway.
|
||||
cmd="create_remote --resume ${target}"
|
||||
wait_turn "${cmd}"
|
||||
if [ "${turn}" = "join" ]; then
|
||||
echo "${cmd} is already running; waiting for it"
|
||||
elif printf '%s\n' "${actions}" | jq -es --arg cmd "${cmd}" \
|
||||
'[.[] | select(.command == $cmd)] | sort_by(.start) | last | .status == "success"' >/dev/null; then
|
||||
echo "${cmd} already succeeded in an earlier attempt"
|
||||
else
|
||||
echo "${cmd}"
|
||||
curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}&resume=true" >/dev/null
|
||||
|
||||
Reference in new issue
Block a user