diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats index 0861da42d..070b6bad5 100644 --- a/hack/altinity-backup-strategy.bats +++ b/hack/altinity-backup-strategy.bats @@ -1,14 +1,28 @@ #!/usr/bin/env bats -# Behavioural tests for the shipped Altinity (ClickHouse) backup/restore -# strategy script, extracted from the rendered cozy-default-altinity Strategy -# CR and EXECUTED against a stubbed clickhouse-backup HTTP API, with the real -# jq doing the archive selection. Which archive a restore reads back is -# decided by that selection over list/remote, so it is exercised against -# several archives rather than string-matched. +# Behavioural tests for the shipped ClickHouse (Altinity) strategy script, +# extracted from the rendered cozy-default-altinity Strategy CR and EXECUTED +# against a stubbed clickhouse-backup sidecar API. # -# The script runs with `bash`, not `sh`, for the reason given in -# hack/redis-backup-strategy.bats: it opens `set -euo pipefail`, which the -# production image's busybox ash supports and the CI runner's dash rejects. +# The contract pinned here is the start gate: the sidecar serialises its +# operations and answers any new one with HTTP 423 while another is "in +# progress", so a strategy Pod that starts its operation without waiting for +# an idle sidecar burns a Job attempt in seconds. The gate has to fail the Pod +# rather than proceed when the sidecar never becomes idle. Once started, an +# operation the sidecar no longer lists (its action log lives in memory and a +# restart empties it) has to fail the Pod too, since the Job has no deadline. +# A Job retry must not start its operation twice: it takes over the +# restore_remote, or the create_remote of the archive the controller named, +# that the attempt before it left running, and a few lost polls do not end an +# attempt in the first place. +# +# Which archive a restore reads back is decided by the script's own jq +# selection over list/remote, so it is exercised against several archives +# rather than string-matched: exactly the archive the Backup recorded, or for +# a Backup that records none, the newest one stamped no later than its takenAt. +# +# Run from the repo root with make bats-unit-tests. The rendered script is run +# with `bash`, not `sh`: it opens `set -euo pipefail`, which the production +# image's busybox ash supports but the CI runner's /bin/sh (dash) rejects. load test_helper @@ -16,12 +30,13 @@ CHART=packages/system/backupstrategy-controller # Render the shipped strategy's in-Pod script into $1 and substitute the # controller-render placeholders the way the driver fills them: $2 mode, $3 -# app (release and source application), $4 the archive name recorded on the -# Backup (empty for a Backup that records none), $5 the Backup name, $6 its -# takenAt in the archive-name layout. An empty render fails early, as does a -# placeholder left unsubstituted. +# app (release and source application), $4 the archive name the controller +# chose or the Backup recorded (empty, the default, for none), $5 the Backup +# name, $6 its takenAt in the archive-name layout (default: later than every +# archive the stub lists). An empty render fails early, as does a placeholder +# left unsubstituted: these tests EXECUTE it. render_script() { - out=$1; mode=$2; app=$3; archive=$4; backup=$5; takenat=$6 + out=$1; mode=$2; app=$3; archive=${4-}; backup=${5-bkp}; takenat=${6-99991231T235959Z} helm template t "$CHART" -n cozy-backup-controller \ --set backupStorage.bucketNameOverride=test-bucket 2>/dev/null \ | yq eval 'select(.kind == "Altinity") | .spec.template.spec.containers[0].args[0]' - \ @@ -39,56 +54,118 @@ render_script() { fi } -# Stub the clickhouse-backup API behind curl, first on PATH. list/remote -# serves $state/remote; actions serves $state/actions plus one success row per -# operation the script has issued, and after its first read swaps in -# $state/actions.next when a test seeds one (an operation that finishes while -# the script waits). Every request is logged to $state/curl.log. sleep is -# stubbed to fail after a few calls, so a script that never sees its -# operation finish fails the test instead of hanging it. +# A curl stub standing in for the sidecar API, put first on PATH next to a +# no-op sleep. State lives in $1: +# unreachable present: every request fails as a refused connection (7) +# down a number: that many next requests fail the same way, as +# before the sidecar has started +# busy a number: that many GET /backup/actions answers still list +# an operation of someone else "in progress" +# op the command line of the sidecar's own operation, written +# by an accepted POST the way the sidecar logs it ("--resume" +# before the name of a create_remote asking for it), or +# seeded by a test as the operation +# an earlier attempt of the same Job started +# op_left how many more GET /backup/actions answers list op as +# "in progress" before it ends with op_result +# op_duration op_left given to an operation a POST starts (default 0) +# op_result the status op ends with (default success) +# prior extra action-log rows, verbatim, older than op +# remote present: the list/remote answer, verbatim, in place of the +# one archive listed by default +# restarted present: the action log no longer lists op, as after the +# sidecar restarted and lost its in-memory log +# fail_on_post a POST that is accepted makes that many following +# GET /backup/actions requests fail (curl exit 7) +# idle created by an answer that lists nothing in progress; a +# POST is refused (423, curl -f exit 22) unless one was seen +# since the last accepted POST, and while anything runs +# accepted one line per POST the sidecar accepted +# requests one line per request: method and URL +# runaway created when the script exceeds 200 requests; that request +# and every later one fails, so a polling loop that never ends +# stops the test instead of hanging it make_stubs() { bin=$1; state=$2 - mkdir -p "$bin" - : > "$state/remote" - : > "$state/actions" + mkdir -p "$bin" "$state" cat > "$bin/curl" <> "$state/curl.log" +echo "\$method \$url" >> "\$state/requests" +if [ "\$(wc -l < "\$state/requests")" -gt 200 ]; then touch "\$state/runaway"; exit 7; fi +if [ -e "\$state/unreachable" ]; then exit 7; fi +num() { cat "\$state/\$1" 2>/dev/null || echo 0; } +down=\$(num down) +if [ "\$down" -gt 0 ]; then echo \$((down - 1)) > "\$state/down"; exit 7; fi +running=0 +if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ] && [ "\$(num op_left)" -gt 0 ]; then running=1; fi case "\$method \$url" in - "GET "*/backup/status) echo '{}' ;; - "GET "*/backup/list/remote) cat "$state/remote" ;; + "POST "*) + if [ "\$(num busy)" -gt 0 ] || [ "\$running" = 1 ] || [ ! -e "\$state/idle" ]; then exit 22; fi + case "\$url" in + */create_remote\?name=*) + name=\${url#*name=}; name=\${name%%&*} + case "\$url" in *[?\&]resume=true*) cmd="create_remote --resume \$name" ;; *) cmd="create_remote \$name" ;; esac + ;; + */restore_remote/*) cmd="restore_remote \${url##*/}" ;; + *) exit 22 ;; + esac + echo "\$cmd" > "\$state/op" + echo "\$cmd" >> "\$state/accepted" + num op_duration > "\$state/op_left" + num fail_on_post > "\$state/fail_polls" + rm -f "\$state/idle" + ;; "GET "*/backup/actions) - cat "$state/actions" - if [ -f "$state/actions.done" ]; then cat "$state/actions.done"; fi - if [ -f "$state/actions.next" ]; then mv "$state/actions.next" "$state/actions"; fi + fail=\$(num fail_polls) + if [ "\$fail" -gt 0 ]; then echo \$((fail - 1)) > "\$state/fail_polls"; exit 7; fi + echo '{"command":"create_remote clickhouse-myapp-20260101T000000Z","status":"error","start":"2026-01-01 00:00:00","error":"old"}' + cat "\$state/prior" 2>/dev/null || true + inprog=0 + busy=\$(num busy) + if [ "\$busy" -gt 0 ]; then + echo \$((busy - 1)) > "\$state/busy" + echo '{"command":"upload --resumable=1 clickhouse-myapp-20260101T000000Z","status":"in progress","start":"2026-01-01 00:01:00"}' + inprog=1 + fi + if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ]; then + left=\$(num op_left) + if [ "\$left" -gt 0 ]; then + echo \$((left - 1)) > "\$state/op_left" + status="in progress" + inprog=1 + else + status=\$(cat "\$state/op_result" 2>/dev/null || echo success) + fi + echo "{\"command\":\"\$(cat "\$state/op")\",\"status\":\"\$status\",\"start\":\"2026-01-01 00:02:00\",\"error\":\"boom\"}" + fi + if [ "\$inprog" = 0 ]; then touch "\$state/idle"; fi ;; - "POST "*/backup/create_remote*) - name=\${url#*name=}; name=\${name%%&*} - printf '{"command":"create_remote --resume %s","status":"success","start":"2099-01-01 00:00:00"}\n' "\$name" >> "$state/actions.done" + "GET "*/backup/list/remote) + if [ -e "\$state/remote" ]; then cat "\$state/remote"; else + echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}' + fi ;; - "POST "*/backup/restore_remote/*) - printf '{"command":"restore_remote %s","status":"success","start":"2099-01-01 00:00:00"}\n' "\${url##*/}" >> "$state/actions.done" - ;; - *) echo "unexpected request: \$method \$url" >&2; exit 22 ;; esac +exit 0 STUB - cat > "$bin/sleep" </dev/null || echo 0) -echo \$((n + 1)) > "$state/sleeps" -if [ "\$n" -ge 3 ]; then echo "sleep budget spent: the operation never finished" >&2; exit 1; fi + cat > "$bin/sleep" <<'STUB' +#!/bin/sh +exit 0 STUB chmod 0755 "$bin/curl" "$bin/sleep" - export API_USERNAME=u API_PASSWORD=p } # One list/remote row per argument, "name" or "name|desc"; created mirrors -# the name's timestamp, which is the order the pre-fix selection sorted by. +# the name's timestamp. remote_rows() { for row in "$@"; do name=${row%%|*}; desc="tar, regular" @@ -98,49 +175,216 @@ remote_rows() { done } +accepted() { + if [ -e "$1/state/accepted" ]; then wc -l < "$1/state/accepted" | tr -d ' '; else echo 0; fi +} + +run_script() { + tmp=$1 + API_USERNAME=u API_PASSWORD=p PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" +} + @test "an empty render fails the fixture instead of passing as a no-op" { tmp=$(mktemp -d) CHART=$tmp/no-such-chart - if render_script "$tmp/s.sh" restore myapp "" bkp 20260304T050622Z 2>/dev/null; then + if render_script "$tmp/s.sh" backup myapp 2>/dev/null; then echo "FAIL: render_script reported success with no chart" false fi - rm -rf "$tmp" +} + +@test "backup on an idle sidecar starts create_remote and waits for its success" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp + run_script "$tmp" + grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests" +} + +@test "backup waits until the running operation finishes before create_remote" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/busy" + render_script "$tmp/s.sh" backup myapp + run_script "$tmp" + grep -q '^create_remote --resume clickhouse-myapp-' "$tmp/state/accepted" + [ "$(cat "$tmp/state/busy")" = 0 ] +} + +@test "restore on an idle sidecar starts restore_remote of the listed archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + grep -q '^POST .*/backup/restore_remote/clickhouse-myapp-20260101T000000Z$' "$tmp/state/requests" +} + +@test "backup fails without starting create_remote when the sidecar stays busy" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 1000 > "$tmp/state/busy" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on a busy sidecar"; false; fi +} + +@test "backup fails without starting create_remote when the sidecar never answers" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + touch "$tmp/state/unreachable" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: an unreachable sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on an unreachable sidecar"; false; fi +} + +@test "restore fails without starting restore_remote when the sidecar never answers" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + touch "$tmp/state/unreachable" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp" 2> "$tmp/err"; then echo "FAIL: an unreachable sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no restore may start on an unreachable sidecar"; false; fi + grep -q 'list/remote failed 60 times in a row' "$tmp/err" + [ "$(grep -c '^GET .*/backup/list/remote$' "$tmp/state/requests")" = 60 ] +} + +@test "restore fails without starting restore_remote when the sidecar stays busy" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 1000 > "$tmp/state/busy" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no restore may start on a busy sidecar"; false; fi +} + +@test "backup fails instead of polling forever when the sidecar restarts after create_remote" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + touch "$tmp/state/restarted" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp" 2> "$tmp/err"; then echo "FAIL: an operation lost from the action log must fail the Pod"; false; fi + grep -q 'no longer in the sidecar action log' "$tmp/err" + grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests" + if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling for an operation the sidecar no longer lists"; false; fi +} + +@test "restore waits for a sidecar that has not started yet" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 10 > "$tmp/state/down" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "restore waits until another operation finishes, then starts its own" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/busy" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted" +} + +@test "failed polls after restore_remote, short of six in a row, do not fail the attempt" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/op_duration" + echo 5 > "$tmp/state/fail_on_post" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "a retry after an attempt lost its polls joins the running restore instead of starting another" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 20 > "$tmp/state/op_duration" + echo 6 > "$tmp/state/fail_on_post" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: six failed polls in a row must fail the attempt"; false; fi + [ "$(accepted "$tmp")" = 1 ] + echo 0 > "$tmp/state/fail_polls" + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: the retry started restore_remote a second time"; false; fi + [ "$(cat "$tmp/state/op_left")" = 0 ] +} + +@test "restore does not join an operation running on another archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'restore_remote clickhouse-myapp-20251231T000000Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted" +} + +# The sidecar never trims its action log, so a finished restore_remote row of +# the same archive may belong to an earlier RestoreJob; only a running one may +# be taken over. +@test "restore starts restore_remote again when the log holds a finished one of the same archive" { + for status in success error cancel; do + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo "{\"command\":\"restore_remote clickhouse-myapp-20260101T000000Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier restore must not stand in for this one"; false; fi + done +} + +@test "a restore that ends in error fails the Pod" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo error > "$tmp/state/op_result" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: a failed restore must fail the Pod"; false; fi + [ "$(accepted "$tmp")" = 1 ] +} + +@test "a cancelled backup fails the Pod instead of polling forever" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo cancel > "$tmp/state/op_result" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: a cancelled operation must fail the Pod"; false; fi + if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling a cancelled operation"; false; fi } @test "restore reads back the archive the Backup recorded, not the newest one" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \ - clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/remote" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log" - [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ] - rm -rf "$tmp" + clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote" + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests" + [ "$(accepted "$tmp")" = 1 ] } @test "restore fails naming the recorded archive when remote storage lacks it" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z - remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/remote" + remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/state/remote" out="" - if out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1); then + if out=$(run_script "$tmp" 2>&1); then echo "FAIL: restore succeeded without the recorded archive: $out" false fi printf '%s\n' "$out" | grep -qF 'archive clickhouse-myapp-20260304T050607Z recorded on Backup bkp-a is not in remote storage' - if grep -qF '/backup/restore_remote/' "$tmp/curl.log"; then - echo "FAIL: restored another archive: $(cat "$tmp/curl.log")" + if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then + echo "FAIL: restored another archive: $(cat "$tmp/state/requests")" false fi - rm -rf "$tmp" } @test "restore of a Backup without a recorded archive takes the newest one stamped by its takenAt, and warns" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z # Each decoy sorts after the expected archive and is ruled out by one # filter: broken, not a bare timestamp after the prefix, or stamped after @@ -148,71 +392,140 @@ remote_rows() { remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \ "clickhouse-myapp-20260304T050700Z|broken (can't stat metadata.json)" \ clickhouse-myapp-20260304T050710Z-copy \ - clickhouse-myapp-20260304T060000Z > "$tmp/remote" - out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1) - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log" - [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ] + myapp-20260304050720-0f3c9a1e5b7d \ + clickhouse-myapp-20260304T060000Z > "$tmp/state/remote" + out=$(run_script "$tmp" 2>&1) + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests" + [ "$(accepted "$tmp")" = 1 ] printf '%s\n' "$out" | grep -qF 'WARNING: Backup bkp-a records no archive name' - rm -rf "$tmp" } @test "restore of a Backup without a recorded archive fails when no archive is stamped by its takenAt" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z - remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/remote" + remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote" out="" - if out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1); then + if out=$(run_script "$tmp" 2>&1); then echo "FAIL: restore succeeded with every archive newer than takenAt: $out" false fi printf '%s\n' "$out" | grep -qF 'no archive of myapp stamped no later than 20260304T050800Z' - if grep -qF '/backup/restore_remote/' "$tmp/curl.log"; then - echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/curl.log")" + if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then + echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/state/requests")" false fi - rm -rf "$tmp" } @test "backup writes the archive under the name the controller chose" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log" - rm -rf "$tmp" -} - -@test "a retried backup waits on the attempt the sidecar is still running" { - tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" - printf '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"in progress","start":"2026-03-04 05:06:08"}\n' > "$tmp/actions" - printf '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-03-04 05:06:08"}\n' > "$tmp/actions.next" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - # The sidecar answers a second operation with 423 while one runs. - if grep -qF 'POST ' "$tmp/curl.log"; then - echo "FAIL: started another operation over the running one: $(cat "$tmp/curl.log")" - false - fi - rm -rf "$tmp" -} - -@test "a retried backup after a failed attempt resumes the same archive" { - tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" - printf '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-03-04 05:06:08","error":"S3 PutObject 403"}\n' > "$tmp/actions" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log" - rm -rf "$tmp" + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests" } @test "backup falls back to a timestamped name when the controller supplies none" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp "" "" "" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/curl.log" - rm -rf "$tmp" + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp "" + run_script "$tmp" + grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/state/requests" +} + +# The sidecar answers a second operation with 423 while one runs, so a retry +# that POSTed over its predecessor's upload would burn every attempt left. +@test "a retried backup takes over the upload of its archive the sidecar is still running" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")" + false + fi + [ "$(cat "$tmp/state/op_left")" = 0 ] +} + +@test "a taken-over backup that ends in error fails the Pod" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op" + echo 2 > "$tmp/state/op_left" + echo error > "$tmp/state/op_result" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + if run_script "$tmp"; then echo "FAIL: a taken-over upload that failed must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")" + false + fi +} + +@test "a retried backup after a failed attempt resumes the same archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30","error":"S3 PutObject 403"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests" +} + +@test "backup does not take over an upload running for another archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T040000Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'create_remote --resume clickhouse-myapp-20260304T050607Z' "$tmp/state/accepted" +} + +# The archive is named per BackupJob, so a success row of exactly this +# upload is an earlier attempt of the same BackupJob that finished while it +# could not see it. Uploading again with resume would re-upload over the +# finished archive and break it if that second upload dies halfway. +@test "a retried backup takes the upload of its archive an earlier attempt finished as its result" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: uploaded again over a finished archive: $(cat "$tmp/state/requests")" + false + fi +} + +@test "a retried backup resumes an upload of its archive that ended in error or was cancelled" { + for status in error cancel; do + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo "{\"command\":\"create_remote --resume clickhouse-myapp-20260304T050607Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row must lead to a resumed upload"; false; fi + done +} + +@test "a retried backup resumes when its latest upload failed after an earlier one succeeded" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + printf '%s\n' \ + '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:00"}' \ + '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "backup does not take another archive's finished upload as its own" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote --resume clickhouse-myapp-20260304T040000Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] } diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index dab780100..f0c597583 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -2,11 +2,11 @@ package backupcontroller import ( - "bytes" "context" "encoding/json" "fmt" "maps" + "regexp" "strings" "time" @@ -57,6 +57,12 @@ const ( // archive relies on. altinityArchiveTimeLayout = "20060102T150405Z" + // altinityArchiveRunTimeLayout and altinityArchiveUIDLength shape the + // archive name of a run; see altinityArchiveName for why they are this + // short. + altinityArchiveRunTimeLayout = "20060102150405" + altinityArchiveUIDLength = 12 + // Polling cadence for the Altinity Job lifecycle. Mirrors the CNPG / // Velero strategy cadence so behaviour is uniform across drivers. altinityPollInterval = 5 * time.Second @@ -187,11 +193,30 @@ func backupParameters(b *backupsv1alpha1.Backup) map[string]string { } // altinityArchiveName mints the archive name for one BackupJob from its -// persisted StartedAt, so every reconcile and every Pod retry of the Job -// writes the same archive. It keeps the clickhouse-- -// shape, because a Backup that records no archive is restored by the -// timestamp in that shape. +// persisted StartedAt and its UID, so every reconcile and every Pod retry of +// the Job writes the same archive and no other run writes it. StartedAt has +// one-second resolution, and two Plans on one schedule start their runs in +// the same second; the strategy takes a finished upload of its archive as an +// earlier attempt of its own run, so a shared name would let one run report +// the other's archive. +// +// The name is no longer than altinityArchiveNameBeforeUIDs, because it sits in +// every object key, a key is capped at 1024 bytes, and the legacy +// s3PathOverride prefix has no length limit of its own: the UID takes the room +// that dropping "clickhouse-", "T" and "Z" frees. It goes after the timestamp +// so names still sort chronologically, and the restore of a Backup that +// records no archive, which accepts only clickhouse--, +// never picks one of these. func altinityArchiveName(j *backupsv1alpha1.BackupJob) string { + uid := strings.ReplaceAll(string(j.UID), "-", "") + uid = uid[:min(len(uid), altinityArchiveUIDLength)] + return fmt.Sprintf("%s-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveRunTimeLayout), uid) +} + +// altinityArchiveNameBeforeUIDs is the name a controller that did not append +// the UID gave the same BackupJob, which a Job it created before an upgrade +// still writes. +func altinityArchiveNameBeforeUIDs(j *backupsv1alpha1.BackupJob) string { return fmt.Sprintf("clickhouse-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout)) } @@ -199,13 +224,16 @@ func altinityArchiveName(j *backupsv1alpha1.BackupJob) string { // the archive name. A Job created by a controller that predates archive // names, or rendered from a strategy that names its own archives, wrote an // archive under another name, and recording this one would point the Backup -// at an archive that does not exist. +// at an archive that does not exist. The name has to stand on its own: the +// name without the UID starts the name with it, and a strategy may append to +// either. func altinityJobCarriesArchive(job *batchv1.Job, archive string) bool { spec, err := json.Marshal(job.Spec.Template.Spec) if err != nil { return false } - return bytes.Contains(spec, []byte(archive)) + whole := regexp.MustCompile(`(^|[^A-Za-z0-9._-])` + regexp.QuoteMeta(archive) + `($|[^A-Za-z0-9._-])`) + return whole.Match(spec) } // renderAltinityTemplate runs the strategy's PodTemplateSpec through the @@ -365,7 +393,10 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv return ctrl.Result{}, nil } if !altinityJobCarriesArchive(batchJob, archive) { - archive = "" + archive = altinityArchiveNameBeforeUIDs(j) + if !altinityJobCarriesArchive(batchJob, archive) { + archive = "" + } } artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved, archive) if err != nil { diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index 48592ac54..a9022ceb6 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" @@ -733,14 +734,66 @@ func newAltinityArchiveStrategy(name string) *strategyv1alpha1.Altinity { func TestAltinityArchiveName_FromStartedAtInUTC(t *testing.T) { startedAt := metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 7, 890000000, time.FixedZone("UTC+3", 3*3600))) j := &backupsv1alpha1.BackupJob{ - Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, - Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + ObjectMeta: metav1.ObjectMeta{UID: "0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, } - if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z"; got != want { + if got, want := altinityArchiveName(j), "ch-test-20260304050607-0f3c9a1e5b7d"; got != want { t.Errorf("altinityArchiveName = %q, want %q", got, want) } } +// TestAltinityArchiveName_NoLongerThanTheNameWithoutUID pins the S3 key +// budget: the archive name is part of every object key, a key may not exceed +// 1024 bytes, and the legacy s3PathOverride prefix is unbounded, so a prefix +// that fit the name without the UID must still fit this one. +func TestAltinityArchiveName_NoLongerThanTheNameWithoutUID(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + for n := 1; n <= 63; n++ { + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{UID: "0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef(strings.Repeat("a", n))}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + } + name, legacy := altinityArchiveName(j), altinityArchiveNameBeforeUIDs(j) + if len(name) > len(legacy) { + t.Errorf("application name of %d characters: archive %q is longer than %q", n, name, legacy) + } + // The manifest key of a prefix that left the name without the UID + // exactly at the limit. + prefix := strings.Repeat("p", 1024-len("/"+legacy+"/metadata.json")) + if key := prefix + "/" + name + "/metadata.json"; len(key) > 1024 { + t.Errorf("application name of %d characters: manifest key is %d bytes", n, len(key)) + } + } +} + +// TestAltinityArchiveName_DistinctForRunsStartedInTheSameSecond pins that two +// BackupJobs of one application started within the same second, as two Plans +// on the same schedule are, write different archives. With a shared name, the +// strategy takes the other run's finished upload for an earlier attempt of its +// own and records an archive missing what changed since that upload. +func TestAltinityArchiveName_DistinctForRunsStartedInTheSameSecond(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + run := func(name, uid string) *backupsv1alpha1.BackupJob { + return &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tenant-test", UID: types.UID(uid)}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + } + } + daily := altinityArchiveName(run("daily-1", "11111111-1111-1111-1111-111111111111")) + weekly := altinityArchiveName(run("weekly-1", "22222222-2222-2222-2222-222222222222")) + if daily == weekly { + t.Fatalf("two runs started in the same second share archive %q", daily) + } + for _, name := range []string{daily, weekly} { + if !strings.HasPrefix(name, "ch-test-20260304050607-") { + t.Errorf("archive %q does not start with the application and start time, so names stop sorting chronologically", name) + } + } +} + // TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven drives one BackupJob // through Job creation and completion in separate reconciles. The archive // name has to come out of the persisted StartedAt both times: the Pod is @@ -751,7 +804,7 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { strategy := newAltinityArchiveStrategy("clickhouse-strategy") startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) backupJob := &backupsv1alpha1.BackupJob{ - ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: newAltinityClickHouseRef("ch-test"), BackupClassName: "clickhouse-backup", @@ -768,7 +821,7 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { } r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy)) ctx := context.Background() - const want = "clickhouse-ch-test-20260304T050607Z" + const want = "ch-test-20260304050607-testbjuid" if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { t.Fatalf("reconcileAltinity() creating the Job: %v", err) @@ -804,6 +857,90 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { } } +// TestReconcileAltinity_RecordsTheArchiveOfAJobNamedBeforeUIDs covers a backup +// Job created by a controller that named archives without the BackupJob's UID +// and still running when the controller is upgraded: the Job is not +// re-rendered, so its archive carries the old name, and the Backup has to +// record that name rather than none. +func TestReconcileAltinity_RecordsTheArchiveOfAJobNamedBeforeUIDs(t *testing.T) { + const legacy = "clickhouse-ch-test-20260304T050607Z" + cases := []struct { + name string + archive string // archive the Job's Pod was rendered with + want string + }{ + {name: "the name without the UID", archive: legacy, want: legacy}, + {name: "a longer name that starts with it", archive: legacy + "-copy", want: ""}, + {name: "a longer name that ends with it", archive: "copy-" + legacy, want: ""}, + {name: "a longer name that continues it after a period", archive: legacy + ".copy", want: ""}, + {name: "a longer name that leads into it with a period", archive: "copy." + legacy, want: ""}, + {name: "a longer name that continues it after an underscore", archive: legacy + "_copy", want: ""}, + {name: "a longer name that leads into it with an underscore", archive: "copy_" + legacy, want: ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := recordAltinityArchiveOfJobRenderedWith(t, tc.archive); got != tc.want { + t.Errorf("Backup records archive %q, want %q", got, tc.want) + } + }) + } +} + +// recordAltinityArchiveOfJobRenderedWith completes a backup Job whose Pod +// carries the given archive in place of the one the controller rendered, and +// returns the archive the resulting Backup records. +func recordAltinityArchiveOfJobRenderedWith(t *testing.T, archive string) string { + t.Helper() + app := newAltinityClickHouseApp("ch-test", "tenant-test") + strategy := newAltinityArchiveStrategy("clickhouse-strategy") + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy)) + ctx := context.Background() + + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() creating the Job: %v", err) + } + job := &batchv1.Job{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil { + t.Fatalf("get backup Job: %v", err) + } + job.Spec.Template.Spec.Containers[0].Args[1] = "--archive=" + archive + if err := r.Update(ctx, job); err != nil { + t.Fatalf("render the backup Job with archive %q: %v", archive, err) + } + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := r.Status().Update(ctx, job); err != nil { + t.Fatalf("mark backup Job complete: %v", err) + } + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil { + t.Fatalf("refresh BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() on the completed Job: %v", err) + } + + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + return created.Spec.DriverMetadata[altinityArchiveMetaKey] +} + // TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven covers a Job whose // Pod never carried the driver's archive name: one created before the // upgrade that introduced the names, or rendered from a strategy that names @@ -872,13 +1009,13 @@ func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) { // A Backup recording this run's own archive is the one an earlier reconcile // created before its status write was lost, and is adopted. func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) { - const thisRun = "clickhouse-ch-test-20260305T050607Z" + const thisRun = "ch-test-20260305050607-testbjuid" cases := []struct { name string existing string // archive the existing Backup records; empty for none wantSucceeded bool }{ - {name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z"}, + {name: "earlier run's Backup records another archive", existing: "ch-test-20260304050607-earlierbjuid"}, {name: "earlier run's Backup records no archive", existing: ""}, {name: "this run's Backup from a reconcile whose status write was lost", existing: thisRun, wantSucceeded: true}, } @@ -888,7 +1025,7 @@ func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) { strategy := newAltinityArchiveStrategy("clickhouse-strategy") startedAt := metav1.NewTime(time.Date(2026, 3, 5, 5, 6, 7, 0, time.UTC)) backupJob := &backupsv1alpha1.BackupJob{ - ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: newAltinityClickHouseRef("ch-test"), BackupClassName: "clickhouse-backup", @@ -1503,7 +1640,7 @@ func TestReconcileAltinityRestore_UnreadableHoldStateDoesNotStartTheJob(t *testi func TestReconcileAltinity_CredentialsWaitKeepsTheArchiveName(t *testing.T) { startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) backupJob := &backupsv1alpha1.BackupJob{ - ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: newAltinityClickHouseRef("ch-test"), BackupClassName: "clickhouse-backup", @@ -1522,7 +1659,7 @@ func TestReconcileAltinity_CredentialsWaitKeepsTheArchiveName(t *testing.T) { }, } ctx := context.Background() - const want = "clickhouse-ch-test-20260304T050607Z" + const want = "ch-test-20260304050607-0f3c9a1e5b7d" reconcile := func(step string) { t.Helper() j := &backupsv1alpha1.BackupJob{} diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml index 5d0c1c6ab..bc439d40b 100644 --- a/packages/apps/clickhouse/templates/clickhouse.yaml +++ b/packages/apps/clickhouse/templates/clickhouse.yaml @@ -222,6 +222,18 @@ spec: key: backup - name: ALLOW_EMPTY_BACKUPS value: "true" + {{- /* + create_remote keeps the local backup when its upload fails, + and the upstream defaults never prune local backups (0) and + resume unfinished uploads at start, which pushes a failed + attempt to the bucket as an archive no Backup refers to and + holds the API lock (423) while it does. -1 keeps only the + newest local backup and removes it once its upload succeeds. + */}} + - name: BACKUPS_TO_KEEP_LOCAL + value: "-1" + - name: API_COMPLETE_RESUMABLE_AFTER_RESTART + value: "false" - name: REMOTE_STORAGE value: s3 {{- /* diff --git a/packages/apps/clickhouse/tests/backup_test.yaml b/packages/apps/clickhouse/tests/backup_test.yaml index 7954d5d3d..a01e10ea5 100644 --- a/packages/apps/clickhouse/tests/backup_test.yaml +++ b/packages/apps/clickhouse/tests/backup_test.yaml @@ -302,3 +302,42 @@ tests: - notMatchRegex: path: stringData["backup.sh"] pattern: 'echo DB_LIST=' + + # A strategy attempt that fails mid-upload leaves its local backup behind, + # and with the upstream defaults the sidecar would keep it forever and push + # it to the bucket on its next start as an archive no Backup refers to. The + # sidecar must neither resume uploads at start nor keep local copies once an + # upload succeeds, on either S3 flow. + - it: "sidecar drops local backups after upload and does not resume uploads at start" + template: templates/clickhouse.yaml + documentSelector: + path: kind + value: ClickHouseInstallation + set: + backup: + enabled: true + useSystemBucket: true + asserts: + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="BACKUPS_TO_KEEP_LOCAL")].value + value: "-1" + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="API_COMPLETE_RESUMABLE_AFTER_RESTART")].value + value: "false" + + - it: "legacy flow: sidecar drops local backups after upload and does not resume uploads at start" + template: templates/clickhouse.yaml + documentSelector: + path: kind + value: ClickHouseInstallation + set: + backup: + enabled: true + useSystemBucket: false + asserts: + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="BACKUPS_TO_KEEP_LOCAL")].value + value: "-1" + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="API_COMPLETE_RESUMABLE_AFTER_RESTART")].value + value: "false" diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml index 21d05f8a4..3e6747966 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml @@ -58,32 +58,75 @@ spec: # text/template's missing-key marker, when a controller that # predates archive names renders this template during an upgrade. archive="{{ printf `{{ .ArchiveName | default "" }}` }}" - for _ in $(seq 1 60); do - if curl -fsS ${auth} --max-time 2 "${api}/backup/status" >/dev/null 2>&1; then break; fi - sleep 5 - done + # Every request but the POST is retried on failure, every 5s, + # for $2 attempts in all: once the operation is accepted, a + # dropped poll must not fail the attempt, since the Job retry it + # causes cannot tell a lost poll from a lost operation. + api_get() { + n=0 + while true; do + if out=$(curl -fsS ${auth} --max-time 10 "${api}$1"); then + printf '%s\n' "${out}" + return 0 + fi + n=$((n + 1)) + if [ "${n}" -ge "$2" ]; then + echo "GET ${api}$1 failed ${n} times in a row" >&2 + return 1 + fi + sleep 5 + done + } + # The sidecar answers 423 to a new operation while any other is + # in progress, so starting before it is idle burns the attempt. + # The exception is the operation $1 itself already running, as + # left by an earlier attempt of this Job that died: it is taken + # over rather than started twice. + wait_turn() { + for _ in $(seq 1 60); do + if actions=$(curl -fsS ${auth} --max-time 10 "${api}/backup/actions"); then + turn=$(printf '%s\n' "${actions}" | jq -rs --arg cmd "$1" ' + if any(.[]; .status == "in progress" and .command == $cmd) then "join" + elif any(.[]; .status == "in progress") then "busy" + else "idle" end') || turn= + case "${turn}" in join|idle) return 0 ;; esac + fi + sleep 5 + done + echo "clickhouse-backup sidecar at ${api} did not become idle" >&2 + exit 1 + } if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-" - # Only that older controller leaves the name empty, and it - # records no name on the Backup either. + # Only a controller that predates archive names leaves it + # empty, and that one records no name on the Backup either. target="${archive:-${prefix}$(date -u +%Y%m%dT%H%M%SZ)}" - # A retried Pod gets the same name. Wait on an attempt the - # sidecar is still running, which would answer a new one with - # 423, and ask for resume, so a local or partial remote backup - # an earlier attempt left is resumed rather than refused as - # existing, whatever use_resumable_state the sidecar runs with. - state=$(curl -fsS ${auth} "${api}/backup/actions" \ - | jq -rs --arg suffix "${target}" \ - '[.[] | select((.command // "") | endswith($suffix))] | sort_by(.start) | last | .status // ""') - if [ "${state}" != "in progress" ]; then - echo "create_remote ${target}" + # A retried Pod gets the same name, so it takes over an attempt + # the sidecar is still running, and otherwise asks for resume, + # so a local or partial remote backup an earlier attempt left is + # resumed rather than refused as existing, whatever + # use_resumable_state the sidecar runs with. The sidecar logs + # the request as "create_remote --resume ". The name is + # this BackupJob's alone, so an upload of it that last ended in + # success is an earlier attempt's finished result: resuming + # over it would upload the archive again and leave it broken if + # that upload died halfway. + cmd="create_remote --resume ${target}" + wait_turn "${cmd}" + if [ "${turn}" = "join" ]; then + echo "${cmd} is already running; waiting for it" + elif printf '%s\n' "${actions}" | jq -es --arg cmd "${cmd}" \ + '[.[] | select(.command == $cmd)] | sort_by(.start) | last | .status == "success"' >/dev/null; then + echo "${cmd} already succeeded in an earlier attempt" + else + echo "${cmd}" curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}&resume=true" >/dev/null fi else app="{{ printf "{{ .Backup.ApplicationRef.Name }}" }}" backup="{{ printf "{{ .Backup.Name }}" }}" prefix="clickhouse-${app}-" - remote=$(curl -fsS ${auth} "${api}/backup/list/remote") + remote=$(api_get /backup/list/remote 60) if [ -n "${archive}" ]; then if ! printf '%s\n' "${remote}" | jq -es --arg name "${archive}" 'any(.[]; .name == $name)' >/dev/null; then echo "archive ${archive} recorded on Backup ${backup} is not in remote storage; not restoring any other archive" >&2 @@ -104,19 +147,39 @@ spec: exit 1 fi fi - echo "restore_remote ${target}" - curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null + # restore_remote drops every table it restores before + # recreating it, so a second run discards whatever was written + # after the first one finished. Only a running one is taken + # over: the action log is never trimmed, and a finished row of + # this archive may be left by an earlier RestoreJob. + cmd="restore_remote ${target}" + wait_turn "${cmd}" + if [ "${turn}" = "join" ]; then + echo "${cmd} is already running; waiting for it" + else + echo "${cmd}" + curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null + fi fi while true; do - row=$(curl -fsS ${auth} "${api}/backup/actions" \ - | jq -rs --arg suffix "${target}" \ - '[.[] | select((.command // "") | endswith($suffix))] | sort_by(.start) | last') + actions=$(api_get /backup/actions 6) + row=$(printf '%s\n' "${actions}" \ + | jq -rs --arg cmd "${cmd}" \ + '[.[] | select(.command == $cmd)] | sort_by(.start) | last') + # The sidecar logs the operation before acknowledging the POST + # and keeps that log only in memory, so a missing row means it + # restarted and the operation is gone; nothing will resume it. + if [ "${row}" = "null" ]; then + echo "operation ${target} is no longer in the sidecar action log; the sidecar restarted and dropped it" >&2 + exit 1 + fi status=$(echo "${row}" | jq -r '.status // ""') case "${status}" in success) echo "operation succeeded" ; exit 0 ;; - error) + "in progress") ;; + *) err=$(echo "${row}" | jq -r '.error // "unknown"') - echo "operation failed: ${err}" >&2 + echo "operation ended with status ${status}: ${err}" >&2 exit 1 ;; esac