Merge branch 'main' into fix/clickhouse-idle-before-backup

main now names the ClickHouse archive in the controller and restores
the archive a Backup recorded, in the same strategy script this branch
gates on an idle sidecar. The merged script keeps both: the archive
name and its restore selection from main, the idle gate, retried polls
and lost-operation check from this branch.

A retried backup is taken over through the same idle gate as a restore,
matched on the whole command line the sidecar logs. With resume
requested that line reads "create_remote --resume <name>", so the gate
matches that form; matching on the name alone would also take another
operation's row ending in the same name for this one.

Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
Alexey Artamonov committed 2026-11-09 16:39:43 +03:00
commit 89f8cc02ee
29 files changed
+3478 -94

No files matched your search

+8 -1
View File
@@ -37,7 +37,14 @@ type BackupArtifact struct {
// For example: s3://bucket/prefix/file.tar.gz
URI string `json:"uri"`
// SizeBytes is the size of the artifact in bytes, if known.
// SizeBytes is the stored size in bytes of the files this backup owns,
// if known: the dump of a MariaDB backup, for example, or the base backup
// of a CNPG one, which leaves out the WAL archive the server's backups
// share and a point-in-time restore also reads. Strategies that cannot
// attribute stored bytes to a single backup, such as Velero's, whose
// repository deduplicates across backups, leave it unset. Drivers never
// write 0 for a size they cannot measure, so an absent value means
// "unknown", not "empty".
// +optional
SizeBytes int64 `json:"sizeBytes,omitempty"`
+9
View File
@@ -16,6 +16,7 @@ require (
github.com/go-logr/logr v1.4.4
github.com/go-logr/zapr v1.3.0
github.com/go-task/slim-sprig/v3 v3.0.0
github.com/minio/minio-go/v7 v7.0.98
github.com/onsi/ginkgo/v2 v2.33.0
github.com/onsi/gomega v1.43.1
github.com/prometheus/client_golang v1.24.1
@@ -63,6 +64,7 @@ require (
github.com/fsnotify/fsnotify v1.10.1 // indirect
github.com/fxamacker/cbor/v2 v2.9.4 // indirect
github.com/gkampitakis/go-snaps v0.5.23 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v1.0.1 // indirect
github.com/go-openapi/jsonreference v1.0.2 // indirect
@@ -93,18 +95,25 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/moby/spdystream v0.5.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.3 // indirect
github.com/prometheus/common v0.71.0 // indirect
github.com/prometheus/procfs v0.22.0 // indirect
github.com/rogpeppe/go-internal v1.16.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/sirupsen/logrus v1.10.2 // indirect
github.com/tidwall/match v1.2.0 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
github.com/x448/float16 v0.8.4 // indirect
go.etcd.io/etcd/api/v3 v3.7.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.7.0 // indirect
+19
View File
@@ -61,6 +61,8 @@ github.com/gkampitakis/ciinfo v0.3.4 h1:5eBSibVuSMbb/H6Elc0IIEFbkzCJi3lm94n0+U7Z
github.com/gkampitakis/ciinfo v0.3.4/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo=
github.com/gkampitakis/go-snaps v0.5.23 h1:okh5QR48zpUjpWtu65AtqxdCY8huJq+dEDuUzd1PuKg=
github.com/gkampitakis/go-snaps v0.5.23/go.mod h1:uy3lVzCCRRsAwYqSocyw5fY8xRLCYEfqoOJNxr8HonM=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -141,6 +143,11 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -151,6 +158,12 @@ github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg=
github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y=
github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -165,6 +178,8 @@ github.com/onsi/ginkgo/v2 v2.33.0 h1:C8gBA6Uc2ZEubiV+SXiu5tZnMTwEmXHgkJwGozKtZf8
github.com/onsi/ginkgo/v2 v2.33.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
github.com/onsi/gomega v1.43.1 h1:vGIPFuYrIO6/0Z09s0I0QQQgFchiX4+tb1re3MScJYo=
github.com/onsi/gomega v1.43.1/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -180,6 +195,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
@@ -206,6 +223,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
github.com/vmware-tanzu/velero v1.18.3 h1:9uwN5yxTWQl9PsF5ljX4xiG+7VIq3I9mxOQ0DyyZ31A=
+185 -11
View File
@@ -10,10 +10,16 @@
# rather than proceed when the sidecar never becomes idle. Once started, an
# operation the sidecar no longer lists (its action log lives in memory and a
# restart empties it) has to fail the Pod too, since the Job has no deadline.
# A Job retry must not run a restore twice: it takes over the restore_remote
# the attempt before it left running, and a few lost polls do not end an
# A Job retry must not start its operation twice: it takes over the
# restore_remote, or the create_remote of the archive the controller named,
# that the attempt before it left running, and a few lost polls do not end an
# attempt in the first place.
#
# Which archive a restore reads back is decided by the script's own jq
# selection over list/remote, so it is exercised against several archives
# rather than string-matched: exactly the archive the Backup recorded, or for
# a Backup that records none, the newest one stamped no later than its takenAt.
#
# Run from the repo root with make bats-unit-tests. The rendered script is run
# with `bash`, not `sh`: it opens `set -euo pipefail`, which the production
# image's busybox ash supports but the CI runner's /bin/sh (dash) rejects.
@@ -22,18 +28,25 @@ load test_helper
CHART=packages/system/backupstrategy-controller
# Render the shipped strategy's in-Pod script into $1 with the controller-render
# placeholders substituted ($2 mode, $3 application name). An empty render, or
# a placeholder left unsubstituted, is an early failure: these tests EXECUTE it.
# Render the shipped strategy's in-Pod script into $1 and substitute the
# controller-render placeholders the way the driver fills them: $2 mode, $3
# app (release and source application), $4 the archive name the controller
# chose or the Backup recorded (empty, the default, for none), $5 the Backup
# name, $6 its takenAt in the archive-name layout (default: later than every
# archive the stub lists). An empty render fails early, as does a placeholder
# left unsubstituted: these tests EXECUTE it.
render_script() {
out=$1; mode=$2; app=$3
out=$1; mode=$2; app=$3; archive=${4-}; backup=${5-bkp}; takenat=${6-99991231T235959Z}
helm template t "$CHART" -n cozy-backup-controller \
--set backupStorage.bucketNameOverride=test-bucket 2>/dev/null \
| yq eval 'select(.kind == "Altinity") | .spec.template.spec.containers[0].args[0]' - \
| sed -e "s#{{ .Mode }}#${mode}#g" \
-e "s#{{ .Release.Name }}#${app}#g" \
-e "s#{{ .Release.Namespace }}#tenant-x#g" \
-e "s#{{ .Backup.ApplicationRef.Name }}#${app}#g" > "$out"
-e "s#{{ .Backup.ApplicationRef.Name }}#${app}#g" \
-e "s#{{ .Backup.Name }}#${backup}#g" \
-e "s#{{ .ArchiveName | default \"\" }}#${archive}#g" \
-e "s#{{ .Backup.TakenAt | default \"\" }}#${takenat}#g" > "$out"
[ -s "$out" ] || return 1
if grep -q '{{' "$out"; then
echo "rendered script still holds a controller placeholder" >&2
@@ -49,13 +62,17 @@ render_script() {
# busy a number: that many GET /backup/actions answers still list
# an operation of someone else "in progress"
# op the command line of the sidecar's own operation, written
# by an accepted POST or seeded by a test as the operation
# by an accepted POST the way the sidecar logs it ("--resume"
# before the name of a create_remote asking for it), or
# seeded by a test as the operation
# an earlier attempt of the same Job started
# op_left how many more GET /backup/actions answers list op as
# "in progress" before it ends with op_result
# op_duration op_left given to an operation a POST starts (default 0)
# op_result the status op ends with (default success)
# prior extra action-log rows, verbatim, older than op
# remote present: the list/remote answer, verbatim, in place of the
# one archive listed by default
# restarted present: the action log no longer lists op, as after the
# sidecar restarted and lost its in-memory log
# fail_on_post a POST that is accepted makes that many following
@@ -94,7 +111,10 @@ case "\$method \$url" in
"POST "*)
if [ "\$(num busy)" -gt 0 ] || [ "\$running" = 1 ] || [ ! -e "\$state/idle" ]; then exit 22; fi
case "\$url" in
*/create_remote\?name=*) cmd="create_remote \${url##*=}" ;;
*/create_remote\?name=*)
name=\${url#*name=}; name=\${name%%&*}
case "\$url" in *[?\&]resume=true*) cmd="create_remote --resume \$name" ;; *) cmd="create_remote \$name" ;; esac
;;
*/restore_remote/*) cmd="restore_remote \${url##*/}" ;;
*) exit 22 ;;
esac
@@ -130,7 +150,9 @@ case "\$method \$url" in
if [ "\$inprog" = 0 ]; then touch "\$state/idle"; fi
;;
"GET "*/backup/list/remote)
echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}'
if [ -e "\$state/remote" ]; then cat "\$state/remote"; else
echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}'
fi
;;
esac
exit 0
@@ -142,6 +164,17 @@ STUB
chmod 0755 "$bin/curl" "$bin/sleep"
}
# One list/remote row per argument, "name" or "name|desc"; created mirrors
# the name's timestamp.
remote_rows() {
for row in "$@"; do
name=${row%%|*}; desc="tar, regular"
if [ "$row" != "$name" ]; then desc=${row#*|}; fi
ts=${name##*-}
printf '{"name":"%s","created":"%s","location":"remote","required":"","desc":"%s"}\n' "$name" "$ts" "$desc"
done
}
# The number of operations the sidecar accepted.
accepted() {
if [ -e "$1/state/accepted" ]; then wc -l < "$1/state/accepted" | tr -d ' '; else echo 0; fi
@@ -175,7 +208,7 @@ run_script() {
echo 3 > "$tmp/state/busy"
render_script "$tmp/s.sh" backup myapp
run_script "$tmp"
grep -q '^create_remote clickhouse-myapp-' "$tmp/state/accepted"
grep -q '^create_remote --resume clickhouse-myapp-' "$tmp/state/accepted"
[ "$(cat "$tmp/state/busy")" = 0 ]
}
@@ -321,3 +354,144 @@ run_script() {
if run_script "$tmp"; then echo "FAIL: a cancelled operation must fail the Pod"; false; fi
if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling a cancelled operation"; false; fi
}
@test "restore reads back the archive the Backup recorded, not the newest one" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z
remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \
clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote"
run_script "$tmp"
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests"
[ "$(accepted "$tmp")" = 1 ]
}
@test "restore fails naming the recorded archive when remote storage lacks it" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z
remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/state/remote"
out=""
if out=$(run_script "$tmp" 2>&1); then
echo "FAIL: restore succeeded without the recorded archive: $out"
false
fi
printf '%s\n' "$out" | grep -qF 'archive clickhouse-myapp-20260304T050607Z recorded on Backup bkp-a is not in remote storage'
if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then
echo "FAIL: restored another archive: $(cat "$tmp/state/requests")"
false
fi
}
@test "restore of a Backup without a recorded archive takes the newest one stamped by its takenAt, and warns" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z
# Each decoy sorts after the expected archive and is ruled out by one
# filter: broken, not a bare timestamp after the prefix, or stamped after
# takenAt.
remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \
"clickhouse-myapp-20260304T050700Z|broken (can't stat metadata.json)" \
clickhouse-myapp-20260304T050710Z-copy \
clickhouse-myapp-20260304T060000Z > "$tmp/state/remote"
out=$(run_script "$tmp" 2>&1)
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests"
[ "$(accepted "$tmp")" = 1 ]
printf '%s\n' "$out" | grep -qF 'WARNING: Backup bkp-a records no archive name'
}
@test "restore of a Backup without a recorded archive fails when no archive is stamped by its takenAt" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z
remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote"
out=""
if out=$(run_script "$tmp" 2>&1); then
echo "FAIL: restore succeeded with every archive newer than takenAt: $out"
false
fi
printf '%s\n' "$out" | grep -qF 'no archive of myapp stamped no later than 20260304T050800Z'
if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then
echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/state/requests")"
false
fi
}
@test "backup writes the archive under the name the controller chose" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests"
}
@test "backup falls back to a timestamped name when the controller supplies none" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
render_script "$tmp/s.sh" backup myapp ""
run_script "$tmp"
grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/state/requests"
}
# The sidecar answers a second operation with 423 while one runs, so a retry
# that POSTed over its predecessor's upload would burn every attempt left.
@test "a retried backup takes over the upload of its archive the sidecar is still running" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op"
echo 3 > "$tmp/state/op_left"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
if grep -q '^POST ' "$tmp/state/requests"; then
echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")"
false
fi
[ "$(cat "$tmp/state/op_left")" = 0 ]
}
@test "a taken-over backup that ends in error fails the Pod" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op"
echo 2 > "$tmp/state/op_left"
echo error > "$tmp/state/op_result"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
if run_script "$tmp"; then echo "FAIL: a taken-over upload that failed must fail the Pod"; false; fi
if grep -q '^POST ' "$tmp/state/requests"; then
echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")"
false
fi
}
@test "a retried backup after a failed attempt resumes the same archive" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30","error":"S3 PutObject 403"}' > "$tmp/state/prior"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests"
}
@test "backup does not take over an upload running for another archive" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo 'create_remote --resume clickhouse-myapp-20260304T040000Z' > "$tmp/state/op"
echo 3 > "$tmp/state/op_left"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
[ "$(accepted "$tmp")" = 1 ]
grep -qx 'create_remote --resume clickhouse-myapp-20260304T050607Z' "$tmp/state/accepted"
}
# The action log is never trimmed, so a finished row of the same archive may
# be an earlier run's and is not taken for this attempt's result.
@test "backup starts create_remote again when the log holds a finished one of the same archive" {
for status in success error cancel; do
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
echo "{\"command\":\"create_remote --resume clickhouse-myapp-20260304T050607Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior"
render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z
run_script "$tmp"
if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier upload must not stand in for this one"; false; fi
done
}
+27
View File
@@ -488,6 +488,33 @@ pod_cidrs_of() {
rm -rf "$STATE"
}
@test "a CiliumNode list larger than one argument may hold still moves every node and names the pod on an old Ingress IP" {
init_state
add_ip web-0.tenant-c ovn-default 10.244.1.135
add_ip web-1.tenant-c ovn-default 10.244.4.135
add_ip db-0.tenant-c ovn-default 10.244.9.9
# Cilium copies every Node label onto the CiliumNode, which takes six real
# ones past the 128 KiB Linux allows a single argument. One annotation stands
# in for the labels and keeps the list over that limit even printed compact.
for i in 1 2 3 4 5 6; do
add_node "n$i" "10.244.$i.0/24" "10.244.$i.223" "10.244.$i.135"
jq '.metadata.annotations.padding = ("x" * 24576)' "$STATE/ciliumnodes/n$i.json" > "$STATE/node.tmp"
mv "$STATE/node.tmp" "$STATE/ciliumnodes/n$i.json"
done
[ "$(kubectl get ciliumnodes -o json | jq -c . | wc -c)" -gt 131072 ]
"$SCRIPT" > "$STATE/out" 2>&1
# Each pod is named with the node whose Ingress IP it holds, and a pod on an
# address that is no node's Ingress IP is not named at all.
grep -q "web-0.tenant-c holds 10.244.1.135, the Ingress IP of n1 before the move" "$STATE/out"
grep -q "web-1.tenant-c holds 10.244.4.135, the Ingress IP of n4 before the move" "$STATE/out"
if grep -q "db-0.tenant-c" "$STATE/out"; then
cat "$STATE/out"
false
fi
[ "$(pod_cidrs_of n6)" = '["10.244.248.40/29"]' ]
rm -rf "$STATE"
}
@test "a cluster without kube-ovn skips the exclusion and still moves its nodes" {
init_state
rm "$STATE/subnet.json"
+187
View File
@@ -0,0 +1,187 @@
#!/usr/bin/env bats
# -----------------------------------------------------------------------------
# packages/apps/kubernetes: the OIDC bootstrap Job must not carry the users[]
# list inside its container command.
#
# The Job's script reaches the container as one `sh -c` argument, and Linux caps
# one exec argument at 128 KiB (MAX_ARG_STRLEN). With a ClusterRoleBinding per
# user written into that script, about 240 users take it past the cap: the
# container cannot start, the post-install/post-upgrade hook fails, and the
# tenant cluster's HelmRelease stays unready. The bindings reach the Job through
# a mounted ConfigMap instead, so the script is the same size for any list.
#
# The last three tests execute the rendered script against a fake kubectl, since
# no assertion on the script's text says what the prune step does with the list:
# a condition forced either way passes every helm-unittest case.
#
# Requires: helm, yq (mikefarah v4+), jq, like hack/admin-kubeconfig-invariant.bats.
# -----------------------------------------------------------------------------
load test_helper
USERS=300
# render_users <dir> <count> renders the chart in System mode with <count>
# generated users and leaves one JSON document per object in <dir>/rendered.json.
render_users() {
jq -n --argjson n "$2" '
{oidc: {users: [range($n) | {email: "user\(.)@example.org", role: (if . % 7 == 0 then "admin" else "view" end)}]}}' \
> "$1/users.yaml"
helm template bindings packages/apps/kubernetes \
--namespace tenant-test \
--values packages/apps/kubernetes/tests/values/oidc-system.yaml \
--values "$1/users.yaml" \
> "$1/rendered.yaml"
yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json"
}
# render_job <dir> <users JSON> renders the chart in CustomConfig mode and unpacks
# what the bootstrap Job runs and mounts into <dir>: script.sh is its `sh -c`
# script, bindings/ holds the two files of the mounted ConfigMap. -j keeps an
# empty value an empty file, as the kubelet writes it.
render_job() {
jq -n --argjson users "$2" '
{_cluster: {"oidc-enabled": "false"},
oidc: {mode: "CustomConfig",
customConfig: {config: "apiVersion: apiserver.config.k8s.io/v1beta1\nkind: AuthenticationConfiguration\njwt: []\n"},
users: $users}}' > "$1/values.yaml"
helm template bindings packages/apps/kubernetes \
--namespace tenant-test \
--values packages/apps/kubernetes/tests/values/oidc-system.yaml \
--values "$1/values.yaml" \
> "$1/rendered.yaml"
yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json"
jq -s -j '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap")
| .spec.template.spec.containers[0].command[2]' "$1/rendered.json" > "$1/script.sh"
mkdir -p "$1/bindings"
jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
| .data["crbs.yaml"]' "$1/rendered.json" > "$1/bindings/crbs.yaml"
jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
| .data["desired.txt"]' "$1/rendered.json" > "$1/bindings/desired.txt"
}
# fake_kubectl <dir> <names> puts a kubectl first on PATH that logs every call and
# answers the script's list of existing bindings with <names>.
fake_kubectl() {
mkdir -p "$1/bin"
printf '%s' "$2" > "$1/existing"
cat > "$1/bin/kubectl" <<'KEOF'
#!/bin/sh
echo "$*" >> "$KLOG"
case "$*" in
*"get clusterrolebindings"*) cat "$EXISTING" ;;
esac
KEOF
chmod +x "$1/bin/kubectl"
}
# run_job <dir> runs the unpacked script as the container does and leaves its
# status in $rc and its output in <dir>/out. Run by bash, which has the pipefail
# the script sets and which is what the unit lane runs under; the container's own
# shell is not exercised.
run_job() {
: > "$1/kubectl.log"
echo kubeconfig > "$1/kubeconfig"
rc=0
env PATH="$1/bin:$PATH" KLOG="$1/kubectl.log" EXISTING="$1/existing" HOME="$1" \
KUBECONFIG_TENANT="$1/kubeconfig" RELEASE=bindings RELEASE_NAMESPACE=tenant-test \
OIDC_MODE=CustomConfig OIDC_BINDINGS_DIR="$1/bindings" \
bash "$1/script.sh" > "$1/out" 2>&1 || rc=$?
}
@test "the OIDC bootstrap Job script stays under the exec argument cap and carries no user" {
dir=$(mktemp -d)
render_users "$dir" "$USERS"
jq -s -r '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap")
| .spec.template.spec.containers[0].command[2]' "$dir/rendered.json" > "$dir/script"
size=$(wc -c < "$dir/script")
[ "$size" -gt 0 ]
[ "$size" -lt 131072 ]
if grep -q -E 'user[0-9]+@example\.org' "$dir/script"; then
echo "the Job script still carries users[] entries" >&2
false
fi
rm -rf "$dir"
}
@test "every user keeps one ClusterRoleBinding with its old name and role in the bindings ConfigMap" {
dir=$(mktemp -d)
render_users "$dir" "$USERS"
jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
| .data["crbs.yaml"]' "$dir/rendered.json" > "$dir/crbs.yaml"
jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
| .data["desired.txt"]' "$dir/rendered.json" > "$dir/desired.txt"
[ -s "$dir/crbs.yaml" ]
yq --output-format=json eval-all '.' "$dir/crbs.yaml" | jq -s '.' > "$dir/crbs.json"
[ "$(jq 'length' "$dir/crbs.json")" -eq "$USERS" ]
[ "$(jq -r '[.[].subjects[0].name] | unique | length' "$dir/crbs.json")" -eq "$USERS" ]
[ "$(jq -r '[.[] | select(.roleRef.name == "cluster-admin")] | length' "$dir/crbs.json")" -eq $(( (USERS + 6) / 7 )) ]
# The name is what the prune step and every binding already in a tenant cluster
# go by, so it must not move: cozystack-oidc- plus the first 16 hex digits of
# sha256("<release>-<email>").
for i in 0 1 6 7; do
email="user$i@example.org"
name="cozystack-oidc-$(printf '%s' "bindings-$email" | sha256sum | cut -c1-16)"
[ "$(jq -r --arg e "$email" '.[] | select(.subjects[0].name == $e) | .metadata.name' "$dir/crbs.json")" = "$name" ]
grep -q -x -F "$name" "$dir/desired.txt"
done
# The list the prune step compares against is exactly the applied names.
[ "$(jq -r '.[].metadata.name' "$dir/crbs.json" | sort)" = "$(grep . "$dir/desired.txt" | sort)" ]
rm -rf "$dir"
}
@test "the Job applies the mounted bindings and deletes only the bindings no user has" {
dir=$(mktemp -d)
render_job "$dir" '[{"email":"alice@example.org","role":"admin"},{"email":"bob@example.org","role":"view"}]'
alice="cozystack-oidc-$(printf '%s' 'bindings-alice@example.org' | sha256sum | cut -c1-16)"
bob="cozystack-oidc-$(printf '%s' 'bindings-bob@example.org' | sha256sum | cut -c1-16)"
fake_kubectl "$dir" "$alice cozystack-oidc-000000000000dead $bob cozystack-oidc-000000000000beef"
run_job "$dir"
if [ "$rc" -ne 0 ]; then
cat "$dir/out"
false
fi
grep -q -F -- "apply --filename $dir/bindings/crbs.yaml" "$dir/kubectl.log"
grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000dead --ignore-not-found" "$dir/kubectl.log"
grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000beef --ignore-not-found" "$dir/kubectl.log"
if grep -F -e "delete clusterrolebinding $alice" -e "delete clusterrolebinding $bob" "$dir/kubectl.log"; then
echo "the Job deleted a binding a user still has" >&2
false
fi
[ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ]
rm -rf "$dir"
}
@test "a release with no users applies nothing and deletes every binding it left behind" {
dir=$(mktemp -d)
render_job "$dir" '[]'
fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef"
run_job "$dir"
if [ "$rc" -ne 0 ]; then
cat "$dir/out"
false
fi
if grep -q -F -- ' apply ' "$dir/kubectl.log"; then
echo "the Job ran kubectl apply with no bindings to apply" >&2
false
fi
[ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ]
rm -rf "$dir"
}
@test "the Job stops before deleting anything when the desired list is not mounted" {
dir=$(mktemp -d)
render_job "$dir" '[{"email":"alice@example.org","role":"admin"}]'
fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef"
rm "$dir/bindings/desired.txt"
run_job "$dir"
[ "$rc" -ne 0 ]
if grep -q -E 'delete clusterrolebinding| apply ' "$dir/kubectl.log"; then
cat "$dir/kubectl.log"
false
fi
rm -rf "$dir"
}
@@ -2,7 +2,9 @@
package backupcontroller
import (
"bytes"
"context"
"encoding/json"
"fmt"
"maps"
"strings"
@@ -43,6 +45,18 @@ const (
// were in effect at backup time.
altinityParamPrefix = "altinity.strategy.backups.cozystack.io/parameter/"
// altinityArchiveMetaKey records, on the Backup, the clickhouse-backup
// archive its Job wrote, as named by altinityArchiveName. A restore reads
// back exactly that archive: the newest one under the release's prefix
// may belong to a later Backup.
altinityArchiveMetaKey = "altinity.strategy.backups.cozystack.io/archive"
// altinityArchiveTimeLayout is the UTC timestamp that ends an archive
// name. It is fixed-width, so names sharing a prefix sort chronologically
// as plain strings, which is what a restore of a Backup that records no
// archive relies on.
altinityArchiveTimeLayout = "20060102T150405Z"
// Polling cadence for the Altinity Job lifecycle. Mirrors the CNPG /
// Velero strategy cadence so behaviour is uniform across drivers.
altinityPollInterval = 5 * time.Second
@@ -102,14 +116,37 @@ func backupParameters(b *backupsv1alpha1.Backup) map[string]string {
return out
}
// altinityArchiveName mints the archive name for one BackupJob from its
// persisted StartedAt, so every reconcile and every Pod retry of the Job
// writes the same archive. It keeps the clickhouse-<release>-<timestamp>
// shape, because a Backup that records no archive is restored by the
// timestamp in that shape.
func altinityArchiveName(j *backupsv1alpha1.BackupJob) string {
return fmt.Sprintf("clickhouse-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout))
}
// altinityJobCarriesArchive reports whether the Job's Pod was rendered with
// the archive name. A Job created by a controller that predates archive
// names, or rendered from a strategy that names its own archives, wrote an
// archive under another name, and recording this one would point the Backup
// at an archive that does not exist.
func altinityJobCarriesArchive(job *batchv1.Job, archive string) bool {
spec, err := json.Marshal(job.Spec.Template.Spec)
if err != nil {
return false
}
return bytes.Contains(spec, []byte(archive))
}
// renderAltinityTemplate runs the strategy's PodTemplateSpec through the
// repository's text/template engine with the same context shape as the CNPG
// driver: every string field is templated against the application object,
// the release shorthand, the run mode, and the resolved parameters.
// the release shorthand, the run mode, the archive name, and the resolved
// parameters.
func renderAltinityTemplate(
tmpl corev1.PodTemplateSpec,
app map[string]any,
releaseName, releaseNamespace, mode string,
releaseName, releaseNamespace, mode, archiveName string,
parameters map[string]string,
backup *backupsv1alpha1.Backup,
) (*corev1.PodTemplateSpec, error) {
@@ -119,8 +156,9 @@ func renderAltinityTemplate(
"Name": releaseName,
"Namespace": releaseNamespace,
},
"Mode": mode,
"Parameters": parameters,
"Mode": mode,
"ArchiveName": archiveName,
"Parameters": parameters,
}
if backup != nil {
// Expose ApplicationRef so restore strategies can reference the
@@ -143,6 +181,9 @@ func renderAltinityTemplate(
"Kind": backup.Spec.ApplicationRef.Kind,
"Name": backup.Spec.ApplicationRef.Name,
},
// In the archive-name layout, so a strategy can compare it with
// the timestamp an archive name ends in.
"TakenAt": backup.Spec.TakenAt.UTC().Format(altinityArchiveTimeLayout),
}
}
return template.Template(&tmpl, ctxMap)
@@ -209,12 +250,14 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv
return ctrl.Result{}, err
}
archive := altinityArchiveName(j)
rendered, err := renderAltinityTemplate(
strategy.Spec.Template,
app,
j.Spec.ApplicationRef.Name,
j.Namespace,
altinityModeBackup,
archive,
resolved.Parameters,
nil,
)
@@ -239,7 +282,10 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv
if j.Status.BackupRef != nil {
return ctrl.Result{}, nil
}
artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved)
if !altinityJobCarriesArchive(batchJob, archive) {
archive = ""
}
artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved, archive)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
}
@@ -411,16 +457,20 @@ func jobFailureMessage(j *batchv1.Job) string {
}
// createAltinityBackupArtifact materialises a Cozystack Backup carrying the
// strategy reference and the BackupClassStrategy parameters in effect at
// backup time. Parameters round-trip via DriverMetadata under
// altinityParamPrefix so a later RestoreJob can re-render the strategy
// template against the same values.
// strategy reference, the archive the Job wrote (when known) and the
// BackupClassStrategy parameters in effect at backup time. Parameters
// round-trip via DriverMetadata under altinityParamPrefix so a later
// RestoreJob can re-render the strategy template against the same values.
func (r *BackupJobReconciler) createAltinityBackupArtifact(
ctx context.Context,
j *backupsv1alpha1.BackupJob,
resolved *ResolvedBackupConfig,
archive string,
) (*backupsv1alpha1.Backup, error) {
driverMD := map[string]string{}
if archive != "" {
driverMD[altinityArchiveMetaKey] = archive
}
for k, v := range resolved.Parameters {
driverMD[altinityParamPrefix+k] = v
}
@@ -459,6 +509,13 @@ func (r *BackupJobReconciler) createAltinityBackupArtifact(
if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
return nil, getErr
}
// A Backup outlives its BackupJob, so a BackupJob re-created under a
// used name meets the earlier run's Backup here. Only one recording
// this run's archive was created by an earlier reconcile of this run.
if got := existing.Spec.DriverMetadata[altinityArchiveMetaKey]; got != archive {
return nil, fmt.Errorf("Backup %s/%s already exists and records archive %q, not %q that this run wrote",
backup.Namespace, backup.Name, got, archive)
}
return existing, nil
}
return backup, nil
@@ -616,6 +673,7 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res
targetAppName,
targetNamespace,
altinityModeRestore,
backup.Spec.DriverMetadata[altinityArchiveMetaKey],
backupParameters(backup),
backup,
)
@@ -3,7 +3,9 @@ package backupcontroller
import (
"context"
"strings"
"testing"
"time"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
@@ -697,6 +699,7 @@ func TestBackupParameters_RoundTrip(t *testing.T) {
DriverMetadata: map[string]string{
altinityParamPrefix + "bucketName": "b1",
altinityParamPrefix + "endpoint": "https://s3",
altinityArchiveMetaKey: "clickhouse-ch-test-20260304T050607Z",
"job.batch/name": "ignored",
"": "ignored",
},
@@ -706,7 +709,326 @@ func TestBackupParameters_RoundTrip(t *testing.T) {
if got["bucketName"] != "b1" || got["endpoint"] != "https://s3" {
t.Errorf("backupParameters = %v", got)
}
if _, ok := got["job.batch/name"]; ok {
if len(got) != 2 {
t.Errorf("non-parameter key leaked through: %v", got)
}
}
// newAltinityArchiveStrategy renders the archive name and the Backup's
// takenAt where a test can read them back off the Job.
func newAltinityArchiveStrategy(name string) *strategyv1alpha1.Altinity {
s := newAltinityStrategy(name)
s.Spec.Template.Spec.Containers[0].Args = []string{
"--mode={{ .Mode }}",
"--archive={{ .ArchiveName }}",
"--taken-at={{ .Backup.TakenAt }}",
"--bucket={{ .Parameters.bucketName }}",
}
return s
}
func TestAltinityArchiveName_FromStartedAtInUTC(t *testing.T) {
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 7, 890000000, time.FixedZone("UTC+3", 3*3600)))
j := &backupsv1alpha1.BackupJob{
Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")},
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt},
}
if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z"; got != want {
t.Errorf("altinityArchiveName = %q, want %q", got, want)
}
}
// TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven drives one BackupJob
// through Job creation and completion in separate reconciles. The archive
// name has to come out of the persisted StartedAt both times: the Pod is
// told it on the first, the Backup records it on the second, and a restore
// later reads back exactly that archive.
func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) {
app := newAltinityClickHouseApp("ch-test", "tenant-test")
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))
backupJob := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: newAltinityClickHouseRef("ch-test"),
BackupClassName: "clickhouse-backup",
},
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
}
resolved := &ResolvedBackupConfig{
StrategyRef: corev1.TypedLocalObjectReference{
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
Kind: strategyv1alpha1.AltinityStrategyKind,
Name: "clickhouse-strategy",
},
Parameters: map[string]string{"bucketName": "ch-bucket"},
}
r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy))
ctx := context.Background()
const want = "clickhouse-ch-test-20260304T050607Z"
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
t.Fatalf("reconcileAltinity() creating the Job: %v", err)
}
job := &batchv1.Job{}
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil {
t.Fatalf("get backup Job: %v", err)
}
if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+want {
t.Fatalf("backup Pod was rendered with %q, want --archive=%s", got, want)
}
job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
if err := r.Status().Update(ctx, job); err != nil {
t.Fatalf("mark backup Job complete: %v", err)
}
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil {
t.Fatalf("refresh BackupJob: %v", err)
}
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
t.Fatalf("reconcileAltinity() on the completed Job: %v", err)
}
created := &backupsv1alpha1.Backup{}
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil {
t.Fatalf("get Backup: %v", err)
}
if got := created.Spec.DriverMetadata[altinityArchiveMetaKey]; got != want {
t.Errorf("Backup records archive %q, want %q", got, want)
}
if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" {
t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got)
}
}
// TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven covers a Job whose
// Pod never carried the driver's archive name: one created before the
// upgrade that introduced the names, or rendered from a strategy that names
// its own archives. Recording the name would send the restore after an
// archive nobody wrote; recording none leaves it the fallback for Backups
// without a name.
func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) {
app := newAltinityClickHouseApp("ch-test", "tenant-test")
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))
backupJob := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: newAltinityClickHouseRef("ch-test"),
BackupClassName: "clickhouse-backup",
},
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
}
completedK8sJob := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{Name: jobNameForBackupJob(backupJob), Namespace: backupJob.Namespace},
Spec: batchv1.JobSpec{
Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyNever,
Containers: []corev1.Container{{
Name: "clickhouse-backup",
Image: "altinity/clickhouse-backup:test",
Args: []string{`target="clickhouse-ch-test-$(date -u +%Y%m%dT%H%M%SZ)"`},
}},
}},
},
Status: batchv1.JobStatus{
Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}},
},
}
resolved := &ResolvedBackupConfig{
StrategyRef: corev1.TypedLocalObjectReference{
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
Kind: strategyv1alpha1.AltinityStrategyKind,
Name: "clickhouse-strategy",
},
Parameters: map[string]string{"bucketName": "ch-bucket"},
}
r, _, _ := newAltinityTestEnv(t, app,
clientfake.NewClientBuilder().WithObjects(backupJob, strategy, completedK8sJob))
ctx := context.Background()
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
t.Fatalf("reconcileAltinity() error = %v", err)
}
created := &backupsv1alpha1.Backup{}
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil {
t.Fatalf("get Backup: %v", err)
}
if got, ok := created.Spec.DriverMetadata[altinityArchiveMetaKey]; ok {
t.Errorf("Backup records archive %q its Job was never given", got)
}
if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" {
t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got)
}
}
// TestReconcileAltinity_BackupNameAlreadyTaken covers a completed Job whose
// Backup name is already in use. A Backup outlives its BackupJob, so a
// BackupJob re-created under an earlier run's name finds that run's Backup,
// and claiming it would point the new run at an archive it did not write.
// A Backup recording this run's own archive is the one an earlier reconcile
// created before its status write was lost, and is adopted.
func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) {
const thisRun = "clickhouse-ch-test-20260305T050607Z"
cases := []struct {
name string
existing string // archive the existing Backup records; empty for none
wantSucceeded bool
}{
{name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z"},
{name: "earlier run's Backup records no archive", existing: ""},
{name: "this run's Backup from a reconcile whose status write was lost", existing: thisRun, wantSucceeded: true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
app := newAltinityClickHouseApp("ch-test", "tenant-test")
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
startedAt := metav1.NewTime(time.Date(2026, 3, 5, 5, 6, 7, 0, time.UTC))
backupJob := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: newAltinityClickHouseRef("ch-test"),
BackupClassName: "clickhouse-backup",
},
Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
}
driverMD := map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"}
if tc.existing != "" {
driverMD[altinityArchiveMetaKey] = tc.existing
}
existing := &backupsv1alpha1.Backup{
ObjectMeta: metav1.ObjectMeta{Name: backupJob.Name, Namespace: backupJob.Namespace},
Spec: backupsv1alpha1.BackupSpec{
ApplicationRef: newAltinityClickHouseRef("ch-test"),
StrategyRef: corev1.TypedLocalObjectReference{
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
Kind: strategyv1alpha1.AltinityStrategyKind,
Name: "clickhouse-strategy",
},
TakenAt: metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 22, 0, time.UTC)),
DriverMetadata: driverMD,
},
}
resolved := &ResolvedBackupConfig{
StrategyRef: corev1.TypedLocalObjectReference{
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
Kind: strategyv1alpha1.AltinityStrategyKind,
Name: "clickhouse-strategy",
},
Parameters: map[string]string{"bucketName": "ch-bucket"},
}
r, _, _ := newAltinityTestEnv(t, app,
clientfake.NewClientBuilder().WithObjects(backupJob, strategy, existing))
ctx := context.Background()
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
t.Fatalf("reconcileAltinity() creating the Job: %v", err)
}
job := &batchv1.Job{}
if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil {
t.Fatalf("get backup Job: %v", err)
}
if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+thisRun {
t.Fatalf("backup Pod was rendered with %q, want --archive=%s", got, thisRun)
}
job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
if err := r.Status().Update(ctx, job); err != nil {
t.Fatalf("mark backup Job complete: %v", err)
}
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil {
t.Fatalf("refresh BackupJob: %v", err)
}
if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
t.Fatalf("reconcileAltinity() on the completed Job: %v", err)
}
updated := &backupsv1alpha1.BackupJob{}
if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), updated); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if tc.wantSucceeded {
if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded {
t.Errorf("phase = %q (%s), want Succeeded", updated.Status.Phase, updated.Status.Message)
}
if updated.Status.BackupRef == nil || updated.Status.BackupRef.Name != existing.Name {
t.Errorf("BackupRef = %v, want %s", updated.Status.BackupRef, existing.Name)
}
} else {
if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
t.Errorf("phase = %q, want Failed: the run would claim a Backup recording %q instead of %q", updated.Status.Phase, tc.existing, thisRun)
}
if updated.Status.BackupRef != nil {
t.Errorf("BackupRef = %v, want none", updated.Status.BackupRef)
}
if !strings.Contains(updated.Status.Message, thisRun) {
t.Errorf("failure message %q does not name the archive this run wrote", updated.Status.Message)
}
}
kept := &backupsv1alpha1.Backup{}
if err := r.Get(ctx, client.ObjectKeyFromObject(existing), kept); err != nil {
t.Fatalf("get existing Backup: %v", err)
}
if got := kept.Spec.DriverMetadata[altinityArchiveMetaKey]; got != tc.existing {
t.Errorf("existing Backup now records archive %q, want %q left in place", got, tc.existing)
}
})
}
}
func TestReconcileAltinityRestore_RendersTheBackupsArchive(t *testing.T) {
cases := []struct {
name string
archive string // recorded on the Backup; empty for a Backup taken before names were recorded
wantArchive string
}{
{name: "recorded archive", archive: "clickhouse-ch-test-20260304T050607Z", wantArchive: "--archive=clickhouse-ch-test-20260304T050607Z"},
{name: "no recorded archive", archive: "", wantArchive: "--archive="},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
targetApp := newAltinityClickHouseApp("ch-test", "tenant-test")
strategy := newAltinityArchiveStrategy("clickhouse-strategy")
driverMD := map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"}
if tc.archive != "" {
driverMD[altinityArchiveMetaKey] = tc.archive
}
now := metav1.Now()
backup := &backupsv1alpha1.Backup{
ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
Spec: backupsv1alpha1.BackupSpec{
ApplicationRef: newAltinityClickHouseRef("ch-test"),
StrategyRef: corev1.TypedLocalObjectReference{
APIGroup: new(strategyv1alpha1.GroupVersion.Group),
Kind: strategyv1alpha1.AltinityStrategyKind,
Name: "clickhouse-strategy",
},
TakenAt: metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 22, 0, time.FixedZone("UTC+3", 3*3600))),
DriverMetadata: driverMD,
},
}
restoreJob := &backupsv1alpha1.RestoreJob{
ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-test"},
Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
}
_, rr, _ := newAltinityTestEnv(t, targetApp,
clientfake.NewClientBuilder().WithObjects(strategy, backup, restoreJob))
ctx := context.Background()
if _, err := rr.reconcileAltinityRestore(ctx, restoreJob, backup); err != nil {
t.Fatalf("reconcileAltinityRestore: %v", err)
}
job := &batchv1.Job{}
if err := rr.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForRestoreJob(restoreJob)}, job); err != nil {
t.Fatalf("get restore Job: %v", err)
}
args := job.Spec.Template.Spec.Containers[0].Args
want := []string{"--mode=restore", tc.wantArchive, "--taken-at=20260304T050622Z", "--bucket=ch-bucket"}
for i, w := range want {
if i >= len(args) || args[i] != w {
t.Errorf("rendered restore args[%d]: want %q, got %q", i, w, args)
}
}
})
}
}
+216
View File
@@ -0,0 +1,216 @@
// SPDX-License-Identifier: Apache-2.0
package backupcontroller
import (
"context"
"crypto/x509"
"errors"
"fmt"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
)
// artifactSizeProbeTimeout bounds the object-store round trips a driver
// makes to measure Backup.status.artifact.sizeBytes. The probe runs inside
// the reconcile that creates the Backup, and the BackupJob controller has a
// single worker, so an unreachable endpoint stalls every other BackupJob for
// this long. A variable so tests can shorten it.
var artifactSizeProbeTimeout = 10 * time.Second
type s3ObjectInfo struct {
Key string
Size int64
LastModified time.Time
}
type s3ObjectLister interface {
ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error)
}
// s3Connection is what a driver resolves from its rendered strategy to
// read the bucket with the same identity the backup was written with.
type s3Connection struct {
// Endpoint is host[:port], without a scheme.
Endpoint string
Secure bool
Region string
AccessKey string
SecretKey string
SessionToken string
// CAPEM is an endpoint CA bundle. It is trusted on top of the system
// roots, or instead of them when CAOnly is set: the probe must trust
// what the writer of the backup trusted, no more.
CAPEM []byte
CAOnly bool
}
// objectStore reads back what a backup wrote: the Secrets it was written
// with and the bucket itself. It is a value rather than reconciler methods
// so that the restore path, which has a reconciler of its own, can read
// the same objects.
type objectStore struct {
// reader reads Secrets, which the manager's client fetches uncached,
// and lists operator Backups, which it serves from the cache.
reader client.Reader
// newLister overrides the S3 client; nil means a real minio client.
newLister func(conn s3Connection) (s3ObjectLister, error)
}
func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) {
if s.newLister != nil {
return s.newLister(conn)
}
return newMinioObjectLister(conn)
}
func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) {
secret := &corev1.Secret{}
if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil {
return "", err
}
v, ok := secret.Data[key]
if !ok {
return "", fmt.Errorf("secret %s/%s has no key %q", namespace, name, key)
}
return string(v), nil
}
func (r *BackupJobReconciler) objectStore() objectStore {
return objectStore{reader: r.Client, newLister: r.newS3ObjectLister}
}
func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) {
return r.objectStore().lister(conn)
}
type minioObjectLister struct {
client *minio.Client
}
func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) {
opts := &minio.Options{
Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken),
Region: conn.Region,
Secure: conn.Secure,
}
if conn.Secure && len(conn.CAPEM) > 0 {
// Start from minio's transport to keep its proxy and timeout settings.
tr, err := minio.DefaultTransport(true)
if err != nil {
return nil, err
}
if tr.TLSClientConfig.RootCAs, err = endpointTrustPool(conn, tr.TLSClientConfig.RootCAs); err != nil {
return nil, err
}
opts.Transport = tr
}
// minio.New refuses an endpoint with a path prefix, which it cannot
// address, and accepts a bare trailing slash.
client, err := minio.New(conn.Endpoint, opts)
if err != nil {
return nil, err
}
return &minioObjectLister{client: client}, nil
}
// endpointTrustPool returns the roots to verify the endpoint against: the
// CA bundle on top of system, or the CA bundle alone with CAOnly. A nil
// system means the platform's roots.
func endpointTrustPool(conn s3Connection, system *x509.CertPool) (*x509.CertPool, error) {
pool := system
switch {
case conn.CAOnly:
pool = x509.NewCertPool()
case pool == nil:
var err error
if pool, err = x509.SystemCertPool(); err != nil {
pool = x509.NewCertPool()
}
}
if !pool.AppendCertsFromPEM(conn.CAPEM) {
return nil, errors.New("no PEM certificates in the endpoint CA bundle")
}
return pool, nil
}
func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) {
return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive}))
}
// collectObjects drains a minio listing. minio-go ends a listing whose
// context is cancelled between pages by closing the channel without an
// error item, so a cancelled context is checked once the channel closes:
// otherwise a timeout would pass a partial listing off as complete.
func collectObjects(ctx context.Context, objects <-chan minio.ObjectInfo) ([]s3ObjectInfo, error) {
var out []s3ObjectInfo
for obj := range objects {
if obj.Err != nil {
return nil, obj.Err
}
out = append(out, s3ObjectInfo{Key: obj.Key, Size: obj.Size, LastModified: obj.LastModified})
}
if err := ctx.Err(); err != nil {
return nil, err
}
return out, nil
}
// errBackupNotReadBack reports that Create collided with an existing Backup
// that could then not be read. The backup itself succeeded, so callers
// retry the reconcile instead of failing the BackupJob.
var errBackupNotReadBack = errors.New("the Backup already exists but could not be read back")
// failBackupArtifact handles an error from creating the Backup of a
// completed backup: errBackupNotReadBack is retried through the controller's
// backoff, with Ready=False telling why the BackupJob is still Running;
// anything else fails the BackupJob.
func (r *BackupJobReconciler) failBackupArtifact(ctx context.Context, j *backupsv1alpha1.BackupJob, err error) (ctrl.Result, error) {
if errors.Is(err, errBackupNotReadBack) {
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "BackupNotReadBack",
Message: err.Error(),
})
if uerr := r.Status().Update(ctx, j); uerr != nil {
getLogger(ctx).Info("cannot record that the Backup could not be read back", "backupjob", j.Name, "error", uerr)
}
return ctrl.Result{}, err
}
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
}
// existingBackup returns the Backup a previous reconcile already created
// for the job, so a retry skips the size probe whose result it would
// discard. It reads past the cache: a retry typically follows a Create made
// moments earlier, which the cache may not show yet. A failed read is not
// fatal, because the Create that follows handles an existing Backup on its
// own; probe is false then, since the Backup may well exist and a probe
// would only be discarded, while a read that keeps failing would rerun it on
// every retry.
func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, name string) (existing *backupsv1alpha1.Backup, probe bool) {
existing = &backupsv1alpha1.Backup{}
if err := r.apiReader().Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil {
if apierrors.IsNotFound(err) {
return nil, true
}
getLogger(ctx).Info("cannot check for an existing Backup; creating it without measuring its size", "backup", name, "error", err)
return nil, false
}
return existing, false
}
func (r *BackupJobReconciler) secretValue(ctx context.Context, namespace, name, key string) (string, error) {
return r.objectStore().secretValue(ctx, namespace, name, key)
}
@@ -0,0 +1,287 @@
// SPDX-License-Identifier: Apache-2.0
package backupcontroller
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/minio/minio-go/v7"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
)
type fakeS3Lister struct {
objects []s3ObjectInfo
err error
conn s3Connection
bucket, prefix string
recursive bool
calls int
}
func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) {
f.calls++
f.bucket, f.prefix, f.recursive = bucket, prefix, recursive
return f.objects, f.err
}
func s3CredsSecret(namespace, name string) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name},
Data: map[string][]byte{
"AWS_ACCESS_KEY_ID": []byte("AKID"),
"AWS_SECRET_ACCESS_KEY": []byte("secret"),
},
}
}
// reconcilerWithLister returns a reconciler whose S3 reads go to l, backed
// by a fake client seeded with objs (typically the credentials Secret).
func reconcilerWithLister(t *testing.T, l *fakeS3Lister, objs ...client.Object) *BackupJobReconciler {
t.Helper()
c := newMariaDBStrategyTestClient(t, objs...)
return &BackupJobReconciler{
Client: c,
Scheme: c.Scheme(),
newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) {
l.conn = conn
return l, nil
},
}
}
func TestCollectObjects(t *testing.T) {
listing := func() <-chan minio.ObjectInfo {
ch := make(chan minio.ObjectInfo, 1)
ch <- minio.ObjectInfo{Key: "a", Size: 1}
close(ch)
return ch
}
got, err := collectObjects(context.Background(), listing())
if err != nil || len(got) != 1 || got[0].Size != 1 {
t.Fatalf("collectObjects = %v, %v; want one object", got, err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if got, err := collectObjects(ctx, listing()); !errors.Is(err, context.Canceled) {
t.Errorf("listing closed after cancellation: got %v, %v; want context.Canceled", got, err)
}
errCh := make(chan minio.ObjectInfo, 1)
errCh <- minio.ObjectInfo{Err: errors.New("AccessDenied")}
close(errCh)
if _, err := collectObjects(context.Background(), errCh); err == nil {
t.Error("listing error was not returned")
}
}
// TestExistingBackup_ReadsPastTheCache pins that the retry check sees a
// Backup the cache does not show yet: the previous reconcile created it
// moments earlier.
func TestExistingBackup_ReadsPastTheCache(t *testing.T) {
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
r := &BackupJobReconciler{
Client: newMariaDBStrategyTestClient(t),
APIReader: newMariaDBStrategyTestClient(t, backup),
}
if got, _ := r.existingBackup(context.Background(), "tenant", "bj"); got == nil {
t.Fatal("existingBackup = nil; want the Backup the API server holds")
}
r.APIReader = newMariaDBStrategyTestClient(t)
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || !probe {
t.Errorf("existingBackup = %v, probe=%v; want nil and a probe when there is no Backup", got, probe)
}
}
// staleCache wraps a fake client so that Get reports NotFound for Backups,
// the way an informer cache does right after another writer created one.
func staleCache(t *testing.T, c client.Client) client.Client {
t.Helper()
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
}
return cl.Get(ctx, key, obj, opts...)
},
})
}
// notFoundOnce wraps a fake client whose first Backup Get reports NotFound:
// the retry check runs before the Backup exists, the refetch after Create
// collides with it.
func notFoundOnce(t *testing.T, c client.Client) client.Client {
t.Helper()
missed := false
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if _, ok := obj.(*backupsv1alpha1.Backup); ok && !missed {
missed = true
return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name)
}
return cl.Get(ctx, key, obj, opts...)
},
})
}
func TestExistingBackup_ReadErrorIsNotFatal(t *testing.T) {
failing := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
return errors.New("etcdserver: request timed out")
},
})
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t), APIReader: failing}
if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || probe {
t.Errorf("existingBackup = %#v, probe=%v; want nil and no probe on a read error", got, probe)
}
}
// blockingLister answers nothing until the probe's context is done, the way
// a black-holed endpoint does.
type blockingLister struct{}
func (blockingLister) ListObjects(ctx context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) {
<-ctx.Done()
return nil, ctx.Err()
}
// withShortProbeTimeout shortens the probe timeout and fails the test if fn
// is not back well before the default timeout would have expired.
func withShortProbeTimeout(t *testing.T, fn func()) {
t.Helper()
saved := artifactSizeProbeTimeout
artifactSizeProbeTimeout = 50 * time.Millisecond
defer func() { artifactSizeProbeTimeout = saved }()
done := make(chan struct{})
go func() {
fn()
close(done)
}()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("the probe did not give up at its timeout")
}
}
// failingReads wraps a fake client whose every Get fails, like an apiserver
// that times out.
func failingReads(t *testing.T, c client.Client) client.Client {
t.Helper()
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
return errors.New("etcdserver: request timed out")
},
})
}
func TestFailBackupArtifact(t *testing.T) {
newJob := func() *backupsv1alpha1.BackupJob {
return &backupsv1alpha1.BackupJob{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}}
}
t.Run("a Backup that exists but cannot be read is retried", func(t *testing.T) {
j := newJob()
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
_, err := r.failBackupArtifact(context.Background(), j, fmt.Errorf("%w: timeout", errBackupNotReadBack))
if !errors.Is(err, errBackupNotReadBack) {
t.Errorf("err = %v, want it returned so the reconcile is retried", err)
}
persisted := &backupsv1alpha1.BackupJob{}
if err := r.Get(context.Background(), client.ObjectKeyFromObject(j), persisted); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Error("the BackupJob was failed although its Backup exists")
}
if cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready"); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "BackupNotReadBack" {
t.Errorf("Ready condition = %+v, want False/BackupNotReadBack", cond)
}
})
t.Run("any other error fails the BackupJob", func(t *testing.T) {
j := newJob()
r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)}
if _, err := r.failBackupArtifact(context.Background(), j, errors.New("admission webhook denied")); err != nil {
t.Fatalf("failBackupArtifact: %v", err)
}
if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
t.Errorf("phase = %q, want Failed", j.Status.Phase)
}
})
}
// TestEndpointTrustPool_CAOnly tells apart "the CA on top of the system
// roots" and "the CA alone". The system pool holds the endpoint's
// certificate, and the configured CA is an unrelated one, so the endpoint
// must stop being trusted only when CAOnly is set.
func TestEndpointTrustPool_CAOnly(t *testing.T) {
endpoint := httptest.NewTLSServer(http.NotFoundHandler())
defer endpoint.Close()
system := x509.NewCertPool()
system.AddCert(endpoint.Certificate())
// Every httptest server shares one certificate, so the unrelated CA is
// minted here.
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
unrelatedCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
verifies := func(caOnly bool) bool {
pool, err := endpointTrustPool(s3Connection{CAPEM: unrelatedCA, CAOnly: caOnly}, system.Clone())
if err != nil {
t.Fatalf("endpointTrustPool: %v", err)
}
_, err = endpoint.Certificate().Verify(x509.VerifyOptions{Roots: pool})
return err == nil
}
if !verifies(false) {
t.Error("CA on top of the system roots: the endpoint was not trusted")
}
if verifies(true) {
t.Error("CA alone: the endpoint was trusted through the system roots")
}
}
// forbidBackupCreate wraps a fake client whose Create of a Backup is
// rejected, the way an admission webhook rejects one.
func forbidBackupCreate(t *testing.T, c client.Client) client.Client {
t.Helper()
return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
return apierrors.NewForbidden(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), obj.GetName(), errors.New("denied by policy"))
}
return cl.Create(ctx, obj, opts...)
},
})
}
@@ -39,6 +39,10 @@ type BackupJobReconciler struct {
// re-reads the BackupJob before dispatching to a driver. Wired in
// SetupWithManager.
APIReader client.Reader
// newS3ObjectLister overrides the S3 client used to measure artifact
// sizes; nil means a real minio client. Set by unit tests.
newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error)
}
func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -0,0 +1,135 @@
// SPDX-License-Identifier: Apache-2.0
package backupcontroller
import (
"context"
"errors"
"fmt"
"net/url"
"strings"
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
)
// cnpgArtifactSizeBytes sums the objects barman-cloud uploaded for one base
// backup, <destinationPath>/<serverName>/base/<backupID>/. The WAL archive
// is not counted: it is one continuous stream shared by every base backup of
// the server, so no part of it belongs to a single Backup. Returns 0 -
// leaving Artifact.SizeBytes absent - when the backup ID is unknown, the
// bucket cannot be read, or nothing is stored under the prefix.
func (r *BackupJobReconciler) cnpgArtifactSizeBytes(
ctx context.Context,
namespace string,
store *strategyv1alpha1.BarmanObjectStoreTemplate,
serverName, backupID string,
) int64 {
logger := getLogger(ctx)
if backupID == "" {
return 0
}
ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout)
defer cancel()
bucket, path, err := parseBarmanS3DestinationPath(store.DestinationPath)
if err != nil {
logger.Info("cannot measure CNPG backup size", "backupID", backupID, "error", err)
return 0
}
conn, err := r.cnpgS3Connection(ctx, namespace, store)
if err != nil {
logger.Info("cannot resolve S3 credentials to measure CNPG backup size", "backupID", backupID, "error", err)
return 0
}
lister, err := r.s3Lister(conn)
if err != nil {
logger.Info("cannot build S3 client to measure CNPG backup size", "backupID", backupID, "error", err)
return 0
}
prefix := barmanBaseBackupPrefix(path, serverName, backupID)
objects, err := lister.ListObjects(ctx, bucket, prefix, true)
if err != nil {
logger.Info("cannot list CNPG base backup objects to measure size", "backupID", backupID, "error", err)
return 0
}
var total int64
for _, obj := range objects {
total += obj.Size
}
if total == 0 {
logger.Info("no CNPG base backup objects found; leaving size unset", "backupID", backupID, "prefix", prefix)
}
return total
}
func (r *BackupJobReconciler) cnpgS3Connection(ctx context.Context, namespace string, store *strategyv1alpha1.BarmanObjectStoreTemplate) (s3Connection, error) {
var conn s3Connection
if store.S3Credentials == nil {
return conn, errors.New("barmanObjectStore has no s3Credentials")
}
if store.EndpointURL == "" {
conn.Endpoint, conn.Secure = "s3.amazonaws.com", true
} else {
u, err := url.Parse(store.EndpointURL)
if err != nil || u.Host == "" {
return conn, fmt.Errorf("invalid endpointURL %q", store.EndpointURL)
}
// Dropping the path would probe the bucket at another location than
// the one barman writes to.
if strings.Trim(u.Path, "/") != "" {
return conn, fmt.Errorf("endpointURL %q has a path, which the S3 client cannot address", store.EndpointURL)
}
conn.Endpoint, conn.Secure = u.Host, u.Scheme == "https"
}
creds := store.S3Credentials
accessKeyKey, secretKeyKey := creds.AccessKeyIDKey, creds.SecretAccessKeyKey
if accessKeyKey == "" {
accessKeyKey = defaultS3AccessKeyIDKey
}
if secretKeyKey == "" {
secretKeyKey = defaultS3SecretAccessKeyKey
}
var err error
if conn.AccessKey, err = r.secretValue(ctx, namespace, creds.SecretRef.Name, accessKeyKey); err != nil {
return conn, err
}
if conn.SecretKey, err = r.secretValue(ctx, namespace, creds.SecretRef.Name, secretKeyKey); err != nil {
return conn, err
}
if ca := store.EndpointCA; ca != nil && ca.SecretRef.Name != "" && conn.Secure {
key := ca.Key
if key == "" {
key = defaultEndpointCAKey
}
pem, err := r.secretValue(ctx, namespace, ca.SecretRef.Name, key)
if err != nil {
return conn, err
}
// barman-cloud gets endpointCA as AWS_CA_BUNDLE, which replaces the
// default trust store rather than adding to it.
conn.CAPEM, conn.CAOnly = []byte(pem), true
}
return conn, nil
}
// parseBarmanS3DestinationPath splits s3://bucket/path into the bucket and
// the key path barman-cloud prefixes every object with.
func parseBarmanS3DestinationPath(destinationPath string) (bucket, path string, err error) {
u, err := url.Parse(destinationPath)
if err != nil || u.Scheme != "s3" || u.Host == "" {
return "", "", fmt.Errorf("destinationPath %q is not an s3:// URL", destinationPath)
}
return u.Host, strings.Trim(u.Path, "/"), nil
}
// barmanBaseBackupPrefix mirrors barman-cloud's
// os.path.join(path, server_name, "base", backup_id), with a trailing slash
// so one backup ID never matches another that extends it.
func barmanBaseBackupPrefix(path, serverName, backupID string) string {
parts := make([]string, 0, 4)
if path != "" {
parts = append(parts, path)
}
parts = append(parts, serverName, "base", backupID)
return strings.Join(parts, "/") + "/"
}
@@ -0,0 +1,453 @@
// SPDX-License-Identifier: Apache-2.0
package backupcontroller
import (
"context"
"encoding/pem"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
"github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes"
)
func newSizedBarmanObjectStore() *strategyv1alpha1.BarmanObjectStoreTemplate {
return &strategyv1alpha1.BarmanObjectStoreTemplate{
DestinationPath: "s3://bkt/tenant/pg/",
EndpointURL: "https://s3.example:8443",
S3Credentials: &strategyv1alpha1.S3CredentialsTemplate{
SecretRef: corev1.LocalObjectReference{Name: "creds"},
},
}
}
func TestBarmanBaseBackupPrefix(t *testing.T) {
cases := []struct{ path, want string }{
{"tenant/pg", "tenant/pg/srv/base/20261021T100000/"},
{"", "srv/base/20261021T100000/"},
}
for _, tc := range cases {
if got := barmanBaseBackupPrefix(tc.path, "srv", "20261021T100000"); got != tc.want {
t.Errorf("barmanBaseBackupPrefix(%q) = %q, want %q", tc.path, got, tc.want)
}
}
}
func TestParseBarmanS3DestinationPath(t *testing.T) {
cases := []struct {
in, bucket, path string
ok bool
}{
{"s3://bkt/tenant/pg/", "bkt", "tenant/pg", true},
{"s3://bkt", "bkt", "", true},
{"s3://bkt/", "bkt", "", true},
{"gs://bkt/x/", "", "", false},
{"bkt/x/", "", "", false},
}
for _, tc := range cases {
bucket, path, err := parseBarmanS3DestinationPath(tc.in)
if (err == nil) != tc.ok || bucket != tc.bucket || path != tc.path {
t.Errorf("parseBarmanS3DestinationPath(%q) = %q, %q, %v", tc.in, bucket, path, err)
}
}
}
func TestCNPGArtifactSizeBytes(t *testing.T) {
const prefix = "tenant/pg/srv/base/20261021T100000/"
t.Run("sums every object of the base backup", func(t *testing.T) {
l := &fakeS3Lister{objects: []s3ObjectInfo{
{Key: prefix + "backup.info", Size: 1},
{Key: prefix + "data_0000.tar.gz", Size: 1000},
{Key: prefix + "data_0001.tar.gz", Size: 200},
{Key: prefix + "16385.tar.gz", Size: 30},
}}
r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"))
got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", newSizedBarmanObjectStore(), "srv", "20261021T100000")
if got != 1231 {
t.Errorf("size = %d, want 1231", got)
}
if l.bucket != "bkt" || l.prefix != prefix || !l.recursive {
t.Errorf("listed %q/%q recursive=%v, want bkt/%q recursive", l.bucket, l.prefix, l.recursive, prefix)
}
if l.conn.Endpoint != "s3.example:8443" || !l.conn.Secure || l.conn.AccessKey != "AKID" || l.conn.SecretKey != "secret" {
t.Errorf("connection = %+v", l.conn)
}
})
t.Run("a hanging endpoint is given up at the probe timeout", func(t *testing.T) {
r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds"))
r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil }
withShortProbeTimeout(t, func() {
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", newSizedBarmanObjectStore(), "srv", "20261021T100000"); got != 0 {
t.Errorf("size = %d, want 0", got)
}
})
})
t.Run("an empty endpointURL means AWS", func(t *testing.T) {
store := newSizedBarmanObjectStore()
store.EndpointURL = ""
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: prefix + "data.tar", Size: 5}}}
r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"))
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 5 {
t.Errorf("size = %d, want 5", got)
}
if l.conn.Endpoint != "s3.amazonaws.com" || !l.conn.Secure {
t.Errorf("connection = %+v, want s3.amazonaws.com over TLS", l.conn)
}
})
t.Run("honours custom credential keys and the endpoint CA", func(t *testing.T) {
store := newSizedBarmanObjectStore()
store.S3Credentials.AccessKeyIDKey = "id"
store.S3Credentials.SecretAccessKeyKey = "key"
store.EndpointCA = &strategyv1alpha1.EndpointCARef{SecretRef: corev1.LocalObjectReference{Name: "ca"}}
creds := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "creds"},
Data: map[string][]byte{"id": []byte("ID2"), "key": []byte("KEY2")},
}
ca := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"},
Data: map[string][]byte{"ca.crt": []byte("PEM")},
}
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: prefix + "data.tar", Size: 5}}}
r := reconcilerWithLister(t, l, creds, ca)
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 5 {
t.Errorf("size = %d, want 5", got)
}
if l.conn.AccessKey != "ID2" || l.conn.SecretKey != "KEY2" || string(l.conn.CAPEM) != "PEM" || !l.conn.CAOnly {
t.Errorf("connection = %+v", l.conn)
}
})
unset := map[string]struct {
store func() *strategyv1alpha1.BarmanObjectStoreTemplate
backupID string
lister *fakeS3Lister
noSecret bool
}{
"unknown backup ID": {
store: newSizedBarmanObjectStore, lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}},
},
"no objects under the prefix": {
store: newSizedBarmanObjectStore, backupID: "id", lister: &fakeS3Lister{},
},
"list error": {
store: newSizedBarmanObjectStore, backupID: "id", lister: &fakeS3Lister{err: errors.New("AccessDenied")},
},
"missing credentials Secret": {
store: newSizedBarmanObjectStore, backupID: "id", noSecret: true,
lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}},
},
"no s3Credentials": {
store: func() *strategyv1alpha1.BarmanObjectStoreTemplate {
s := newSizedBarmanObjectStore()
s.S3Credentials = nil
return s
},
backupID: "id", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}},
},
"non-s3 destination": {
store: func() *strategyv1alpha1.BarmanObjectStoreTemplate {
s := newSizedBarmanObjectStore()
s.DestinationPath = "azure://container/x"
return s
},
backupID: "id", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}},
},
"endpointURL with a path": {
store: func() *strategyv1alpha1.BarmanObjectStoreTemplate {
s := newSizedBarmanObjectStore()
s.EndpointURL = "https://s3.example:8443/prefix"
return s
},
backupID: "20261021T100000", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}},
},
}
for name, tc := range unset {
t.Run(name+" leaves size unset", func(t *testing.T) {
var r *BackupJobReconciler
if tc.noSecret {
r = reconcilerWithLister(t, tc.lister)
} else {
r = reconcilerWithLister(t, tc.lister, s3CredsSecret("tenant", "creds"))
}
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", tc.store(), "srv", tc.backupID); got != 0 {
t.Errorf("size = %d, want 0", got)
}
})
}
}
// TestCNPGArtifactSizeBytes_MinioClient drives the real minio client against
// a stub ListObjectsV2 endpoint, so the endpoint URL parsing, Secret wiring
// and response parsing are exercised together.
func TestCNPGArtifactSizeBytes_MinioClient(t *testing.T) {
const prefix = "tenant/pg/srv/base/20261021T100000/"
var gotPath, gotPrefix, gotDelimiter, gotAuth string
srv := httptest.NewServer(cnpgListStub(prefix, func(req *http.Request) {
gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization")
gotDelimiter = req.URL.Query().Get("delimiter")
}))
defer srv.Close()
c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds"))
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()}
store := newSizedBarmanObjectStore()
store.EndpointURL = srv.URL
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 4103 {
t.Fatalf("size = %d, want 4103", got)
}
if gotPath != "/bkt/" || gotPrefix != prefix || gotDelimiter != "" {
t.Errorf("request path=%q prefix=%q delimiter=%q, want /bkt/, %q and a recursive listing", gotPath, gotPrefix, gotDelimiter, prefix)
}
if !strings.Contains(gotAuth, "Credential=AKID/") {
t.Errorf("request not signed with the Secret's access key: %q", gotAuth)
}
}
func cnpgListStub(prefix string, seen func(*http.Request)) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "application/xml")
if _, ok := req.URL.Query()["location"]; ok {
fmt.Fprint(w, `<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></LocationConstraint>`)
return
}
if seen != nil {
seen(req)
}
fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>bkt</Name><Prefix>%[1]s</Prefix><KeyCount>2</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>
<Contents><Key>%[1]sbackup.info</Key><Size>7</Size><ETag>"a"</ETag><LastModified>2026-10-21T10:01:00.000Z</LastModified></Contents>
<Contents><Key>%[1]sdata.tar.gz</Key><Size>4096</Size><ETag>"b"</ETag><LastModified>2026-10-21T10:01:00.000Z</LastModified></Contents>
</ListBucketResult>`, prefix)
}
}
// TestCNPGArtifactSizeBytes_EndpointCA serves the listing over TLS with a
// self-signed certificate: the probe must trust it only through endpointCA.
func TestCNPGArtifactSizeBytes_EndpointCA(t *testing.T) {
const prefix = "tenant/pg/srv/base/20261021T100000/"
srv := httptest.NewTLSServer(cnpgListStub(prefix, nil))
defer srv.Close()
caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw})
ca := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "s3-ca"},
Data: map[string][]byte{"ca.crt": caPEM},
}
c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds"), ca)
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()}
store := newSizedBarmanObjectStore()
store.EndpointURL = srv.URL
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 0 {
t.Errorf("without endpointCA: size = %d, want 0 (untrusted certificate)", got)
}
store.EndpointCA = &strategyv1alpha1.EndpointCARef{SecretRef: corev1.LocalObjectReference{Name: "s3-ca"}}
if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 4103 {
t.Errorf("with endpointCA: size = %d, want 4103", got)
}
}
func TestCreateCNPGBackupArtifact_SizeBytes(t *testing.T) {
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
strategyGroup := strategyv1alpha1.GroupVersion.Group
j := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-size"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
},
}
resolved := &ResolvedBackupConfig{
StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "strat"},
}
cnpgBk := &cnpgtypes.Backup{
ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"},
Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"},
}
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()}
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "tenant/pg/srv/base/20261021T100000/data.tar.gz", Size: 777}}}
r := reconcilerWithLister(t, l)
r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds"))
got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant"))
if err != nil {
t.Fatalf("createCNPGBackupArtifact: %v", err)
}
if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 777 {
t.Fatalf("Artifact = %#v, want SizeBytes 777", got.Status.Artifact)
}
if l.prefix != "tenant/pg/srv/base/20261021T100000/" {
t.Errorf("listed prefix %q does not use the effective serverName", l.prefix)
}
}
func TestCreateCNPGBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) {
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
j := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
},
}
existing := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"}}
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}
r := reconcilerWithLister(t, l)
r.Client = newCNPGStrategyTestClient(t, existing, s3CredsSecret("tenant", "creds"))
cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}}
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()}
if _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")); err != nil {
t.Fatalf("createCNPGBackupArtifact: %v", err)
}
if l.calls != 0 {
t.Errorf("S3 listed %d times for a Backup that already exists", l.calls)
}
}
func TestCreateCNPGBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testing.T) {
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
j := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
},
}
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}}
store := newCNPGStrategyTestClient(t, backup)
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store)}
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}}
got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant"))
if err != nil || got == nil || got.Name != "bj-race" {
t.Fatalf("createCNPGBackupArtifact = %v, %v; want the Backup that already exists", got, err)
}
}
func TestCreateCNPGBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T) {
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
j := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
},
}
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}}
store := newCNPGStrategyTestClient(t, backup)
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store)}
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}}
_, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant"))
if !errors.Is(err, errBackupNotReadBack) {
t.Errorf("err = %v, want errBackupNotReadBack", err)
}
}
// TestReconcileCNPG_UnreadableBackupIsRetried drives a completed backup
// through the reconcile: when the Backup exists but cannot be read back,
// the reconcile is retried and the BackupJob is not failed.
func TestReconcileCNPG_UnreadableBackupIsRetried(t *testing.T) {
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Minute)),
newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)),
newRolloutInstancePod(time.Now().Add(-time.Hour)))
_, backups := f.pass()
if len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups))
}
ctx := context.Background()
completed := backups[0].DeepCopy()
completed.Status.Phase = cnpgBackupPhaseComplete
completed.Status.BackupID = "20261021T100000"
if err := f.c.Update(ctx, completed); err != nil {
t.Fatalf("complete cnpg.io/Backup: %v", err)
}
if err := f.c.Create(ctx, &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "bj"}}); err != nil {
t.Fatalf("seed Backup: %v", err)
}
f.r.APIReader = failingReads(t, f.c)
j := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); !errors.Is(err, errBackupNotReadBack) {
t.Fatalf("reconcileCNPG err = %v, want errBackupNotReadBack so the reconcile is retried", err)
}
persisted := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Error("the BackupJob was failed although its Backup exists")
}
}
func TestCreateCNPGBackupArtifact_FailedExistenceCheckSkipsProbe(t *testing.T) {
apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
j := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-transient"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"},
},
}
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}
r := reconcilerWithLister(t, l)
r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds"))
r.APIReader = failingReads(t, newCNPGStrategyTestClient(t))
cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}}
rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()}
got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant"))
if err != nil || got == nil {
t.Fatalf("createCNPGBackupArtifact = %v, %v; a transient read must not fail the backup", got, err)
}
if l.calls != 0 {
t.Errorf("S3 probed %d times after the existence check failed", l.calls)
}
}
// TestReconcileCNPG_RejectedCreateFailsTheJob pins that only the read-back
// is retried: a Create the apiserver rejects still fails the BackupJob.
func TestReconcileCNPG_RejectedCreateFailsTheJob(t *testing.T) {
f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Minute)),
newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)),
newRolloutInstancePod(time.Now().Add(-time.Hour)))
_, backups := f.pass()
if len(backups) != 1 {
t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups))
}
ctx := context.Background()
completed := backups[0].DeepCopy()
completed.Status.Phase = cnpgBackupPhaseComplete
if err := f.c.Update(ctx, completed); err != nil {
t.Fatalf("complete cnpg.io/Backup: %v", err)
}
f.r.Client = forbidBackupCreate(t, f.c)
j := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); err != nil {
t.Fatalf("reconcileCNPG err = %v, want the BackupJob failed rather than retried", err)
}
persisted := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if persisted.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase)
}
}
@@ -435,7 +435,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
}
artifact, err := r.createCNPGBackupArtifact(ctx, j, resolved, cnpgBackup, clusterName, serverName, attachment.flavor, rendered, app)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
return r.failBackupArtifact(ctx, j, err)
}
now := metav1.Now()
j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
@@ -815,6 +815,10 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact(
rendered *strategyv1alpha1.CNPGTemplate,
sourceApp *postgresapp.Postgres,
) (*backupsv1alpha1.Backup, error) {
existing, probe := r.existingBackup(ctx, j.Namespace, j.Name)
if existing != nil {
return existing, nil
}
takenAt := metav1.Now()
if cnpgBackup.Status.StartedAt != nil && !cnpgBackup.Status.StartedAt.IsZero() {
takenAt = *cnpgBackup.Status.StartedAt
@@ -860,6 +864,9 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact(
UnderlyingResources: underlyingResources,
},
}
if probe {
backup.Status.Artifact.SizeBytes = r.cnpgArtifactSizeBytes(ctx, j.Namespace, &rendered.BarmanObjectStore, serverName, cnpgBackup.Status.BackupID)
}
if j.Spec.PlanRef != nil {
backup.Spec.PlanRef = j.Spec.PlanRef
}
@@ -872,8 +879,8 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact(
return nil, err
}
existing := &backupsv1alpha1.Backup{}
if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
return nil, getErr
if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr)
}
return existing, nil
}
@@ -0,0 +1,261 @@
// SPDX-License-Identifier: Apache-2.0
package backupcontroller
import (
"context"
"errors"
"fmt"
"strings"
"time"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes"
)
// Slack around the [created, completed] window of the operator Backup when
// matching a dump: the timestamp in its file name comes from the backup Job
// pod's clock and is truncated to seconds, and its LastModified comes from
// the object store's clock, neither of which is the apiserver's.
const mariadbSizeProbeClockSkew = time.Minute
var (
errMariaDBDumpNotFound = errors.New("no dump in the bucket belongs to the operator Backup")
errMariaDBDumpAmbiguous = errors.New("the dump of the operator Backup cannot be told apart from another run's")
)
// mariadbDump is a logical dump mariadb-operator wrote to the bucket.
type mariadbDump struct {
// Key is the object key; Name is the file name, Key without the prefix.
Key string
Name string
// Timestamp is the one in the file name, taken by the backup Job pod's
// clock as the dump started.
Timestamp time.Time
Size int64
// LastModified is the object store's clock as the upload finished.
LastModified time.Time
}
// mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for
// mdbBackup. Returns 0 - leaving Artifact.SizeBytes absent - when the dump
// cannot be found or told apart: an unknown size must not be reported as a
// measured one.
func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) int64 {
ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout)
defer cancel()
dump, err := findMariaDBDump(ctx, r.objectStore(), mdbBackup, completedAt)
if err != nil {
getLogger(ctx).Info("MariaDB backup size not measured; leaving it unset", "backup", mdbBackup.Name, "error", err)
return 0
}
return dump.Size
}
// findMariaDBDump lists the bucket mdbBackup wrote to and returns the one
// dump that belongs to it, as matchMariaDBDump decides. The location and
// credentials come from mdbBackup's own spec: the strategy it was rendered
// from can have been edited since, and pointing at another bucket now says
// nothing about where this dump went.
func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) {
s3 := mdbBackup.Spec.Storage.S3
if s3 == nil {
return mariadbDump{}, errors.New("the operator Backup does not store to S3")
}
others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup)
if err != nil {
return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err)
}
conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3)
if err != nil {
return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err)
}
lister, err := store.lister(conn)
if err != nil {
return mariadbDump{}, fmt.Errorf("build S3 client: %w", err)
}
// Non-recursive, as mariadb-operator lists it: the default prefix is also
// the default CNPG destinationPath, so an app of each kind with the same
// name shares it and a recursive listing would walk the WAL archive.
prefix := mariadbOperatorS3Prefix(s3.Prefix)
objects, err := lister.ListObjects(ctx, s3.Bucket, prefix, false)
if err != nil {
return mariadbDump{}, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, prefix, err)
}
own := newMariaDBRun(mdbBackup, mdbBackup.CreationTimestamp.Time, completedAt)
return matchMariaDBDump(objects, prefix, own, others)
}
// mariadbRun is the span in which an operator Backup can have written its
// dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side.
type mariadbRun struct {
name string
lower, upper time.Time
}
func newMariaDBRun(b *mariadbtypes.Backup, created, completed time.Time) mariadbRun {
return mariadbRun{
name: b.Namespace + "/" + b.Name,
lower: created.Add(-mariadbSizeProbeClockSkew),
upper: completed.Add(mariadbSizeProbeClockSkew),
}
}
func (r mariadbRun) placesOnBothClocks(d mariadbDump) bool {
return inWindow(d.Timestamp, r.lower, r.upper) && inWindow(d.LastModified, r.lower, r.upper)
}
func (r mariadbRun) placesOnEitherClock(d mariadbDump) bool {
return inWindow(d.Timestamp, r.lower, r.upper) || inWindow(d.LastModified, r.lower, r.upper)
}
// matchMariaDBDump picks the dump of run own among objects, which share a
// prefix with every other run of the application. A dump carries two
// clocks, the backup pod's in its file name and the object store's in its
// LastModified, and either can be skewed. A dump is taken only when it is
// the one dump either clock places in own's window, both clocks place it
// there, and neither places it in the window of another finished run: a
// dump in the window on one clock only may be this run's own, misplaced by
// a skewed clock, which leaves any other candidate unproven. A neighbour's
// dump is still taken if this run's own is missing from the listing, or
// misplaced on both clocks, and the neighbour's operator Backup is gone or
// has not completed.
func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, others []mariadbRun) (mariadbDump, error) {
var near []mariadbDump
for _, obj := range objects {
name := strings.TrimPrefix(obj.Key, prefix)
ts, ok := parseMariaDBLogicalBackupFile(name)
if !ok {
continue
}
d := mariadbDump{Key: obj.Key, Name: name, Timestamp: ts, Size: obj.Size, LastModified: obj.LastModified}
if own.placesOnEitherClock(d) {
near = append(near, d)
}
}
if len(near) == 0 {
return mariadbDump{}, errMariaDBDumpNotFound
}
if len(near) > 1 {
return mariadbDump{}, fmt.Errorf("%w: %d dumps fall in its lifetime on at least one clock", errMariaDBDumpAmbiguous, len(near))
}
d := near[0]
if !own.placesOnBothClocks(d) {
return mariadbDump{}, fmt.Errorf("%w: %s falls in its lifetime on one clock only", errMariaDBDumpAmbiguous, d.Name)
}
for _, other := range others {
if other.placesOnEitherClock(d) {
return mariadbDump{}, fmt.Errorf("%w: %s also falls in the lifetime of operator Backup %s", errMariaDBDumpAmbiguous, d.Name, other.name)
}
}
return d, nil
}
// finishedMariaDBRuns returns the runs of the other operator Backups that
// write under the same bucket and prefix as b and have completed, whether
// their Job succeeded or failed. One still running is left out: its window
// has no end yet, and an operator Backup stuck short of Complete, as one
// whose Job pod never starts is, would otherwise contest every later run of
// the application for good. The endpoint is not compared, since one has
// several spellings, and a false neighbour only leaves a size unset.
func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup) ([]mariadbRun, error) {
list := &mariadbtypes.BackupList{}
if err := reader.List(ctx, list); err != nil {
return nil, err
}
var runs []mariadbRun
for i := range list.Items {
other := &list.Items[i]
if other.Namespace == b.Namespace && other.Name == b.Name {
continue
}
s3 := other.Spec.Storage.S3
if s3 == nil || s3.Bucket != b.Spec.Storage.S3.Bucket || mariadbOperatorS3Prefix(s3.Prefix) != mariadbOperatorS3Prefix(b.Spec.Storage.S3.Prefix) {
continue
}
cond := apimeta.FindStatusCondition(other.Status.Conditions, mariadbtypes.ConditionTypeComplete)
if cond == nil || cond.Status != metav1.ConditionTrue {
continue
}
runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, cond.LastTransitionTime.Time))
}
return runs, nil
}
// mariadbS3Connection resolves the Secrets and TLS settings of the operator
// Backup's storage, so the bucket is read with the tenant's credentials
// rather than the controller's.
func mariadbS3Connection(ctx context.Context, store objectStore, namespace string, s3 *mariadbtypes.S3Storage) (s3Connection, error) {
conn := s3Connection{
Endpoint: s3.Endpoint,
Region: s3.Region,
Secure: s3.TLS != nil && s3.TLS.Enabled,
}
var err error
if conn.AccessKey, err = store.secretValue(ctx, namespace, s3.AccessKeyIdSecretKeyRef.Name, s3.AccessKeyIdSecretKeyRef.Key); err != nil {
return conn, err
}
if conn.SecretKey, err = store.secretValue(ctx, namespace, s3.SecretAccessKeySecretKeyRef.Name, s3.SecretAccessKeySecretKeyRef.Key); err != nil {
return conn, err
}
if ref := s3.SessionTokenSecretKeyRef; ref != nil {
if conn.SessionToken, err = store.secretValue(ctx, namespace, ref.Name, ref.Key); err != nil {
return conn, err
}
}
if conn.Secure && s3.TLS.CASecretKeyRef != nil {
ca, err := store.secretValue(ctx, namespace, s3.TLS.CASecretKeyRef.Name, s3.TLS.CASecretKeyRef.Key)
if err != nil {
return conn, err
}
conn.CAPEM = []byte(ca)
}
return conn, nil
}
// mariadbOperatorS3Prefix mirrors how mariadb-operator normalises
// spec.storage.s3.prefix before prepending it to object keys.
func mariadbOperatorS3Prefix(prefix string) string {
if prefix == "" || prefix == "/" {
return ""
}
if !strings.HasSuffix(prefix, "/") {
return prefix + "/"
}
return prefix
}
func inWindow(t, lower, upper time.Time) bool {
return !t.Before(lower) && !t.After(upper)
}
// Compression tokens mariadb-operator puts in a dump file name: the
// algorithm before .sql in v25.10 (backup.<ts>.gzip.sql), the extension
// after it in later releases (backup.<ts>.sql.gz).
var (
mariadbDumpAlgorithms = map[string]bool{"none": true, "bzip2": true, "gzip": true, "zstd": true}
mariadbDumpExtensions = map[string]bool{"bz2": true, "gz": true, "zst": true}
)
// parseMariaDBLogicalBackupFile accepts the logical-backup file names
// mariadb-operator writes and returns the embedded timestamp.
func parseMariaDBLogicalBackupFile(name string) (time.Time, bool) {
parts := strings.Split(name, ".")
if parts[0] != "backup" {
return time.Time{}, false
}
switch {
case len(parts) == 3 && parts[2] == "sql":
case len(parts) == 4 && parts[2] == "sql" && mariadbDumpExtensions[parts[3]]:
case len(parts) == 4 && parts[3] == "sql" && mariadbDumpAlgorithms[parts[2]]:
default:
return time.Time{}, false
}
ts, err := time.Parse(time.RFC3339, parts[1])
if err != nil {
return time.Time{}, false
}
return ts, true
}
@@ -0,0 +1,750 @@
// SPDX-License-Identifier: Apache-2.0
package backupcontroller
import (
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp"
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes"
)
func dumpKey(prefix string, ts time.Time, compression string) string {
if compression == "" {
return fmt.Sprintf("%sbackup.%s.sql", prefix, ts.UTC().Format(time.RFC3339))
}
return fmt.Sprintf("%sbackup.%s.%s.sql", prefix, ts.UTC().Format(time.RFC3339), compression)
}
// mariadbBackupSucceeded is the Complete condition mariadb-operator sets on a
// Backup whose Job succeeded. A failed Job also gets Status=True, with
// Reason=JobFailed, so a fixture without the reason is not a success.
func mariadbBackupSucceeded(at metav1.Time) metav1.Condition {
return metav1.Condition{
Type: mariadbtypes.ConditionTypeComplete,
Status: metav1.ConditionTrue,
Reason: mariadbtypes.ConditionReasonJobComplete,
Message: "Success",
LastTransitionTime: at,
}
}
func TestParseMariaDBLogicalBackupFile(t *testing.T) {
cases := map[string]bool{
"backup.2026-10-21T10:00:00Z.sql": true,
"backup.2026-10-21T10:00:00Z.gzip.sql": true,
"backup.2026-10-21T10:00:00+04:00.sql": true,
"backup.2026-10-21T10:00:00Z.sql.tmp": false,
"backup.2026-10-21T10:00:00Z.sql.gz": true,
"backup.2026-10-21T10:00:00Z.sql.zst": true,
"backup.2026-10-21T10:00:00Z.zstd.sql": true,
"backup.2026-10-21T10:00:00Z.xz.sql": false,
"backup.2026-10-21T10:00:00Z.sql.gz.tmp": false,
"physicalbackup-20261021100000.xb": false,
"backup.not-a-date.sql": false,
"sub/backup.2026-10-21T10:00:00Z.sql": false,
"backup.2026-10-21T10:00:00Z.a.gzip.sql": false,
"backups.2026-10-21T10:00:00Z.gzip.sql": false,
"backup.2026-10-21T10:00:00Z.gzip.dump": false,
"": false,
}
for name, want := range cases {
if _, got := parseMariaDBLogicalBackupFile(name); got != want {
t.Errorf("parseMariaDBLogicalBackupFile(%q) = %v, want %v", name, got, want)
}
}
}
func TestMariaDBOperatorS3Prefix(t *testing.T) {
cases := map[string]string{"": "", "/": "", "ns/app": "ns/app/", "ns/app/": "ns/app/"}
for in, want := range cases {
if got := mariadbOperatorS3Prefix(in); got != want {
t.Errorf("mariadbOperatorS3Prefix(%q) = %q, want %q", in, got, want)
}
}
}
// operatorBackupStoringTo returns the operator Backup a strategy with s3
// storage renders, created at created in namespace tenant.
func operatorBackupStoringTo(t *testing.T, s3 *strategyv1alpha1.MariaDBS3Template, created time.Time) *mariadbtypes.Backup {
t.Helper()
storage, err := buildMariaDBBackupStorage(strategyv1alpha1.MariaDBStorageTemplate{S3: s3})
if err != nil {
t.Fatal(err)
}
return &mariadbtypes.Backup{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk", CreationTimestamp: metav1.NewTime(created)},
Spec: mariadbtypes.BackupSpec{Storage: storage},
}
}
func TestMariaDBArtifactSizeBytes(t *testing.T) {
created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
completed := created.Add(3 * time.Minute)
s3 := newRenderedMariaDBTemplate().Storage.S3
s3.Prefix = "tenant/db"
mdbBackup := operatorBackupStoringTo(t, s3, created)
const prefix = "tenant/db/"
t.Run("single dump inside the backup window", func(t *testing.T) {
l := &fakeS3Lister{objects: []s3ObjectInfo{
{Key: dumpKey(prefix, created.Add(-24*time.Hour), "gzip"), Size: 111, LastModified: created.Add(-24 * time.Hour)},
{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)},
{Key: prefix + "0-backup-target.txt", Size: 7},
}}
got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed)
if got != 4242 {
t.Errorf("size = %d, want 4242", got)
}
if l.conn.AccessKey != "AKID" || l.conn.Endpoint != "s3.example:9000" {
t.Errorf("connection = %+v, want the Secret's key and the rendered endpoint", l.conn)
}
if l.bucket != "bkt" || l.prefix != prefix || l.recursive {
t.Errorf("listed %q/%q recursive=%v, want bkt/%q non-recursive", l.bucket, l.prefix, l.recursive, prefix)
}
})
t.Run("reads the endpoint CA from its Secret", func(t *testing.T) {
tlsS3 := newRenderedMariaDBTemplate().Storage.S3
tlsS3.Prefix = "tenant/db"
tlsS3.TLS = &strategyv1alpha1.MariaDBS3TLS{
Enabled: true,
CASecretKeyRef: &strategyv1alpha1.MariaDBSecretKeySelector{Name: "ca", Key: "ca.crt"},
}
ca := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"},
Data: map[string][]byte{"ca.crt": []byte("PEM")},
}
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3, LastModified: created.Add(time.Minute)}}}
r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), ca)
if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed); got != 3 {
t.Errorf("size = %d, want 3", got)
}
if !l.conn.Secure || string(l.conn.CAPEM) != "PEM" {
t.Errorf("connection = %+v, want TLS with the Secret's CA", l.conn)
}
})
t.Run("reads the session token from its Secret", func(t *testing.T) {
stsS3 := newRenderedMariaDBTemplate().Storage.S3
stsS3.Prefix = "tenant/db"
stsS3.SessionTokenSecretKeyRef = &strategyv1alpha1.MariaDBSecretKeySelector{Name: "sts", Key: "token"}
sts := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "sts"},
Data: map[string][]byte{"token": []byte("TOKEN")},
}
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4, LastModified: created.Add(time.Minute)}}}
r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), sts)
if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed); got != 4 {
t.Errorf("size = %d, want 4", got)
}
if l.conn.SessionToken != "TOKEN" {
t.Errorf("session token = %q, want the Secret's", l.conn.SessionToken)
}
})
t.Run("pod clock skew within tolerance", func(t *testing.T) {
l := &fakeS3Lister{objects: []s3ObjectInfo{
{Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9, LastModified: created.Add(time.Minute)},
}}
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 9 {
t.Errorf("size = %d, want 9", got)
}
})
t.Run("a skewed pod clock does not attribute a neighbouring run's dump", func(t *testing.T) {
// The pod clock runs 5m behind. This run's dump is named outside the
// window; the previous run's dump, uploaded before this run started,
// is named inside it.
l := &fakeS3Lister{objects: []s3ObjectInfo{
{Key: dumpKey(prefix, created.Add(-4*time.Minute), "gzip"), Size: 7, LastModified: created.Add(time.Minute)},
{Key: dumpKey(prefix, created.Add(30*time.Second), "gzip"), Size: 666, LastModified: created.Add(-4 * time.Minute)},
}}
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
t.Errorf("size = %d, want 0: no dump is inside the window on both clocks", got)
}
})
t.Run("a dump uploaded just after Complete is inside the slack", func(t *testing.T) {
l := &fakeS3Lister{objects: []s3ObjectInfo{
{Key: dumpKey(prefix, completed.Add(30*time.Second), "gzip"), Size: 5, LastModified: completed.Add(30 * time.Second)},
}}
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 5 {
t.Errorf("size = %d, want 5", got)
}
})
t.Run("a hanging endpoint is given up at the probe timeout", func(t *testing.T) {
r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds"))
r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil }
withShortProbeTimeout(t, func() {
if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
t.Errorf("size = %d, want 0", got)
}
})
})
t.Run("two dumps in the window leave size unset", func(t *testing.T) {
l := &fakeS3Lister{objects: []s3ObjectInfo{
{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)},
{Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2, LastModified: created.Add(2 * time.Minute)},
}}
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
t.Errorf("size = %d, want 0 (ambiguous)", got)
}
})
t.Run("no dump in the window leaves size unset", func(t *testing.T) {
l := &fakeS3Lister{objects: []s3ObjectInfo{
{Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1, LastModified: completed.Add(time.Hour)},
}}
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
t.Errorf("size = %d, want 0", got)
}
})
t.Run("list error leaves size unset", func(t *testing.T) {
l := &fakeS3Lister{err: errors.New("AccessDenied")}
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
t.Errorf("size = %d, want 0", got)
}
})
t.Run("missing credentials Secret leaves size unset", func(t *testing.T) {
c := newMariaDBStrategyTestClient(t)
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()}
if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
t.Errorf("size = %d, want 0", got)
}
})
}
// TestMatchMariaDBDump pins what a restore reads off a match: the object
// itself, and errors it can tell apart.
func TestMatchMariaDBDump(t *testing.T) {
created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
completed := created.Add(3 * time.Minute)
const prefix = "tenant/db/"
own := s3ObjectInfo{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(2 * time.Minute)}
run := newMariaDBRun(&mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk"}}, created, completed)
got, err := matchMariaDBDump([]s3ObjectInfo{own}, prefix, run, nil)
if err != nil {
t.Fatalf("matchMariaDBDump: %v", err)
}
want := mariadbDump{
Key: own.Key,
Name: strings.TrimPrefix(own.Key, prefix),
Timestamp: created.Add(time.Minute),
Size: 4242,
LastModified: own.LastModified,
}
if got != want {
t.Errorf("matchMariaDBDump = %+v, want %+v", got, want)
}
if _, err := matchMariaDBDump(nil, prefix, run, nil); !errors.Is(err, errMariaDBDumpNotFound) {
t.Errorf("no dump: err = %v, want errMariaDBDumpNotFound", err)
}
second := s3ObjectInfo{Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 1, LastModified: created.Add(2 * time.Minute)}
if _, err := matchMariaDBDump([]s3ObjectInfo{own, second}, prefix, run, nil); !errors.Is(err, errMariaDBDumpAmbiguous) {
t.Errorf("two dumps: err = %v, want errMariaDBDumpAmbiguous", err)
}
}
// TestFindMariaDBDump pins that a dump is read with nothing but a client,
// as a reconciler other than BackupJobReconciler has, and that an operator
// Backup that does not store to S3 is refused without a listing.
func TestFindMariaDBDump(t *testing.T) {
created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
completed := created.Add(3 * time.Minute)
s3 := newRenderedMariaDBTemplate().Storage.S3
s3.Prefix = "tenant/db"
l := &fakeS3Lister{objects: []s3ObjectInfo{
{Key: dumpKey("tenant/db/", created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)},
}}
store := objectStore{
reader: newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds")),
newLister: func(conn s3Connection) (s3ObjectLister, error) {
l.conn = conn
return l, nil
},
}
got, err := findMariaDBDump(context.Background(), store, operatorBackupStoringTo(t, s3, created), completed)
if err != nil || got.Size != 4242 {
t.Fatalf("findMariaDBDump = %+v, %v; want the 4242-byte dump", got, err)
}
if l.conn.AccessKey != "AKID" {
t.Errorf("connection = %+v, want the Secret's key", l.conn)
}
pvc := &mariadbtypes.Backup{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-pvc", CreationTimestamp: metav1.NewTime(created)},
Spec: mariadbtypes.BackupSpec{Storage: mariadbtypes.BackupStorage{PersistentVolumeClaim: &corev1.PersistentVolumeClaimSpec{}}},
}
if _, err := findMariaDBDump(context.Background(), store, pvc, completed); err == nil {
t.Error("findMariaDBDump on a PVC-backed operator Backup returned no error")
}
if l.calls != 1 {
t.Errorf("S3 listed %d times, want once: a PVC-backed Backup has nothing to list", l.calls)
}
}
// TestFindMariaDBDump_Ownership drives runs of one application that share
// a prefix. This run lives from t0 to t0+40s, so its window with the
// one-minute slack is [t0-1m, t0+1m40s]; its own dump is 123 bytes and a
// neighbour's is 666 or 777. A neighbour's dump must never be returned.
func TestFindMariaDBDump_Ownership(t *testing.T) {
t0 := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
s3 := newRenderedMariaDBTemplate().Storage.S3
s3.Prefix = "tenant/db"
const prefix = "tenant/db/"
run := func(name string, created, completed time.Time) *mariadbtypes.Backup {
b := operatorBackupStoringTo(t, s3, created)
b.Name = name
b.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(completed))}
return b
}
dump := func(named, uploaded time.Time, size int64) s3ObjectInfo {
return s3ObjectInfo{Key: dumpKey(prefix, named, "gzip"), Size: size, LastModified: uploaded}
}
own := run("op-bk", t0, t0.Add(40*time.Second))
find := func(t *testing.T, objects []s3ObjectInfo, operatorBackups ...client.Object) (mariadbDump, error) {
t.Helper()
store := objectStore{
reader: newMariaDBStrategyTestClient(t, append(operatorBackups, own, s3CredsSecret("tenant", "creds"))...),
newLister: func(s3Connection) (s3ObjectLister, error) {
return &fakeS3Lister{objects: objects}, nil
},
}
return findMariaDBDump(context.Background(), store, own, t0.Add(40*time.Second))
}
wantAmbiguous := func(t *testing.T, got mariadbDump, err error) {
t.Helper()
if !errors.Is(err, errMariaDBDumpAmbiguous) {
t.Errorf("findMariaDBDump = %+v, %v; want errMariaDBDumpAmbiguous", got, err)
}
}
wantOwn := func(t *testing.T, got mariadbDump, err error) {
t.Helper()
if err != nil || got.Size != 123 {
t.Errorf("findMariaDBDump = %+v, %v; want this run's 123-byte dump", got, err)
}
}
t.Run("a pod clock behind does not hand this run the previous run's dump", func(t *testing.T) {
// The previous run finished 20s before this one was created and its
// operator Backup is gone. This run's pod clock is 5m behind, so its
// own dump is named outside the window and uploaded inside it.
got, err := find(t, []s3ObjectInfo{
dump(t0.Add(-30*time.Second), t0.Add(-20*time.Second), 666),
dump(t0.Add(5*time.Second-5*time.Minute), t0.Add(30*time.Second), 123),
})
wantAmbiguous(t, got, err)
})
t.Run("a pod clock ahead does not hand this run the next run's dump", func(t *testing.T) {
// The next run started 5s after this one completed and has uploaded
// its dump by the time this one is probed; it has not completed yet.
next := operatorBackupStoringTo(t, s3, t0.Add(45*time.Second))
next.Name = "op-next"
next.Status.Conditions = []metav1.Condition{{
Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse,
Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(45 * time.Second)),
}}
got, err := find(t, []s3ObjectInfo{
dump(t0.Add(5*time.Second+5*time.Minute), t0.Add(30*time.Second), 123),
dump(t0.Add(50*time.Second), t0.Add(80*time.Second), 777),
}, next)
wantAmbiguous(t, got, err)
})
t.Run("an object store clock behind does not hand this run the next run's dump", func(t *testing.T) {
// The store clock is 2m behind: this run's upload is stamped before
// the window opens, the next run's is pulled back into it.
got, err := find(t, []s3ObjectInfo{
dump(t0.Add(5*time.Second), t0.Add(30*time.Second-2*time.Minute), 123),
dump(t0.Add(50*time.Second), t0.Add(3*time.Minute-2*time.Minute), 777),
})
wantAmbiguous(t, got, err)
})
t.Run("a dump another finished run wrote is not taken when this run's own is gone", func(t *testing.T) {
// This run's dump was pruned; the next run's falls in this run's
// window on both clocks, and in its own.
next := run("op-next", t0.Add(45*time.Second), t0.Add(85*time.Second))
got, err := find(t, []s3ObjectInfo{
dump(t0.Add(50*time.Second), t0.Add(75*time.Second), 777),
}, next)
wantAmbiguous(t, got, err)
})
t.Run("a dump the window holds on one clock only is not taken", func(t *testing.T) {
// This run's dump is gone; the previous run's operator Backup is too,
// and its dump, started 2m before this run, finished uploading 20s
// before it.
got, err := find(t, []s3ObjectInfo{
dump(t0.Add(-2*time.Minute), t0.Add(-20*time.Second), 666),
})
if err == nil {
t.Errorf("findMariaDBDump = %+v; want no dump attributed", got)
}
})
t.Run("a run still in progress does not contest this run's dump", func(t *testing.T) {
// A concurrent run created 30s before this one has not uploaded yet.
running := operatorBackupStoringTo(t, s3, t0.Add(-30*time.Second))
running.Name = "op-running"
running.Status.Conditions = []metav1.Condition{{
Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse,
Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(-30 * time.Second)),
}}
got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, running)
wantOwn(t, got, err)
})
t.Run("runs minutes apart each keep their own dump", func(t *testing.T) {
previous := run("op-previous", t0.Add(-4*time.Minute), t0.Add(-3*time.Minute))
next := run("op-next", t0.Add(3*time.Minute), t0.Add(4*time.Minute))
got, err := find(t, []s3ObjectInfo{
dump(t0.Add(-230*time.Second), t0.Add(-190*time.Second), 666),
dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123),
dump(t0.Add(185*time.Second), t0.Add(220*time.Second), 777),
}, previous, next)
wantOwn(t, got, err)
})
t.Run("a run that writes elsewhere does not contest this run's dump", func(t *testing.T) {
other := newRenderedMariaDBTemplate().Storage.S3
other.Prefix = "tenant/other"
elsewhere := operatorBackupStoringTo(t, other, t0)
elsewhere.Name = "op-elsewhere"
elsewhere.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(t0.Add(40 * time.Second)))}
got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, elsewhere)
wantOwn(t, got, err)
})
t.Run("a run stuck short of Complete does not contest later runs", func(t *testing.T) {
// A backup Job whose pod never starts leaves its operator Backup
// JobRunning for good.
stuck := operatorBackupStoringTo(t, s3, t0.Add(-3*time.Hour))
stuck.Name = "op-stuck"
stuck.Status.Conditions = []metav1.Condition{{
Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse,
Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(-3 * time.Hour)),
}}
got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, stuck)
wantOwn(t, got, err)
})
}
// TestMariaDBArtifactSizeBytes_MinioClient drives the real minio client
// against a stub ListObjectsV2 endpoint, so the Secret wiring, prefix and
// response parsing are exercised together.
func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) {
created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
key := dumpKey("tenant/db/", created.Add(time.Minute), "gzip")
var gotPath, gotPrefix, gotDelimiter, gotAuth string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization")
gotDelimiter = req.URL.Query().Get("delimiter")
w.Header().Set("Content-Type", "application/xml")
fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>bkt</Name><Prefix>tenant/db/</Prefix><KeyCount>1</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>
<Contents><Key>%s</Key><Size>31337</Size><ETag>"x"</ETag><LastModified>2026-10-21T10:01:00.000Z</LastModified></Contents>
<CommonPrefixes><Prefix>tenant/db/srv/</Prefix></CommonPrefixes>
</ListBucketResult>`, key)
}))
defer srv.Close()
c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds"))
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()}
s3 := newRenderedMariaDBTemplate().Storage.S3
s3.Endpoint = strings.TrimPrefix(srv.URL, "http://")
s3.Prefix = "tenant/db/"
s3.Region = "us-east-1"
if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute)); got != 31337 {
t.Fatalf("size = %d, want 31337", got)
}
if gotPath != "/bkt/" || gotPrefix != "tenant/db/" || gotDelimiter != "/" {
t.Errorf("request path=%q prefix=%q delimiter=%q, want /bkt/, tenant/db/ and a non-recursive listing", gotPath, gotPrefix, gotDelimiter)
}
if !strings.Contains(gotAuth, "Credential=AKID/") {
t.Errorf("request not signed with the Secret's access key: %q", gotAuth)
}
}
func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) {
apps := mariadbapp.GroupName
completed := metav1.NewTime(time.Date(2026, 10, 21, 10, 3, 0, 0, time.UTC))
rendered := newRenderedMariaDBTemplate()
rendered.Storage.S3.Prefix = "src/"
mdbBackup := operatorBackupStoringTo(t, rendered.Storage.S3, completed.Add(-3*time.Minute))
mdbBackup.Name = "src-op-bk"
mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)}
resolved := &ResolvedBackupConfig{
StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"},
Parameters: map[string]string{},
}
job := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-size"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
},
}
r := reconcilerWithLister(t, &fakeS3Lister{objects: []s3ObjectInfo{
{Key: dumpKey("src/", completed.Add(-time.Minute), "gzip"), Size: 2048, LastModified: completed.Add(-time.Minute)},
}}, job, s3CredsSecret("tenant", "creds"))
artefact, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, rendered)
if err != nil {
t.Fatalf("createMariaDBBackupArtifact: %v", err)
}
if artefact.Status.Artifact == nil || artefact.Status.Artifact.SizeBytes != 2048 {
t.Fatalf("Artifact = %#v, want SizeBytes 2048", artefact.Status.Artifact)
}
}
// TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup pins that the
// probe reads where the operator Backup wrote its dump, not where the
// strategy, edited while the backup ran, now points.
func TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup(t *testing.T) {
apps := mariadbapp.GroupName
completed := metav1.NewTime(time.Date(2026, 10, 21, 10, 3, 0, 0, time.UTC))
original := newRenderedMariaDBTemplate().Storage.S3
original.Prefix = "original/"
mdbBackup := operatorBackupStoringTo(t, original, completed.Add(-3*time.Minute))
mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)}
edited := newRenderedMariaDBTemplate()
edited.Storage.S3.Bucket = "replacement-bkt"
edited.Storage.S3.Endpoint = "replacement.example:9000"
edited.Storage.S3.Prefix = "replacement/"
job := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-edited"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
},
}
l := &fakeS3Lister{objects: []s3ObjectInfo{
{Key: dumpKey("original/", completed.Add(-time.Minute), "gzip"), Size: 2048, LastModified: completed.Add(-time.Minute)},
}}
r := reconcilerWithLister(t, l, job, s3CredsSecret("tenant", "creds"))
resolved := &ResolvedBackupConfig{
StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"},
Parameters: map[string]string{},
}
artefact, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, edited)
if err != nil {
t.Fatalf("createMariaDBBackupArtifact: %v", err)
}
if l.bucket != "bkt" || l.prefix != "original/" || l.conn.Endpoint != "s3.example:9000" {
t.Errorf("listed %s/%s at %s, want bkt/original/ at s3.example:9000, where the operator Backup wrote", l.bucket, l.prefix, l.conn.Endpoint)
}
if artefact.Status.Artifact == nil || artefact.Status.Artifact.SizeBytes != 2048 {
t.Errorf("Artifact = %#v, want SizeBytes 2048", artefact.Status.Artifact)
}
}
func TestCreateMariaDBBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) {
apps := mariadbapp.GroupName
job := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
},
}
existing := &backupsv1alpha1.Backup{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"},
Status: backupsv1alpha1.BackupStatus{Artifact: &backupsv1alpha1.BackupArtifact{URI: "s3://bkt/src/op-bk", SizeBytes: 99}},
}
l := &fakeS3Lister{}
r := reconcilerWithLister(t, l, job, existing, s3CredsSecret("tenant", "creds"))
rendered := newRenderedMariaDBTemplate()
rendered.Storage.S3.Prefix = "src/"
resolved := &ResolvedBackupConfig{
StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"},
Parameters: map[string]string{},
}
got, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk"}}, rendered)
if err != nil {
t.Fatalf("createMariaDBBackupArtifact: %v", err)
}
if l.calls != 0 {
t.Errorf("S3 listed %d times for a Backup that already exists", l.calls)
}
if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 99 {
t.Errorf("Artifact = %#v, want the existing one", got.Status.Artifact)
}
}
func TestCreateMariaDBBackupArtifact_TransientRetryReadStillCreates(t *testing.T) {
apps := mariadbapp.GroupName
job := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-transient"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
},
}
l := &fakeS3Lister{}
r := reconcilerWithLister(t, l, job, s3CredsSecret("tenant", "creds"))
r.APIReader = failingReads(t, newMariaDBStrategyTestClient(t))
got, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, newRenderedMariaDBTemplate())
if err != nil || got == nil || got.Name != "bj-transient" {
t.Fatalf("createMariaDBBackupArtifact = %v, %v; a transient read must not fail the backup", got, err)
}
if err := r.Get(context.Background(), client.ObjectKeyFromObject(got), &backupsv1alpha1.Backup{}); err != nil {
t.Errorf("the Backup was not created: %v", err)
}
if l.calls != 0 {
t.Errorf("S3 probed %d times after the existence check failed", l.calls)
}
}
func TestCreateMariaDBBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testing.T) {
apps := mariadbapp.GroupName
job := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
},
}
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}}
store := newMariaDBStrategyTestClient(t, job, backup)
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store), Scheme: store.Scheme()}
got, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, &strategyv1alpha1.MariaDBTemplate{})
if err != nil || got == nil || got.Name != "bj-race" {
t.Fatalf("createMariaDBBackupArtifact = %v, %v; want the Backup that already exists", got, err)
}
}
func TestCreateMariaDBBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T) {
apps := mariadbapp.GroupName
job := &backupsv1alpha1.BackupJob{
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"},
Spec: backupsv1alpha1.BackupJobSpec{
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
},
}
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}}
store := newMariaDBStrategyTestClient(t, job, backup)
r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store), Scheme: store.Scheme()}
_, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, &strategyv1alpha1.MariaDBTemplate{})
if !errors.Is(err, errBackupNotReadBack) {
t.Errorf("err = %v, want errBackupNotReadBack", err)
}
}
// TestReconcileMariaDB_UnreadableBackupIsRetried drives a completed backup
// through the reconcile: when the Backup exists but cannot be read back,
// the reconcile is retried and the BackupJob is not failed.
func TestReconcileMariaDB_UnreadableBackupIsRetried(t *testing.T) {
strategyGroup := strategyv1alpha1.GroupVersion.Group
now := metav1.Now()
bj := newMariaDBBackupJob("bj-unreadable", "tenant")
bj.Status.StartedAt = &now
bj.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
strategy := &strategyv1alpha1.MariaDB{
ObjectMeta: metav1.ObjectMeta{Name: "mdb-strategy"},
Spec: strategyv1alpha1.MariaDBSpec{Template: *newRenderedMariaDBTemplate()},
}
operatorMariaDB := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("mariadb-src")}}
operatorBackup := &mariadbtypes.Backup{
ObjectMeta: metav1.ObjectMeta{
Namespace: "tenant", Name: "op-bk",
Labels: map[string]string{
backupsv1alpha1.OwningJobNameLabel: bj.Name,
backupsv1alpha1.OwningJobNamespaceLabel: bj.Namespace,
},
},
Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{mariadbBackupSucceeded(now)}},
}
backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: bj.Name}}
store := newMariaDBStrategyTestClient(t, bj, strategy, newMariaDBApp("mariadb-src", "tenant"), operatorMariaDB, operatorBackup, backup)
r := &BackupJobReconciler{
Client: store,
APIReader: failingReads(t, store),
Scheme: store.Scheme(),
Recorder: record.NewFakeRecorder(10),
}
resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{
APIGroup: &strategyGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: strategy.Name,
}}
_, err := r.reconcileMariaDB(context.Background(), bj, resolved)
if !errors.Is(err, errBackupNotReadBack) {
t.Fatalf("reconcileMariaDB err = %v, want errBackupNotReadBack so the reconcile is retried", err)
}
persisted := &backupsv1alpha1.BackupJob{}
if err := store.Get(context.Background(), client.ObjectKeyFromObject(bj), persisted); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed {
t.Error("the BackupJob was failed although its Backup exists")
}
}
// TestReconcileMariaDB_RejectedCreateFailsTheJob pins that only the
// read-back is retried: a Create the apiserver rejects still fails the
// BackupJob.
func TestReconcileMariaDB_RejectedCreateFailsTheJob(t *testing.T) {
strategyGroup := strategyv1alpha1.GroupVersion.Group
now := metav1.Now()
bj := newMariaDBBackupJob("bj-rejected", "tenant")
bj.Status.StartedAt = &now
bj.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning
strategy := &strategyv1alpha1.MariaDB{
ObjectMeta: metav1.ObjectMeta{Name: "mdb-strategy"},
Spec: strategyv1alpha1.MariaDBSpec{Template: *newRenderedMariaDBTemplate()},
}
operatorMariaDB := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("mariadb-src")}}
operatorBackup := &mariadbtypes.Backup{
ObjectMeta: metav1.ObjectMeta{
Namespace: "tenant", Name: "op-bk",
Labels: map[string]string{
backupsv1alpha1.OwningJobNameLabel: bj.Name,
backupsv1alpha1.OwningJobNamespaceLabel: bj.Namespace,
},
},
Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{mariadbBackupSucceeded(now)}},
}
store := newMariaDBStrategyTestClient(t, bj, strategy, newMariaDBApp("mariadb-src", "tenant"), operatorMariaDB, operatorBackup)
r := &BackupJobReconciler{
Client: forbidBackupCreate(t, store),
Scheme: store.Scheme(),
Recorder: record.NewFakeRecorder(10),
}
resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{
APIGroup: &strategyGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: strategy.Name,
}}
if _, err := r.reconcileMariaDB(context.Background(), bj, resolved); err != nil {
t.Fatalf("reconcileMariaDB err = %v, want the BackupJob failed rather than retried", err)
}
persisted := &backupsv1alpha1.BackupJob{}
if err := store.Get(context.Background(), client.ObjectKeyFromObject(bj), persisted); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if persisted.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed {
t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase)
}
}
@@ -214,7 +214,7 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1
}
artifact, err := r.createMariaDBBackupArtifact(ctx, j, resolved, mdbBackup, rendered)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
return r.failBackupArtifact(ctx, j, err)
}
now := metav1.Now()
j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name}
@@ -365,6 +365,10 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact(
mdbBackup *mariadbtypes.Backup,
rendered *strategyv1alpha1.MariaDBTemplate,
) (*backupsv1alpha1.Backup, error) {
existing, probe := r.existingBackup(ctx, j.Namespace, j.Name)
if existing != nil {
return existing, nil
}
takenAt := metav1.Now()
if cond := apimeta.FindStatusCondition(mdbBackup.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && !cond.LastTransitionTime.IsZero() {
takenAt = cond.LastTransitionTime
@@ -393,6 +397,9 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact(
// no Artifact at all - leave it nil instead.
if uri := driverMD[mariadbStorageURIKey]; uri != "" {
status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri}
if probe {
status.Artifact.SizeBytes = r.mariadbArtifactSizeBytes(ctx, mdbBackup, takenAt.Time)
}
}
backup := &backupsv1alpha1.Backup{
ObjectMeta: metav1.ObjectMeta{
@@ -415,8 +422,8 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact(
return nil, err
}
existing := &backupsv1alpha1.Backup{}
if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
return nil, getErr
if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil {
return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr)
}
return existing, nil
}
@@ -558,7 +558,9 @@ func (r *BackupJobReconciler) createBackupResource(ctx context.Context, backupJo
veleroBackupNamespaceMetadataKey: veleroBackup.Namespace,
}
// Create a basic artifact referencing the Velero backup
// SizeBytes stays unset: data lands in a content-addressed kopia
// repository shared across backups, so there is no per-backup stored
// size, and DataUpload totalBytes counts bytes read, not bytes stored.
artifact := &backupsv1alpha1.BackupArtifact{
URI: fmt.Sprintf("velero://%s/%s", veleroBackup.Namespace, veleroBackup.Name),
}
@@ -0,0 +1,42 @@
{{- /*
The ClusterRoleBindings the OIDC bootstrap Job (oidc-rbac-job.yaml) applies
inside the tenant cluster, one per `oidc.users` entry, and the list of their
names it prunes against. They travel in a ConfigMap the Job mounts and not as
text inside its script: the script reaches the container as a single `sh -c`
argument, Linux caps one argument at 128 KiB, and a binding per user crosses
that cap at about 240 users, after which the container cannot start.
A release-owned object, not a hook resource, so it exists before the
post-install and post-upgrade hooks run. It is rendered only where users are
bound; at `mode: None` the Job takes its early branch and the volume is
optional.
*/}}
{{- if ne .Values.oidc.mode "None" }}
{{- $desired := "" }}
{{- $bindings := "" }}
{{- range .Values.oidc.users }}
{{- $name := printf "cozystack-oidc-%s" (printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16) }}
{{- $role := "view" }}
{{- if eq .role "admin" }}
{{- $role = "cluster-admin" }}
{{- end }}
{{- $desired = printf "%s%s\n" $desired $name }}
{{- $binding := dict
"apiVersion" "rbac.authorization.k8s.io/v1"
"kind" "ClusterRoleBinding"
"metadata" (dict "name" $name "labels" (dict "app.kubernetes.io/managed-by" "cozystack-oidc" "app.kubernetes.io/instance" $.Release.Name))
"roleRef" (dict "apiGroup" "rbac.authorization.k8s.io" "kind" "ClusterRole" "name" $role)
"subjects" (list (dict "kind" "User" "apiGroup" "rbac.authorization.k8s.io" "name" .email)) }}
{{- $bindings = printf "%s---\n%s\n" $bindings (toYaml $binding) }}
{{- end }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .Release.Name }}-oidc-bindings
namespace: {{ .Release.Namespace }}
labels:
{{- include "kubernetes.labels" . | nindent 4 }}
data:
{{- dict "crbs.yaml" $bindings "desired.txt" $desired | toYaml | nindent 2 }}
{{- end }}
@@ -25,7 +25,9 @@
- `mode: System | CustomConfig`:
- Apply one ClusterRoleBinding per `users[]` entry inside the tenant
cluster (`admin` → `cluster-admin`, `view` → `view`, subject = the
literal email matched against the `email` claim).
literal email matched against the `email` claim). The bindings are
rendered by oidc-rbac-bindings.yaml into a ConfigMap this Job mounts,
not into the script below.
- Prune stale CRBs labelled by this release.
- `mode: System`: patch the pre-rendered `<release>-oidc-kubeconfig`
Secret with a real kubeconfig wired to `kubectl oidc-login`
@@ -168,6 +170,8 @@ spec:
value: {{ $issuer | quote }}
- name: OIDC_SERVER_URL
value: {{ printf "https://%s:443" $externalHost | quote }}
- name: OIDC_BINDINGS_DIR
value: /etc/oidc-bindings
resources:
requests:
cpu: 10m
@@ -254,46 +258,32 @@ spec:
sleep 5
done
# A missing list must stop the Job here: grep on a file that is not
# there exits 2, `! grep` is then true, and the prune below would
# delete every binding of the release.
BINDINGS=$OIDC_BINDINGS_DIR
if [ ! -f "$BINDINGS/desired.txt" ]; then
echo "$BINDINGS/desired.txt is not mounted (ConfigMap $RELEASE-oidc-bindings); refusing to prune against a list that is not there" >&2
exit 1
fi
echo "Applying ${OIDC_MODE} ClusterRoleBindings to the tenant cluster..."
DESIRED_FILE=$(mktemp)
{{- range .Values.oidc.users }}
{{- $hash := printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16 }}
{{- $crbName := printf "cozystack-oidc-%s" $hash }}
{{- $role := "view" }}
{{- if eq .role "admin" }}
{{- $role = "cluster-admin" }}
{{- end }}
echo "{{ $crbName }}" >> "$DESIRED_FILE"
cat <<'EOF' | kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ $crbName }}
labels:
app.kubernetes.io/managed-by: cozystack-oidc
app.kubernetes.io/instance: {{ $.Release.Name }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: {{ $role }}
subjects:
- kind: User
apiGroup: rbac.authorization.k8s.io
name: {{ .email | quote }}
EOF
{{- end }}
# kubectl apply fails on a file with no objects, and a release with
# no users has none.
if [ -s "$BINDINGS/crbs.yaml" ]; then
kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename "$BINDINGS/crbs.yaml"
fi
echo "Pruning stale ClusterRoleBindings labelled by this release..."
EXISTING=$(kubectl --kubeconfig "$KUBECONFIG_TENANT" get clusterrolebindings \
--selector=app.kubernetes.io/managed-by=cozystack-oidc,app.kubernetes.io/instance=$RELEASE \
--output=jsonpath='{.items[*].metadata.name}')
for crb in $EXISTING; do
if ! grep -x -q -F "$crb" "$DESIRED_FILE"; then
if ! grep -x -q -F "$crb" "$BINDINGS/desired.txt"; then
echo " removing $crb"
kubectl --kubeconfig "$KUBECONFIG_TENANT" delete clusterrolebinding "$crb" --ignore-not-found
fi
done
rm -f "$DESIRED_FILE"
if [ "$OIDC_MODE" = "System" ]; then
echo "Patching the OIDC kubeconfig Secret on the management cluster..."
@@ -349,6 +339,9 @@ spec:
- name: kubeconfig
mountPath: /etc/kubernetes/kubeconfig
readOnly: true
- name: oidc-bindings
mountPath: /etc/oidc-bindings
readOnly: true
- name: tmp
mountPath: /tmp
volumes:
@@ -356,6 +349,10 @@ spec:
secret:
secretName: {{ .Release.Name }}-admin-kubeconfig
optional: true
- name: oidc-bindings
configMap:
name: {{ .Release.Name }}-oidc-bindings
optional: true
- name: tmp
emptyDir: {}
{{- /*
+183 -10
View File
@@ -2561,7 +2561,73 @@ tests:
value: "true"
documentIndex: 3
- it: bootstrap Job command builds a CRB for each users[] entry
- it: the bindings ConfigMap holds a CRB for each users[] entry
release:
name: kubernetes-app
namespace: tenant-foo
template: templates/oidc-rbac-bindings.yaml
set:
_cluster.oidc-enabled: "true"
_cluster.root-host: example.org
oidc.mode: System
oidc.users:
- email: alice@example.org
role: admin
- email: bob@example.org
role: view
asserts:
- isKind:
of: ConfigMap
- equal:
path: metadata.name
value: kubernetes-app-oidc-bindings
- matchRegex:
path: data["crbs.yaml"]
pattern: 'name: alice@example.org'
- matchRegex:
path: data["crbs.yaml"]
pattern: 'name: bob@example.org'
- matchRegex:
path: data["crbs.yaml"]
pattern: 'app.kubernetes.io/managed-by: cozystack-oidc'
- matchRegex:
path: data["crbs.yaml"]
pattern: 'app.kubernetes.io/instance: kubernetes-app'
- it: the binding names and roles stay what the Job script used to render
# The prune step and every binding already in a tenant cluster go by these
# names: cozystack-oidc- plus the first 16 hex digits of
# sha256("<release>-<email>"). A different name leaves the old binding
# behind next to the new one.
release:
name: kubernetes-app
namespace: tenant-foo
template: templates/oidc-rbac-bindings.yaml
set:
_cluster.oidc-enabled: "true"
_cluster.root-host: example.org
oidc.mode: System
oidc.users:
- email: alice@example.org
role: admin
- email: bob@example.org
role: view
asserts:
- equal:
path: data["desired.txt"]
value: "cozystack-oidc-0001a28136ca1a02\ncozystack-oidc-ec8dfb1cbc75e699\n"
- matchRegex:
path: data["crbs.yaml"]
pattern: 'name: cozystack-oidc-0001a28136ca1a02\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cluster-admin\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: User\n name: alice@example.org\n'
- matchRegex:
path: data["crbs.yaml"]
pattern: 'name: cozystack-oidc-ec8dfb1cbc75e699\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: view\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: User\n name: bob@example.org\n'
- it: the bootstrap Job mounts the bindings ConfigMap and carries no users[] entry in its script
# The script is one `sh -c` argument and Linux caps one argument at 128 KiB:
# a binding per user in it stops the container from starting at about 240
# users. hack/kubernetes-oidc-bindings.bats renders 300 users to hold the
# size itself.
release:
name: kubernetes-app
namespace: tenant-foo
@@ -2575,23 +2641,130 @@ tests:
role: admin
- email: bob@example.org
role: view
asserts:
- contains:
path: spec.template.spec.volumes
content:
name: oidc-bindings
configMap:
name: kubernetes-app-oidc-bindings
optional: true
documentIndex: 3
- contains:
path: spec.template.spec.containers[0].volumeMounts
content:
name: oidc-bindings
mountPath: /etc/oidc-bindings
readOnly: true
documentIndex: 3
# The script reads the directory from the environment, which is also how
# hack/kubernetes-oidc-bindings.bats points it at a scratch directory.
- contains:
path: spec.template.spec.containers[0].env
content:
name: OIDC_BINDINGS_DIR
value: /etc/oidc-bindings
documentIndex: 3
- notMatchRegex:
path: spec.template.spec.containers[0].command[2]
pattern: 'alice@example\.org'
documentIndex: 3
- notMatchRegex:
path: spec.template.spec.containers[0].command[2]
pattern: 'bob@example\.org'
documentIndex: 3
- it: the bootstrap Job stops before pruning when the desired list is not mounted
# `! grep -F` on a file that is not there is true, so a missing mount would
# delete every binding of the release.
release:
name: kubernetes-app
namespace: tenant-foo
template: templates/oidc-rbac-job.yaml
set:
_cluster.oidc-enabled: "true"
_cluster.root-host: example.org
oidc.mode: System
asserts:
- matchRegex:
path: spec.template.spec.containers[0].command[2]
pattern: 'name: "alice@example.org"'
pattern: '(?s)if \[ ! -f "\$BINDINGS/desired\.txt" \]; then.*?exit 1\n\s*fi.*Pruning stale ClusterRoleBindings'
documentIndex: 3
- it: the bootstrap Job applies the bindings file only when it has objects in it
# kubectl apply exits non-zero on a file with no objects, and a release
# with no users has none.
release:
name: kubernetes-app
namespace: tenant-foo
template: templates/oidc-rbac-job.yaml
set:
_cluster.oidc-enabled: "true"
_cluster.root-host: example.org
oidc.mode: System
asserts:
- matchRegex:
path: spec.template.spec.containers[0].command[2]
pattern: 'name: "bob@example.org"'
pattern: 'if \[ -s "\$BINDINGS/crbs\.yaml" \]; then\n\s+kubectl --kubeconfig "\$KUBECONFIG_TENANT" apply --filename "\$BINDINGS/crbs\.yaml"\n\s*fi'
documentIndex: 3
- it: a release with no users renders an empty bindings ConfigMap
release:
name: kubernetes-app
namespace: tenant-foo
template: templates/oidc-rbac-bindings.yaml
set:
_cluster.oidc-enabled: "true"
_cluster.root-host: example.org
oidc.mode: System
oidc.users: []
asserts:
- equal:
path: data["crbs.yaml"]
value: ""
- equal:
path: data["desired.txt"]
value: ""
- it: CustomConfig mode renders the bindings ConfigMap for its users
release:
name: kubernetes-app
namespace: tenant-foo
template: templates/oidc-rbac-bindings.yaml
set:
_cluster.oidc-enabled: "false"
oidc.mode: CustomConfig
oidc.customConfig.config: |
apiVersion: apiserver.config.k8s.io/v1beta1
kind: AuthenticationConfiguration
jwt:
- issuer:
url: https://idp.example.com
audiences: [my-cluster]
oidc.users:
- email: alice@example.org
role: admin
asserts:
- isKind:
of: ConfigMap
- matchRegex:
path: spec.template.spec.containers[0].command[2]
pattern: 'name: cluster-admin'
documentIndex: 3
- matchRegex:
path: spec.template.spec.containers[0].command[2]
pattern: 'name: view'
documentIndex: 3
path: data["crbs.yaml"]
pattern: 'name: alice@example.org'
- it: mode=None renders no bindings ConfigMap even when users are set
# The Job takes its early branch at mode None and never reads the file.
release:
name: kubernetes-app
namespace: tenant-foo
template: templates/oidc-rbac-bindings.yaml
set:
oidc.mode: None
oidc.users:
- email: alice@example.org
role: admin
asserts:
- hasDocuments:
count: 0
- it: mode=System aborts the Job instead of leaving the OIDC kubeconfig Secret as the install-time placeholder when the tenant CA is missing
# A placeholder kubeconfig cannot authenticate, so the release must not go Ready with it.
@@ -131,8 +131,15 @@ spec:
For example: "sha256:<hex>".
type: string
sizeBytes:
description: SizeBytes is the size of the artifact in bytes, if
known.
description: |-
SizeBytes is the stored size in bytes of the files this backup owns,
if known: the dump of a MariaDB backup, for example, or the base backup
of a CNPG one, which leaves out the WAL archive the server's backups
share and a point-in-time restore also reads. Strategies that cannot
attribute stored bytes to a single backup, such as Velero's, whose
repository deduplicates across backups, leave it unset. Drivers never
write 0 for a size they cannot measure, so an absent value means
"unknown", not "empty".
format: int64
type: integer
uri:
@@ -3,8 +3,8 @@
1. Helm renders this file at chart install time. Anything inside
`{{ ... }}` is interpreted by Helm.
2. The strategy controller renders the resulting Strategy CR
per-BackupJob, expanding `{{ .Application | .Release | .Mode }}`
against the live application object.
per-BackupJob, expanding `{{ .Application | .Release | .Mode |
.ArchiveName | .Backup }}` against the live application object.
To emit a literal "{{ .Foo }}" that survives Helm, we pass it as the
string argument of `printf` — Helm does not interpret inner template
delimiters inside printf's quoted strings. Do NOT simplify by inlining
@@ -51,6 +51,11 @@ spec:
host="chi-clickhouse-{{ printf "{{ .Release.Name }}" }}-clickhouse-0-0.{{ printf "{{ .Release.Namespace }}" }}.svc"
api="http://${host}:7171"
auth="-u ${API_USERNAME}:${API_PASSWORD}"
# The archive the controller named for this BackupJob and records
# on its Backup. The default renders it empty, rather than as
# text/template's missing-key marker, when a controller that
# predates archive names renders this template during an upgrade.
archive="{{ printf `{{ .ArchiveName | default "" }}` }}"
# Every request but the POST is retried on failure, every 5s,
# for $2 attempts in all: once the operation is accepted, a
# dropped poll must not fail the attempt, since the Job retry it
@@ -91,23 +96,47 @@ spec:
}
if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then
prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-"
# The name is new on every attempt, so a retry never takes over
# the create_remote of the attempt it replaces; that needs a
# name fixed for the whole BackupJob.
target="${prefix}$(date -u +%Y%m%dT%H%M%SZ)"
cmd="create_remote ${target}"
# Only a controller that predates archive names leaves it
# empty, and that one records no name on the Backup either.
target="${archive:-${prefix}$(date -u +%Y%m%dT%H%M%SZ)}"
# A retried Pod gets the same name, so it takes over an attempt
# the sidecar is still running, and otherwise asks for resume,
# so a local or partial remote backup an earlier attempt left is
# resumed rather than refused as existing, whatever
# use_resumable_state the sidecar runs with. The sidecar logs
# the request as "create_remote --resume <name>".
cmd="create_remote --resume ${target}"
wait_turn "${cmd}"
echo "${cmd}"
curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}" >/dev/null
if [ "${turn}" = "join" ]; then
echo "${cmd} is already running; waiting for it"
else
echo "${cmd}"
curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}&resume=true" >/dev/null
fi
else
prefix="clickhouse-{{ printf "{{ .Backup.ApplicationRef.Name }}" }}-"
listing=$(api_get /backup/list/remote 60)
target=$(printf '%s\n' "${listing}" \
| jq -rs --arg prefix "${prefix}" \
'[.[] | select((.name // "") | startswith($prefix))] | sort_by(.created // "") | last.name // ""')
if [ -z "${target}" ] || [ "${target}" = "null" ]; then
echo "no remote backup found" >&2
exit 1
app="{{ printf "{{ .Backup.ApplicationRef.Name }}" }}"
backup="{{ printf "{{ .Backup.Name }}" }}"
prefix="clickhouse-${app}-"
remote=$(api_get /backup/list/remote 60)
if [ -n "${archive}" ]; then
if ! printf '%s\n' "${remote}" | jq -es --arg name "${archive}" 'any(.[]; .name == $name)' >/dev/null; then
echo "archive ${archive} recorded on Backup ${backup} is not in remote storage; not restoring any other archive" >&2
exit 1
fi
target="${archive}"
else
# takenAt is set after the backup Job finished, so the
# archive that Job wrote is stamped no later than it.
notafter="{{ printf `{{ .Backup.TakenAt | default "" }}` }}"
echo "WARNING: Backup ${backup} records no archive name; restoring the newest archive of ${app} stamped no later than its takenAt (${notafter}), a best-effort choice that may not be the archive this Backup was taken as" >&2
target=$(printf '%s\n' "${remote}" | jq -rs --arg prefix "${prefix}" --arg notafter "${notafter}" '
[.[] | select((.desc // "") | startswith("broken") | not) | .name // ""
| select(startswith($prefix) and (ltrimstr($prefix) | test("^[0-9]{8}T[0-9]{6}Z$")) and ltrimstr($prefix) <= $notafter)]
| sort | last // ""')
if [ -z "${target}" ]; then
echo "no archive of ${app} stamped no later than ${notafter} in remote storage" >&2
exit 1
fi
fi
# restore_remote drops every table it restores before
# recreating it, so a second run discards whatever was written
@@ -240,10 +240,12 @@ exclude_from_kube_ovn() {
fi
# A pod that holds a node's current Ingress IP keeps the address after the
# move, and that node keeps it in its ipcache as reserved:ingress, so the pod
# stays unreachable from there until it is recreated.
jq -r --argjson cn "$ciliumnodes" '
([$cn.items[] | select(.spec.ingress.ipv4) | {key: .spec.ingress.ipv4, value: .metadata.name}] | from_entries) as $ingress
| .[] | select($ingress[.ip]) | "\(.name) \(.ip) \($ingress[.ip])"' <<<"$ipobjects" \
# stays unreachable from there until it is recreated. The CiliumNode list goes
# in on stdin, because the Node labels Cilium copies onto every CiliumNode take
# it past the 128 KiB a single argument may hold within a handful of nodes.
jq -rn 'input as $cn
| ([$cn.items[] | select(.spec.ingress.ipv4) | {key: .spec.ingress.ipv4, value: .metadata.name}] | from_entries) as $ingress
| input[] | select($ingress[.ip]) | "\(.name) \(.ip) \($ingress[.ip])"' <<<"$ciliumnodes"$'\n'"$ipobjects" \
| while read -r ipname ip node; do
if ! in_any "$POOL" "$ip"; then
log "$ipname holds $ip, the Ingress IP of $node before the move. Recreate it once the run is done, $node keeps that address as reserved:ingress"
+17
View File
@@ -55,6 +55,23 @@ type ApplicationStatus struct {
// ExternalIPsCount holds the number of LoadBalancer services with assigned external IPs for Tenant applications.
// +optional
ExternalIPsCount int32 `json:"externalIPsCount,omitempty"`
// InstallFailures is the install failure count of the underlying HelmRelease.
// It counts only attempts that recorded a Helm release, so a failure before
// Helm records one, such as a chart that fails to render, shows only as
// Ready=False; a failure to apply the rendered resources is counted. It
// resets when the desired state (generation, chart version, values) changes,
// and on success only under the RetryOnFailure strategy, so a Ready release
// may carry it.
// +optional
InstallFailures int64 `json:"installFailures,omitempty"`
// UpgradeFailures is the upgrade failure count of the underlying HelmRelease,
// with the same counting and reset rules as InstallFailures.
// +optional
UpgradeFailures int64 `json:"upgradeFailures,omitempty"`
// LastAttemptedReleaseAction is the last release action (install or upgrade)
// the underlying HelmRelease attempted.
// +optional
LastAttemptedReleaseAction string `json:"lastAttemptedReleaseAction,omitempty"`
}
// SchedulingClass returns the scheduling class requested by this Application.
@@ -34,6 +34,20 @@ type ApplicationStatusApplyConfiguration struct {
Namespace *string `json:"namespace,omitempty"`
// ExternalIPsCount holds the number of LoadBalancer services with assigned external IPs for Tenant applications.
ExternalIPsCount *int32 `json:"externalIPsCount,omitempty"`
// InstallFailures is the install failure count of the underlying HelmRelease.
// It counts only attempts that recorded a Helm release, so a failure before
// Helm records one, such as a chart that fails to render, shows only as
// Ready=False; a failure to apply the rendered resources is counted. It
// resets when the desired state (generation, chart version, values) changes,
// and on success only under the RetryOnFailure strategy, so a Ready release
// may carry it.
InstallFailures *int64 `json:"installFailures,omitempty"`
// UpgradeFailures is the upgrade failure count of the underlying HelmRelease,
// with the same counting and reset rules as InstallFailures.
UpgradeFailures *int64 `json:"upgradeFailures,omitempty"`
// LastAttemptedReleaseAction is the last release action (install or upgrade)
// the underlying HelmRelease attempted.
LastAttemptedReleaseAction *string `json:"lastAttemptedReleaseAction,omitempty"`
}
// ApplicationStatusApplyConfiguration constructs a declarative configuration of the ApplicationStatus type for use with
@@ -78,3 +92,27 @@ func (b *ApplicationStatusApplyConfiguration) WithExternalIPsCount(value int32)
b.ExternalIPsCount = &value
return b
}
// WithInstallFailures sets the InstallFailures field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the InstallFailures field is set to the value of the last call.
func (b *ApplicationStatusApplyConfiguration) WithInstallFailures(value int64) *ApplicationStatusApplyConfiguration {
b.InstallFailures = &value
return b
}
// WithUpgradeFailures sets the UpgradeFailures field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the UpgradeFailures field is set to the value of the last call.
func (b *ApplicationStatusApplyConfiguration) WithUpgradeFailures(value int64) *ApplicationStatusApplyConfiguration {
b.UpgradeFailures = &value
return b
}
// WithLastAttemptedReleaseAction sets the LastAttemptedReleaseAction field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the LastAttemptedReleaseAction field is set to the value of the last call.
func (b *ApplicationStatusApplyConfiguration) WithLastAttemptedReleaseAction(value string) *ApplicationStatusApplyConfiguration {
b.LastAttemptedReleaseAction = &value
return b
}
@@ -290,6 +290,27 @@ func schema_pkg_apis_apps_v1alpha1_ApplicationStatus(ref common.ReferenceCallbac
Format: "int32",
},
},
"installFailures": {
SchemaProps: spec.SchemaProps{
Description: "InstallFailures is the install failure count of the underlying HelmRelease. It counts only attempts that recorded a Helm release, so a failure before Helm records one, such as a chart that fails to render, shows only as Ready=False; a failure to apply the rendered resources is counted. It resets when the desired state (generation, chart version, values) changes, and on success only under the RetryOnFailure strategy, so a Ready release may carry it.",
Type: []string{"integer"},
Format: "int64",
},
},
"upgradeFailures": {
SchemaProps: spec.SchemaProps{
Description: "UpgradeFailures is the upgrade failure count of the underlying HelmRelease, with the same counting and reset rules as InstallFailures.",
Type: []string{"integer"},
Format: "int64",
},
},
"lastAttemptedReleaseAction": {
SchemaProps: spec.SchemaProps{
Description: "LastAttemptedReleaseAction is the last release action (install or upgrade) the underlying HelmRelease attempted.",
Type: []string{"string"},
Format: "",
},
},
},
},
},
+12 -2
View File
@@ -1573,13 +1573,23 @@ func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.H
},
Spec: filteredSpec,
Status: appsv1alpha1.ApplicationStatus{
Version: hr.Status.LastAttemptedRevision,
Version: hr.Status.LastAttemptedRevision,
InstallFailures: hr.Status.InstallFailures,
UpgradeFailures: hr.Status.UpgradeFailures,
LastAttemptedReleaseAction: string(hr.Status.LastAttemptedReleaseAction),
},
}
var conditions []metav1.Condition
for _, hrCondition := range hr.GetConditions() {
if hrCondition.Type == "Ready" || hrCondition.Type == "Released" {
// The Application carries no generation, so a client cannot tell a Stalled
// left over from the previous spec from one about the current spec. Compare
// the condition's own ObservedGeneration, not Status.ObservedGeneration: the
// question is which spec this condition describes.
if hrCondition.Type == "Stalled" && hrCondition.ObservedGeneration < hr.Generation {
continue
}
if hrCondition.Type == "Ready" || hrCondition.Type == "Released" || hrCondition.Type == "Stalled" {
conditions = append(conditions, metav1.Condition{
LastTransitionTime: hrCondition.LastTransitionTime,
Reason: hrCondition.Reason,
@@ -0,0 +1,123 @@
package application
import (
"context"
"testing"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestConvertReleaseFailure_StalledAndCountersExposed(t *testing.T) {
r := newTestRESTWithSchemes()
hr := makeHelmRelease("postgresql-mydb", "default")
hr.Generation = 2
hr.Status.Conditions = []metav1.Condition{
{Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 2},
{Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2},
{Type: "Released", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2},
}
hr.Status.Failures = 7
hr.Status.InstallFailures = 1
hr.Status.UpgradeFailures = 3
hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
sc := findCondition(app.GetConditions(), "Stalled")
if sc == nil {
t.Fatal("expected Stalled condition to be copied from the HelmRelease")
}
if sc.Status != metav1.ConditionTrue || sc.Reason != "RetriesExceeded" {
t.Errorf("expected Stalled=True/RetriesExceeded, got %s/%s", sc.Status, sc.Reason)
}
if app.Status.InstallFailures != 1 {
t.Errorf("expected installFailures=1, got %d", app.Status.InstallFailures)
}
if app.Status.UpgradeFailures != 3 {
t.Errorf("expected upgradeFailures=3, got %d", app.Status.UpgradeFailures)
}
if app.Status.LastAttemptedReleaseAction != "install" {
t.Errorf("expected lastAttemptedReleaseAction=install, got %q", app.Status.LastAttemptedReleaseAction)
}
}
// After a spec update the HelmRelease keeps the Stalled condition of the
// previous generation until helm-controller reconciles it; reported as is, it
// would read as a terminal failure of the new values.
func TestConvertReleaseFailure_StalledFromOlderGenerationOmitted(t *testing.T) {
r := newTestRESTWithSchemes()
hr := makeHelmRelease("postgresql-mydb", "default")
hr.Generation = 2
hr.Status.Conditions = []metav1.Condition{
{Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 1},
{Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 1},
}
hr.Status.InstallFailures = 1
hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if findCondition(app.GetConditions(), "Stalled") != nil {
t.Error("expected Stalled observed at an older generation not to be copied")
}
if findCondition(app.GetConditions(), "Ready") == nil {
t.Error("expected Ready to still be copied")
}
}
// helm-controller clears the counters on success only under the RetryOnFailure
// strategy, so a Ready release can still carry them; they must not be hidden.
func TestConvertReleaseFailure_CountersKeptOnReadyRelease(t *testing.T) {
r := newTestRESTWithSchemes()
hr := makeHelmRelease("postgresql-mydb", "default")
hr.Status.Conditions = []metav1.Condition{
{Type: "Ready", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"},
{Type: "Released", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"},
}
hr.Status.InstallFailures = 2
hr.Status.UpgradeFailures = 1
hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionUpgrade
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if findCondition(app.GetConditions(), "Stalled") != nil {
t.Error("expected no Stalled condition on a release that does not carry one")
}
if app.Status.InstallFailures != 2 || app.Status.UpgradeFailures != 1 {
t.Errorf("expected install=2 upgrade=1, got install=%d upgrade=%d", app.Status.InstallFailures, app.Status.UpgradeFailures)
}
if app.Status.LastAttemptedReleaseAction != "upgrade" {
t.Errorf("expected lastAttemptedReleaseAction=upgrade, got %q", app.Status.LastAttemptedReleaseAction)
}
}
func TestConvertReleaseFailure_ReconcilingNotCopied(t *testing.T) {
r := newTestRESTWithSchemes()
hr := makeHelmRelease("postgresql-mydb", "default")
hr.Status.Conditions = []metav1.Condition{
{Type: "Reconciling", Status: metav1.ConditionTrue, Reason: "Progressing", Message: "Running 'install' action"},
{Type: "Ready", Status: metav1.ConditionUnknown, Reason: "Progressing", Message: "Running 'install' action"},
}
app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if findCondition(app.GetConditions(), "Reconciling") != nil {
t.Error("expected Reconciling not to be copied")
}
if findCondition(app.GetConditions(), "Ready") == nil {
t.Error("expected Ready to be copied")
}
}