diff --git a/api/backups/v1alpha1/backup_types.go b/api/backups/v1alpha1/backup_types.go index fd76ac47e..541488202 100644 --- a/api/backups/v1alpha1/backup_types.go +++ b/api/backups/v1alpha1/backup_types.go @@ -37,7 +37,14 @@ type BackupArtifact struct { // For example: s3://bucket/prefix/file.tar.gz URI string `json:"uri"` - // SizeBytes is the size of the artifact in bytes, if known. + // SizeBytes is the stored size in bytes of the files this backup owns, + // if known: the dump of a MariaDB backup, for example, or the base backup + // of a CNPG one, which leaves out the WAL archive the server's backups + // share and a point-in-time restore also reads. Strategies that cannot + // attribute stored bytes to a single backup, such as Velero's, whose + // repository deduplicates across backups, leave it unset. Drivers never + // write 0 for a size they cannot measure, so an absent value means + // "unknown", not "empty". // +optional SizeBytes int64 `json:"sizeBytes,omitempty"` diff --git a/go.mod b/go.mod index 6109faf15..34352e583 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-logr/logr v1.4.4 github.com/go-logr/zapr v1.3.0 github.com/go-task/slim-sprig/v3 v3.0.0 + github.com/minio/minio-go/v7 v7.0.98 github.com/onsi/ginkgo/v2 v2.33.0 github.com/onsi/gomega v1.43.1 github.com/prometheus/client_golang v1.24.1 @@ -63,6 +64,7 @@ require ( github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.4 // indirect github.com/gkampitakis/go-snaps v0.5.23 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v1.0.1 // indirect github.com/go-openapi/jsonreference v1.0.2 // indirect @@ -93,18 +95,25 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.20.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/moby/spdystream v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.3 // indirect github.com/prometheus/common v0.71.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect github.com/rogpeppe/go-internal v1.16.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sirupsen/logrus v1.10.2 // indirect github.com/tidwall/match v1.2.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.etcd.io/etcd/api/v3 v3.7.0 // indirect go.etcd.io/etcd/client/pkg/v3 v3.7.0 // indirect diff --git a/go.sum b/go.sum index dea6aaf08..667ca71ee 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ github.com/gkampitakis/ciinfo v0.3.4 h1:5eBSibVuSMbb/H6Elc0IIEFbkzCJi3lm94n0+U7Z github.com/gkampitakis/ciinfo v0.3.4/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-snaps v0.5.23 h1:okh5QR48zpUjpWtu65AtqxdCY8huJq+dEDuUzd1PuKg= github.com/gkampitakis/go-snaps v0.5.23/go.mod h1:uy3lVzCCRRsAwYqSocyw5fY8xRLCYEfqoOJNxr8HonM= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -141,6 +143,11 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -151,6 +158,12 @@ github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= +github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -165,6 +178,8 @@ github.com/onsi/ginkgo/v2 v2.33.0 h1:C8gBA6Uc2ZEubiV+SXiu5tZnMTwEmXHgkJwGozKtZf8 github.com/onsi/ginkgo/v2 v2.33.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.43.1 h1:vGIPFuYrIO6/0Z09s0I0QQQgFchiX4+tb1re3MScJYo= github.com/onsi/gomega v1.43.1/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -180,6 +195,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= @@ -206,6 +223,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/vmware-tanzu/velero v1.18.3 h1:9uwN5yxTWQl9PsF5ljX4xiG+7VIq3I9mxOQ0DyyZ31A= diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats index 9b31158d9..2597338a2 100644 --- a/hack/altinity-backup-strategy.bats +++ b/hack/altinity-backup-strategy.bats @@ -10,10 +10,16 @@ # rather than proceed when the sidecar never becomes idle. Once started, an # operation the sidecar no longer lists (its action log lives in memory and a # restart empties it) has to fail the Pod too, since the Job has no deadline. -# A Job retry must not run a restore twice: it takes over the restore_remote -# the attempt before it left running, and a few lost polls do not end an +# A Job retry must not start its operation twice: it takes over the +# restore_remote, or the create_remote of the archive the controller named, +# that the attempt before it left running, and a few lost polls do not end an # attempt in the first place. # +# Which archive a restore reads back is decided by the script's own jq +# selection over list/remote, so it is exercised against several archives +# rather than string-matched: exactly the archive the Backup recorded, or for +# a Backup that records none, the newest one stamped no later than its takenAt. +# # Run from the repo root with make bats-unit-tests. The rendered script is run # with `bash`, not `sh`: it opens `set -euo pipefail`, which the production # image's busybox ash supports but the CI runner's /bin/sh (dash) rejects. @@ -22,18 +28,25 @@ load test_helper CHART=packages/system/backupstrategy-controller -# Render the shipped strategy's in-Pod script into $1 with the controller-render -# placeholders substituted ($2 mode, $3 application name). An empty render, or -# a placeholder left unsubstituted, is an early failure: these tests EXECUTE it. +# Render the shipped strategy's in-Pod script into $1 and substitute the +# controller-render placeholders the way the driver fills them: $2 mode, $3 +# app (release and source application), $4 the archive name the controller +# chose or the Backup recorded (empty, the default, for none), $5 the Backup +# name, $6 its takenAt in the archive-name layout (default: later than every +# archive the stub lists). An empty render fails early, as does a placeholder +# left unsubstituted: these tests EXECUTE it. render_script() { - out=$1; mode=$2; app=$3 + out=$1; mode=$2; app=$3; archive=${4-}; backup=${5-bkp}; takenat=${6-99991231T235959Z} helm template t "$CHART" -n cozy-backup-controller \ --set backupStorage.bucketNameOverride=test-bucket 2>/dev/null \ | yq eval 'select(.kind == "Altinity") | .spec.template.spec.containers[0].args[0]' - \ | sed -e "s#{{ .Mode }}#${mode}#g" \ -e "s#{{ .Release.Name }}#${app}#g" \ -e "s#{{ .Release.Namespace }}#tenant-x#g" \ - -e "s#{{ .Backup.ApplicationRef.Name }}#${app}#g" > "$out" + -e "s#{{ .Backup.ApplicationRef.Name }}#${app}#g" \ + -e "s#{{ .Backup.Name }}#${backup}#g" \ + -e "s#{{ .ArchiveName | default \"\" }}#${archive}#g" \ + -e "s#{{ .Backup.TakenAt | default \"\" }}#${takenat}#g" > "$out" [ -s "$out" ] || return 1 if grep -q '{{' "$out"; then echo "rendered script still holds a controller placeholder" >&2 @@ -49,13 +62,17 @@ render_script() { # busy a number: that many GET /backup/actions answers still list # an operation of someone else "in progress" # op the command line of the sidecar's own operation, written -# by an accepted POST or seeded by a test as the operation +# by an accepted POST the way the sidecar logs it ("--resume" +# before the name of a create_remote asking for it), or +# seeded by a test as the operation # an earlier attempt of the same Job started # op_left how many more GET /backup/actions answers list op as # "in progress" before it ends with op_result # op_duration op_left given to an operation a POST starts (default 0) # op_result the status op ends with (default success) # prior extra action-log rows, verbatim, older than op +# remote present: the list/remote answer, verbatim, in place of the +# one archive listed by default # restarted present: the action log no longer lists op, as after the # sidecar restarted and lost its in-memory log # fail_on_post a POST that is accepted makes that many following @@ -94,7 +111,10 @@ case "\$method \$url" in "POST "*) if [ "\$(num busy)" -gt 0 ] || [ "\$running" = 1 ] || [ ! -e "\$state/idle" ]; then exit 22; fi case "\$url" in - */create_remote\?name=*) cmd="create_remote \${url##*=}" ;; + */create_remote\?name=*) + name=\${url#*name=}; name=\${name%%&*} + case "\$url" in *[?\&]resume=true*) cmd="create_remote --resume \$name" ;; *) cmd="create_remote \$name" ;; esac + ;; */restore_remote/*) cmd="restore_remote \${url##*/}" ;; *) exit 22 ;; esac @@ -130,7 +150,9 @@ case "\$method \$url" in if [ "\$inprog" = 0 ]; then touch "\$state/idle"; fi ;; "GET "*/backup/list/remote) - echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}' + if [ -e "\$state/remote" ]; then cat "\$state/remote"; else + echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}' + fi ;; esac exit 0 @@ -142,6 +164,17 @@ STUB chmod 0755 "$bin/curl" "$bin/sleep" } +# One list/remote row per argument, "name" or "name|desc"; created mirrors +# the name's timestamp. +remote_rows() { + for row in "$@"; do + name=${row%%|*}; desc="tar, regular" + if [ "$row" != "$name" ]; then desc=${row#*|}; fi + ts=${name##*-} + printf '{"name":"%s","created":"%s","location":"remote","required":"","desc":"%s"}\n' "$name" "$ts" "$desc" + done +} + # The number of operations the sidecar accepted. accepted() { if [ -e "$1/state/accepted" ]; then wc -l < "$1/state/accepted" | tr -d ' '; else echo 0; fi @@ -175,7 +208,7 @@ run_script() { echo 3 > "$tmp/state/busy" render_script "$tmp/s.sh" backup myapp run_script "$tmp" - grep -q '^create_remote clickhouse-myapp-' "$tmp/state/accepted" + grep -q '^create_remote --resume clickhouse-myapp-' "$tmp/state/accepted" [ "$(cat "$tmp/state/busy")" = 0 ] } @@ -321,3 +354,144 @@ run_script() { if run_script "$tmp"; then echo "FAIL: a cancelled operation must fail the Pod"; false; fi if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling a cancelled operation"; false; fi } + +@test "restore reads back the archive the Backup recorded, not the newest one" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z + remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \ + clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote" + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "restore fails naming the recorded archive when remote storage lacks it" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z + remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/state/remote" + out="" + if out=$(run_script "$tmp" 2>&1); then + echo "FAIL: restore succeeded without the recorded archive: $out" + false + fi + printf '%s\n' "$out" | grep -qF 'archive clickhouse-myapp-20260304T050607Z recorded on Backup bkp-a is not in remote storage' + if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then + echo "FAIL: restored another archive: $(cat "$tmp/state/requests")" + false + fi +} + +@test "restore of a Backup without a recorded archive takes the newest one stamped by its takenAt, and warns" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z + # Each decoy sorts after the expected archive and is ruled out by one + # filter: broken, not a bare timestamp after the prefix, or stamped after + # takenAt. + remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \ + "clickhouse-myapp-20260304T050700Z|broken (can't stat metadata.json)" \ + clickhouse-myapp-20260304T050710Z-copy \ + clickhouse-myapp-20260304T060000Z > "$tmp/state/remote" + out=$(run_script "$tmp" 2>&1) + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests" + [ "$(accepted "$tmp")" = 1 ] + printf '%s\n' "$out" | grep -qF 'WARNING: Backup bkp-a records no archive name' +} + +@test "restore of a Backup without a recorded archive fails when no archive is stamped by its takenAt" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z + remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote" + out="" + if out=$(run_script "$tmp" 2>&1); then + echo "FAIL: restore succeeded with every archive newer than takenAt: $out" + false + fi + printf '%s\n' "$out" | grep -qF 'no archive of myapp stamped no later than 20260304T050800Z' + if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then + echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/state/requests")" + false + fi +} + +@test "backup writes the archive under the name the controller chose" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests" +} + +@test "backup falls back to a timestamped name when the controller supplies none" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp "" + run_script "$tmp" + grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/state/requests" +} + +# The sidecar answers a second operation with 423 while one runs, so a retry +# that POSTed over its predecessor's upload would burn every attempt left. +@test "a retried backup takes over the upload of its archive the sidecar is still running" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")" + false + fi + [ "$(cat "$tmp/state/op_left")" = 0 ] +} + +@test "a taken-over backup that ends in error fails the Pod" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op" + echo 2 > "$tmp/state/op_left" + echo error > "$tmp/state/op_result" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + if run_script "$tmp"; then echo "FAIL: a taken-over upload that failed must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")" + false + fi +} + +@test "a retried backup after a failed attempt resumes the same archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30","error":"S3 PutObject 403"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests" +} + +@test "backup does not take over an upload running for another archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T040000Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'create_remote --resume clickhouse-myapp-20260304T050607Z' "$tmp/state/accepted" +} + +# The action log is never trimmed, so a finished row of the same archive may +# be an earlier run's and is not taken for this attempt's result. +@test "backup starts create_remote again when the log holds a finished one of the same archive" { + for status in success error cancel; do + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo "{\"command\":\"create_remote --resume clickhouse-myapp-20260304T050607Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier upload must not stand in for this one"; false; fi + done +} diff --git a/hack/cilium-ipam-migration.bats b/hack/cilium-ipam-migration.bats index c54db50ab..53053747a 100644 --- a/hack/cilium-ipam-migration.bats +++ b/hack/cilium-ipam-migration.bats @@ -488,6 +488,33 @@ pod_cidrs_of() { rm -rf "$STATE" } +@test "a CiliumNode list larger than one argument may hold still moves every node and names the pod on an old Ingress IP" { + init_state + add_ip web-0.tenant-c ovn-default 10.244.1.135 + add_ip web-1.tenant-c ovn-default 10.244.4.135 + add_ip db-0.tenant-c ovn-default 10.244.9.9 + # Cilium copies every Node label onto the CiliumNode, which takes six real + # ones past the 128 KiB Linux allows a single argument. One annotation stands + # in for the labels and keeps the list over that limit even printed compact. + for i in 1 2 3 4 5 6; do + add_node "n$i" "10.244.$i.0/24" "10.244.$i.223" "10.244.$i.135" + jq '.metadata.annotations.padding = ("x" * 24576)' "$STATE/ciliumnodes/n$i.json" > "$STATE/node.tmp" + mv "$STATE/node.tmp" "$STATE/ciliumnodes/n$i.json" + done + [ "$(kubectl get ciliumnodes -o json | jq -c . | wc -c)" -gt 131072 ] + "$SCRIPT" > "$STATE/out" 2>&1 + # Each pod is named with the node whose Ingress IP it holds, and a pod on an + # address that is no node's Ingress IP is not named at all. + grep -q "web-0.tenant-c holds 10.244.1.135, the Ingress IP of n1 before the move" "$STATE/out" + grep -q "web-1.tenant-c holds 10.244.4.135, the Ingress IP of n4 before the move" "$STATE/out" + if grep -q "db-0.tenant-c" "$STATE/out"; then + cat "$STATE/out" + false + fi + [ "$(pod_cidrs_of n6)" = '["10.244.248.40/29"]' ] + rm -rf "$STATE" +} + @test "a cluster without kube-ovn skips the exclusion and still moves its nodes" { init_state rm "$STATE/subnet.json" diff --git a/hack/kubernetes-oidc-bindings.bats b/hack/kubernetes-oidc-bindings.bats new file mode 100644 index 000000000..f37273802 --- /dev/null +++ b/hack/kubernetes-oidc-bindings.bats @@ -0,0 +1,187 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# packages/apps/kubernetes: the OIDC bootstrap Job must not carry the users[] +# list inside its container command. +# +# The Job's script reaches the container as one `sh -c` argument, and Linux caps +# one exec argument at 128 KiB (MAX_ARG_STRLEN). With a ClusterRoleBinding per +# user written into that script, about 240 users take it past the cap: the +# container cannot start, the post-install/post-upgrade hook fails, and the +# tenant cluster's HelmRelease stays unready. The bindings reach the Job through +# a mounted ConfigMap instead, so the script is the same size for any list. +# +# The last three tests execute the rendered script against a fake kubectl, since +# no assertion on the script's text says what the prune step does with the list: +# a condition forced either way passes every helm-unittest case. +# +# Requires: helm, yq (mikefarah v4+), jq, like hack/admin-kubeconfig-invariant.bats. +# ----------------------------------------------------------------------------- + +load test_helper + +USERS=300 + +# render_users renders the chart in System mode with +# generated users and leaves one JSON document per object in /rendered.json. +render_users() { + jq -n --argjson n "$2" ' + {oidc: {users: [range($n) | {email: "user\(.)@example.org", role: (if . % 7 == 0 then "admin" else "view" end)}]}}' \ + > "$1/users.yaml" + helm template bindings packages/apps/kubernetes \ + --namespace tenant-test \ + --values packages/apps/kubernetes/tests/values/oidc-system.yaml \ + --values "$1/users.yaml" \ + > "$1/rendered.yaml" + yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json" +} + +# render_job renders the chart in CustomConfig mode and unpacks +# what the bootstrap Job runs and mounts into : script.sh is its `sh -c` +# script, bindings/ holds the two files of the mounted ConfigMap. -j keeps an +# empty value an empty file, as the kubelet writes it. +render_job() { + jq -n --argjson users "$2" ' + {_cluster: {"oidc-enabled": "false"}, + oidc: {mode: "CustomConfig", + customConfig: {config: "apiVersion: apiserver.config.k8s.io/v1beta1\nkind: AuthenticationConfiguration\njwt: []\n"}, + users: $users}}' > "$1/values.yaml" + helm template bindings packages/apps/kubernetes \ + --namespace tenant-test \ + --values packages/apps/kubernetes/tests/values/oidc-system.yaml \ + --values "$1/values.yaml" \ + > "$1/rendered.yaml" + yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json" + jq -s -j '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap") + | .spec.template.spec.containers[0].command[2]' "$1/rendered.json" > "$1/script.sh" + mkdir -p "$1/bindings" + jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings") + | .data["crbs.yaml"]' "$1/rendered.json" > "$1/bindings/crbs.yaml" + jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings") + | .data["desired.txt"]' "$1/rendered.json" > "$1/bindings/desired.txt" +} + +# fake_kubectl puts a kubectl first on PATH that logs every call and +# answers the script's list of existing bindings with . +fake_kubectl() { + mkdir -p "$1/bin" + printf '%s' "$2" > "$1/existing" + cat > "$1/bin/kubectl" <<'KEOF' +#!/bin/sh +echo "$*" >> "$KLOG" +case "$*" in + *"get clusterrolebindings"*) cat "$EXISTING" ;; +esac +KEOF + chmod +x "$1/bin/kubectl" +} + +# run_job runs the unpacked script as the container does and leaves its +# status in $rc and its output in /out. Run by bash, which has the pipefail +# the script sets and which is what the unit lane runs under; the container's own +# shell is not exercised. +run_job() { + : > "$1/kubectl.log" + echo kubeconfig > "$1/kubeconfig" + rc=0 + env PATH="$1/bin:$PATH" KLOG="$1/kubectl.log" EXISTING="$1/existing" HOME="$1" \ + KUBECONFIG_TENANT="$1/kubeconfig" RELEASE=bindings RELEASE_NAMESPACE=tenant-test \ + OIDC_MODE=CustomConfig OIDC_BINDINGS_DIR="$1/bindings" \ + bash "$1/script.sh" > "$1/out" 2>&1 || rc=$? +} + +@test "the OIDC bootstrap Job script stays under the exec argument cap and carries no user" { + dir=$(mktemp -d) + render_users "$dir" "$USERS" + jq -s -r '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap") + | .spec.template.spec.containers[0].command[2]' "$dir/rendered.json" > "$dir/script" + size=$(wc -c < "$dir/script") + [ "$size" -gt 0 ] + [ "$size" -lt 131072 ] + if grep -q -E 'user[0-9]+@example\.org' "$dir/script"; then + echo "the Job script still carries users[] entries" >&2 + false + fi + rm -rf "$dir" +} + +@test "every user keeps one ClusterRoleBinding with its old name and role in the bindings ConfigMap" { + dir=$(mktemp -d) + render_users "$dir" "$USERS" + jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings") + | .data["crbs.yaml"]' "$dir/rendered.json" > "$dir/crbs.yaml" + jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings") + | .data["desired.txt"]' "$dir/rendered.json" > "$dir/desired.txt" + [ -s "$dir/crbs.yaml" ] + yq --output-format=json eval-all '.' "$dir/crbs.yaml" | jq -s '.' > "$dir/crbs.json" + + [ "$(jq 'length' "$dir/crbs.json")" -eq "$USERS" ] + [ "$(jq -r '[.[].subjects[0].name] | unique | length' "$dir/crbs.json")" -eq "$USERS" ] + [ "$(jq -r '[.[] | select(.roleRef.name == "cluster-admin")] | length' "$dir/crbs.json")" -eq $(( (USERS + 6) / 7 )) ] + + # The name is what the prune step and every binding already in a tenant cluster + # go by, so it must not move: cozystack-oidc- plus the first 16 hex digits of + # sha256("-"). + for i in 0 1 6 7; do + email="user$i@example.org" + name="cozystack-oidc-$(printf '%s' "bindings-$email" | sha256sum | cut -c1-16)" + [ "$(jq -r --arg e "$email" '.[] | select(.subjects[0].name == $e) | .metadata.name' "$dir/crbs.json")" = "$name" ] + grep -q -x -F "$name" "$dir/desired.txt" + done + + # The list the prune step compares against is exactly the applied names. + [ "$(jq -r '.[].metadata.name' "$dir/crbs.json" | sort)" = "$(grep . "$dir/desired.txt" | sort)" ] + rm -rf "$dir" +} + +@test "the Job applies the mounted bindings and deletes only the bindings no user has" { + dir=$(mktemp -d) + render_job "$dir" '[{"email":"alice@example.org","role":"admin"},{"email":"bob@example.org","role":"view"}]' + alice="cozystack-oidc-$(printf '%s' 'bindings-alice@example.org' | sha256sum | cut -c1-16)" + bob="cozystack-oidc-$(printf '%s' 'bindings-bob@example.org' | sha256sum | cut -c1-16)" + fake_kubectl "$dir" "$alice cozystack-oidc-000000000000dead $bob cozystack-oidc-000000000000beef" + run_job "$dir" + if [ "$rc" -ne 0 ]; then + cat "$dir/out" + false + fi + grep -q -F -- "apply --filename $dir/bindings/crbs.yaml" "$dir/kubectl.log" + grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000dead --ignore-not-found" "$dir/kubectl.log" + grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000beef --ignore-not-found" "$dir/kubectl.log" + if grep -F -e "delete clusterrolebinding $alice" -e "delete clusterrolebinding $bob" "$dir/kubectl.log"; then + echo "the Job deleted a binding a user still has" >&2 + false + fi + [ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ] + rm -rf "$dir" +} + +@test "a release with no users applies nothing and deletes every binding it left behind" { + dir=$(mktemp -d) + render_job "$dir" '[]' + fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef" + run_job "$dir" + if [ "$rc" -ne 0 ]; then + cat "$dir/out" + false + fi + if grep -q -F -- ' apply ' "$dir/kubectl.log"; then + echo "the Job ran kubectl apply with no bindings to apply" >&2 + false + fi + [ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ] + rm -rf "$dir" +} + +@test "the Job stops before deleting anything when the desired list is not mounted" { + dir=$(mktemp -d) + render_job "$dir" '[{"email":"alice@example.org","role":"admin"}]' + fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef" + rm "$dir/bindings/desired.txt" + run_job "$dir" + [ "$rc" -ne 0 ] + if grep -q -E 'delete clusterrolebinding| apply ' "$dir/kubectl.log"; then + cat "$dir/kubectl.log" + false + fi + rm -rf "$dir" +} diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index b96d571f0..a0868ec38 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -2,7 +2,9 @@ package backupcontroller import ( + "bytes" "context" + "encoding/json" "fmt" "maps" "strings" @@ -43,6 +45,18 @@ const ( // were in effect at backup time. altinityParamPrefix = "altinity.strategy.backups.cozystack.io/parameter/" + // altinityArchiveMetaKey records, on the Backup, the clickhouse-backup + // archive its Job wrote, as named by altinityArchiveName. A restore reads + // back exactly that archive: the newest one under the release's prefix + // may belong to a later Backup. + altinityArchiveMetaKey = "altinity.strategy.backups.cozystack.io/archive" + + // altinityArchiveTimeLayout is the UTC timestamp that ends an archive + // name. It is fixed-width, so names sharing a prefix sort chronologically + // as plain strings, which is what a restore of a Backup that records no + // archive relies on. + altinityArchiveTimeLayout = "20060102T150405Z" + // Polling cadence for the Altinity Job lifecycle. Mirrors the CNPG / // Velero strategy cadence so behaviour is uniform across drivers. altinityPollInterval = 5 * time.Second @@ -102,14 +116,37 @@ func backupParameters(b *backupsv1alpha1.Backup) map[string]string { return out } +// altinityArchiveName mints the archive name for one BackupJob from its +// persisted StartedAt, so every reconcile and every Pod retry of the Job +// writes the same archive. It keeps the clickhouse-- +// shape, because a Backup that records no archive is restored by the +// timestamp in that shape. +func altinityArchiveName(j *backupsv1alpha1.BackupJob) string { + return fmt.Sprintf("clickhouse-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout)) +} + +// altinityJobCarriesArchive reports whether the Job's Pod was rendered with +// the archive name. A Job created by a controller that predates archive +// names, or rendered from a strategy that names its own archives, wrote an +// archive under another name, and recording this one would point the Backup +// at an archive that does not exist. +func altinityJobCarriesArchive(job *batchv1.Job, archive string) bool { + spec, err := json.Marshal(job.Spec.Template.Spec) + if err != nil { + return false + } + return bytes.Contains(spec, []byte(archive)) +} + // renderAltinityTemplate runs the strategy's PodTemplateSpec through the // repository's text/template engine with the same context shape as the CNPG // driver: every string field is templated against the application object, -// the release shorthand, the run mode, and the resolved parameters. +// the release shorthand, the run mode, the archive name, and the resolved +// parameters. func renderAltinityTemplate( tmpl corev1.PodTemplateSpec, app map[string]any, - releaseName, releaseNamespace, mode string, + releaseName, releaseNamespace, mode, archiveName string, parameters map[string]string, backup *backupsv1alpha1.Backup, ) (*corev1.PodTemplateSpec, error) { @@ -119,8 +156,9 @@ func renderAltinityTemplate( "Name": releaseName, "Namespace": releaseNamespace, }, - "Mode": mode, - "Parameters": parameters, + "Mode": mode, + "ArchiveName": archiveName, + "Parameters": parameters, } if backup != nil { // Expose ApplicationRef so restore strategies can reference the @@ -143,6 +181,9 @@ func renderAltinityTemplate( "Kind": backup.Spec.ApplicationRef.Kind, "Name": backup.Spec.ApplicationRef.Name, }, + // In the archive-name layout, so a strategy can compare it with + // the timestamp an archive name ends in. + "TakenAt": backup.Spec.TakenAt.UTC().Format(altinityArchiveTimeLayout), } } return template.Template(&tmpl, ctxMap) @@ -209,12 +250,14 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv return ctrl.Result{}, err } + archive := altinityArchiveName(j) rendered, err := renderAltinityTemplate( strategy.Spec.Template, app, j.Spec.ApplicationRef.Name, j.Namespace, altinityModeBackup, + archive, resolved.Parameters, nil, ) @@ -239,7 +282,10 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv if j.Status.BackupRef != nil { return ctrl.Result{}, nil } - artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved) + if !altinityJobCarriesArchive(batchJob, archive) { + archive = "" + } + artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved, archive) if err != nil { return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err)) } @@ -411,16 +457,20 @@ func jobFailureMessage(j *batchv1.Job) string { } // createAltinityBackupArtifact materialises a Cozystack Backup carrying the -// strategy reference and the BackupClassStrategy parameters in effect at -// backup time. Parameters round-trip via DriverMetadata under -// altinityParamPrefix so a later RestoreJob can re-render the strategy -// template against the same values. +// strategy reference, the archive the Job wrote (when known) and the +// BackupClassStrategy parameters in effect at backup time. Parameters +// round-trip via DriverMetadata under altinityParamPrefix so a later +// RestoreJob can re-render the strategy template against the same values. func (r *BackupJobReconciler) createAltinityBackupArtifact( ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig, + archive string, ) (*backupsv1alpha1.Backup, error) { driverMD := map[string]string{} + if archive != "" { + driverMD[altinityArchiveMetaKey] = archive + } for k, v := range resolved.Parameters { driverMD[altinityParamPrefix+k] = v } @@ -459,6 +509,13 @@ func (r *BackupJobReconciler) createAltinityBackupArtifact( if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { return nil, getErr } + // A Backup outlives its BackupJob, so a BackupJob re-created under a + // used name meets the earlier run's Backup here. Only one recording + // this run's archive was created by an earlier reconcile of this run. + if got := existing.Spec.DriverMetadata[altinityArchiveMetaKey]; got != archive { + return nil, fmt.Errorf("Backup %s/%s already exists and records archive %q, not %q that this run wrote", + backup.Namespace, backup.Name, got, archive) + } return existing, nil } return backup, nil @@ -616,6 +673,7 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res targetAppName, targetNamespace, altinityModeRestore, + backup.Spec.DriverMetadata[altinityArchiveMetaKey], backupParameters(backup), backup, ) diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index 68f18fa28..e5f1e1695 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -3,7 +3,9 @@ package backupcontroller import ( "context" + "strings" "testing" + "time" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -697,6 +699,7 @@ func TestBackupParameters_RoundTrip(t *testing.T) { DriverMetadata: map[string]string{ altinityParamPrefix + "bucketName": "b1", altinityParamPrefix + "endpoint": "https://s3", + altinityArchiveMetaKey: "clickhouse-ch-test-20260304T050607Z", "job.batch/name": "ignored", "": "ignored", }, @@ -706,7 +709,326 @@ func TestBackupParameters_RoundTrip(t *testing.T) { if got["bucketName"] != "b1" || got["endpoint"] != "https://s3" { t.Errorf("backupParameters = %v", got) } - if _, ok := got["job.batch/name"]; ok { + if len(got) != 2 { t.Errorf("non-parameter key leaked through: %v", got) } } + +// newAltinityArchiveStrategy renders the archive name and the Backup's +// takenAt where a test can read them back off the Job. +func newAltinityArchiveStrategy(name string) *strategyv1alpha1.Altinity { + s := newAltinityStrategy(name) + s.Spec.Template.Spec.Containers[0].Args = []string{ + "--mode={{ .Mode }}", + "--archive={{ .ArchiveName }}", + "--taken-at={{ .Backup.TakenAt }}", + "--bucket={{ .Parameters.bucketName }}", + } + return s +} + +func TestAltinityArchiveName_FromStartedAtInUTC(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 7, 890000000, time.FixedZone("UTC+3", 3*3600))) + j := &backupsv1alpha1.BackupJob{ + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + } + if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z"; got != want { + t.Errorf("altinityArchiveName = %q, want %q", got, want) + } +} + +// TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven drives one BackupJob +// through Job creation and completion in separate reconciles. The archive +// name has to come out of the persisted StartedAt both times: the Pod is +// told it on the first, the Backup records it on the second, and a restore +// later reads back exactly that archive. +func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { + app := newAltinityClickHouseApp("ch-test", "tenant-test") + strategy := newAltinityArchiveStrategy("clickhouse-strategy") + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + Parameters: map[string]string{"bucketName": "ch-bucket"}, + } + r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy)) + ctx := context.Background() + const want = "clickhouse-ch-test-20260304T050607Z" + + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() creating the Job: %v", err) + } + job := &batchv1.Job{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil { + t.Fatalf("get backup Job: %v", err) + } + if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+want { + t.Fatalf("backup Pod was rendered with %q, want --archive=%s", got, want) + } + + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := r.Status().Update(ctx, job); err != nil { + t.Fatalf("mark backup Job complete: %v", err) + } + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil { + t.Fatalf("refresh BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() on the completed Job: %v", err) + } + + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + if got := created.Spec.DriverMetadata[altinityArchiveMetaKey]; got != want { + t.Errorf("Backup records archive %q, want %q", got, want) + } + if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" { + t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got) + } +} + +// TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven covers a Job whose +// Pod never carried the driver's archive name: one created before the +// upgrade that introduced the names, or rendered from a strategy that names +// its own archives. Recording the name would send the restore after an +// archive nobody wrote; recording none leaves it the fallback for Backups +// without a name. +func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) { + app := newAltinityClickHouseApp("ch-test", "tenant-test") + strategy := newAltinityArchiveStrategy("clickhouse-strategy") + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + completedK8sJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: jobNameForBackupJob(backupJob), Namespace: backupJob.Namespace}, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{{ + Name: "clickhouse-backup", + Image: "altinity/clickhouse-backup:test", + Args: []string{`target="clickhouse-ch-test-$(date -u +%Y%m%dT%H%M%SZ)"`}, + }}, + }}, + }, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}, + }, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + Parameters: map[string]string{"bucketName": "ch-bucket"}, + } + r, _, _ := newAltinityTestEnv(t, app, + clientfake.NewClientBuilder().WithObjects(backupJob, strategy, completedK8sJob)) + ctx := context.Background() + + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() error = %v", err) + } + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + if got, ok := created.Spec.DriverMetadata[altinityArchiveMetaKey]; ok { + t.Errorf("Backup records archive %q its Job was never given", got) + } + if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" { + t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got) + } +} + +// TestReconcileAltinity_BackupNameAlreadyTaken covers a completed Job whose +// Backup name is already in use. A Backup outlives its BackupJob, so a +// BackupJob re-created under an earlier run's name finds that run's Backup, +// and claiming it would point the new run at an archive it did not write. +// A Backup recording this run's own archive is the one an earlier reconcile +// created before its status write was lost, and is adopted. +func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) { + const thisRun = "clickhouse-ch-test-20260305T050607Z" + cases := []struct { + name string + existing string // archive the existing Backup records; empty for none + wantSucceeded bool + }{ + {name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z"}, + {name: "earlier run's Backup records no archive", existing: ""}, + {name: "this run's Backup from a reconcile whose status write was lost", existing: thisRun, wantSucceeded: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + app := newAltinityClickHouseApp("ch-test", "tenant-test") + strategy := newAltinityArchiveStrategy("clickhouse-strategy") + startedAt := metav1.NewTime(time.Date(2026, 3, 5, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + driverMD := map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"} + if tc.existing != "" { + driverMD[altinityArchiveMetaKey] = tc.existing + } + existing := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: backupJob.Name, Namespace: backupJob.Namespace}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 22, 0, time.UTC)), + DriverMetadata: driverMD, + }, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + Parameters: map[string]string{"bucketName": "ch-bucket"}, + } + r, _, _ := newAltinityTestEnv(t, app, + clientfake.NewClientBuilder().WithObjects(backupJob, strategy, existing)) + ctx := context.Background() + + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() creating the Job: %v", err) + } + job := &batchv1.Job{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil { + t.Fatalf("get backup Job: %v", err) + } + if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+thisRun { + t.Fatalf("backup Pod was rendered with %q, want --archive=%s", got, thisRun) + } + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := r.Status().Update(ctx, job); err != nil { + t.Fatalf("mark backup Job complete: %v", err) + } + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil { + t.Fatalf("refresh BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() on the completed Job: %v", err) + } + + updated := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), updated); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if tc.wantSucceeded { + if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Errorf("phase = %q (%s), want Succeeded", updated.Status.Phase, updated.Status.Message) + } + if updated.Status.BackupRef == nil || updated.Status.BackupRef.Name != existing.Name { + t.Errorf("BackupRef = %v, want %s", updated.Status.BackupRef, existing.Name) + } + } else { + if updated.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Errorf("phase = %q, want Failed: the run would claim a Backup recording %q instead of %q", updated.Status.Phase, tc.existing, thisRun) + } + if updated.Status.BackupRef != nil { + t.Errorf("BackupRef = %v, want none", updated.Status.BackupRef) + } + if !strings.Contains(updated.Status.Message, thisRun) { + t.Errorf("failure message %q does not name the archive this run wrote", updated.Status.Message) + } + } + kept := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKeyFromObject(existing), kept); err != nil { + t.Fatalf("get existing Backup: %v", err) + } + if got := kept.Spec.DriverMetadata[altinityArchiveMetaKey]; got != tc.existing { + t.Errorf("existing Backup now records archive %q, want %q left in place", got, tc.existing) + } + }) + } +} + +func TestReconcileAltinityRestore_RendersTheBackupsArchive(t *testing.T) { + cases := []struct { + name string + archive string // recorded on the Backup; empty for a Backup taken before names were recorded + wantArchive string + }{ + {name: "recorded archive", archive: "clickhouse-ch-test-20260304T050607Z", wantArchive: "--archive=clickhouse-ch-test-20260304T050607Z"}, + {name: "no recorded archive", archive: "", wantArchive: "--archive="}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + targetApp := newAltinityClickHouseApp("ch-test", "tenant-test") + strategy := newAltinityArchiveStrategy("clickhouse-strategy") + driverMD := map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"} + if tc.archive != "" { + driverMD[altinityArchiveMetaKey] = tc.archive + } + now := metav1.Now() + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + TakenAt: metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 22, 0, time.FixedZone("UTC+3", 3*3600))), + DriverMetadata: driverMD, + }, + } + restoreJob := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-test"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + _, rr, _ := newAltinityTestEnv(t, targetApp, + clientfake.NewClientBuilder().WithObjects(strategy, backup, restoreJob)) + ctx := context.Background() + + if _, err := rr.reconcileAltinityRestore(ctx, restoreJob, backup); err != nil { + t.Fatalf("reconcileAltinityRestore: %v", err) + } + job := &batchv1.Job{} + if err := rr.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForRestoreJob(restoreJob)}, job); err != nil { + t.Fatalf("get restore Job: %v", err) + } + args := job.Spec.Template.Spec.Containers[0].Args + want := []string{"--mode=restore", tc.wantArchive, "--taken-at=20260304T050622Z", "--bucket=ch-bucket"} + for i, w := range want { + if i >= len(args) || args[i] != w { + t.Errorf("rendered restore args[%d]: want %q, got %q", i, w, args) + } + } + }) + } +} diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go new file mode 100644 index 000000000..941b00323 --- /dev/null +++ b/internal/backupcontroller/artifact_size.go @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +// artifactSizeProbeTimeout bounds the object-store round trips a driver +// makes to measure Backup.status.artifact.sizeBytes. The probe runs inside +// the reconcile that creates the Backup, and the BackupJob controller has a +// single worker, so an unreachable endpoint stalls every other BackupJob for +// this long. A variable so tests can shorten it. +var artifactSizeProbeTimeout = 10 * time.Second + +type s3ObjectInfo struct { + Key string + Size int64 + LastModified time.Time +} + +type s3ObjectLister interface { + ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) +} + +// s3Connection is what a driver resolves from its rendered strategy to +// read the bucket with the same identity the backup was written with. +type s3Connection struct { + // Endpoint is host[:port], without a scheme. + Endpoint string + Secure bool + Region string + AccessKey string + SecretKey string + SessionToken string + // CAPEM is an endpoint CA bundle. It is trusted on top of the system + // roots, or instead of them when CAOnly is set: the probe must trust + // what the writer of the backup trusted, no more. + CAPEM []byte + CAOnly bool +} + +// objectStore reads back what a backup wrote: the Secrets it was written +// with and the bucket itself. It is a value rather than reconciler methods +// so that the restore path, which has a reconciler of its own, can read +// the same objects. +type objectStore struct { + // reader reads Secrets, which the manager's client fetches uncached, + // and lists operator Backups, which it serves from the cache. + reader client.Reader + // newLister overrides the S3 client; nil means a real minio client. + newLister func(conn s3Connection) (s3ObjectLister, error) +} + +func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { + if s.newLister != nil { + return s.newLister(conn) + } + return newMinioObjectLister(conn) +} + +func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) { + secret := &corev1.Secret{} + if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { + return "", err + } + v, ok := secret.Data[key] + if !ok { + return "", fmt.Errorf("secret %s/%s has no key %q", namespace, name, key) + } + return string(v), nil +} + +func (r *BackupJobReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} +} + +func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { + return r.objectStore().lister(conn) +} + +type minioObjectLister struct { + client *minio.Client +} + +func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { + opts := &minio.Options{ + Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken), + Region: conn.Region, + Secure: conn.Secure, + } + if conn.Secure && len(conn.CAPEM) > 0 { + // Start from minio's transport to keep its proxy and timeout settings. + tr, err := minio.DefaultTransport(true) + if err != nil { + return nil, err + } + if tr.TLSClientConfig.RootCAs, err = endpointTrustPool(conn, tr.TLSClientConfig.RootCAs); err != nil { + return nil, err + } + opts.Transport = tr + } + // minio.New refuses an endpoint with a path prefix, which it cannot + // address, and accepts a bare trailing slash. + client, err := minio.New(conn.Endpoint, opts) + if err != nil { + return nil, err + } + return &minioObjectLister{client: client}, nil +} + +// endpointTrustPool returns the roots to verify the endpoint against: the +// CA bundle on top of system, or the CA bundle alone with CAOnly. A nil +// system means the platform's roots. +func endpointTrustPool(conn s3Connection, system *x509.CertPool) (*x509.CertPool, error) { + pool := system + switch { + case conn.CAOnly: + pool = x509.NewCertPool() + case pool == nil: + var err error + if pool, err = x509.SystemCertPool(); err != nil { + pool = x509.NewCertPool() + } + } + if !pool.AppendCertsFromPEM(conn.CAPEM) { + return nil, errors.New("no PEM certificates in the endpoint CA bundle") + } + return pool, nil +} + +func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { + return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive})) +} + +// collectObjects drains a minio listing. minio-go ends a listing whose +// context is cancelled between pages by closing the channel without an +// error item, so a cancelled context is checked once the channel closes: +// otherwise a timeout would pass a partial listing off as complete. +func collectObjects(ctx context.Context, objects <-chan minio.ObjectInfo) ([]s3ObjectInfo, error) { + var out []s3ObjectInfo + for obj := range objects { + if obj.Err != nil { + return nil, obj.Err + } + out = append(out, s3ObjectInfo{Key: obj.Key, Size: obj.Size, LastModified: obj.LastModified}) + } + if err := ctx.Err(); err != nil { + return nil, err + } + return out, nil +} + +// errBackupNotReadBack reports that Create collided with an existing Backup +// that could then not be read. The backup itself succeeded, so callers +// retry the reconcile instead of failing the BackupJob. +var errBackupNotReadBack = errors.New("the Backup already exists but could not be read back") + +// failBackupArtifact handles an error from creating the Backup of a +// completed backup: errBackupNotReadBack is retried through the controller's +// backoff, with Ready=False telling why the BackupJob is still Running; +// anything else fails the BackupJob. +func (r *BackupJobReconciler) failBackupArtifact(ctx context.Context, j *backupsv1alpha1.BackupJob, err error) (ctrl.Result, error) { + if errors.Is(err, errBackupNotReadBack) { + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "BackupNotReadBack", + Message: err.Error(), + }) + if uerr := r.Status().Update(ctx, j); uerr != nil { + getLogger(ctx).Info("cannot record that the Backup could not be read back", "backupjob", j.Name, "error", uerr) + } + return ctrl.Result{}, err + } + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err)) +} + +// existingBackup returns the Backup a previous reconcile already created +// for the job, so a retry skips the size probe whose result it would +// discard. It reads past the cache: a retry typically follows a Create made +// moments earlier, which the cache may not show yet. A failed read is not +// fatal, because the Create that follows handles an existing Backup on its +// own; probe is false then, since the Backup may well exist and a probe +// would only be discarded, while a read that keeps failing would rerun it on +// every retry. +func (r *BackupJobReconciler) existingBackup(ctx context.Context, namespace, name string) (existing *backupsv1alpha1.Backup, probe bool) { + existing = &backupsv1alpha1.Backup{} + if err := r.apiReader().Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, existing); err != nil { + if apierrors.IsNotFound(err) { + return nil, true + } + getLogger(ctx).Info("cannot check for an existing Backup; creating it without measuring its size", "backup", name, "error", err) + return nil, false + } + return existing, false +} + +func (r *BackupJobReconciler) secretValue(ctx context.Context, namespace, name, key string) (string, error) { + return r.objectStore().secretValue(ctx, namespace, name, key) +} diff --git a/internal/backupcontroller/artifact_size_test.go b/internal/backupcontroller/artifact_size_test.go new file mode 100644 index 000000000..3d9c35b23 --- /dev/null +++ b/internal/backupcontroller/artifact_size_test.go @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/minio/minio-go/v7" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +type fakeS3Lister struct { + objects []s3ObjectInfo + err error + + conn s3Connection + bucket, prefix string + recursive bool + calls int +} + +func (f *fakeS3Lister) ListObjects(_ context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { + f.calls++ + f.bucket, f.prefix, f.recursive = bucket, prefix, recursive + return f.objects, f.err +} + +func s3CredsSecret(namespace, name string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + Data: map[string][]byte{ + "AWS_ACCESS_KEY_ID": []byte("AKID"), + "AWS_SECRET_ACCESS_KEY": []byte("secret"), + }, + } +} + +// reconcilerWithLister returns a reconciler whose S3 reads go to l, backed +// by a fake client seeded with objs (typically the credentials Secret). +func reconcilerWithLister(t *testing.T, l *fakeS3Lister, objs ...client.Object) *BackupJobReconciler { + t.Helper() + c := newMariaDBStrategyTestClient(t, objs...) + return &BackupJobReconciler{ + Client: c, + Scheme: c.Scheme(), + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + l.conn = conn + return l, nil + }, + } +} + +func TestCollectObjects(t *testing.T) { + listing := func() <-chan minio.ObjectInfo { + ch := make(chan minio.ObjectInfo, 1) + ch <- minio.ObjectInfo{Key: "a", Size: 1} + close(ch) + return ch + } + + got, err := collectObjects(context.Background(), listing()) + if err != nil || len(got) != 1 || got[0].Size != 1 { + t.Fatalf("collectObjects = %v, %v; want one object", got, err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if got, err := collectObjects(ctx, listing()); !errors.Is(err, context.Canceled) { + t.Errorf("listing closed after cancellation: got %v, %v; want context.Canceled", got, err) + } + + errCh := make(chan minio.ObjectInfo, 1) + errCh <- minio.ObjectInfo{Err: errors.New("AccessDenied")} + close(errCh) + if _, err := collectObjects(context.Background(), errCh); err == nil { + t.Error("listing error was not returned") + } +} + +// TestExistingBackup_ReadsPastTheCache pins that the retry check sees a +// Backup the cache does not show yet: the previous reconcile created it +// moments earlier. +func TestExistingBackup_ReadsPastTheCache(t *testing.T) { + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}} + r := &BackupJobReconciler{ + Client: newMariaDBStrategyTestClient(t), + APIReader: newMariaDBStrategyTestClient(t, backup), + } + if got, _ := r.existingBackup(context.Background(), "tenant", "bj"); got == nil { + t.Fatal("existingBackup = nil; want the Backup the API server holds") + } + r.APIReader = newMariaDBStrategyTestClient(t) + if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || !probe { + t.Errorf("existingBackup = %v, probe=%v; want nil and a probe when there is no Backup", got, probe) + } +} + +// staleCache wraps a fake client so that Get reports NotFound for Backups, +// the way an informer cache does right after another writer created one. +func staleCache(t *testing.T, c client.Client) client.Client { + t.Helper() + return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name) + } + return cl.Get(ctx, key, obj, opts...) + }, + }) +} + +// notFoundOnce wraps a fake client whose first Backup Get reports NotFound: +// the retry check runs before the Backup exists, the refetch after Create +// collides with it. +func notFoundOnce(t *testing.T, c client.Client) client.Client { + t.Helper() + missed := false + return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok && !missed { + missed = true + return apierrors.NewNotFound(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), key.Name) + } + return cl.Get(ctx, key, obj, opts...) + }, + }) +} + +func TestExistingBackup_ReadErrorIsNotFatal(t *testing.T) { + failing := interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("etcdserver: request timed out") + }, + }) + r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t), APIReader: failing} + if got, probe := r.existingBackup(context.Background(), "tenant", "bj"); got != nil || probe { + t.Errorf("existingBackup = %#v, probe=%v; want nil and no probe on a read error", got, probe) + } +} + +// blockingLister answers nothing until the probe's context is done, the way +// a black-holed endpoint does. +type blockingLister struct{} + +func (blockingLister) ListObjects(ctx context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +// withShortProbeTimeout shortens the probe timeout and fails the test if fn +// is not back well before the default timeout would have expired. +func withShortProbeTimeout(t *testing.T, fn func()) { + t.Helper() + saved := artifactSizeProbeTimeout + artifactSizeProbeTimeout = 50 * time.Millisecond + defer func() { artifactSizeProbeTimeout = saved }() + done := make(chan struct{}) + go func() { + fn() + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the probe did not give up at its timeout") + } +} + +// failingReads wraps a fake client whose every Get fails, like an apiserver +// that times out. +func failingReads(t *testing.T, c client.Client) client.Client { + t.Helper() + return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("etcdserver: request timed out") + }, + }) +} + +func TestFailBackupArtifact(t *testing.T) { + newJob := func() *backupsv1alpha1.BackupJob { + return &backupsv1alpha1.BackupJob{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj"}} + } + + t.Run("a Backup that exists but cannot be read is retried", func(t *testing.T) { + j := newJob() + r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)} + _, err := r.failBackupArtifact(context.Background(), j, fmt.Errorf("%w: timeout", errBackupNotReadBack)) + if !errors.Is(err, errBackupNotReadBack) { + t.Errorf("err = %v, want it returned so the reconcile is retried", err) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := r.Get(context.Background(), client.ObjectKeyFromObject(j), persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Error("the BackupJob was failed although its Backup exists") + } + if cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready"); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "BackupNotReadBack" { + t.Errorf("Ready condition = %+v, want False/BackupNotReadBack", cond) + } + }) + + t.Run("any other error fails the BackupJob", func(t *testing.T) { + j := newJob() + r := &BackupJobReconciler{Client: newMariaDBStrategyTestClient(t, j)} + if _, err := r.failBackupArtifact(context.Background(), j, errors.New("admission webhook denied")); err != nil { + t.Fatalf("failBackupArtifact: %v", err) + } + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Errorf("phase = %q, want Failed", j.Status.Phase) + } + }) +} + +// TestEndpointTrustPool_CAOnly tells apart "the CA on top of the system +// roots" and "the CA alone". The system pool holds the endpoint's +// certificate, and the configured CA is an unrelated one, so the endpoint +// must stop being trusted only when CAOnly is set. +func TestEndpointTrustPool_CAOnly(t *testing.T) { + endpoint := httptest.NewTLSServer(http.NotFoundHandler()) + defer endpoint.Close() + system := x509.NewCertPool() + system.AddCert(endpoint.Certificate()) + + // Every httptest server shares one certificate, so the unrelated CA is + // minted here. + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "unrelated CA"}, NotBefore: time.Now(), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true} + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + unrelatedCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + + verifies := func(caOnly bool) bool { + pool, err := endpointTrustPool(s3Connection{CAPEM: unrelatedCA, CAOnly: caOnly}, system.Clone()) + if err != nil { + t.Fatalf("endpointTrustPool: %v", err) + } + _, err = endpoint.Certificate().Verify(x509.VerifyOptions{Roots: pool}) + return err == nil + } + if !verifies(false) { + t.Error("CA on top of the system roots: the endpoint was not trusted") + } + if verifies(true) { + t.Error("CA alone: the endpoint was trusted through the system roots") + } +} + +// forbidBackupCreate wraps a fake client whose Create of a Backup is +// rejected, the way an admission webhook rejects one. +func forbidBackupCreate(t *testing.T, c client.Client) client.Client { + t.Helper() + return interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return apierrors.NewForbidden(backupsv1alpha1.GroupVersion.WithResource("backups").GroupResource(), obj.GetName(), errors.New("denied by policy")) + } + return cl.Create(ctx, obj, opts...) + }, + }) +} diff --git a/internal/backupcontroller/backupjob_controller.go b/internal/backupcontroller/backupjob_controller.go index c6b743356..b86278649 100644 --- a/internal/backupcontroller/backupjob_controller.go +++ b/internal/backupcontroller/backupjob_controller.go @@ -39,6 +39,10 @@ type BackupJobReconciler struct { // re-reads the BackupJob before dispatching to a driver. Wired in // SetupWithManager. APIReader client.Reader + + // newS3ObjectLister overrides the S3 client used to measure artifact + // sizes; nil means a real minio client. Set by unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) } func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { diff --git a/internal/backupcontroller/cnpg_artifact_size.go b/internal/backupcontroller/cnpg_artifact_size.go new file mode 100644 index 000000000..0756e2bb9 --- /dev/null +++ b/internal/backupcontroller/cnpg_artifact_size.go @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" +) + +// cnpgArtifactSizeBytes sums the objects barman-cloud uploaded for one base +// backup, //base//. The WAL archive +// is not counted: it is one continuous stream shared by every base backup of +// the server, so no part of it belongs to a single Backup. Returns 0 - +// leaving Artifact.SizeBytes absent - when the backup ID is unknown, the +// bucket cannot be read, or nothing is stored under the prefix. +func (r *BackupJobReconciler) cnpgArtifactSizeBytes( + ctx context.Context, + namespace string, + store *strategyv1alpha1.BarmanObjectStoreTemplate, + serverName, backupID string, +) int64 { + logger := getLogger(ctx) + if backupID == "" { + return 0 + } + ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) + defer cancel() + + bucket, path, err := parseBarmanS3DestinationPath(store.DestinationPath) + if err != nil { + logger.Info("cannot measure CNPG backup size", "backupID", backupID, "error", err) + return 0 + } + conn, err := r.cnpgS3Connection(ctx, namespace, store) + if err != nil { + logger.Info("cannot resolve S3 credentials to measure CNPG backup size", "backupID", backupID, "error", err) + return 0 + } + lister, err := r.s3Lister(conn) + if err != nil { + logger.Info("cannot build S3 client to measure CNPG backup size", "backupID", backupID, "error", err) + return 0 + } + prefix := barmanBaseBackupPrefix(path, serverName, backupID) + objects, err := lister.ListObjects(ctx, bucket, prefix, true) + if err != nil { + logger.Info("cannot list CNPG base backup objects to measure size", "backupID", backupID, "error", err) + return 0 + } + var total int64 + for _, obj := range objects { + total += obj.Size + } + if total == 0 { + logger.Info("no CNPG base backup objects found; leaving size unset", "backupID", backupID, "prefix", prefix) + } + return total +} + +func (r *BackupJobReconciler) cnpgS3Connection(ctx context.Context, namespace string, store *strategyv1alpha1.BarmanObjectStoreTemplate) (s3Connection, error) { + var conn s3Connection + if store.S3Credentials == nil { + return conn, errors.New("barmanObjectStore has no s3Credentials") + } + if store.EndpointURL == "" { + conn.Endpoint, conn.Secure = "s3.amazonaws.com", true + } else { + u, err := url.Parse(store.EndpointURL) + if err != nil || u.Host == "" { + return conn, fmt.Errorf("invalid endpointURL %q", store.EndpointURL) + } + // Dropping the path would probe the bucket at another location than + // the one barman writes to. + if strings.Trim(u.Path, "/") != "" { + return conn, fmt.Errorf("endpointURL %q has a path, which the S3 client cannot address", store.EndpointURL) + } + conn.Endpoint, conn.Secure = u.Host, u.Scheme == "https" + } + + creds := store.S3Credentials + accessKeyKey, secretKeyKey := creds.AccessKeyIDKey, creds.SecretAccessKeyKey + if accessKeyKey == "" { + accessKeyKey = defaultS3AccessKeyIDKey + } + if secretKeyKey == "" { + secretKeyKey = defaultS3SecretAccessKeyKey + } + var err error + if conn.AccessKey, err = r.secretValue(ctx, namespace, creds.SecretRef.Name, accessKeyKey); err != nil { + return conn, err + } + if conn.SecretKey, err = r.secretValue(ctx, namespace, creds.SecretRef.Name, secretKeyKey); err != nil { + return conn, err + } + if ca := store.EndpointCA; ca != nil && ca.SecretRef.Name != "" && conn.Secure { + key := ca.Key + if key == "" { + key = defaultEndpointCAKey + } + pem, err := r.secretValue(ctx, namespace, ca.SecretRef.Name, key) + if err != nil { + return conn, err + } + // barman-cloud gets endpointCA as AWS_CA_BUNDLE, which replaces the + // default trust store rather than adding to it. + conn.CAPEM, conn.CAOnly = []byte(pem), true + } + return conn, nil +} + +// parseBarmanS3DestinationPath splits s3://bucket/path into the bucket and +// the key path barman-cloud prefixes every object with. +func parseBarmanS3DestinationPath(destinationPath string) (bucket, path string, err error) { + u, err := url.Parse(destinationPath) + if err != nil || u.Scheme != "s3" || u.Host == "" { + return "", "", fmt.Errorf("destinationPath %q is not an s3:// URL", destinationPath) + } + return u.Host, strings.Trim(u.Path, "/"), nil +} + +// barmanBaseBackupPrefix mirrors barman-cloud's +// os.path.join(path, server_name, "base", backup_id), with a trailing slash +// so one backup ID never matches another that extends it. +func barmanBaseBackupPrefix(path, serverName, backupID string) string { + parts := make([]string, 0, 4) + if path != "" { + parts = append(parts, path) + } + parts = append(parts, serverName, "base", backupID) + return strings.Join(parts, "/") + "/" +} diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go new file mode 100644 index 000000000..85e55e8c2 --- /dev/null +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "encoding/pem" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/cnpgtypes" +) + +func newSizedBarmanObjectStore() *strategyv1alpha1.BarmanObjectStoreTemplate { + return &strategyv1alpha1.BarmanObjectStoreTemplate{ + DestinationPath: "s3://bkt/tenant/pg/", + EndpointURL: "https://s3.example:8443", + S3Credentials: &strategyv1alpha1.S3CredentialsTemplate{ + SecretRef: corev1.LocalObjectReference{Name: "creds"}, + }, + } +} + +func TestBarmanBaseBackupPrefix(t *testing.T) { + cases := []struct{ path, want string }{ + {"tenant/pg", "tenant/pg/srv/base/20261021T100000/"}, + {"", "srv/base/20261021T100000/"}, + } + for _, tc := range cases { + if got := barmanBaseBackupPrefix(tc.path, "srv", "20261021T100000"); got != tc.want { + t.Errorf("barmanBaseBackupPrefix(%q) = %q, want %q", tc.path, got, tc.want) + } + } +} + +func TestParseBarmanS3DestinationPath(t *testing.T) { + cases := []struct { + in, bucket, path string + ok bool + }{ + {"s3://bkt/tenant/pg/", "bkt", "tenant/pg", true}, + {"s3://bkt", "bkt", "", true}, + {"s3://bkt/", "bkt", "", true}, + {"gs://bkt/x/", "", "", false}, + {"bkt/x/", "", "", false}, + } + for _, tc := range cases { + bucket, path, err := parseBarmanS3DestinationPath(tc.in) + if (err == nil) != tc.ok || bucket != tc.bucket || path != tc.path { + t.Errorf("parseBarmanS3DestinationPath(%q) = %q, %q, %v", tc.in, bucket, path, err) + } + } +} + +func TestCNPGArtifactSizeBytes(t *testing.T) { + const prefix = "tenant/pg/srv/base/20261021T100000/" + + t.Run("sums every object of the base backup", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: prefix + "backup.info", Size: 1}, + {Key: prefix + "data_0000.tar.gz", Size: 1000}, + {Key: prefix + "data_0001.tar.gz", Size: 200}, + {Key: prefix + "16385.tar.gz", Size: 30}, + }} + r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")) + got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", newSizedBarmanObjectStore(), "srv", "20261021T100000") + if got != 1231 { + t.Errorf("size = %d, want 1231", got) + } + if l.bucket != "bkt" || l.prefix != prefix || !l.recursive { + t.Errorf("listed %q/%q recursive=%v, want bkt/%q recursive", l.bucket, l.prefix, l.recursive, prefix) + } + if l.conn.Endpoint != "s3.example:8443" || !l.conn.Secure || l.conn.AccessKey != "AKID" || l.conn.SecretKey != "secret" { + t.Errorf("connection = %+v", l.conn) + } + }) + + t.Run("a hanging endpoint is given up at the probe timeout", func(t *testing.T) { + r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds")) + r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + withShortProbeTimeout(t, func() { + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", newSizedBarmanObjectStore(), "srv", "20261021T100000"); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + }) + + t.Run("an empty endpointURL means AWS", func(t *testing.T) { + store := newSizedBarmanObjectStore() + store.EndpointURL = "" + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: prefix + "data.tar", Size: 5}}} + r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")) + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 5 { + t.Errorf("size = %d, want 5", got) + } + if l.conn.Endpoint != "s3.amazonaws.com" || !l.conn.Secure { + t.Errorf("connection = %+v, want s3.amazonaws.com over TLS", l.conn) + } + }) + + t.Run("honours custom credential keys and the endpoint CA", func(t *testing.T) { + store := newSizedBarmanObjectStore() + store.S3Credentials.AccessKeyIDKey = "id" + store.S3Credentials.SecretAccessKeyKey = "key" + store.EndpointCA = &strategyv1alpha1.EndpointCARef{SecretRef: corev1.LocalObjectReference{Name: "ca"}} + creds := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "creds"}, + Data: map[string][]byte{"id": []byte("ID2"), "key": []byte("KEY2")}, + } + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, + Data: map[string][]byte{"ca.crt": []byte("PEM")}, + } + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: prefix + "data.tar", Size: 5}}} + r := reconcilerWithLister(t, l, creds, ca) + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 5 { + t.Errorf("size = %d, want 5", got) + } + if l.conn.AccessKey != "ID2" || l.conn.SecretKey != "KEY2" || string(l.conn.CAPEM) != "PEM" || !l.conn.CAOnly { + t.Errorf("connection = %+v", l.conn) + } + }) + + unset := map[string]struct { + store func() *strategyv1alpha1.BarmanObjectStoreTemplate + backupID string + lister *fakeS3Lister + noSecret bool + }{ + "unknown backup ID": { + store: newSizedBarmanObjectStore, lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}, + }, + "no objects under the prefix": { + store: newSizedBarmanObjectStore, backupID: "id", lister: &fakeS3Lister{}, + }, + "list error": { + store: newSizedBarmanObjectStore, backupID: "id", lister: &fakeS3Lister{err: errors.New("AccessDenied")}, + }, + "missing credentials Secret": { + store: newSizedBarmanObjectStore, backupID: "id", noSecret: true, + lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}, + }, + "no s3Credentials": { + store: func() *strategyv1alpha1.BarmanObjectStoreTemplate { + s := newSizedBarmanObjectStore() + s.S3Credentials = nil + return s + }, + backupID: "id", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}, + }, + "non-s3 destination": { + store: func() *strategyv1alpha1.BarmanObjectStoreTemplate { + s := newSizedBarmanObjectStore() + s.DestinationPath = "azure://container/x" + return s + }, + backupID: "id", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}, + }, + "endpointURL with a path": { + store: func() *strategyv1alpha1.BarmanObjectStoreTemplate { + s := newSizedBarmanObjectStore() + s.EndpointURL = "https://s3.example:8443/prefix" + return s + }, + backupID: "20261021T100000", lister: &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}}, + }, + } + for name, tc := range unset { + t.Run(name+" leaves size unset", func(t *testing.T) { + var r *BackupJobReconciler + if tc.noSecret { + r = reconcilerWithLister(t, tc.lister) + } else { + r = reconcilerWithLister(t, tc.lister, s3CredsSecret("tenant", "creds")) + } + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", tc.store(), "srv", tc.backupID); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + } +} + +// TestCNPGArtifactSizeBytes_MinioClient drives the real minio client against +// a stub ListObjectsV2 endpoint, so the endpoint URL parsing, Secret wiring +// and response parsing are exercised together. +func TestCNPGArtifactSizeBytes_MinioClient(t *testing.T) { + const prefix = "tenant/pg/srv/base/20261021T100000/" + var gotPath, gotPrefix, gotDelimiter, gotAuth string + srv := httptest.NewServer(cnpgListStub(prefix, func(req *http.Request) { + gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization") + gotDelimiter = req.URL.Query().Get("delimiter") + })) + defer srv.Close() + + c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds")) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + store := newSizedBarmanObjectStore() + store.EndpointURL = srv.URL + + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 4103 { + t.Fatalf("size = %d, want 4103", got) + } + if gotPath != "/bkt/" || gotPrefix != prefix || gotDelimiter != "" { + t.Errorf("request path=%q prefix=%q delimiter=%q, want /bkt/, %q and a recursive listing", gotPath, gotPrefix, gotDelimiter, prefix) + } + if !strings.Contains(gotAuth, "Credential=AKID/") { + t.Errorf("request not signed with the Secret's access key: %q", gotAuth) + } +} + +func cnpgListStub(prefix string, seen func(*http.Request)) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/xml") + if _, ok := req.URL.Query()["location"]; ok { + fmt.Fprint(w, ``) + return + } + if seen != nil { + seen(req) + } + fmt.Fprintf(w, ` + + bkt%[1]s21000false + %[1]sbackup.info7"a"2026-10-21T10:01:00.000Z + %[1]sdata.tar.gz4096"b"2026-10-21T10:01:00.000Z +`, prefix) + } +} + +// TestCNPGArtifactSizeBytes_EndpointCA serves the listing over TLS with a +// self-signed certificate: the probe must trust it only through endpointCA. +func TestCNPGArtifactSizeBytes_EndpointCA(t *testing.T) { + const prefix = "tenant/pg/srv/base/20261021T100000/" + srv := httptest.NewTLSServer(cnpgListStub(prefix, nil)) + defer srv.Close() + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}) + + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "s3-ca"}, + Data: map[string][]byte{"ca.crt": caPEM}, + } + c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds"), ca) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + store := newSizedBarmanObjectStore() + store.EndpointURL = srv.URL + + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 0 { + t.Errorf("without endpointCA: size = %d, want 0 (untrusted certificate)", got) + } + store.EndpointCA = &strategyv1alpha1.EndpointCARef{SecretRef: corev1.LocalObjectReference{Name: "s3-ca"}} + if got := r.cnpgArtifactSizeBytes(context.Background(), "tenant", store, "srv", "20261021T100000"); got != 4103 { + t.Errorf("with endpointCA: size = %d, want 4103", got) + } +} + +func TestCreateCNPGBackupArtifact_SizeBytes(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + strategyGroup := strategyv1alpha1.GroupVersion.Group + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-size"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "strat"}, + } + cnpgBk := &cnpgtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, + Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}, + } + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "tenant/pg/srv/base/20261021T100000/data.tar.gz", Size: 777}}} + r := reconcilerWithLister(t, l) + r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds")) + + got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + if err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 777 { + t.Fatalf("Artifact = %#v, want SizeBytes 777", got.Status.Artifact) + } + if l.prefix != "tenant/pg/srv/base/20261021T100000/" { + t.Errorf("listed prefix %q does not use the effective serverName", l.prefix) + } +} + +func TestCreateCNPGBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + existing := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"}} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}} + r := reconcilerWithLister(t, l) + r.Client = newCNPGStrategyTestClient(t, existing, s3CredsSecret("tenant", "creds")) + cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} + + if _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")); err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + if l.calls != 0 { + t.Errorf("S3 listed %d times for a Backup that already exists", l.calls) + } +} + +func TestCreateCNPGBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}} + store := newCNPGStrategyTestClient(t, backup) + r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store)} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}} + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + if err != nil || got == nil || got.Name != "bj-race" { + t.Fatalf("createCNPGBackupArtifact = %v, %v; want the Backup that already exists", got, err) + } +} + +func TestCreateCNPGBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}} + store := newCNPGStrategyTestClient(t, backup) + r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store)} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}} + _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + if !errors.Is(err, errBackupNotReadBack) { + t.Errorf("err = %v, want errBackupNotReadBack", err) + } +} + +// TestReconcileCNPG_UnreadableBackupIsRetried drives a completed backup +// through the reconcile: when the Backup exists but cannot be read back, +// the reconcile is retried and the BackupJob is not failed. +func TestReconcileCNPG_UnreadableBackupIsRetried(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Minute)), + newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(time.Now().Add(-time.Hour))) + _, backups := f.pass() + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups)) + } + ctx := context.Background() + completed := backups[0].DeepCopy() + completed.Status.Phase = cnpgBackupPhaseComplete + completed.Status.BackupID = "20261021T100000" + if err := f.c.Update(ctx, completed); err != nil { + t.Fatalf("complete cnpg.io/Backup: %v", err) + } + if err := f.c.Create(ctx, &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "bj"}}); err != nil { + t.Fatalf("seed Backup: %v", err) + } + f.r.APIReader = failingReads(t, f.c) + + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); !errors.Is(err, errBackupNotReadBack) { + t.Fatalf("reconcileCNPG err = %v, want errBackupNotReadBack so the reconcile is retried", err) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Error("the BackupJob was failed although its Backup exists") + } +} + +func TestCreateCNPGBackupArtifact_FailedExistenceCheckSkipsProbe(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-transient"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}} + r := reconcilerWithLister(t, l) + r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds")) + r.APIReader = failingReads(t, newCNPGStrategyTestClient(t)) + cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} + + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + if err != nil || got == nil { + t.Fatalf("createCNPGBackupArtifact = %v, %v; a transient read must not fail the backup", got, err) + } + if l.calls != 0 { + t.Errorf("S3 probed %d times after the existence check failed", l.calls) + } +} + +// TestReconcileCNPG_RejectedCreateFailsTheJob pins that only the read-back +// is retried: a Create the apiserver rejects still fails the BackupJob. +func TestReconcileCNPG_RejectedCreateFailsTheJob(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Minute)), + newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(time.Now().Add(-time.Hour))) + _, backups := f.pass() + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups)) + } + ctx := context.Background() + completed := backups[0].DeepCopy() + completed.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(ctx, completed); err != nil { + t.Fatalf("complete cnpg.io/Backup: %v", err) + } + f.r.Client = forbidBackupCreate(t, f.c) + + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); err != nil { + t.Fatalf("reconcileCNPG err = %v, want the BackupJob failed rather than retried", err) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase) + } +} diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 99e6b03f6..819efc10f 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -435,7 +435,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp } artifact, err := r.createCNPGBackupArtifact(ctx, j, resolved, cnpgBackup, clusterName, serverName, attachment.flavor, rendered, app) if err != nil { - return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err)) + return r.failBackupArtifact(ctx, j, err) } now := metav1.Now() j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name} @@ -815,6 +815,10 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( rendered *strategyv1alpha1.CNPGTemplate, sourceApp *postgresapp.Postgres, ) (*backupsv1alpha1.Backup, error) { + existing, probe := r.existingBackup(ctx, j.Namespace, j.Name) + if existing != nil { + return existing, nil + } takenAt := metav1.Now() if cnpgBackup.Status.StartedAt != nil && !cnpgBackup.Status.StartedAt.IsZero() { takenAt = *cnpgBackup.Status.StartedAt @@ -860,6 +864,9 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( UnderlyingResources: underlyingResources, }, } + if probe { + backup.Status.Artifact.SizeBytes = r.cnpgArtifactSizeBytes(ctx, j.Namespace, &rendered.BarmanObjectStore, serverName, cnpgBackup.Status.BackupID) + } if j.Spec.PlanRef != nil { backup.Spec.PlanRef = j.Spec.PlanRef } @@ -872,8 +879,8 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( return nil, err } existing := &backupsv1alpha1.Backup{} - if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { - return nil, getErr + if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { + return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr) } return existing, nil } diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go new file mode 100644 index 000000000..6397325ca --- /dev/null +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// Slack around the [created, completed] window of the operator Backup when +// matching a dump: the timestamp in its file name comes from the backup Job +// pod's clock and is truncated to seconds, and its LastModified comes from +// the object store's clock, neither of which is the apiserver's. +const mariadbSizeProbeClockSkew = time.Minute + +var ( + errMariaDBDumpNotFound = errors.New("no dump in the bucket belongs to the operator Backup") + errMariaDBDumpAmbiguous = errors.New("the dump of the operator Backup cannot be told apart from another run's") +) + +// mariadbDump is a logical dump mariadb-operator wrote to the bucket. +type mariadbDump struct { + // Key is the object key; Name is the file name, Key without the prefix. + Key string + Name string + // Timestamp is the one in the file name, taken by the backup Job pod's + // clock as the dump started. + Timestamp time.Time + Size int64 + // LastModified is the object store's clock as the upload finished. + LastModified time.Time +} + +// mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for +// mdbBackup. Returns 0 - leaving Artifact.SizeBytes absent - when the dump +// cannot be found or told apart: an unknown size must not be reported as a +// measured one. +func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) int64 { + ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) + defer cancel() + dump, err := findMariaDBDump(ctx, r.objectStore(), mdbBackup, completedAt) + if err != nil { + getLogger(ctx).Info("MariaDB backup size not measured; leaving it unset", "backup", mdbBackup.Name, "error", err) + return 0 + } + return dump.Size +} + +// findMariaDBDump lists the bucket mdbBackup wrote to and returns the one +// dump that belongs to it, as matchMariaDBDump decides. The location and +// credentials come from mdbBackup's own spec: the strategy it was rendered +// from can have been edited since, and pointing at another bucket now says +// nothing about where this dump went. +func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { + s3 := mdbBackup.Spec.Storage.S3 + if s3 == nil { + return mariadbDump{}, errors.New("the operator Backup does not store to S3") + } + others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup) + if err != nil { + return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) + } + conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) + if err != nil { + return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err) + } + lister, err := store.lister(conn) + if err != nil { + return mariadbDump{}, fmt.Errorf("build S3 client: %w", err) + } + // Non-recursive, as mariadb-operator lists it: the default prefix is also + // the default CNPG destinationPath, so an app of each kind with the same + // name shares it and a recursive listing would walk the WAL archive. + prefix := mariadbOperatorS3Prefix(s3.Prefix) + objects, err := lister.ListObjects(ctx, s3.Bucket, prefix, false) + if err != nil { + return mariadbDump{}, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, prefix, err) + } + own := newMariaDBRun(mdbBackup, mdbBackup.CreationTimestamp.Time, completedAt) + return matchMariaDBDump(objects, prefix, own, others) +} + +// mariadbRun is the span in which an operator Backup can have written its +// dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side. +type mariadbRun struct { + name string + lower, upper time.Time +} + +func newMariaDBRun(b *mariadbtypes.Backup, created, completed time.Time) mariadbRun { + return mariadbRun{ + name: b.Namespace + "/" + b.Name, + lower: created.Add(-mariadbSizeProbeClockSkew), + upper: completed.Add(mariadbSizeProbeClockSkew), + } +} + +func (r mariadbRun) placesOnBothClocks(d mariadbDump) bool { + return inWindow(d.Timestamp, r.lower, r.upper) && inWindow(d.LastModified, r.lower, r.upper) +} + +func (r mariadbRun) placesOnEitherClock(d mariadbDump) bool { + return inWindow(d.Timestamp, r.lower, r.upper) || inWindow(d.LastModified, r.lower, r.upper) +} + +// matchMariaDBDump picks the dump of run own among objects, which share a +// prefix with every other run of the application. A dump carries two +// clocks, the backup pod's in its file name and the object store's in its +// LastModified, and either can be skewed. A dump is taken only when it is +// the one dump either clock places in own's window, both clocks place it +// there, and neither places it in the window of another finished run: a +// dump in the window on one clock only may be this run's own, misplaced by +// a skewed clock, which leaves any other candidate unproven. A neighbour's +// dump is still taken if this run's own is missing from the listing, or +// misplaced on both clocks, and the neighbour's operator Backup is gone or +// has not completed. +func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, others []mariadbRun) (mariadbDump, error) { + var near []mariadbDump + for _, obj := range objects { + name := strings.TrimPrefix(obj.Key, prefix) + ts, ok := parseMariaDBLogicalBackupFile(name) + if !ok { + continue + } + d := mariadbDump{Key: obj.Key, Name: name, Timestamp: ts, Size: obj.Size, LastModified: obj.LastModified} + if own.placesOnEitherClock(d) { + near = append(near, d) + } + } + if len(near) == 0 { + return mariadbDump{}, errMariaDBDumpNotFound + } + if len(near) > 1 { + return mariadbDump{}, fmt.Errorf("%w: %d dumps fall in its lifetime on at least one clock", errMariaDBDumpAmbiguous, len(near)) + } + d := near[0] + if !own.placesOnBothClocks(d) { + return mariadbDump{}, fmt.Errorf("%w: %s falls in its lifetime on one clock only", errMariaDBDumpAmbiguous, d.Name) + } + for _, other := range others { + if other.placesOnEitherClock(d) { + return mariadbDump{}, fmt.Errorf("%w: %s also falls in the lifetime of operator Backup %s", errMariaDBDumpAmbiguous, d.Name, other.name) + } + } + return d, nil +} + +// finishedMariaDBRuns returns the runs of the other operator Backups that +// write under the same bucket and prefix as b and have completed, whether +// their Job succeeded or failed. One still running is left out: its window +// has no end yet, and an operator Backup stuck short of Complete, as one +// whose Job pod never starts is, would otherwise contest every later run of +// the application for good. The endpoint is not compared, since one has +// several spellings, and a false neighbour only leaves a size unset. +func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup) ([]mariadbRun, error) { + list := &mariadbtypes.BackupList{} + if err := reader.List(ctx, list); err != nil { + return nil, err + } + var runs []mariadbRun + for i := range list.Items { + other := &list.Items[i] + if other.Namespace == b.Namespace && other.Name == b.Name { + continue + } + s3 := other.Spec.Storage.S3 + if s3 == nil || s3.Bucket != b.Spec.Storage.S3.Bucket || mariadbOperatorS3Prefix(s3.Prefix) != mariadbOperatorS3Prefix(b.Spec.Storage.S3.Prefix) { + continue + } + cond := apimeta.FindStatusCondition(other.Status.Conditions, mariadbtypes.ConditionTypeComplete) + if cond == nil || cond.Status != metav1.ConditionTrue { + continue + } + runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, cond.LastTransitionTime.Time)) + } + return runs, nil +} + +// mariadbS3Connection resolves the Secrets and TLS settings of the operator +// Backup's storage, so the bucket is read with the tenant's credentials +// rather than the controller's. +func mariadbS3Connection(ctx context.Context, store objectStore, namespace string, s3 *mariadbtypes.S3Storage) (s3Connection, error) { + conn := s3Connection{ + Endpoint: s3.Endpoint, + Region: s3.Region, + Secure: s3.TLS != nil && s3.TLS.Enabled, + } + var err error + if conn.AccessKey, err = store.secretValue(ctx, namespace, s3.AccessKeyIdSecretKeyRef.Name, s3.AccessKeyIdSecretKeyRef.Key); err != nil { + return conn, err + } + if conn.SecretKey, err = store.secretValue(ctx, namespace, s3.SecretAccessKeySecretKeyRef.Name, s3.SecretAccessKeySecretKeyRef.Key); err != nil { + return conn, err + } + if ref := s3.SessionTokenSecretKeyRef; ref != nil { + if conn.SessionToken, err = store.secretValue(ctx, namespace, ref.Name, ref.Key); err != nil { + return conn, err + } + } + if conn.Secure && s3.TLS.CASecretKeyRef != nil { + ca, err := store.secretValue(ctx, namespace, s3.TLS.CASecretKeyRef.Name, s3.TLS.CASecretKeyRef.Key) + if err != nil { + return conn, err + } + conn.CAPEM = []byte(ca) + } + return conn, nil +} + +// mariadbOperatorS3Prefix mirrors how mariadb-operator normalises +// spec.storage.s3.prefix before prepending it to object keys. +func mariadbOperatorS3Prefix(prefix string) string { + if prefix == "" || prefix == "/" { + return "" + } + if !strings.HasSuffix(prefix, "/") { + return prefix + "/" + } + return prefix +} + +func inWindow(t, lower, upper time.Time) bool { + return !t.Before(lower) && !t.After(upper) +} + +// Compression tokens mariadb-operator puts in a dump file name: the +// algorithm before .sql in v25.10 (backup..gzip.sql), the extension +// after it in later releases (backup..sql.gz). +var ( + mariadbDumpAlgorithms = map[string]bool{"none": true, "bzip2": true, "gzip": true, "zstd": true} + mariadbDumpExtensions = map[string]bool{"bz2": true, "gz": true, "zst": true} +) + +// parseMariaDBLogicalBackupFile accepts the logical-backup file names +// mariadb-operator writes and returns the embedded timestamp. +func parseMariaDBLogicalBackupFile(name string) (time.Time, bool) { + parts := strings.Split(name, ".") + if parts[0] != "backup" { + return time.Time{}, false + } + switch { + case len(parts) == 3 && parts[2] == "sql": + case len(parts) == 4 && parts[2] == "sql" && mariadbDumpExtensions[parts[3]]: + case len(parts) == 4 && parts[3] == "sql" && mariadbDumpAlgorithms[parts[2]]: + default: + return time.Time{}, false + } + ts, err := time.Parse(time.RFC3339, parts[1]) + if err != nil { + return time.Time{}, false + } + return ts, true +} diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go new file mode 100644 index 000000000..9d649c15c --- /dev/null +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -0,0 +1,750 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +func dumpKey(prefix string, ts time.Time, compression string) string { + if compression == "" { + return fmt.Sprintf("%sbackup.%s.sql", prefix, ts.UTC().Format(time.RFC3339)) + } + return fmt.Sprintf("%sbackup.%s.%s.sql", prefix, ts.UTC().Format(time.RFC3339), compression) +} + +// mariadbBackupSucceeded is the Complete condition mariadb-operator sets on a +// Backup whose Job succeeded. A failed Job also gets Status=True, with +// Reason=JobFailed, so a fixture without the reason is not a success. +func mariadbBackupSucceeded(at metav1.Time) metav1.Condition { + return metav1.Condition{ + Type: mariadbtypes.ConditionTypeComplete, + Status: metav1.ConditionTrue, + Reason: mariadbtypes.ConditionReasonJobComplete, + Message: "Success", + LastTransitionTime: at, + } +} + +func TestParseMariaDBLogicalBackupFile(t *testing.T) { + cases := map[string]bool{ + "backup.2026-10-21T10:00:00Z.sql": true, + "backup.2026-10-21T10:00:00Z.gzip.sql": true, + "backup.2026-10-21T10:00:00+04:00.sql": true, + "backup.2026-10-21T10:00:00Z.sql.tmp": false, + "backup.2026-10-21T10:00:00Z.sql.gz": true, + "backup.2026-10-21T10:00:00Z.sql.zst": true, + "backup.2026-10-21T10:00:00Z.zstd.sql": true, + "backup.2026-10-21T10:00:00Z.xz.sql": false, + "backup.2026-10-21T10:00:00Z.sql.gz.tmp": false, + "physicalbackup-20261021100000.xb": false, + "backup.not-a-date.sql": false, + "sub/backup.2026-10-21T10:00:00Z.sql": false, + "backup.2026-10-21T10:00:00Z.a.gzip.sql": false, + "backups.2026-10-21T10:00:00Z.gzip.sql": false, + "backup.2026-10-21T10:00:00Z.gzip.dump": false, + "": false, + } + for name, want := range cases { + if _, got := parseMariaDBLogicalBackupFile(name); got != want { + t.Errorf("parseMariaDBLogicalBackupFile(%q) = %v, want %v", name, got, want) + } + } +} + +func TestMariaDBOperatorS3Prefix(t *testing.T) { + cases := map[string]string{"": "", "/": "", "ns/app": "ns/app/", "ns/app/": "ns/app/"} + for in, want := range cases { + if got := mariadbOperatorS3Prefix(in); got != want { + t.Errorf("mariadbOperatorS3Prefix(%q) = %q, want %q", in, got, want) + } + } +} + +// operatorBackupStoringTo returns the operator Backup a strategy with s3 +// storage renders, created at created in namespace tenant. +func operatorBackupStoringTo(t *testing.T, s3 *strategyv1alpha1.MariaDBS3Template, created time.Time) *mariadbtypes.Backup { + t.Helper() + storage, err := buildMariaDBBackupStorage(strategyv1alpha1.MariaDBStorageTemplate{S3: s3}) + if err != nil { + t.Fatal(err) + } + return &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk", CreationTimestamp: metav1.NewTime(created)}, + Spec: mariadbtypes.BackupSpec{Storage: storage}, + } +} + +func TestMariaDBArtifactSizeBytes(t *testing.T) { + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + completed := created.Add(3 * time.Minute) + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + mdbBackup := operatorBackupStoringTo(t, s3, created) + const prefix = "tenant/db/" + + t.Run("single dump inside the backup window", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, created.Add(-24*time.Hour), "gzip"), Size: 111, LastModified: created.Add(-24 * time.Hour)}, + {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)}, + {Key: prefix + "0-backup-target.txt", Size: 7}, + }} + got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed) + if got != 4242 { + t.Errorf("size = %d, want 4242", got) + } + if l.conn.AccessKey != "AKID" || l.conn.Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the Secret's key and the rendered endpoint", l.conn) + } + if l.bucket != "bkt" || l.prefix != prefix || l.recursive { + t.Errorf("listed %q/%q recursive=%v, want bkt/%q non-recursive", l.bucket, l.prefix, l.recursive, prefix) + } + }) + + t.Run("reads the endpoint CA from its Secret", func(t *testing.T) { + tlsS3 := newRenderedMariaDBTemplate().Storage.S3 + tlsS3.Prefix = "tenant/db" + tlsS3.TLS = &strategyv1alpha1.MariaDBS3TLS{ + Enabled: true, + CASecretKeyRef: &strategyv1alpha1.MariaDBSecretKeySelector{Name: "ca", Key: "ca.crt"}, + } + ca := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, + Data: map[string][]byte{"ca.crt": []byte("PEM")}, + } + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3, LastModified: created.Add(time.Minute)}}} + r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), ca) + if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed); got != 3 { + t.Errorf("size = %d, want 3", got) + } + if !l.conn.Secure || string(l.conn.CAPEM) != "PEM" { + t.Errorf("connection = %+v, want TLS with the Secret's CA", l.conn) + } + }) + + t.Run("reads the session token from its Secret", func(t *testing.T) { + stsS3 := newRenderedMariaDBTemplate().Storage.S3 + stsS3.Prefix = "tenant/db" + stsS3.SessionTokenSecretKeyRef = &strategyv1alpha1.MariaDBSecretKeySelector{Name: "sts", Key: "token"} + sts := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "sts"}, + Data: map[string][]byte{"token": []byte("TOKEN")}, + } + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4, LastModified: created.Add(time.Minute)}}} + r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), sts) + if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed); got != 4 { + t.Errorf("size = %d, want 4", got) + } + if l.conn.SessionToken != "TOKEN" { + t.Errorf("session token = %q, want the Secret's", l.conn.SessionToken) + } + }) + + t.Run("pod clock skew within tolerance", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9, LastModified: created.Add(time.Minute)}, + }} + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 9 { + t.Errorf("size = %d, want 9", got) + } + }) + + t.Run("a skewed pod clock does not attribute a neighbouring run's dump", func(t *testing.T) { + // The pod clock runs 5m behind. This run's dump is named outside the + // window; the previous run's dump, uploaded before this run started, + // is named inside it. + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, created.Add(-4*time.Minute), "gzip"), Size: 7, LastModified: created.Add(time.Minute)}, + {Key: dumpKey(prefix, created.Add(30*time.Second), "gzip"), Size: 666, LastModified: created.Add(-4 * time.Minute)}, + }} + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0: no dump is inside the window on both clocks", got) + } + }) + + t.Run("a dump uploaded just after Complete is inside the slack", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, completed.Add(30*time.Second), "gzip"), Size: 5, LastModified: completed.Add(30 * time.Second)}, + }} + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 5 { + t.Errorf("size = %d, want 5", got) + } + }) + + t.Run("a hanging endpoint is given up at the probe timeout", func(t *testing.T) { + r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds")) + r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + withShortProbeTimeout(t, func() { + if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + }) + + t.Run("two dumps in the window leave size unset", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)}, + {Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2, LastModified: created.Add(2 * time.Minute)}, + }} + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0 (ambiguous)", got) + } + }) + + t.Run("no dump in the window leaves size unset", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1, LastModified: completed.Add(time.Hour)}, + }} + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + + t.Run("list error leaves size unset", func(t *testing.T) { + l := &fakeS3Lister{err: errors.New("AccessDenied")} + if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) + + t.Run("missing credentials Secret leaves size unset", func(t *testing.T) { + c := newMariaDBStrategyTestClient(t) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + t.Errorf("size = %d, want 0", got) + } + }) +} + +// TestMatchMariaDBDump pins what a restore reads off a match: the object +// itself, and errors it can tell apart. +func TestMatchMariaDBDump(t *testing.T) { + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + completed := created.Add(3 * time.Minute) + const prefix = "tenant/db/" + own := s3ObjectInfo{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(2 * time.Minute)} + run := newMariaDBRun(&mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk"}}, created, completed) + + got, err := matchMariaDBDump([]s3ObjectInfo{own}, prefix, run, nil) + if err != nil { + t.Fatalf("matchMariaDBDump: %v", err) + } + want := mariadbDump{ + Key: own.Key, + Name: strings.TrimPrefix(own.Key, prefix), + Timestamp: created.Add(time.Minute), + Size: 4242, + LastModified: own.LastModified, + } + if got != want { + t.Errorf("matchMariaDBDump = %+v, want %+v", got, want) + } + + if _, err := matchMariaDBDump(nil, prefix, run, nil); !errors.Is(err, errMariaDBDumpNotFound) { + t.Errorf("no dump: err = %v, want errMariaDBDumpNotFound", err) + } + second := s3ObjectInfo{Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 1, LastModified: created.Add(2 * time.Minute)} + if _, err := matchMariaDBDump([]s3ObjectInfo{own, second}, prefix, run, nil); !errors.Is(err, errMariaDBDumpAmbiguous) { + t.Errorf("two dumps: err = %v, want errMariaDBDumpAmbiguous", err) + } +} + +// TestFindMariaDBDump pins that a dump is read with nothing but a client, +// as a reconciler other than BackupJobReconciler has, and that an operator +// Backup that does not store to S3 is refused without a listing. +func TestFindMariaDBDump(t *testing.T) { + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + completed := created.Add(3 * time.Minute) + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey("tenant/db/", created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)}, + }} + store := objectStore{ + reader: newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds")), + newLister: func(conn s3Connection) (s3ObjectLister, error) { + l.conn = conn + return l, nil + }, + } + + got, err := findMariaDBDump(context.Background(), store, operatorBackupStoringTo(t, s3, created), completed) + if err != nil || got.Size != 4242 { + t.Fatalf("findMariaDBDump = %+v, %v; want the 4242-byte dump", got, err) + } + if l.conn.AccessKey != "AKID" { + t.Errorf("connection = %+v, want the Secret's key", l.conn) + } + + pvc := &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-pvc", CreationTimestamp: metav1.NewTime(created)}, + Spec: mariadbtypes.BackupSpec{Storage: mariadbtypes.BackupStorage{PersistentVolumeClaim: &corev1.PersistentVolumeClaimSpec{}}}, + } + if _, err := findMariaDBDump(context.Background(), store, pvc, completed); err == nil { + t.Error("findMariaDBDump on a PVC-backed operator Backup returned no error") + } + if l.calls != 1 { + t.Errorf("S3 listed %d times, want once: a PVC-backed Backup has nothing to list", l.calls) + } +} + +// TestFindMariaDBDump_Ownership drives runs of one application that share +// a prefix. This run lives from t0 to t0+40s, so its window with the +// one-minute slack is [t0-1m, t0+1m40s]; its own dump is 123 bytes and a +// neighbour's is 666 or 777. A neighbour's dump must never be returned. +func TestFindMariaDBDump_Ownership(t *testing.T) { + t0 := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + const prefix = "tenant/db/" + run := func(name string, created, completed time.Time) *mariadbtypes.Backup { + b := operatorBackupStoringTo(t, s3, created) + b.Name = name + b.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(completed))} + return b + } + dump := func(named, uploaded time.Time, size int64) s3ObjectInfo { + return s3ObjectInfo{Key: dumpKey(prefix, named, "gzip"), Size: size, LastModified: uploaded} + } + own := run("op-bk", t0, t0.Add(40*time.Second)) + find := func(t *testing.T, objects []s3ObjectInfo, operatorBackups ...client.Object) (mariadbDump, error) { + t.Helper() + store := objectStore{ + reader: newMariaDBStrategyTestClient(t, append(operatorBackups, own, s3CredsSecret("tenant", "creds"))...), + newLister: func(s3Connection) (s3ObjectLister, error) { + return &fakeS3Lister{objects: objects}, nil + }, + } + return findMariaDBDump(context.Background(), store, own, t0.Add(40*time.Second)) + } + wantAmbiguous := func(t *testing.T, got mariadbDump, err error) { + t.Helper() + if !errors.Is(err, errMariaDBDumpAmbiguous) { + t.Errorf("findMariaDBDump = %+v, %v; want errMariaDBDumpAmbiguous", got, err) + } + } + wantOwn := func(t *testing.T, got mariadbDump, err error) { + t.Helper() + if err != nil || got.Size != 123 { + t.Errorf("findMariaDBDump = %+v, %v; want this run's 123-byte dump", got, err) + } + } + + t.Run("a pod clock behind does not hand this run the previous run's dump", func(t *testing.T) { + // The previous run finished 20s before this one was created and its + // operator Backup is gone. This run's pod clock is 5m behind, so its + // own dump is named outside the window and uploaded inside it. + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(-30*time.Second), t0.Add(-20*time.Second), 666), + dump(t0.Add(5*time.Second-5*time.Minute), t0.Add(30*time.Second), 123), + }) + wantAmbiguous(t, got, err) + }) + + t.Run("a pod clock ahead does not hand this run the next run's dump", func(t *testing.T) { + // The next run started 5s after this one completed and has uploaded + // its dump by the time this one is probed; it has not completed yet. + next := operatorBackupStoringTo(t, s3, t0.Add(45*time.Second)) + next.Name = "op-next" + next.Status.Conditions = []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(45 * time.Second)), + }} + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(5*time.Second+5*time.Minute), t0.Add(30*time.Second), 123), + dump(t0.Add(50*time.Second), t0.Add(80*time.Second), 777), + }, next) + wantAmbiguous(t, got, err) + }) + + t.Run("an object store clock behind does not hand this run the next run's dump", func(t *testing.T) { + // The store clock is 2m behind: this run's upload is stamped before + // the window opens, the next run's is pulled back into it. + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(5*time.Second), t0.Add(30*time.Second-2*time.Minute), 123), + dump(t0.Add(50*time.Second), t0.Add(3*time.Minute-2*time.Minute), 777), + }) + wantAmbiguous(t, got, err) + }) + + t.Run("a dump another finished run wrote is not taken when this run's own is gone", func(t *testing.T) { + // This run's dump was pruned; the next run's falls in this run's + // window on both clocks, and in its own. + next := run("op-next", t0.Add(45*time.Second), t0.Add(85*time.Second)) + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(50*time.Second), t0.Add(75*time.Second), 777), + }, next) + wantAmbiguous(t, got, err) + }) + + t.Run("a dump the window holds on one clock only is not taken", func(t *testing.T) { + // This run's dump is gone; the previous run's operator Backup is too, + // and its dump, started 2m before this run, finished uploading 20s + // before it. + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(-2*time.Minute), t0.Add(-20*time.Second), 666), + }) + if err == nil { + t.Errorf("findMariaDBDump = %+v; want no dump attributed", got) + } + }) + + t.Run("a run still in progress does not contest this run's dump", func(t *testing.T) { + // A concurrent run created 30s before this one has not uploaded yet. + running := operatorBackupStoringTo(t, s3, t0.Add(-30*time.Second)) + running.Name = "op-running" + running.Status.Conditions = []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(-30 * time.Second)), + }} + got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, running) + wantOwn(t, got, err) + }) + + t.Run("runs minutes apart each keep their own dump", func(t *testing.T) { + previous := run("op-previous", t0.Add(-4*time.Minute), t0.Add(-3*time.Minute)) + next := run("op-next", t0.Add(3*time.Minute), t0.Add(4*time.Minute)) + got, err := find(t, []s3ObjectInfo{ + dump(t0.Add(-230*time.Second), t0.Add(-190*time.Second), 666), + dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123), + dump(t0.Add(185*time.Second), t0.Add(220*time.Second), 777), + }, previous, next) + wantOwn(t, got, err) + }) + + t.Run("a run that writes elsewhere does not contest this run's dump", func(t *testing.T) { + other := newRenderedMariaDBTemplate().Storage.S3 + other.Prefix = "tenant/other" + elsewhere := operatorBackupStoringTo(t, other, t0) + elsewhere.Name = "op-elsewhere" + elsewhere.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(t0.Add(40 * time.Second)))} + got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, elsewhere) + wantOwn(t, got, err) + }) + + t.Run("a run stuck short of Complete does not contest later runs", func(t *testing.T) { + // A backup Job whose pod never starts leaves its operator Backup + // JobRunning for good. + stuck := operatorBackupStoringTo(t, s3, t0.Add(-3*time.Hour)) + stuck.Name = "op-stuck" + stuck.Status.Conditions = []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(t0.Add(-3 * time.Hour)), + }} + got, err := find(t, []s3ObjectInfo{dump(t0.Add(5*time.Second), t0.Add(30*time.Second), 123)}, stuck) + wantOwn(t, got, err) + }) +} + +// TestMariaDBArtifactSizeBytes_MinioClient drives the real minio client +// against a stub ListObjectsV2 endpoint, so the Secret wiring, prefix and +// response parsing are exercised together. +func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + key := dumpKey("tenant/db/", created.Add(time.Minute), "gzip") + + var gotPath, gotPrefix, gotDelimiter, gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization") + gotDelimiter = req.URL.Query().Get("delimiter") + w.Header().Set("Content-Type", "application/xml") + fmt.Fprintf(w, ` + + bkttenant/db/11000false + %s31337"x"2026-10-21T10:01:00.000Z + tenant/db/srv/ +`, key) + })) + defer srv.Close() + + c := newMariaDBStrategyTestClient(t, s3CredsSecret("tenant", "creds")) + r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} + + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Endpoint = strings.TrimPrefix(srv.URL, "http://") + s3.Prefix = "tenant/db/" + s3.Region = "us-east-1" + + if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute)); got != 31337 { + t.Fatalf("size = %d, want 31337", got) + } + if gotPath != "/bkt/" || gotPrefix != "tenant/db/" || gotDelimiter != "/" { + t.Errorf("request path=%q prefix=%q delimiter=%q, want /bkt/, tenant/db/ and a non-recursive listing", gotPath, gotPrefix, gotDelimiter) + } + if !strings.Contains(gotAuth, "Credential=AKID/") { + t.Errorf("request not signed with the Secret's access key: %q", gotAuth) + } +} + +func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { + apps := mariadbapp.GroupName + completed := metav1.NewTime(time.Date(2026, 10, 21, 10, 3, 0, 0, time.UTC)) + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "src/" + mdbBackup := operatorBackupStoringTo(t, rendered.Storage.S3, completed.Add(-3*time.Minute)) + mdbBackup.Name = "src-op-bk" + mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)} + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-size"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + r := reconcilerWithLister(t, &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey("src/", completed.Add(-time.Minute), "gzip"), Size: 2048, LastModified: completed.Add(-time.Minute)}, + }}, job, s3CredsSecret("tenant", "creds")) + + artefact, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, rendered) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if artefact.Status.Artifact == nil || artefact.Status.Artifact.SizeBytes != 2048 { + t.Fatalf("Artifact = %#v, want SizeBytes 2048", artefact.Status.Artifact) + } +} + +// TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup pins that the +// probe reads where the operator Backup wrote its dump, not where the +// strategy, edited while the backup ran, now points. +func TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup(t *testing.T) { + apps := mariadbapp.GroupName + completed := metav1.NewTime(time.Date(2026, 10, 21, 10, 3, 0, 0, time.UTC)) + original := newRenderedMariaDBTemplate().Storage.S3 + original.Prefix = "original/" + mdbBackup := operatorBackupStoringTo(t, original, completed.Add(-3*time.Minute)) + mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)} + edited := newRenderedMariaDBTemplate() + edited.Storage.S3.Bucket = "replacement-bkt" + edited.Storage.S3.Endpoint = "replacement.example:9000" + edited.Storage.S3.Prefix = "replacement/" + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-edited"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: dumpKey("original/", completed.Add(-time.Minute), "gzip"), Size: 2048, LastModified: completed.Add(-time.Minute)}, + }} + r := reconcilerWithLister(t, l, job, s3CredsSecret("tenant", "creds")) + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + + artefact, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, edited) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if l.bucket != "bkt" || l.prefix != "original/" || l.conn.Endpoint != "s3.example:9000" { + t.Errorf("listed %s/%s at %s, want bkt/original/ at s3.example:9000, where the operator Backup wrote", l.bucket, l.prefix, l.conn.Endpoint) + } + if artefact.Status.Artifact == nil || artefact.Status.Artifact.SizeBytes != 2048 { + t.Errorf("Artifact = %#v, want SizeBytes 2048", artefact.Status.Artifact) + } +} + +func TestCreateMariaDBBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) { + apps := mariadbapp.GroupName + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + existing := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-retry"}, + Status: backupsv1alpha1.BackupStatus{Artifact: &backupsv1alpha1.BackupArtifact{URI: "s3://bkt/src/op-bk", SizeBytes: 99}}, + } + l := &fakeS3Lister{} + r := reconcilerWithLister(t, l, job, existing, s3CredsSecret("tenant", "creds")) + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "src/" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + + got, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op-bk"}}, rendered) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if l.calls != 0 { + t.Errorf("S3 listed %d times for a Backup that already exists", l.calls) + } + if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 99 { + t.Errorf("Artifact = %#v, want the existing one", got.Status.Artifact) + } +} + +func TestCreateMariaDBBackupArtifact_TransientRetryReadStillCreates(t *testing.T) { + apps := mariadbapp.GroupName + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-transient"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + l := &fakeS3Lister{} + r := reconcilerWithLister(t, l, job, s3CredsSecret("tenant", "creds")) + r.APIReader = failingReads(t, newMariaDBStrategyTestClient(t)) + got, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, newRenderedMariaDBTemplate()) + if err != nil || got == nil || got.Name != "bj-transient" { + t.Fatalf("createMariaDBBackupArtifact = %v, %v; a transient read must not fail the backup", got, err) + } + if err := r.Get(context.Background(), client.ObjectKeyFromObject(got), &backupsv1alpha1.Backup{}); err != nil { + t.Errorf("the Backup was not created: %v", err) + } + if l.calls != 0 { + t.Errorf("S3 probed %d times after the existence check failed", l.calls) + } +} + +func TestCreateMariaDBBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testing.T) { + apps := mariadbapp.GroupName + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-race"}} + store := newMariaDBStrategyTestClient(t, job, backup) + r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store), Scheme: store.Scheme()} + got, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, &strategyv1alpha1.MariaDBTemplate{}) + if err != nil || got == nil || got.Name != "bj-race" { + t.Fatalf("createMariaDBBackupArtifact = %v, %v; want the Backup that already exists", got, err) + } +} + +func TestCreateMariaDBBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T) { + apps := mariadbapp.GroupName + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-unreadable"}} + store := newMariaDBStrategyTestClient(t, job, backup) + r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store), Scheme: store.Scheme()} + _, err := r.createMariaDBBackupArtifact(context.Background(), job, &ResolvedBackupConfig{}, &mariadbtypes.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "op"}}, &strategyv1alpha1.MariaDBTemplate{}) + if !errors.Is(err, errBackupNotReadBack) { + t.Errorf("err = %v, want errBackupNotReadBack", err) + } +} + +// TestReconcileMariaDB_UnreadableBackupIsRetried drives a completed backup +// through the reconcile: when the Backup exists but cannot be read back, +// the reconcile is retried and the BackupJob is not failed. +func TestReconcileMariaDB_UnreadableBackupIsRetried(t *testing.T) { + strategyGroup := strategyv1alpha1.GroupVersion.Group + now := metav1.Now() + bj := newMariaDBBackupJob("bj-unreadable", "tenant") + bj.Status.StartedAt = &now + bj.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning + strategy := &strategyv1alpha1.MariaDB{ + ObjectMeta: metav1.ObjectMeta{Name: "mdb-strategy"}, + Spec: strategyv1alpha1.MariaDBSpec{Template: *newRenderedMariaDBTemplate()}, + } + operatorMariaDB := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("mariadb-src")}} + operatorBackup := &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "op-bk", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: bj.Name, + backupsv1alpha1.OwningJobNamespaceLabel: bj.Namespace, + }, + }, + Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{mariadbBackupSucceeded(now)}}, + } + backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: bj.Name}} + store := newMariaDBStrategyTestClient(t, bj, strategy, newMariaDBApp("mariadb-src", "tenant"), operatorMariaDB, operatorBackup, backup) + r := &BackupJobReconciler{ + Client: store, + APIReader: failingReads(t, store), + Scheme: store.Scheme(), + Recorder: record.NewFakeRecorder(10), + } + resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &strategyGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: strategy.Name, + }} + + _, err := r.reconcileMariaDB(context.Background(), bj, resolved) + if !errors.Is(err, errBackupNotReadBack) { + t.Fatalf("reconcileMariaDB err = %v, want errBackupNotReadBack so the reconcile is retried", err) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := store.Get(context.Background(), client.ObjectKeyFromObject(bj), persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Error("the BackupJob was failed although its Backup exists") + } +} + +// TestReconcileMariaDB_RejectedCreateFailsTheJob pins that only the +// read-back is retried: a Create the apiserver rejects still fails the +// BackupJob. +func TestReconcileMariaDB_RejectedCreateFailsTheJob(t *testing.T) { + strategyGroup := strategyv1alpha1.GroupVersion.Group + now := metav1.Now() + bj := newMariaDBBackupJob("bj-rejected", "tenant") + bj.Status.StartedAt = &now + bj.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning + strategy := &strategyv1alpha1.MariaDB{ + ObjectMeta: metav1.ObjectMeta{Name: "mdb-strategy"}, + Spec: strategyv1alpha1.MariaDBSpec{Template: *newRenderedMariaDBTemplate()}, + } + operatorMariaDB := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("mariadb-src")}} + operatorBackup := &mariadbtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "op-bk", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: bj.Name, + backupsv1alpha1.OwningJobNamespaceLabel: bj.Namespace, + }, + }, + Status: mariadbtypes.BackupStatus{Conditions: []metav1.Condition{mariadbBackupSucceeded(now)}}, + } + store := newMariaDBStrategyTestClient(t, bj, strategy, newMariaDBApp("mariadb-src", "tenant"), operatorMariaDB, operatorBackup) + r := &BackupJobReconciler{ + Client: forbidBackupCreate(t, store), + Scheme: store.Scheme(), + Recorder: record.NewFakeRecorder(10), + } + resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &strategyGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: strategy.Name, + }} + + if _, err := r.reconcileMariaDB(context.Background(), bj, resolved); err != nil { + t.Fatalf("reconcileMariaDB err = %v, want the BackupJob failed rather than retried", err) + } + persisted := &backupsv1alpha1.BackupJob{} + if err := store.Get(context.Background(), client.ObjectKeyFromObject(bj), persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase) + } +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index a5bd77094..4d88ab8a1 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -214,7 +214,7 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1 } artifact, err := r.createMariaDBBackupArtifact(ctx, j, resolved, mdbBackup, rendered) if err != nil { - return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err)) + return r.failBackupArtifact(ctx, j, err) } now := metav1.Now() j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name} @@ -365,6 +365,10 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( mdbBackup *mariadbtypes.Backup, rendered *strategyv1alpha1.MariaDBTemplate, ) (*backupsv1alpha1.Backup, error) { + existing, probe := r.existingBackup(ctx, j.Namespace, j.Name) + if existing != nil { + return existing, nil + } takenAt := metav1.Now() if cond := apimeta.FindStatusCondition(mdbBackup.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && !cond.LastTransitionTime.IsZero() { takenAt = cond.LastTransitionTime @@ -393,6 +397,9 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( // no Artifact at all - leave it nil instead. if uri := driverMD[mariadbStorageURIKey]; uri != "" { status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri} + if probe { + status.Artifact.SizeBytes = r.mariadbArtifactSizeBytes(ctx, mdbBackup, takenAt.Time) + } } backup := &backupsv1alpha1.Backup{ ObjectMeta: metav1.ObjectMeta{ @@ -415,8 +422,8 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( return nil, err } existing := &backupsv1alpha1.Backup{} - if getErr := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { - return nil, getErr + if getErr := r.apiReader().Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); getErr != nil { + return nil, fmt.Errorf("%w: %v", errBackupNotReadBack, getErr) } return existing, nil } diff --git a/internal/backupcontroller/velerostrategy_controller.go b/internal/backupcontroller/velerostrategy_controller.go index e08b2f818..abedc97be 100644 --- a/internal/backupcontroller/velerostrategy_controller.go +++ b/internal/backupcontroller/velerostrategy_controller.go @@ -558,7 +558,9 @@ func (r *BackupJobReconciler) createBackupResource(ctx context.Context, backupJo veleroBackupNamespaceMetadataKey: veleroBackup.Namespace, } - // Create a basic artifact referencing the Velero backup + // SizeBytes stays unset: data lands in a content-addressed kopia + // repository shared across backups, so there is no per-backup stored + // size, and DataUpload totalBytes counts bytes read, not bytes stored. artifact := &backupsv1alpha1.BackupArtifact{ URI: fmt.Sprintf("velero://%s/%s", veleroBackup.Namespace, veleroBackup.Name), } diff --git a/packages/apps/kubernetes/templates/oidc-rbac-bindings.yaml b/packages/apps/kubernetes/templates/oidc-rbac-bindings.yaml new file mode 100644 index 000000000..a8517da4f --- /dev/null +++ b/packages/apps/kubernetes/templates/oidc-rbac-bindings.yaml @@ -0,0 +1,42 @@ +{{- /* + The ClusterRoleBindings the OIDC bootstrap Job (oidc-rbac-job.yaml) applies + inside the tenant cluster, one per `oidc.users` entry, and the list of their + names it prunes against. They travel in a ConfigMap the Job mounts and not as + text inside its script: the script reaches the container as a single `sh -c` + argument, Linux caps one argument at 128 KiB, and a binding per user crosses + that cap at about 240 users, after which the container cannot start. + + A release-owned object, not a hook resource, so it exists before the + post-install and post-upgrade hooks run. It is rendered only where users are + bound; at `mode: None` the Job takes its early branch and the volume is + optional. +*/}} +{{- if ne .Values.oidc.mode "None" }} +{{- $desired := "" }} +{{- $bindings := "" }} +{{- range .Values.oidc.users }} +{{- $name := printf "cozystack-oidc-%s" (printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16) }} +{{- $role := "view" }} +{{- if eq .role "admin" }} +{{- $role = "cluster-admin" }} +{{- end }} +{{- $desired = printf "%s%s\n" $desired $name }} +{{- $binding := dict + "apiVersion" "rbac.authorization.k8s.io/v1" + "kind" "ClusterRoleBinding" + "metadata" (dict "name" $name "labels" (dict "app.kubernetes.io/managed-by" "cozystack-oidc" "app.kubernetes.io/instance" $.Release.Name)) + "roleRef" (dict "apiGroup" "rbac.authorization.k8s.io" "kind" "ClusterRole" "name" $role) + "subjects" (list (dict "kind" "User" "apiGroup" "rbac.authorization.k8s.io" "name" .email)) }} +{{- $bindings = printf "%s---\n%s\n" $bindings (toYaml $binding) }} +{{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-oidc-bindings + namespace: {{ .Release.Namespace }} + labels: + {{- include "kubernetes.labels" . | nindent 4 }} +data: + {{- dict "crbs.yaml" $bindings "desired.txt" $desired | toYaml | nindent 2 }} +{{- end }} diff --git a/packages/apps/kubernetes/templates/oidc-rbac-job.yaml b/packages/apps/kubernetes/templates/oidc-rbac-job.yaml index 8672ce124..8556d092b 100644 --- a/packages/apps/kubernetes/templates/oidc-rbac-job.yaml +++ b/packages/apps/kubernetes/templates/oidc-rbac-job.yaml @@ -25,7 +25,9 @@ - `mode: System | CustomConfig`: - Apply one ClusterRoleBinding per `users[]` entry inside the tenant cluster (`admin` → `cluster-admin`, `view` → `view`, subject = the - literal email matched against the `email` claim). + literal email matched against the `email` claim). The bindings are + rendered by oidc-rbac-bindings.yaml into a ConfigMap this Job mounts, + not into the script below. - Prune stale CRBs labelled by this release. - `mode: System`: patch the pre-rendered `-oidc-kubeconfig` Secret with a real kubeconfig wired to `kubectl oidc-login` @@ -168,6 +170,8 @@ spec: value: {{ $issuer | quote }} - name: OIDC_SERVER_URL value: {{ printf "https://%s:443" $externalHost | quote }} + - name: OIDC_BINDINGS_DIR + value: /etc/oidc-bindings resources: requests: cpu: 10m @@ -254,46 +258,32 @@ spec: sleep 5 done + # A missing list must stop the Job here: grep on a file that is not + # there exits 2, `! grep` is then true, and the prune below would + # delete every binding of the release. + BINDINGS=$OIDC_BINDINGS_DIR + if [ ! -f "$BINDINGS/desired.txt" ]; then + echo "$BINDINGS/desired.txt is not mounted (ConfigMap $RELEASE-oidc-bindings); refusing to prune against a list that is not there" >&2 + exit 1 + fi + echo "Applying ${OIDC_MODE} ClusterRoleBindings to the tenant cluster..." - DESIRED_FILE=$(mktemp) - {{- range .Values.oidc.users }} - {{- $hash := printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16 }} - {{- $crbName := printf "cozystack-oidc-%s" $hash }} - {{- $role := "view" }} - {{- if eq .role "admin" }} - {{- $role = "cluster-admin" }} - {{- end }} - echo "{{ $crbName }}" >> "$DESIRED_FILE" - cat <<'EOF' | kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: {{ $crbName }} - labels: - app.kubernetes.io/managed-by: cozystack-oidc - app.kubernetes.io/instance: {{ $.Release.Name }} - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ $role }} - subjects: - - kind: User - apiGroup: rbac.authorization.k8s.io - name: {{ .email | quote }} - EOF - {{- end }} + # kubectl apply fails on a file with no objects, and a release with + # no users has none. + if [ -s "$BINDINGS/crbs.yaml" ]; then + kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename "$BINDINGS/crbs.yaml" + fi echo "Pruning stale ClusterRoleBindings labelled by this release..." EXISTING=$(kubectl --kubeconfig "$KUBECONFIG_TENANT" get clusterrolebindings \ --selector=app.kubernetes.io/managed-by=cozystack-oidc,app.kubernetes.io/instance=$RELEASE \ --output=jsonpath='{.items[*].metadata.name}') for crb in $EXISTING; do - if ! grep -x -q -F "$crb" "$DESIRED_FILE"; then + if ! grep -x -q -F "$crb" "$BINDINGS/desired.txt"; then echo " removing $crb" kubectl --kubeconfig "$KUBECONFIG_TENANT" delete clusterrolebinding "$crb" --ignore-not-found fi done - rm -f "$DESIRED_FILE" if [ "$OIDC_MODE" = "System" ]; then echo "Patching the OIDC kubeconfig Secret on the management cluster..." @@ -349,6 +339,9 @@ spec: - name: kubeconfig mountPath: /etc/kubernetes/kubeconfig readOnly: true + - name: oidc-bindings + mountPath: /etc/oidc-bindings + readOnly: true - name: tmp mountPath: /tmp volumes: @@ -356,6 +349,10 @@ spec: secret: secretName: {{ .Release.Name }}-admin-kubeconfig optional: true + - name: oidc-bindings + configMap: + name: {{ .Release.Name }}-oidc-bindings + optional: true - name: tmp emptyDir: {} {{- /* diff --git a/packages/apps/kubernetes/tests/oidc_test.yaml b/packages/apps/kubernetes/tests/oidc_test.yaml index 0623063e7..8511e392c 100644 --- a/packages/apps/kubernetes/tests/oidc_test.yaml +++ b/packages/apps/kubernetes/tests/oidc_test.yaml @@ -2561,7 +2561,73 @@ tests: value: "true" documentIndex: 3 - - it: bootstrap Job command builds a CRB for each users[] entry + - it: the bindings ConfigMap holds a CRB for each users[] entry + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-bindings.yaml + set: + _cluster.oidc-enabled: "true" + _cluster.root-host: example.org + oidc.mode: System + oidc.users: + - email: alice@example.org + role: admin + - email: bob@example.org + role: view + asserts: + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: kubernetes-app-oidc-bindings + - matchRegex: + path: data["crbs.yaml"] + pattern: 'name: alice@example.org' + - matchRegex: + path: data["crbs.yaml"] + pattern: 'name: bob@example.org' + - matchRegex: + path: data["crbs.yaml"] + pattern: 'app.kubernetes.io/managed-by: cozystack-oidc' + - matchRegex: + path: data["crbs.yaml"] + pattern: 'app.kubernetes.io/instance: kubernetes-app' + + - it: the binding names and roles stay what the Job script used to render + # The prune step and every binding already in a tenant cluster go by these + # names: cozystack-oidc- plus the first 16 hex digits of + # sha256("-"). A different name leaves the old binding + # behind next to the new one. + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-bindings.yaml + set: + _cluster.oidc-enabled: "true" + _cluster.root-host: example.org + oidc.mode: System + oidc.users: + - email: alice@example.org + role: admin + - email: bob@example.org + role: view + asserts: + - equal: + path: data["desired.txt"] + value: "cozystack-oidc-0001a28136ca1a02\ncozystack-oidc-ec8dfb1cbc75e699\n" + - matchRegex: + path: data["crbs.yaml"] + pattern: 'name: cozystack-oidc-0001a28136ca1a02\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cluster-admin\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: User\n name: alice@example.org\n' + - matchRegex: + path: data["crbs.yaml"] + pattern: 'name: cozystack-oidc-ec8dfb1cbc75e699\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: view\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: User\n name: bob@example.org\n' + + - it: the bootstrap Job mounts the bindings ConfigMap and carries no users[] entry in its script + # The script is one `sh -c` argument and Linux caps one argument at 128 KiB: + # a binding per user in it stops the container from starting at about 240 + # users. hack/kubernetes-oidc-bindings.bats renders 300 users to hold the + # size itself. release: name: kubernetes-app namespace: tenant-foo @@ -2575,23 +2641,130 @@ tests: role: admin - email: bob@example.org role: view + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: oidc-bindings + configMap: + name: kubernetes-app-oidc-bindings + optional: true + documentIndex: 3 + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: oidc-bindings + mountPath: /etc/oidc-bindings + readOnly: true + documentIndex: 3 + # The script reads the directory from the environment, which is also how + # hack/kubernetes-oidc-bindings.bats points it at a scratch directory. + - contains: + path: spec.template.spec.containers[0].env + content: + name: OIDC_BINDINGS_DIR + value: /etc/oidc-bindings + documentIndex: 3 + - notMatchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'alice@example\.org' + documentIndex: 3 + - notMatchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'bob@example\.org' + documentIndex: 3 + + - it: the bootstrap Job stops before pruning when the desired list is not mounted + # `! grep -F` on a file that is not there is true, so a missing mount would + # delete every binding of the release. + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-job.yaml + set: + _cluster.oidc-enabled: "true" + _cluster.root-host: example.org + oidc.mode: System asserts: - matchRegex: path: spec.template.spec.containers[0].command[2] - pattern: 'name: "alice@example.org"' + pattern: '(?s)if \[ ! -f "\$BINDINGS/desired\.txt" \]; then.*?exit 1\n\s*fi.*Pruning stale ClusterRoleBindings' documentIndex: 3 + + - it: the bootstrap Job applies the bindings file only when it has objects in it + # kubectl apply exits non-zero on a file with no objects, and a release + # with no users has none. + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-job.yaml + set: + _cluster.oidc-enabled: "true" + _cluster.root-host: example.org + oidc.mode: System + asserts: - matchRegex: path: spec.template.spec.containers[0].command[2] - pattern: 'name: "bob@example.org"' + pattern: 'if \[ -s "\$BINDINGS/crbs\.yaml" \]; then\n\s+kubectl --kubeconfig "\$KUBECONFIG_TENANT" apply --filename "\$BINDINGS/crbs\.yaml"\n\s*fi' documentIndex: 3 + + - it: a release with no users renders an empty bindings ConfigMap + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-bindings.yaml + set: + _cluster.oidc-enabled: "true" + _cluster.root-host: example.org + oidc.mode: System + oidc.users: [] + asserts: + - equal: + path: data["crbs.yaml"] + value: "" + - equal: + path: data["desired.txt"] + value: "" + + - it: CustomConfig mode renders the bindings ConfigMap for its users + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-bindings.yaml + set: + _cluster.oidc-enabled: "false" + oidc.mode: CustomConfig + oidc.customConfig.config: | + apiVersion: apiserver.config.k8s.io/v1beta1 + kind: AuthenticationConfiguration + jwt: + - issuer: + url: https://idp.example.com + audiences: [my-cluster] + oidc.users: + - email: alice@example.org + role: admin + asserts: + - isKind: + of: ConfigMap - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: 'name: cluster-admin' - documentIndex: 3 - - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: 'name: view' - documentIndex: 3 + path: data["crbs.yaml"] + pattern: 'name: alice@example.org' + + - it: mode=None renders no bindings ConfigMap even when users are set + # The Job takes its early branch at mode None and never reads the file. + release: + name: kubernetes-app + namespace: tenant-foo + template: templates/oidc-rbac-bindings.yaml + set: + oidc.mode: None + oidc.users: + - email: alice@example.org + role: admin + asserts: + - hasDocuments: + count: 0 - it: mode=System aborts the Job instead of leaving the OIDC kubeconfig Secret as the install-time placeholder when the tenant CA is missing # A placeholder kubeconfig cannot authenticate, so the release must not go Ready with it. diff --git a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml index 28ef57fb7..dfa25dbc5 100644 --- a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml +++ b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml @@ -131,8 +131,15 @@ spec: For example: "sha256:". type: string sizeBytes: - description: SizeBytes is the size of the artifact in bytes, if - known. + description: |- + SizeBytes is the stored size in bytes of the files this backup owns, + if known: the dump of a MariaDB backup, for example, or the base backup + of a CNPG one, which leaves out the WAL archive the server's backups + share and a point-in-time restore also reads. Strategies that cannot + attribute stored bytes to a single backup, such as Velero's, whose + repository deduplicates across backups, leave it unset. Drivers never + write 0 for a size they cannot measure, so an absent value means + "unknown", not "empty". format: int64 type: integer uri: diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml index a176044d6..18174d61a 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml @@ -3,8 +3,8 @@ 1. Helm renders this file at chart install time. Anything inside `{{ ... }}` is interpreted by Helm. 2. The strategy controller renders the resulting Strategy CR - per-BackupJob, expanding `{{ .Application | .Release | .Mode }}` - against the live application object. + per-BackupJob, expanding `{{ .Application | .Release | .Mode | + .ArchiveName | .Backup }}` against the live application object. To emit a literal "{{ .Foo }}" that survives Helm, we pass it as the string argument of `printf` — Helm does not interpret inner template delimiters inside printf's quoted strings. Do NOT simplify by inlining @@ -51,6 +51,11 @@ spec: host="chi-clickhouse-{{ printf "{{ .Release.Name }}" }}-clickhouse-0-0.{{ printf "{{ .Release.Namespace }}" }}.svc" api="http://${host}:7171" auth="-u ${API_USERNAME}:${API_PASSWORD}" + # The archive the controller named for this BackupJob and records + # on its Backup. The default renders it empty, rather than as + # text/template's missing-key marker, when a controller that + # predates archive names renders this template during an upgrade. + archive="{{ printf `{{ .ArchiveName | default "" }}` }}" # Every request but the POST is retried on failure, every 5s, # for $2 attempts in all: once the operation is accepted, a # dropped poll must not fail the attempt, since the Job retry it @@ -91,23 +96,47 @@ spec: } if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-" - # The name is new on every attempt, so a retry never takes over - # the create_remote of the attempt it replaces; that needs a - # name fixed for the whole BackupJob. - target="${prefix}$(date -u +%Y%m%dT%H%M%SZ)" - cmd="create_remote ${target}" + # Only a controller that predates archive names leaves it + # empty, and that one records no name on the Backup either. + target="${archive:-${prefix}$(date -u +%Y%m%dT%H%M%SZ)}" + # A retried Pod gets the same name, so it takes over an attempt + # the sidecar is still running, and otherwise asks for resume, + # so a local or partial remote backup an earlier attempt left is + # resumed rather than refused as existing, whatever + # use_resumable_state the sidecar runs with. The sidecar logs + # the request as "create_remote --resume ". + cmd="create_remote --resume ${target}" wait_turn "${cmd}" - echo "${cmd}" - curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}" >/dev/null + if [ "${turn}" = "join" ]; then + echo "${cmd} is already running; waiting for it" + else + echo "${cmd}" + curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}&resume=true" >/dev/null + fi else - prefix="clickhouse-{{ printf "{{ .Backup.ApplicationRef.Name }}" }}-" - listing=$(api_get /backup/list/remote 60) - target=$(printf '%s\n' "${listing}" \ - | jq -rs --arg prefix "${prefix}" \ - '[.[] | select((.name // "") | startswith($prefix))] | sort_by(.created // "") | last.name // ""') - if [ -z "${target}" ] || [ "${target}" = "null" ]; then - echo "no remote backup found" >&2 - exit 1 + app="{{ printf "{{ .Backup.ApplicationRef.Name }}" }}" + backup="{{ printf "{{ .Backup.Name }}" }}" + prefix="clickhouse-${app}-" + remote=$(api_get /backup/list/remote 60) + if [ -n "${archive}" ]; then + if ! printf '%s\n' "${remote}" | jq -es --arg name "${archive}" 'any(.[]; .name == $name)' >/dev/null; then + echo "archive ${archive} recorded on Backup ${backup} is not in remote storage; not restoring any other archive" >&2 + exit 1 + fi + target="${archive}" + else + # takenAt is set after the backup Job finished, so the + # archive that Job wrote is stamped no later than it. + notafter="{{ printf `{{ .Backup.TakenAt | default "" }}` }}" + echo "WARNING: Backup ${backup} records no archive name; restoring the newest archive of ${app} stamped no later than its takenAt (${notafter}), a best-effort choice that may not be the archive this Backup was taken as" >&2 + target=$(printf '%s\n' "${remote}" | jq -rs --arg prefix "${prefix}" --arg notafter "${notafter}" ' + [.[] | select((.desc // "") | startswith("broken") | not) | .name // "" + | select(startswith($prefix) and (ltrimstr($prefix) | test("^[0-9]{8}T[0-9]{6}Z$")) and ltrimstr($prefix) <= $notafter)] + | sort | last // ""') + if [ -z "${target}" ]; then + echo "no archive of ${app} stamped no later than ${notafter} in remote storage" >&2 + exit 1 + fi fi # restore_remote drops every table it restores before # recreating it, so a second run discards whatever was written diff --git a/packages/system/cilium/files/ipam-migration.sh b/packages/system/cilium/files/ipam-migration.sh index 002a1bf78..9345c19f6 100755 --- a/packages/system/cilium/files/ipam-migration.sh +++ b/packages/system/cilium/files/ipam-migration.sh @@ -240,10 +240,12 @@ exclude_from_kube_ovn() { fi # A pod that holds a node's current Ingress IP keeps the address after the # move, and that node keeps it in its ipcache as reserved:ingress, so the pod - # stays unreachable from there until it is recreated. - jq -r --argjson cn "$ciliumnodes" ' - ([$cn.items[] | select(.spec.ingress.ipv4) | {key: .spec.ingress.ipv4, value: .metadata.name}] | from_entries) as $ingress - | .[] | select($ingress[.ip]) | "\(.name) \(.ip) \($ingress[.ip])"' <<<"$ipobjects" \ + # stays unreachable from there until it is recreated. The CiliumNode list goes + # in on stdin, because the Node labels Cilium copies onto every CiliumNode take + # it past the 128 KiB a single argument may hold within a handful of nodes. + jq -rn 'input as $cn + | ([$cn.items[] | select(.spec.ingress.ipv4) | {key: .spec.ingress.ipv4, value: .metadata.name}] | from_entries) as $ingress + | input[] | select($ingress[.ip]) | "\(.name) \(.ip) \($ingress[.ip])"' <<<"$ciliumnodes"$'\n'"$ipobjects" \ | while read -r ipname ip node; do if ! in_any "$POOL" "$ip"; then log "$ipname holds $ip, the Ingress IP of $node before the move. Recreate it once the run is done, $node keeps that address as reserved:ingress" diff --git a/pkg/apis/apps/v1alpha1/types.go b/pkg/apis/apps/v1alpha1/types.go index 15bab146f..13270b0f1 100644 --- a/pkg/apis/apps/v1alpha1/types.go +++ b/pkg/apis/apps/v1alpha1/types.go @@ -55,6 +55,23 @@ type ApplicationStatus struct { // ExternalIPsCount holds the number of LoadBalancer services with assigned external IPs for Tenant applications. // +optional ExternalIPsCount int32 `json:"externalIPsCount,omitempty"` + // InstallFailures is the install failure count of the underlying HelmRelease. + // It counts only attempts that recorded a Helm release, so a failure before + // Helm records one, such as a chart that fails to render, shows only as + // Ready=False; a failure to apply the rendered resources is counted. It + // resets when the desired state (generation, chart version, values) changes, + // and on success only under the RetryOnFailure strategy, so a Ready release + // may carry it. + // +optional + InstallFailures int64 `json:"installFailures,omitempty"` + // UpgradeFailures is the upgrade failure count of the underlying HelmRelease, + // with the same counting and reset rules as InstallFailures. + // +optional + UpgradeFailures int64 `json:"upgradeFailures,omitempty"` + // LastAttemptedReleaseAction is the last release action (install or upgrade) + // the underlying HelmRelease attempted. + // +optional + LastAttemptedReleaseAction string `json:"lastAttemptedReleaseAction,omitempty"` } // SchedulingClass returns the scheduling class requested by this Application. diff --git a/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go b/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go index e20e08a51..77db6b0e7 100644 --- a/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go +++ b/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go @@ -34,6 +34,20 @@ type ApplicationStatusApplyConfiguration struct { Namespace *string `json:"namespace,omitempty"` // ExternalIPsCount holds the number of LoadBalancer services with assigned external IPs for Tenant applications. ExternalIPsCount *int32 `json:"externalIPsCount,omitempty"` + // InstallFailures is the install failure count of the underlying HelmRelease. + // It counts only attempts that recorded a Helm release, so a failure before + // Helm records one, such as a chart that fails to render, shows only as + // Ready=False; a failure to apply the rendered resources is counted. It + // resets when the desired state (generation, chart version, values) changes, + // and on success only under the RetryOnFailure strategy, so a Ready release + // may carry it. + InstallFailures *int64 `json:"installFailures,omitempty"` + // UpgradeFailures is the upgrade failure count of the underlying HelmRelease, + // with the same counting and reset rules as InstallFailures. + UpgradeFailures *int64 `json:"upgradeFailures,omitempty"` + // LastAttemptedReleaseAction is the last release action (install or upgrade) + // the underlying HelmRelease attempted. + LastAttemptedReleaseAction *string `json:"lastAttemptedReleaseAction,omitempty"` } // ApplicationStatusApplyConfiguration constructs a declarative configuration of the ApplicationStatus type for use with @@ -78,3 +92,27 @@ func (b *ApplicationStatusApplyConfiguration) WithExternalIPsCount(value int32) b.ExternalIPsCount = &value return b } + +// WithInstallFailures sets the InstallFailures field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the InstallFailures field is set to the value of the last call. +func (b *ApplicationStatusApplyConfiguration) WithInstallFailures(value int64) *ApplicationStatusApplyConfiguration { + b.InstallFailures = &value + return b +} + +// WithUpgradeFailures sets the UpgradeFailures field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UpgradeFailures field is set to the value of the last call. +func (b *ApplicationStatusApplyConfiguration) WithUpgradeFailures(value int64) *ApplicationStatusApplyConfiguration { + b.UpgradeFailures = &value + return b +} + +// WithLastAttemptedReleaseAction sets the LastAttemptedReleaseAction field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the LastAttemptedReleaseAction field is set to the value of the last call. +func (b *ApplicationStatusApplyConfiguration) WithLastAttemptedReleaseAction(value string) *ApplicationStatusApplyConfiguration { + b.LastAttemptedReleaseAction = &value + return b +} diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 68bb1713f..6f5b41465 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -290,6 +290,27 @@ func schema_pkg_apis_apps_v1alpha1_ApplicationStatus(ref common.ReferenceCallbac Format: "int32", }, }, + "installFailures": { + SchemaProps: spec.SchemaProps{ + Description: "InstallFailures is the install failure count of the underlying HelmRelease. It counts only attempts that recorded a Helm release, so a failure before Helm records one, such as a chart that fails to render, shows only as Ready=False; a failure to apply the rendered resources is counted. It resets when the desired state (generation, chart version, values) changes, and on success only under the RetryOnFailure strategy, so a Ready release may carry it.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "upgradeFailures": { + SchemaProps: spec.SchemaProps{ + Description: "UpgradeFailures is the upgrade failure count of the underlying HelmRelease, with the same counting and reset rules as InstallFailures.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "lastAttemptedReleaseAction": { + SchemaProps: spec.SchemaProps{ + Description: "LastAttemptedReleaseAction is the last release action (install or upgrade) the underlying HelmRelease attempted.", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 322ef6c6d..5f618b4e3 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -1573,13 +1573,23 @@ func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.H }, Spec: filteredSpec, Status: appsv1alpha1.ApplicationStatus{ - Version: hr.Status.LastAttemptedRevision, + Version: hr.Status.LastAttemptedRevision, + InstallFailures: hr.Status.InstallFailures, + UpgradeFailures: hr.Status.UpgradeFailures, + LastAttemptedReleaseAction: string(hr.Status.LastAttemptedReleaseAction), }, } var conditions []metav1.Condition for _, hrCondition := range hr.GetConditions() { - if hrCondition.Type == "Ready" || hrCondition.Type == "Released" { + // The Application carries no generation, so a client cannot tell a Stalled + // left over from the previous spec from one about the current spec. Compare + // the condition's own ObservedGeneration, not Status.ObservedGeneration: the + // question is which spec this condition describes. + if hrCondition.Type == "Stalled" && hrCondition.ObservedGeneration < hr.Generation { + continue + } + if hrCondition.Type == "Ready" || hrCondition.Type == "Released" || hrCondition.Type == "Stalled" { conditions = append(conditions, metav1.Condition{ LastTransitionTime: hrCondition.LastTransitionTime, Reason: hrCondition.Reason, diff --git a/pkg/registry/apps/application/rest_release_failure_test.go b/pkg/registry/apps/application/rest_release_failure_test.go new file mode 100644 index 000000000..fc28f1553 --- /dev/null +++ b/pkg/registry/apps/application/rest_release_failure_test.go @@ -0,0 +1,123 @@ +package application + +import ( + "context" + "testing" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestConvertReleaseFailure_StalledAndCountersExposed(t *testing.T) { + r := newTestRESTWithSchemes() + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Generation = 2 + hr.Status.Conditions = []metav1.Condition{ + {Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 2}, + {Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2}, + {Type: "Released", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2}, + } + hr.Status.Failures = 7 + hr.Status.InstallFailures = 1 + hr.Status.UpgradeFailures = 3 + hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + sc := findCondition(app.GetConditions(), "Stalled") + if sc == nil { + t.Fatal("expected Stalled condition to be copied from the HelmRelease") + } + if sc.Status != metav1.ConditionTrue || sc.Reason != "RetriesExceeded" { + t.Errorf("expected Stalled=True/RetriesExceeded, got %s/%s", sc.Status, sc.Reason) + } + if app.Status.InstallFailures != 1 { + t.Errorf("expected installFailures=1, got %d", app.Status.InstallFailures) + } + if app.Status.UpgradeFailures != 3 { + t.Errorf("expected upgradeFailures=3, got %d", app.Status.UpgradeFailures) + } + if app.Status.LastAttemptedReleaseAction != "install" { + t.Errorf("expected lastAttemptedReleaseAction=install, got %q", app.Status.LastAttemptedReleaseAction) + } +} + +// After a spec update the HelmRelease keeps the Stalled condition of the +// previous generation until helm-controller reconciles it; reported as is, it +// would read as a terminal failure of the new values. +func TestConvertReleaseFailure_StalledFromOlderGenerationOmitted(t *testing.T) { + r := newTestRESTWithSchemes() + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Generation = 2 + hr.Status.Conditions = []metav1.Condition{ + {Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 1}, + {Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 1}, + } + hr.Status.InstallFailures = 1 + hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if findCondition(app.GetConditions(), "Stalled") != nil { + t.Error("expected Stalled observed at an older generation not to be copied") + } + if findCondition(app.GetConditions(), "Ready") == nil { + t.Error("expected Ready to still be copied") + } +} + +// helm-controller clears the counters on success only under the RetryOnFailure +// strategy, so a Ready release can still carry them; they must not be hidden. +func TestConvertReleaseFailure_CountersKeptOnReadyRelease(t *testing.T) { + r := newTestRESTWithSchemes() + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Status.Conditions = []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"}, + {Type: "Released", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"}, + } + hr.Status.InstallFailures = 2 + hr.Status.UpgradeFailures = 1 + hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionUpgrade + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if findCondition(app.GetConditions(), "Stalled") != nil { + t.Error("expected no Stalled condition on a release that does not carry one") + } + if app.Status.InstallFailures != 2 || app.Status.UpgradeFailures != 1 { + t.Errorf("expected install=2 upgrade=1, got install=%d upgrade=%d", app.Status.InstallFailures, app.Status.UpgradeFailures) + } + if app.Status.LastAttemptedReleaseAction != "upgrade" { + t.Errorf("expected lastAttemptedReleaseAction=upgrade, got %q", app.Status.LastAttemptedReleaseAction) + } +} + +func TestConvertReleaseFailure_ReconcilingNotCopied(t *testing.T) { + r := newTestRESTWithSchemes() + hr := makeHelmRelease("postgresql-mydb", "default") + hr.Status.Conditions = []metav1.Condition{ + {Type: "Reconciling", Status: metav1.ConditionTrue, Reason: "Progressing", Message: "Running 'install' action"}, + {Type: "Ready", Status: metav1.ConditionUnknown, Reason: "Progressing", Message: "Running 'install' action"}, + } + + app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if findCondition(app.GetConditions(), "Reconciling") != nil { + t.Error("expected Reconciling not to be copied") + } + if findCondition(app.GetConditions(), "Ready") == nil { + t.Error("expected Ready to be copied") + } +}